mirror of
https://github.com/ScuroNeko/mtg.git
synced 2026-08-31 16:24:03 +03:00
@@ -174,6 +174,27 @@ urls = [
|
|||||||
# How often do we need to update a blocklist set.
|
# How often do we need to update a blocklist set.
|
||||||
update-each = "24h"
|
update-each = "24h"
|
||||||
|
|
||||||
|
# Allowlist is an opposite to a blocklist. Only those IPs that are coming from
|
||||||
|
# subnets defined in these lists are allowed. All others will be rejected.
|
||||||
|
#
|
||||||
|
# If this feature is disabled, then there won't be any check performed by this
|
||||||
|
# validator. It is possible to combine both blocklist and whitelist.
|
||||||
|
[defense.allowlist]
|
||||||
|
# You can enable/disable this feature.
|
||||||
|
enabled = false
|
||||||
|
# This is a limiter for concurrency. In order to protect website
|
||||||
|
# from overloading, we download files in this number of threads.
|
||||||
|
download-concurrency = 2
|
||||||
|
# A list of URLs in FireHOL format (https://iplists.firehol.org/)
|
||||||
|
# You can provider links here (starts with https:// or http://) or
|
||||||
|
# path to a local file, but in this case it should be absolute.
|
||||||
|
urls = [
|
||||||
|
# "https://iplists.firehol.org/files/firehol_level1.netset",
|
||||||
|
# "/local.file"
|
||||||
|
|
||||||
|
]
|
||||||
|
update-each = "24h"
|
||||||
|
|
||||||
# statsd statistics integration.
|
# statsd statistics integration.
|
||||||
[stats.statsd]
|
[stats.statsd]
|
||||||
# enabled/disabled
|
# enabled/disabled
|
||||||
|
|||||||
@@ -86,15 +86,15 @@ func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeIPBlocklist(conf *config.Config, logger mtglib.Logger, ntw mtglib.Network) (mtglib.IPBlocklist, error) {
|
func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Network) (mtglib.IPBlocklist, error) {
|
||||||
if !conf.Defense.Blocklist.Enabled.Get(false) {
|
if !conf.Enabled.Get(false) {
|
||||||
return ipblocklist.NewNoop(), nil
|
return ipblocklist.NewNoop(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
remoteURLs := []string{}
|
remoteURLs := []string{}
|
||||||
localFiles := []string{}
|
localFiles := []string{}
|
||||||
|
|
||||||
for _, v := range conf.Defense.Blocklist.URLs {
|
for _, v := range conf.URLs {
|
||||||
if v.IsRemote() {
|
if v.IsRemote() {
|
||||||
remoteURLs = append(remoteURLs, v.String())
|
remoteURLs = append(remoteURLs, v.String())
|
||||||
} else {
|
} else {
|
||||||
@@ -104,7 +104,7 @@ func makeIPBlocklist(conf *config.Config, logger mtglib.Logger, ntw mtglib.Netwo
|
|||||||
|
|
||||||
firehol, err := ipblocklist.NewFirehol(logger.Named("ipblockist"),
|
firehol, err := ipblocklist.NewFirehol(logger.Named("ipblockist"),
|
||||||
ntw,
|
ntw,
|
||||||
conf.Defense.Blocklist.DownloadConcurrency.Get(1),
|
conf.DownloadConcurrency.Get(1),
|
||||||
remoteURLs,
|
remoteURLs,
|
||||||
localFiles)
|
localFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -153,7 +153,7 @@ func makeEventStream(conf *config.Config, logger mtglib.Logger) (mtglib.EventStr
|
|||||||
return events.NewNoopStream(), nil
|
return events.NewNoopStream(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runProxy(conf *config.Config, version string) error {
|
func runProxy(conf *config.Config, version string) error { // nolint: funlen
|
||||||
logger := makeLogger(conf)
|
logger := makeLogger(conf)
|
||||||
|
|
||||||
logger.BindJSON("configuration", conf.String()).Debug("configuration")
|
logger.BindJSON("configuration", conf.String()).Debug("configuration")
|
||||||
@@ -163,11 +163,22 @@ func runProxy(conf *config.Config, version string) error {
|
|||||||
return fmt.Errorf("cannot build network: %w", err)
|
return fmt.Errorf("cannot build network: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklist, err := makeIPBlocklist(conf, logger, ntw)
|
blocklist, err := makeIPBlocklist(conf.Defense.Blocklist, logger, ntw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot build ip blocklist: %w", err)
|
return fmt.Errorf("cannot build ip blocklist: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var whitelist mtglib.IPBlocklist
|
||||||
|
|
||||||
|
if conf.Defense.Allowlist.Enabled.Get(false) {
|
||||||
|
whlist, err := makeIPBlocklist(conf.Defense.Allowlist, logger, ntw)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot build ip blocklist: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
whitelist = whlist
|
||||||
|
}
|
||||||
|
|
||||||
eventStream, err := makeEventStream(conf, logger)
|
eventStream, err := makeEventStream(conf, logger)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot build event stream: %w", err)
|
return fmt.Errorf("cannot build event stream: %w", err)
|
||||||
@@ -178,6 +189,7 @@ func runProxy(conf *config.Config, version string) error {
|
|||||||
Network: ntw,
|
Network: ntw,
|
||||||
AntiReplayCache: makeAntiReplayCache(conf),
|
AntiReplayCache: makeAntiReplayCache(conf),
|
||||||
IPBlocklist: blocklist,
|
IPBlocklist: blocklist,
|
||||||
|
IPWhitelist: whitelist,
|
||||||
EventStream: eventStream,
|
EventStream: eventStream,
|
||||||
|
|
||||||
Secret: conf.Secret,
|
Secret: conf.Secret,
|
||||||
|
|||||||
@@ -8,6 +8,18 @@ import (
|
|||||||
"github.com/9seconds/mtg/v2/mtglib"
|
"github.com/9seconds/mtg/v2/mtglib"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type Optional struct {
|
||||||
|
Enabled TypeBool `json:"enabled"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ListConfig struct {
|
||||||
|
Optional
|
||||||
|
|
||||||
|
DownloadConcurrency TypeConcurrency `json:"downloadConcurrency"`
|
||||||
|
URLs []TypeBlocklistURI `json:"urls"`
|
||||||
|
UpdateEach TypeDuration `json:"updateEach"`
|
||||||
|
}
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Debug TypeBool `json:"debug"`
|
Debug TypeBool `json:"debug"`
|
||||||
AllowFallbackOnUnknownDC TypeBool `json:"allowFallbackOnUnknownDc"`
|
AllowFallbackOnUnknownDC TypeBool `json:"allowFallbackOnUnknownDc"`
|
||||||
@@ -20,16 +32,13 @@ type Config struct {
|
|||||||
Concurrency TypeConcurrency `json:"concurrency"`
|
Concurrency TypeConcurrency `json:"concurrency"`
|
||||||
Defense struct {
|
Defense struct {
|
||||||
AntiReplay struct {
|
AntiReplay struct {
|
||||||
Enabled TypeBool `json:"enabled"`
|
Optional
|
||||||
|
|
||||||
MaxSize TypeBytes `json:"maxSize"`
|
MaxSize TypeBytes `json:"maxSize"`
|
||||||
ErrorRate TypeErrorRate `json:"errorRate"`
|
ErrorRate TypeErrorRate `json:"errorRate"`
|
||||||
} `json:"antiReplay"`
|
} `json:"antiReplay"`
|
||||||
Blocklist struct {
|
Blocklist ListConfig `json:"blocklist"`
|
||||||
Enabled TypeBool `json:"enabled"`
|
Allowlist ListConfig `json:"allowlist"`
|
||||||
DownloadConcurrency TypeConcurrency `json:"downloadConcurrency"`
|
|
||||||
URLs []TypeBlocklistURI `json:"urls"`
|
|
||||||
UpdateEach TypeDuration `json:"updateEach"`
|
|
||||||
} `json:"blocklist"`
|
|
||||||
} `json:"defense"`
|
} `json:"defense"`
|
||||||
Network struct {
|
Network struct {
|
||||||
Timeout struct {
|
Timeout struct {
|
||||||
@@ -42,13 +51,15 @@ type Config struct {
|
|||||||
} `json:"network"`
|
} `json:"network"`
|
||||||
Stats struct {
|
Stats struct {
|
||||||
StatsD struct {
|
StatsD struct {
|
||||||
Enabled TypeBool `json:"enabled"`
|
Optional
|
||||||
|
|
||||||
Address TypeHostPort `json:"address"`
|
Address TypeHostPort `json:"address"`
|
||||||
MetricPrefix TypeMetricPrefix `json:"metricPrefix"`
|
MetricPrefix TypeMetricPrefix `json:"metricPrefix"`
|
||||||
TagFormat TypeStatsdTagFormat `json:"tagFormat"`
|
TagFormat TypeStatsdTagFormat `json:"tagFormat"`
|
||||||
} `json:"statsd"`
|
} `json:"statsd"`
|
||||||
Prometheus struct {
|
Prometheus struct {
|
||||||
Enabled TypeBool `json:"enabled"`
|
Optional
|
||||||
|
|
||||||
BindTo TypeHostPort `json:"bindTo"`
|
BindTo TypeHostPort `json:"bindTo"`
|
||||||
HTTPPath TypeHTTPPath `json:"httpPath"`
|
HTTPPath TypeHTTPPath `json:"httpPath"`
|
||||||
MetricPrefix TypeMetricPrefix `json:"metricPrefix"`
|
MetricPrefix TypeMetricPrefix `json:"metricPrefix"`
|
||||||
|
|||||||
@@ -30,6 +30,12 @@ type tomlConfig struct {
|
|||||||
URLs []string `toml:"urls" json:"urls,omitempty"`
|
URLs []string `toml:"urls" json:"urls,omitempty"`
|
||||||
UpdateEach string `toml:"update-each" json:"updateEach,omitempty"`
|
UpdateEach string `toml:"update-each" json:"updateEach,omitempty"`
|
||||||
} `toml:"blocklist" json:"blocklist,omitempty"`
|
} `toml:"blocklist" json:"blocklist,omitempty"`
|
||||||
|
Allowlist struct {
|
||||||
|
Enabled bool `toml:"enabled" json:"enabled,omitempty"`
|
||||||
|
DownloadConcurrency uint `toml:"download-concurrency" json:"downloadConcurrency,omitempty"`
|
||||||
|
URLs []string `toml:"urls" json:"urls,omitempty"`
|
||||||
|
UpdateEach string `toml:"update-each" json:"updateEach,omitempty"`
|
||||||
|
} `toml:"allowlist" json:"allowlist,omitempty"`
|
||||||
} `toml:"defense" json:"defense,omitempty"`
|
} `toml:"defense" json:"defense,omitempty"`
|
||||||
Network struct {
|
Network struct {
|
||||||
Timeout struct {
|
Timeout struct {
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
)
|
||||||
|
|
||||||
|
type httpFile struct {
|
||||||
|
http *http.Client
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h httpFile) Open(ctx context.Context) (io.ReadCloser, error) {
|
||||||
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, h.url, nil)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
response, err := h.http.Do(request)
|
||||||
|
if err != nil {
|
||||||
|
if response != nil {
|
||||||
|
io.Copy(io.Discard, response.Body) // nolint: errcheck
|
||||||
|
response.Body.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("cannot get url %s: %w", h.url, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if response.StatusCode >= http.StatusBadRequest {
|
||||||
|
return nil, fmt.Errorf("unexpected status code %d", response.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h httpFile) String() string {
|
||||||
|
return h.url
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHTTP(client *http.Client, endpoint string) (File, error) {
|
||||||
|
if client == nil {
|
||||||
|
return nil, ErrBadHTTPClient
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("incorrect url %s: %w", endpoint, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch parsed.Scheme {
|
||||||
|
case "http", "https":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported url %s", endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
|
return httpFile{
|
||||||
|
http: client,
|
||||||
|
url: endpoint,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package files_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/9seconds/mtg/v2/ipblocklist/files"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HTTPTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
|
||||||
|
httpClient *http.Client
|
||||||
|
httpServer *httptest.Server
|
||||||
|
ctx context.Context
|
||||||
|
ctxCancel context.CancelFunc
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) makeFile(path string) (files.File, error) {
|
||||||
|
return files.NewHTTP(suite.httpClient, suite.httpServer.URL+"/"+path) // nolint: wrapcheck
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) SetupSuite() {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
mux.Handle("/", http.FileServer(http.Dir("testdata")))
|
||||||
|
|
||||||
|
suite.httpServer = httptest.NewServer(mux)
|
||||||
|
suite.httpClient = suite.httpServer.Client()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) SetupTest() {
|
||||||
|
suite.ctx, suite.ctxCancel = context.WithCancel(context.Background())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TearDownTest() {
|
||||||
|
suite.ctxCancel()
|
||||||
|
suite.httpServer.CloseClientConnections()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TearDownSuite() {
|
||||||
|
suite.httpServer.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TestBadURL() {
|
||||||
|
_, err := files.NewHTTP(suite.httpClient, "sdfsdf")
|
||||||
|
suite.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TestBadSchema() {
|
||||||
|
_, err := files.NewHTTP(suite.httpClient, "gopher://lala")
|
||||||
|
suite.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TestNilHTTPClient() {
|
||||||
|
_, err := files.NewHTTP(nil, "")
|
||||||
|
suite.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TestAbsentFile() {
|
||||||
|
file, err := suite.makeFile("absent")
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
_, err = file.Open(suite.ctx)
|
||||||
|
suite.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *HTTPTestSuite) TestOk() {
|
||||||
|
file, err := suite.makeFile("readable")
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
readCloser, err := file.Open(suite.ctx)
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
defer readCloser.Close()
|
||||||
|
|
||||||
|
data, err := io.ReadAll(readCloser)
|
||||||
|
suite.NoError(err)
|
||||||
|
suite.Equal("Hooray!", strings.TrimSpace(string(data)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHTTP(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
suite.Run(t, &HTTPTestSuite{})
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrBadHTTPClient = errors.New("incorrect http client")
|
||||||
|
|
||||||
|
type File interface {
|
||||||
|
Open(context.Context) (io.ReadCloser, error)
|
||||||
|
String() string
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package files
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type localFile struct {
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l localFile) Open(ctx context.Context) (io.ReadCloser, error) {
|
||||||
|
return os.Open(l.path) // nolint: wrapcheck
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l localFile) String() string {
|
||||||
|
return l.path
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewLocal(path string) (File, error) {
|
||||||
|
if stat, err := os.Stat(path); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 {
|
||||||
|
return nil, fmt.Errorf("%s is not a readable file", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return localFile{
|
||||||
|
path: path,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package files_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/9seconds/mtg/v2/ipblocklist/files"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/suite"
|
||||||
|
)
|
||||||
|
|
||||||
|
type LocalTestSuite struct {
|
||||||
|
suite.Suite
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *LocalTestSuite) getLocalFile(name string) string {
|
||||||
|
return filepath.Join("testdata", name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *LocalTestSuite) TestIncorrect() {
|
||||||
|
names := []string{
|
||||||
|
"absent",
|
||||||
|
"directory",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, v := range names {
|
||||||
|
value := v
|
||||||
|
|
||||||
|
suite.T().Run(v, func(t *testing.T) {
|
||||||
|
_, err := files.NewLocal(suite.getLocalFile(value))
|
||||||
|
assert.Error(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (suite *LocalTestSuite) TestOk() {
|
||||||
|
file, err := files.NewLocal(suite.getLocalFile("readable"))
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
reader, err := file.Open(context.Background())
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
data, err := io.ReadAll(reader)
|
||||||
|
suite.NoError(err)
|
||||||
|
|
||||||
|
suite.Equal("Hooray!", strings.TrimSpace(string(data)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocal(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
suite.Run(t, &LocalTestSuite{})
|
||||||
|
}
|
||||||
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
Hooray!
|
||||||
+60
-136
@@ -4,16 +4,13 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/9seconds/mtg/v2/ipblocklist/files"
|
||||||
"github.com/9seconds/mtg/v2/mtglib"
|
"github.com/9seconds/mtg/v2/mtglib"
|
||||||
"github.com/kentik/patricia"
|
"github.com/kentik/patricia"
|
||||||
"github.com/kentik/patricia/bool_tree"
|
"github.com/kentik/patricia/bool_tree"
|
||||||
@@ -41,20 +38,16 @@ var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
|
|||||||
// 127.0.0.1 # you can specify an IP
|
// 127.0.0.1 # you can specify an IP
|
||||||
// 10.0.0.0/8 # or cidr
|
// 10.0.0.0/8 # or cidr
|
||||||
type Firehol struct {
|
type Firehol struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
ctxCancel context.CancelFunc
|
ctxCancel context.CancelFunc
|
||||||
logger mtglib.Logger
|
logger mtglib.Logger
|
||||||
|
updateMutex sync.RWMutex
|
||||||
|
|
||||||
rwMutex sync.RWMutex
|
blocklists []files.File
|
||||||
|
|
||||||
remoteURLs []string
|
|
||||||
localFiles []string
|
|
||||||
|
|
||||||
httpClient *http.Client
|
|
||||||
workerPool *ants.Pool
|
workerPool *ants.Pool
|
||||||
|
treeV4 *bool_tree.TreeV4
|
||||||
treeV4 *bool_tree.TreeV4
|
treeV6 *bool_tree.TreeV6
|
||||||
treeV6 *bool_tree.TreeV6
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shutdown stop a background update process.
|
// Shutdown stop a background update process.
|
||||||
@@ -68,8 +61,8 @@ func (f *Firehol) Contains(ip net.IP) bool {
|
|||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
f.rwMutex.RLock()
|
f.updateMutex.RLock()
|
||||||
defer f.rwMutex.RUnlock()
|
defer f.updateMutex.RUnlock()
|
||||||
|
|
||||||
if ip4 := ip.To4(); ip4 != nil {
|
if ip4 := ip.To4(); ip4 != nil {
|
||||||
return f.containsIPv4(ip4)
|
return f.containsIPv4(ip4)
|
||||||
@@ -98,22 +91,14 @@ func (f *Firehol) Run(updateEach time.Duration) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if err := f.update(); err != nil {
|
f.update()
|
||||||
f.logger.WarningError("cannot update blocklist", err)
|
|
||||||
} else {
|
|
||||||
f.logger.Info("blocklist was updated")
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-f.ctx.Done():
|
case <-f.ctx.Done():
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
if err := f.update(); err != nil {
|
f.update()
|
||||||
f.logger.WarningError("cannot update blocklist", err)
|
|
||||||
} else {
|
|
||||||
f.logger.Info("blocklist was updated")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -138,121 +123,53 @@ func (f *Firehol) containsIPv6(addr net.IP) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Firehol) update() error { // nolint: funlen, cyclop
|
func (f *Firehol) update() {
|
||||||
ctx, cancel := context.WithCancel(f.ctx)
|
ctx, cancel := context.WithCancel(f.ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
wg := &sync.WaitGroup{}
|
wg := &sync.WaitGroup{}
|
||||||
wg.Add(len(f.remoteURLs) + len(f.localFiles))
|
wg.Add(len(f.blocklists))
|
||||||
|
|
||||||
treeMutex := &sync.Mutex{}
|
treeMutex := &sync.Mutex{}
|
||||||
v4tree := bool_tree.NewTreeV4()
|
v4tree := bool_tree.NewTreeV4()
|
||||||
v6tree := bool_tree.NewTreeV6()
|
v6tree := bool_tree.NewTreeV6()
|
||||||
|
|
||||||
errorChan := make(chan error, 1)
|
for _, v := range f.blocklists {
|
||||||
defer close(errorChan)
|
go func(file files.File) {
|
||||||
|
|
||||||
for _, v := range f.localFiles {
|
|
||||||
go func(filename string) {
|
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
if err := f.updateLocalFile(ctx, filename, treeMutex, v4tree, v6tree); err != nil {
|
logger := f.logger.BindStr("filename", file.String())
|
||||||
cancel()
|
|
||||||
f.logger.BindStr("filename", filename).WarningError("cannot update", err)
|
|
||||||
|
|
||||||
select {
|
fileContent, err := file.Open(ctx)
|
||||||
case errorChan <- err:
|
if err != nil {
|
||||||
default:
|
logger.WarningError("update has failed", err)
|
||||||
}
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer fileContent.Close()
|
||||||
|
|
||||||
|
if err := f.updateFromFile(treeMutex, v4tree, v6tree, bufio.NewScanner(fileContent)); err != nil {
|
||||||
|
logger.WarningError("update has failed", err)
|
||||||
}
|
}
|
||||||
}(v)
|
}(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, v := range f.remoteURLs {
|
|
||||||
value := v
|
|
||||||
|
|
||||||
f.workerPool.Submit(func() { // nolint: errcheck
|
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
if err := f.updateRemoteURL(ctx, value, treeMutex, v4tree, v6tree); err != nil {
|
|
||||||
cancel()
|
|
||||||
f.logger.BindStr("url", value).WarningError("cannot update", err)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case errorChan <- err:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
select {
|
f.updateMutex.Lock()
|
||||||
case err := <-errorChan:
|
defer f.updateMutex.Unlock()
|
||||||
return fmt.Errorf("cannot update trees: %w", err)
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
f.rwMutex.Lock()
|
|
||||||
defer f.rwMutex.Unlock()
|
|
||||||
|
|
||||||
f.treeV4 = v4tree
|
f.treeV4 = v4tree
|
||||||
f.treeV6 = v6tree
|
f.treeV6 = v6tree
|
||||||
|
|
||||||
return nil
|
f.logger.Info("blocklist was updated")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Firehol) updateLocalFile(ctx context.Context, filename string,
|
func (f *Firehol) updateFromFile(mutex sync.Locker,
|
||||||
mutex sync.Locker,
|
|
||||||
v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
|
|
||||||
filefp, err := os.Open(filename)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cannot open file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
go func(ctx context.Context, closer io.Closer) {
|
|
||||||
<-ctx.Done()
|
|
||||||
closer.Close()
|
|
||||||
}(ctx, filefp)
|
|
||||||
|
|
||||||
defer filefp.Close()
|
|
||||||
|
|
||||||
return f.updateTrees(mutex, filefp, v4tree, v6tree)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Firehol) updateRemoteURL(ctx context.Context, url string,
|
|
||||||
mutex sync.Locker,
|
|
||||||
v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cannot build a request: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := f.httpClient.Do(req) // nolint: bodyclose
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("cannot request a remote URL %s: %w", url, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
go func(ctx context.Context, closer io.Closer) {
|
|
||||||
<-ctx.Done()
|
|
||||||
closer.Close()
|
|
||||||
}(ctx, resp.Body)
|
|
||||||
|
|
||||||
defer func(rc io.ReadCloser) {
|
|
||||||
io.Copy(io.Discard, rc) // nolint: errcheck
|
|
||||||
rc.Close()
|
|
||||||
}(resp.Body)
|
|
||||||
|
|
||||||
return f.updateTrees(mutex, resp.Body, v4tree, v6tree)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *Firehol) updateTrees(mutex sync.Locker,
|
|
||||||
reader io.Reader,
|
|
||||||
v4tree *bool_tree.TreeV4,
|
v4tree *bool_tree.TreeV4,
|
||||||
v6tree *bool_tree.TreeV6) error {
|
v6tree *bool_tree.TreeV6,
|
||||||
scanner := bufio.NewScanner(reader)
|
scanner *bufio.Scanner) error {
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
text := scanner.Text()
|
text := scanner.Text()
|
||||||
text = fireholRegexpComment.ReplaceAllLiteralString(text, "")
|
text = fireholRegexpComment.ReplaceAllLiteralString(text, "")
|
||||||
@@ -271,7 +188,7 @@ func (f *Firehol) updateTrees(mutex sync.Locker,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if scanner.Err() != nil {
|
if scanner.Err() != nil {
|
||||||
return fmt.Errorf("cannot parse a response: %w", scanner.Err())
|
return fmt.Errorf("cannot parse a file: %w", scanner.Err())
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -317,27 +234,36 @@ func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint,
|
|||||||
// when it is necessary.
|
// when it is necessary.
|
||||||
func NewFirehol(logger mtglib.Logger, network mtglib.Network,
|
func NewFirehol(logger mtglib.Logger, network mtglib.Network,
|
||||||
downloadConcurrency uint,
|
downloadConcurrency uint,
|
||||||
remoteURLs []string,
|
urls []string,
|
||||||
localFiles []string) (*Firehol, error) {
|
localFiles []string) (*Firehol, error) {
|
||||||
for _, v := range remoteURLs {
|
blocklists := []files.File{}
|
||||||
parsed, err := url.Parse(v)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("incorrect url %s: %w", v, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch parsed.Scheme {
|
|
||||||
case "http", "https":
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported url %s", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, v := range localFiles {
|
for _, v := range localFiles {
|
||||||
if stat, err := os.Stat(v); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 {
|
file, err := files.NewLocal(v)
|
||||||
return nil, fmt.Errorf("%s is not a readable file", v)
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot create a local file %s: %w", v, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
blocklists = append(blocklists, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
httpClient := network.MakeHTTPClient(nil)
|
||||||
|
|
||||||
|
for _, v := range urls {
|
||||||
|
file, err := files.NewHTTP(httpClient, v)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot create a HTTP file %s: %w", v, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
blocklists = append(blocklists, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
return NewFireholFromFiles(logger, downloadConcurrency, blocklists)
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewFireholFromFiles(logger mtglib.Logger,
|
||||||
|
downloadConcurrency uint,
|
||||||
|
blocklists []files.File) (*Firehol, error) {
|
||||||
if downloadConcurrency == 0 {
|
if downloadConcurrency == 0 {
|
||||||
downloadConcurrency = DefaultFireholDownloadConcurrency
|
downloadConcurrency = DefaultFireholDownloadConcurrency
|
||||||
}
|
}
|
||||||
@@ -349,11 +275,9 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network,
|
|||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
ctxCancel: cancel,
|
ctxCancel: cancel,
|
||||||
logger: logger.Named("firehol"),
|
logger: logger.Named("firehol"),
|
||||||
httpClient: network.MakeHTTPClient(nil),
|
|
||||||
treeV4: bool_tree.NewTreeV4(),
|
treeV4: bool_tree.NewTreeV4(),
|
||||||
treeV6: bool_tree.NewTreeV6(),
|
treeV6: bool_tree.NewTreeV6(),
|
||||||
workerPool: workerPool,
|
workerPool: workerPool,
|
||||||
remoteURLs: remoteURLs,
|
blocklists: blocklists,
|
||||||
localFiles: localFiles,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
+14
-4
@@ -33,7 +33,8 @@ type Proxy struct {
|
|||||||
secret Secret
|
secret Secret
|
||||||
network Network
|
network Network
|
||||||
antiReplayCache AntiReplayCache
|
antiReplayCache AntiReplayCache
|
||||||
ipBlocklist IPBlocklist
|
blocklist IPBlocklist
|
||||||
|
whitelist IPBlocklist
|
||||||
eventStream EventStream
|
eventStream EventStream
|
||||||
logger Logger
|
logger Logger
|
||||||
}
|
}
|
||||||
@@ -91,7 +92,7 @@ func (p *Proxy) ServeConn(conn net.Conn) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Serve starts a proxy on a given listener.
|
// Serve starts a proxy on a given listener.
|
||||||
func (p *Proxy) Serve(listener net.Listener) error {
|
func (p *Proxy) Serve(listener net.Listener) error { // nolint: cyclop
|
||||||
p.streamWaitGroup.Add(1)
|
p.streamWaitGroup.Add(1)
|
||||||
defer p.streamWaitGroup.Done()
|
defer p.streamWaitGroup.Done()
|
||||||
|
|
||||||
@@ -109,7 +110,15 @@ func (p *Proxy) Serve(listener net.Listener) error {
|
|||||||
ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP
|
ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP
|
||||||
logger := p.logger.BindStr("ip", ipAddr.String())
|
logger := p.logger.BindStr("ip", ipAddr.String())
|
||||||
|
|
||||||
if p.ipBlocklist.Contains(ipAddr) {
|
if p.whitelist != nil && !p.whitelist.Contains(ipAddr) {
|
||||||
|
conn.Close()
|
||||||
|
logger.Info("ip was rejected by whitelist")
|
||||||
|
p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr))
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if p.blocklist.Contains(ipAddr) {
|
||||||
conn.Close()
|
conn.Close()
|
||||||
logger.Info("ip was blacklisted")
|
logger.Info("ip was blacklisted")
|
||||||
p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr))
|
p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr))
|
||||||
@@ -291,7 +300,8 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {
|
|||||||
secret: opts.Secret,
|
secret: opts.Secret,
|
||||||
network: opts.Network,
|
network: opts.Network,
|
||||||
antiReplayCache: opts.AntiReplayCache,
|
antiReplayCache: opts.AntiReplayCache,
|
||||||
ipBlocklist: opts.IPBlocklist,
|
blocklist: opts.IPBlocklist,
|
||||||
|
whitelist: opts.IPWhitelist,
|
||||||
eventStream: opts.EventStream,
|
eventStream: opts.EventStream,
|
||||||
logger: opts.getLogger("proxy"),
|
logger: opts.getLogger("proxy"),
|
||||||
domainFrontingPort: opts.getDomainFrontingPort(),
|
domainFrontingPort: opts.getDomainFrontingPort(),
|
||||||
|
|||||||
@@ -28,6 +28,11 @@ type ProxyOpts struct {
|
|||||||
// This is a mandatory setting.
|
// This is a mandatory setting.
|
||||||
IPBlocklist IPBlocklist
|
IPBlocklist IPBlocklist
|
||||||
|
|
||||||
|
// IPWhitelist defines a whitelist of IPs to allow to use proxy.
|
||||||
|
//
|
||||||
|
// This is an optional setting, ignored by default (no restrictions).
|
||||||
|
IPWhitelist IPBlocklist
|
||||||
|
|
||||||
// EventStream defines an instance of event stream.
|
// EventStream defines an instance of event stream.
|
||||||
//
|
//
|
||||||
// This ia a mandatory setting.
|
// This ia a mandatory setting.
|
||||||
|
|||||||
Reference in New Issue
Block a user