diff --git a/go.mod b/go.mod index 1438615..f9caefe 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,11 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/jarcoal/httpmock v1.0.8 + github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 + github.com/panjf2000/ants v1.3.0 // indirect github.com/pelletier/go-toml v1.8.1 github.com/rs/zerolog v1.20.0 // indirect github.com/stretchr/objx v0.3.0 // indirect diff --git a/go.sum b/go.sum index 5f3cd5b..bcb02ae 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 h1:D7qhJP3R49ZjUzpzKQ6B2H3lgejPs6DTO5gRomhhOpE= +github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -23,6 +25,8 @@ github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyC github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE= github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I= +github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= +github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -36,6 +40,7 @@ github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go new file mode 100644 index 0000000..e4c0370 --- /dev/null +++ b/ipblocklist/firehol.go @@ -0,0 +1,327 @@ +package ipblocklist + +import ( + "bufio" + "context" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/kentik/patricia" + "github.com/kentik/patricia/bool_tree" + "github.com/panjf2000/ants" +) + +const ( + fireholIPv4DefaultCIDR = 32 + fireholIPv6DefaultCIDR = 128 +) + +var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) + +type Firehol struct { + logger mtglib.Logger + rwMutex sync.RWMutex + remoteURLs []string + localFiles []string + httpClient *http.Client + workerPool *ants.Pool + treeV4 *bool_tree.TreeV4 + treeV6 *bool_tree.TreeV6 +} + +func (f *Firehol) Contains(ip net.IP) bool { + if ip == nil { + return true + } + + ip4 := ip.To4() + + f.rwMutex.RLock() + defer f.rwMutex.RUnlock() + + if ip4 != nil { + return f.containsIPv4(ip4) + } + + return f.containsIPv6(ip.To16()) +} + +func (f *Firehol) containsIPv4(addr net.IP) bool { + ip := patricia.NewIPv4AddressFromBytes(addr, 32) + + if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) containsIPv6(addr net.IP) bool { + ip := patricia.NewIPv6Address(addr, 128) + + if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) Run(ctx context.Context, updateEach time.Duration) { + ticker := time.NewTicker(updateEach) + + defer func() { + ticker.Stop() + + select { + case <-ticker.C: + default: + } + }() + + if err := f.update(ctx); err != nil { + f.logger.WarningError("cannot update blocklist", err) + } + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := f.update(ctx); err != nil { + f.logger.WarningError("cannot update blocklist", err) + } + } + } +} + +func (f *Firehol) update(ctx context.Context) error { // nolint: funlen, cyclop + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + wg := &sync.WaitGroup{} + wg.Add(len(f.remoteURLs) + len(f.localFiles)) + + treeMutex := &sync.Mutex{} + v4tree := bool_tree.NewTreeV4() + v6tree := bool_tree.NewTreeV6() + + errorChan := make(chan error, 1) + defer close(errorChan) + + for _, v := range f.localFiles { + go func(filename string) { + defer wg.Done() + + if err := f.updateLocalFile(ctx, filename, treeMutex, v4tree, v6tree); err != nil { + cancel() + f.logger.BindStr("filename", filename).WarningError("cannot update", err) + + select { + case errorChan <- err: + default: + } + } + }(v) + } + + for _, v := range f.remoteURLs { + value := v + + f.workerPool.Submit(func() { // nolint: errcheck + defer wg.Done() + + if err := f.updateRemoteURL(ctx, value, treeMutex, v4tree, v6tree); err != nil { + cancel() + f.logger.BindStr("url", value).WarningError("cannot update", err) + + select { + case errorChan <- err: + default: + } + } + }) + } + + wg.Wait() + + select { + case err := <-errorChan: + return fmt.Errorf("cannot update trees: %w", err) + default: + } + + f.rwMutex.Lock() + defer f.rwMutex.Unlock() + + f.treeV4 = v4tree + f.treeV6 = v6tree + + return nil +} + +func (f *Firehol) updateLocalFile(ctx context.Context, filename string, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + filefp, err := os.Open(filename) + if err != nil { + return fmt.Errorf("cannot open file: %w", err) + } + + defer filefp.Close() + + return f.updateTrees(ctx, mutex, filefp, v4tree, v6tree) +} + +func (f *Firehol) updateRemoteURL(ctx context.Context, url string, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("cannot build a request: %w", err) + } + + resp, err := f.httpClient.Do(req) + if err != nil { + return fmt.Errorf("cannot request a remote URL %s: %w", url, err) + } + + defer func() { + io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck + resp.Body.Close() + }() + + return f.updateTrees(ctx, mutex, resp.Body, v4tree, v6tree) +} + +func (f *Firehol) updateTrees(ctx context.Context, + mutex sync.Locker, + reader io.Reader, + v4tree *bool_tree.TreeV4, + v6tree *bool_tree.TreeV6) error { + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + text := scanner.Text() + text = fireholRegexpComment.ReplaceAllLiteralString(text, "") + text = strings.TrimSpace(text) + + if text == "" { + continue + } + + ip, cidr, err := f.updateParseLine(text) + if err != nil { + return fmt.Errorf("cannot parse a line: %w", err) + } + + if err := f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree); err != nil { + return fmt.Errorf("cannot add a node to the tree: %w", err) + } + } + + if scanner.Err() != nil { + return fmt.Errorf("cannot parse a response: %w", scanner.Err()) + } + + return nil +} + +func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) { + _, ipnet, err := net.ParseCIDR(text) + if err != nil { + ipaddr := net.ParseIP(text) + if ipaddr == nil { + return nil, 0, fmt.Errorf("incorrect ip address %s", text) + } + + ip4 := ipaddr.To4() + if ip4 != nil { + return ip4, fireholIPv4DefaultCIDR, nil + } + + return ipaddr.To16(), fireholIPv6DefaultCIDR, nil + } + + ones, _ := ipnet.Mask.Size() + + return ipnet.IP, uint(ones), nil +} + +func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + mutex.Lock() + defer mutex.Unlock() + + if ip.To4() != nil { + addr := patricia.NewIPv4AddressFromBytes(ip, cidr) + + if _, _, err := v4tree.Set(addr, true); err != nil { + return err // nolint: wrapcheck + } + } else { + addr := patricia.NewIPv6Address(ip, cidr) + + if _, _, err := v6tree.Set(addr, true); err != nil { + return err // nolint: wrapcheck + } + } + + return nil +} + +func NewFirehol(logger mtglib.Logger, network mtglib.Network, + downloadConcurrency uint, + remoteURLs []string, + localFiles []string) (*Firehol, error) { + for _, v := range remoteURLs { + parsed, err := url.Parse(v) + if err != nil { + return nil, fmt.Errorf("incorrect url %s: %w", v, err) + } + + switch parsed.Scheme { + case "http", "https": + default: + return nil, fmt.Errorf("unsupported url %s", v) + } + } + + for _, v := range localFiles { + if stat, err := os.Stat(v); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 { + return nil, fmt.Errorf("%s is not a readable file", v) + } + } + + if downloadConcurrency == 0 { + downloadConcurrency = 1 + } + + workerPool, _ := ants.NewPool(int(downloadConcurrency)) + + return &Firehol{ + logger: logger.Named("firehol"), + httpClient: network.MakeHTTPClient(nil), + treeV4: bool_tree.NewTreeV4(), + treeV6: bool_tree.NewTreeV6(), + workerPool: workerPool, + remoteURLs: remoteURLs, + localFiles: localFiles, + }, nil +} diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go new file mode 100644 index 0000000..5b18820 --- /dev/null +++ b/ipblocklist/firehol_test.go @@ -0,0 +1,137 @@ +package ipblocklist_test + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/network" + "github.com/9seconds/mtg/v2/testlib" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type FireholTestSuite struct { + suite.Suite + + networkMock *testlib.MtglibNetworkMock + httpServer *httptest.Server +} + +func (suite *FireholTestSuite) SetupSuite() { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + filefp, err := os.Open(filepath.Join("testdata", "remote_ipset.ipset")) + if err != nil { + panic(err) + } + + defer filefp.Close() + + io.Copy(w, filefp) // nolint: errcheck + }) + + suite.httpServer = httptest.NewServer(mux) +} + +func (suite *FireholTestSuite) SetupTest() { + httpClient := &http.Client{} + suite.networkMock = &testlib.MtglibNetworkMock{} + + httpmock.ActivateNonDefault(httpClient) + + suite.networkMock. + On("MakeHTTPClient", mock.Anything). + Maybe(). + Return(httpClient) +} + +func (suite *FireholTestSuite) TearDownTest() { + suite.networkMock.AssertExpectations(suite.T()) + httpmock.DeactivateAndReset() +} + +func (suite *FireholTestSuite) TearDownSuite() { + suite.httpServer.Close() +} + +func (suite *FireholTestSuite) TestLocalFail() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + nil, []string{filepath.Join("testdata", "broken_ipset.ipset")}) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.False(blocklist.Contains(net.ParseIP("10.0.0.10"))) + suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) +} + +func (suite *FireholTestSuite) TestLocalOk() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + nil, []string{filepath.Join("testdata", "good_ipset.ipset")}) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.True(blocklist.Contains(net.ParseIP("10.0.0.10"))) + suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) +} + +func (suite *FireholTestSuite) TestRemoteFail() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + []string{"https://google.com"}, nil) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.False(blocklist.Contains(net.ParseIP("10.2.2.2"))) +} + +func (suite *FireholTestSuite) TestMixed() { + dialer, _ := network.NewDefaultDialer(0, 0) + ntw, _ := network.NewNetwork(dialer, "mtg", "1.1.1.1", 0, 0) + + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + ntw, 2, + []string{ + suite.httpServer.URL, + }, []string{ + filepath.Join("testdata", "good_ipset.ipset"), + }) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.True(blocklist.Contains(net.ParseIP("10.2.2.2"))) + suite.True(blocklist.Contains(net.ParseIP("10.1.0.100"))) +} + +func TestFirehol(t *testing.T) { + t.Parallel() + suite.Run(t, &FireholTestSuite{}) +} diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go new file mode 100644 index 0000000..dcbed19 --- /dev/null +++ b/ipblocklist/noop.go @@ -0,0 +1,15 @@ +package ipblocklist + +import ( + "net" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type noop struct{} + +func (n noop) Contains(ip net.IP) bool { return false } + +func NewNoop() mtglib.IPBlocklist { + return noop{} +} diff --git a/ipblocklist/noop_test.go b/ipblocklist/noop_test.go new file mode 100644 index 0000000..f5bb346 --- /dev/null +++ b/ipblocklist/noop_test.go @@ -0,0 +1,23 @@ +package ipblocklist_test + +import ( + "net" + "testing" + + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite +} + +func (suite *NoopTestSuite) TestOp() { + suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) + suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/ipblocklist/testdata/broken_ipset.ipset b/ipblocklist/testdata/broken_ipset.ipset new file mode 100644 index 0000000..2758925 --- /dev/null +++ b/ipblocklist/testdata/broken_ipset.ipset @@ -0,0 +1,5 @@ +# +# This is an intentionally broken ipset. +# + +ajsdkfbd diff --git a/ipblocklist/testdata/good_ipset.ipset b/ipblocklist/testdata/good_ipset.ipset new file mode 100644 index 0000000..411ba01 --- /dev/null +++ b/ipblocklist/testdata/good_ipset.ipset @@ -0,0 +1,7 @@ +# +# This is very good ipset +# + +10.0.0.10 # just an example +10.1.0.0/24 +2001:0db8:85a3:0000:0000:8a2e:0370:7334 diff --git a/ipblocklist/testdata/remote_ipset.ipset b/ipblocklist/testdata/remote_ipset.ipset new file mode 100644 index 0000000..80d49e4 --- /dev/null +++ b/ipblocklist/testdata/remote_ipset.ipset @@ -0,0 +1 @@ +10.2.2.2 diff --git a/mtglib/init.go b/mtglib/init.go index 2bb6975..12e1393 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -21,6 +21,10 @@ type AntiReplayCache interface { SeenBefore(data []byte) bool } +type IPBlocklist interface { + Contains(net.IP) bool +} + type Logger interface { Named(name string) Logger diff --git a/network/network.go b/network/network.go index a897899..585ff3c 100644 --- a/network/network.go +++ b/network/network.go @@ -132,6 +132,13 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { func NewNetwork(dialer Dialer, userAgent, dohHostname string, httpTimeout, idleTimeout time.Duration) (mtglib.Network, error) { + switch { + case httpTimeout < 0: + return nil, fmt.Errorf("timeout should be positive number %s", httpTimeout) + case httpTimeout == 0: + httpTimeout = DefaultHTTPTimeout + } + switch { case idleTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout)