From c14a2329c51275fa13731f246dd862a854dc2d99 Mon Sep 17 00:00:00 2001
From: 9seconds <nineseconds@yandex.ru>
Date: Mon, 29 Nov 2021 07:26:27 +0300
Subject: [PATCH 1/5] Add local file abstraction

---
 ipblocklist/files/http.go                     | 51 +++++++++++++++++
 ipblocklist/files/init.go                     | 10 ++++
 ipblocklist/files/local.go                    | 30 ++++++++++
 ipblocklist/files/local_test.go               | 55 +++++++++++++++++++
 ipblocklist/files/testdata/directory/.gitkeep |  0
 ipblocklist/files/testdata/readable           |  1 +
 6 files changed, 147 insertions(+)
 create mode 100644 ipblocklist/files/http.go
 create mode 100644 ipblocklist/files/init.go
 create mode 100644 ipblocklist/files/local.go
 create mode 100644 ipblocklist/files/local_test.go
 create mode 100644 ipblocklist/files/testdata/directory/.gitkeep
 create mode 100644 ipblocklist/files/testdata/readable

diff --git a/ipblocklist/files/http.go b/ipblocklist/files/http.go
new file mode 100644
index 0000000..c024399
--- /dev/null
+++ b/ipblocklist/files/http.go
@@ -0,0 +1,51 @@
+package files
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+type httpFile struct {
+	http *http.Client
+	url  string
+}
+
+func (h httpFile) Open(ctx context.Context) (io.ReadCloser, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, h.url, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	response, err := h.http.Do(request)
+	if err != nil {
+		if response != nil {
+			io.Copy(io.Discard, response.Body)
+			response.Body.Close()
+		}
+
+		return nil, fmt.Errorf("cannot get url %s: %w", h.url, err)
+	}
+
+	return response.Body, nil
+}
+
+func NewHTTP(client *http.Client, endpoint string) (File, error) {
+	parsed, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("incorrect url %s: %w", endpoint, err)
+	}
+
+	switch parsed.Scheme {
+	case "http", "https":
+	default:
+		return nil, fmt.Errorf("unsupported url %s", endpoint)
+	}
+
+	return httpFile{
+		http: client,
+		url:  endpoint,
+	}, nil
+}
diff --git a/ipblocklist/files/init.go b/ipblocklist/files/init.go
new file mode 100644
index 0000000..922b211
--- /dev/null
+++ b/ipblocklist/files/init.go
@@ -0,0 +1,10 @@
+package files
+
+import (
+	"context"
+	"io"
+)
+
+type File interface {
+	Open(context.Context) (io.ReadCloser, error)
+}
diff --git a/ipblocklist/files/local.go b/ipblocklist/files/local.go
new file mode 100644
index 0000000..8e3fe64
--- /dev/null
+++ b/ipblocklist/files/local.go
@@ -0,0 +1,30 @@
+package files
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+type localFile struct {
+	root fs.FS
+	name string
+}
+
+func (l localFile) Open(ctx context.Context) (io.ReadCloser, error) {
+	return l.root.Open(l.name)
+}
+
+func NewLocal(path string) (File, error) {
+	if stat, err := os.Stat(path); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 {
+		return nil, fmt.Errorf("%s is not a readable file", path)
+	}
+
+	return localFile{
+		root: os.DirFS(filepath.Dir(path)),
+		name: filepath.Base(path),
+	}, nil
+}
diff --git a/ipblocklist/files/local_test.go b/ipblocklist/files/local_test.go
new file mode 100644
index 0000000..3108dab
--- /dev/null
+++ b/ipblocklist/files/local_test.go
@@ -0,0 +1,55 @@
+package files_test
+
+import (
+	"context"
+	"io"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/9seconds/mtg/v2/ipblocklist/files"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type LocalTestSuite struct {
+	suite.Suite
+}
+
+func (suite *LocalTestSuite) GetLocalFile(name string) string {
+	return filepath.Join("testdata", name)
+}
+
+func (suite *LocalTestSuite) TestIncorrect() {
+	names := []string{
+		"absent",
+		"directory",
+	}
+
+	for _, v := range names {
+		value := v
+
+		suite.T().Run(v, func(t *testing.T) {
+			_, err := files.NewLocal(suite.GetLocalFile(value))
+			assert.Error(t, err)
+		})
+	}
+}
+
+func (suite *LocalTestSuite) TestOk() {
+	file, err := files.NewLocal(suite.GetLocalFile("readable"))
+	suite.NoError(err)
+
+	reader, err := file.Open(context.Background())
+	suite.NoError(err)
+
+	data, err := io.ReadAll(reader)
+	suite.NoError(err)
+
+	suite.Equal("Hooray!", strings.TrimSpace(string(data)))
+}
+
+func TestLocal(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &LocalTestSuite{})
+}
diff --git a/ipblocklist/files/testdata/directory/.gitkeep b/ipblocklist/files/testdata/directory/.gitkeep
new file mode 100644
index 0000000..e69de29
diff --git a/ipblocklist/files/testdata/readable b/ipblocklist/files/testdata/readable
new file mode 100644
index 0000000..715fcb7
--- /dev/null
+++ b/ipblocklist/files/testdata/readable
@@ -0,0 +1 @@
+Hooray!

From e6fa69d28890ebe1ee287f8338b19de54fe9862e Mon Sep 17 00:00:00 2001
From: 9seconds <nineseconds@yandex.ru>
Date: Mon, 29 Nov 2021 15:56:53 +0300
Subject: [PATCH 2/5] Add tests for HTTP file abstraction

---
 ipblocklist/files/http.go       |  8 +++
 ipblocklist/files/http_test.go  | 90 +++++++++++++++++++++++++++++++++
 ipblocklist/files/init.go       |  3 ++
 ipblocklist/files/local_test.go |  6 +--
 4 files changed, 104 insertions(+), 3 deletions(-)
 create mode 100644 ipblocklist/files/http_test.go

diff --git a/ipblocklist/files/http.go b/ipblocklist/files/http.go
index c024399..69b4cfc 100644
--- a/ipblocklist/files/http.go
+++ b/ipblocklist/files/http.go
@@ -29,10 +29,18 @@ func (h httpFile) Open(ctx context.Context) (io.ReadCloser, error) {
 		return nil, fmt.Errorf("cannot get url %s: %w", h.url, err)
 	}
 
+	if response.StatusCode >= http.StatusBadRequest {
+		return nil, fmt.Errorf("unexpected status code %d", response.StatusCode)
+	}
+
 	return response.Body, nil
 }
 
 func NewHTTP(client *http.Client, endpoint string) (File, error) {
+	if client == nil {
+		return nil, ErrBadHTTPClient
+	}
+
 	parsed, err := url.Parse(endpoint)
 	if err != nil {
 		return nil, fmt.Errorf("incorrect url %s: %w", endpoint, err)
diff --git a/ipblocklist/files/http_test.go b/ipblocklist/files/http_test.go
new file mode 100644
index 0000000..294739e
--- /dev/null
+++ b/ipblocklist/files/http_test.go
@@ -0,0 +1,90 @@
+package files_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/9seconds/mtg/v2/ipblocklist/files"
+	"github.com/stretchr/testify/suite"
+)
+
+type HTTPTestSuite struct {
+	suite.Suite
+
+	httpClient *http.Client
+	httpServer *httptest.Server
+	ctx        context.Context
+	ctxCancel  context.CancelFunc
+}
+
+func (suite *HTTPTestSuite) makeFile(path string) (files.File, error) {
+	return files.NewHTTP(suite.httpClient, suite.httpServer.URL+"/"+path)
+}
+
+func (suite *HTTPTestSuite) SetupSuite() {
+	mux := http.NewServeMux()
+
+	mux.Handle("/", http.FileServer(http.Dir("testdata")))
+
+	suite.httpServer = httptest.NewServer(mux)
+	suite.httpClient = suite.httpServer.Client()
+}
+
+func (suite *HTTPTestSuite) SetupTest() {
+	suite.ctx, suite.ctxCancel = context.WithCancel(context.Background())
+}
+
+func (suite *HTTPTestSuite) TearDownTest() {
+	suite.ctxCancel()
+	suite.httpServer.CloseClientConnections()
+}
+
+func (suite *HTTPTestSuite) TearDownSuite() {
+	suite.httpServer.Close()
+}
+
+func (suite *HTTPTestSuite) TestBadURL() {
+	_, err := files.NewHTTP(suite.httpClient, "sdfsdf")
+	suite.Error(err)
+}
+
+func (suite *HTTPTestSuite) TestBadSchema() {
+	_, err := files.NewHTTP(suite.httpClient, "gopher://lala")
+	suite.Error(err)
+}
+
+func (suite *HTTPTestSuite) TestNilHTTPClient() {
+	_, err := files.NewHTTP(nil, "")
+	suite.Error(err)
+}
+
+func (suite *HTTPTestSuite) TestAbsentFile() {
+	file, err := suite.makeFile("absent")
+	suite.NoError(err)
+
+	_, err = file.Open(suite.ctx)
+	suite.Error(err)
+}
+
+func (suite *HTTPTestSuite) TestOk() {
+	file, err := suite.makeFile("readable")
+	suite.NoError(err)
+
+	readCloser, err := file.Open(suite.ctx)
+	suite.NoError(err)
+
+	defer readCloser.Close()
+
+	data, err := io.ReadAll(readCloser)
+	suite.NoError(err)
+	suite.Equal("Hooray!", strings.TrimSpace(string(data)))
+}
+
+func TestHTTP(t *testing.T) {
+	t.Parallel()
+	suite.Run(t, &HTTPTestSuite{})
+}
diff --git a/ipblocklist/files/init.go b/ipblocklist/files/init.go
index 922b211..520b2c5 100644
--- a/ipblocklist/files/init.go
+++ b/ipblocklist/files/init.go
@@ -2,9 +2,12 @@ package files
 
 import (
 	"context"
+	"errors"
 	"io"
 )
 
+var ErrBadHTTPClient = errors.New("incorrect http client")
+
 type File interface {
 	Open(context.Context) (io.ReadCloser, error)
 }
diff --git a/ipblocklist/files/local_test.go b/ipblocklist/files/local_test.go
index 3108dab..f3dba38 100644
--- a/ipblocklist/files/local_test.go
+++ b/ipblocklist/files/local_test.go
@@ -16,7 +16,7 @@ type LocalTestSuite struct {
 	suite.Suite
 }
 
-func (suite *LocalTestSuite) GetLocalFile(name string) string {
+func (suite *LocalTestSuite) getLocalFile(name string) string {
 	return filepath.Join("testdata", name)
 }
 
@@ -30,14 +30,14 @@ func (suite *LocalTestSuite) TestIncorrect() {
 		value := v
 
 		suite.T().Run(v, func(t *testing.T) {
-			_, err := files.NewLocal(suite.GetLocalFile(value))
+			_, err := files.NewLocal(suite.getLocalFile(value))
 			assert.Error(t, err)
 		})
 	}
 }
 
 func (suite *LocalTestSuite) TestOk() {
-	file, err := files.NewLocal(suite.GetLocalFile("readable"))
+	file, err := files.NewLocal(suite.getLocalFile("readable"))
 	suite.NoError(err)
 
 	reader, err := file.Open(context.Background())

From cc101c9a47c868967c29e4271b9e5faf4e73173b Mon Sep 17 00:00:00 2001
From: 9seconds <nineseconds@yandex.ru>
Date: Mon, 29 Nov 2021 15:58:14 +0300
Subject: [PATCH 3/5] Rename rwMutex to updateMutex

---
 ipblocklist/firehol.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go
index 828afe0..23f4cdd 100644
--- a/ipblocklist/firehol.go
+++ b/ipblocklist/firehol.go
@@ -45,7 +45,7 @@ type Firehol struct {
 	ctxCancel context.CancelFunc
 	logger    mtglib.Logger
 
-	rwMutex sync.RWMutex
+	updateMutex sync.RWMutex
 
 	remoteURLs []string
 	localFiles []string
@@ -68,8 +68,8 @@ func (f *Firehol) Contains(ip net.IP) bool {
 		return true
 	}
 
-	f.rwMutex.RLock()
-	defer f.rwMutex.RUnlock()
+	f.updateMutex.RLock()
+	defer f.updateMutex.RUnlock()
 
 	if ip4 := ip.To4(); ip4 != nil {
 		return f.containsIPv4(ip4)
@@ -194,8 +194,8 @@ func (f *Firehol) update() error { // nolint: funlen, cyclop
 	default:
 	}
 
-	f.rwMutex.Lock()
-	defer f.rwMutex.Unlock()
+	f.updateMutex.Lock()
+	defer f.updateMutex.Unlock()
 
 	f.treeV4 = v4tree
 	f.treeV6 = v6tree

From 558fec60de4c32e009a3dded49481c389af0844e Mon Sep 17 00:00:00 2001
From: 9seconds <nineseconds@yandex.ru>
Date: Mon, 29 Nov 2021 16:25:33 +0300
Subject: [PATCH 4/5] Refactor firehol

---
 ipblocklist/files/http.go      |   6 +-
 ipblocklist/files/http_test.go |   2 +-
 ipblocklist/files/init.go      |   1 +
 ipblocklist/files/local.go     |  14 +--
 ipblocklist/firehol.go         | 186 ++++++++++-----------------------
 5 files changed, 69 insertions(+), 140 deletions(-)

diff --git a/ipblocklist/files/http.go b/ipblocklist/files/http.go
index 69b4cfc..6a60ede 100644
--- a/ipblocklist/files/http.go
+++ b/ipblocklist/files/http.go
@@ -22,7 +22,7 @@ func (h httpFile) Open(ctx context.Context) (io.ReadCloser, error) {
 	response, err := h.http.Do(request)
 	if err != nil {
 		if response != nil {
-			io.Copy(io.Discard, response.Body)
+			io.Copy(io.Discard, response.Body) // nolint: errcheck
 			response.Body.Close()
 		}
 
@@ -36,6 +36,10 @@ func (h httpFile) Open(ctx context.Context) (io.ReadCloser, error) {
 	return response.Body, nil
 }
 
+func (h httpFile) String() string {
+	return h.url
+}
+
 func NewHTTP(client *http.Client, endpoint string) (File, error) {
 	if client == nil {
 		return nil, ErrBadHTTPClient
diff --git a/ipblocklist/files/http_test.go b/ipblocklist/files/http_test.go
index 294739e..6559969 100644
--- a/ipblocklist/files/http_test.go
+++ b/ipblocklist/files/http_test.go
@@ -22,7 +22,7 @@ type HTTPTestSuite struct {
 }
 
 func (suite *HTTPTestSuite) makeFile(path string) (files.File, error) {
-	return files.NewHTTP(suite.httpClient, suite.httpServer.URL+"/"+path)
+	return files.NewHTTP(suite.httpClient, suite.httpServer.URL+"/"+path) // nolint: wrapcheck
 }
 
 func (suite *HTTPTestSuite) SetupSuite() {
diff --git a/ipblocklist/files/init.go b/ipblocklist/files/init.go
index 520b2c5..97570af 100644
--- a/ipblocklist/files/init.go
+++ b/ipblocklist/files/init.go
@@ -10,4 +10,5 @@ var ErrBadHTTPClient = errors.New("incorrect http client")
 
 type File interface {
 	Open(context.Context) (io.ReadCloser, error)
+	String() string
 }
diff --git a/ipblocklist/files/local.go b/ipblocklist/files/local.go
index 8e3fe64..3cd08c7 100644
--- a/ipblocklist/files/local.go
+++ b/ipblocklist/files/local.go
@@ -4,18 +4,19 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/fs"
 	"os"
-	"path/filepath"
 )
 
 type localFile struct {
-	root fs.FS
-	name string
+	path string
 }
 
 func (l localFile) Open(ctx context.Context) (io.ReadCloser, error) {
-	return l.root.Open(l.name)
+	return os.Open(l.path) // nolint: wrapcheck
+}
+
+func (l localFile) String() string {
+	return l.path
 }
 
 func NewLocal(path string) (File, error) {
@@ -24,7 +25,6 @@ func NewLocal(path string) (File, error) {
 	}
 
 	return localFile{
-		root: os.DirFS(filepath.Dir(path)),
-		name: filepath.Base(path),
+		path: path,
 	}, nil
 }
diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go
index 23f4cdd..41726a7 100644
--- a/ipblocklist/firehol.go
+++ b/ipblocklist/firehol.go
@@ -4,16 +4,13 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"io"
 	"net"
-	"net/http"
-	"net/url"
-	"os"
 	"regexp"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/9seconds/mtg/v2/ipblocklist/files"
 	"github.com/9seconds/mtg/v2/mtglib"
 	"github.com/kentik/patricia"
 	"github.com/kentik/patricia/bool_tree"
@@ -41,20 +38,16 @@ var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
 //     127.0.0.1   # you can specify an IP
 //     10.0.0.0/8  # or cidr
 type Firehol struct {
-	ctx       context.Context
-	ctxCancel context.CancelFunc
-	logger    mtglib.Logger
-
+	ctx         context.Context
+	ctxCancel   context.CancelFunc
+	logger      mtglib.Logger
 	updateMutex sync.RWMutex
 
-	remoteURLs []string
-	localFiles []string
+	blocklists []files.File
 
-	httpClient *http.Client
 	workerPool *ants.Pool
-
-	treeV4 *bool_tree.TreeV4
-	treeV6 *bool_tree.TreeV6
+	treeV4     *bool_tree.TreeV4
+	treeV6     *bool_tree.TreeV6
 }
 
 // Shutdown stop a background update process.
@@ -98,22 +91,14 @@ func (f *Firehol) Run(updateEach time.Duration) {
 		}
 	}()
 
-	if err := f.update(); err != nil {
-		f.logger.WarningError("cannot update blocklist", err)
-	} else {
-		f.logger.Info("blocklist was updated")
-	}
+	f.update()
 
 	for {
 		select {
 		case <-f.ctx.Done():
 			return
 		case <-ticker.C:
-			if err := f.update(); err != nil {
-				f.logger.WarningError("cannot update blocklist", err)
-			} else {
-				f.logger.Info("blocklist was updated")
-			}
+			f.update()
 		}
 	}
 }
@@ -138,121 +123,53 @@ func (f *Firehol) containsIPv6(addr net.IP) bool {
 	return false
 }
 
-func (f *Firehol) update() error { // nolint: funlen, cyclop
+func (f *Firehol) update() {
 	ctx, cancel := context.WithCancel(f.ctx)
 	defer cancel()
 
 	wg := &sync.WaitGroup{}
-	wg.Add(len(f.remoteURLs) + len(f.localFiles))
+	wg.Add(len(f.blocklists))
 
 	treeMutex := &sync.Mutex{}
 	v4tree := bool_tree.NewTreeV4()
 	v6tree := bool_tree.NewTreeV6()
 
-	errorChan := make(chan error, 1)
-	defer close(errorChan)
-
-	for _, v := range f.localFiles {
-		go func(filename string) {
+	for _, v := range f.blocklists {
+		go func(file files.File) {
 			defer wg.Done()
 
-			if err := f.updateLocalFile(ctx, filename, treeMutex, v4tree, v6tree); err != nil {
-				cancel()
-				f.logger.BindStr("filename", filename).WarningError("cannot update", err)
+			logger := f.logger.BindStr("filename", file.String())
 
-				select {
-				case errorChan <- err:
-				default:
-				}
+			fileContent, err := file.Open(ctx)
+			if err != nil {
+				logger.WarningError("update has failed", err)
+
+				return
+			}
+
+			defer fileContent.Close()
+
+			if err := f.updateFromFile(treeMutex, v4tree, v6tree, bufio.NewScanner(fileContent)); err != nil {
+				logger.WarningError("update has failed", err)
 			}
 		}(v)
 	}
 
-	for _, v := range f.remoteURLs {
-		value := v
-
-		f.workerPool.Submit(func() { // nolint: errcheck
-			defer wg.Done()
-
-			if err := f.updateRemoteURL(ctx, value, treeMutex, v4tree, v6tree); err != nil {
-				cancel()
-				f.logger.BindStr("url", value).WarningError("cannot update", err)
-
-				select {
-				case errorChan <- err:
-				default:
-				}
-			}
-		})
-	}
-
 	wg.Wait()
 
-	select {
-	case err := <-errorChan:
-		return fmt.Errorf("cannot update trees: %w", err)
-	default:
-	}
-
 	f.updateMutex.Lock()
 	defer f.updateMutex.Unlock()
 
 	f.treeV4 = v4tree
 	f.treeV6 = v6tree
 
-	return nil
+	f.logger.Info("blocklist was updated")
 }
 
-func (f *Firehol) updateLocalFile(ctx context.Context, filename string,
-	mutex sync.Locker,
-	v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
-	filefp, err := os.Open(filename)
-	if err != nil {
-		return fmt.Errorf("cannot open file: %w", err)
-	}
-
-	go func(ctx context.Context, closer io.Closer) {
-		<-ctx.Done()
-		closer.Close()
-	}(ctx, filefp)
-
-	defer filefp.Close()
-
-	return f.updateTrees(mutex, filefp, v4tree, v6tree)
-}
-
-func (f *Firehol) updateRemoteURL(ctx context.Context, url string,
-	mutex sync.Locker,
-	v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return fmt.Errorf("cannot build a request: %w", err)
-	}
-
-	resp, err := f.httpClient.Do(req) // nolint: bodyclose
-	if err != nil {
-		return fmt.Errorf("cannot request a remote URL %s: %w", url, err)
-	}
-
-	go func(ctx context.Context, closer io.Closer) {
-		<-ctx.Done()
-		closer.Close()
-	}(ctx, resp.Body)
-
-	defer func(rc io.ReadCloser) {
-		io.Copy(io.Discard, rc) // nolint: errcheck
-		rc.Close()
-	}(resp.Body)
-
-	return f.updateTrees(mutex, resp.Body, v4tree, v6tree)
-}
-
-func (f *Firehol) updateTrees(mutex sync.Locker,
-	reader io.Reader,
+func (f *Firehol) updateFromFile(mutex sync.Locker,
 	v4tree *bool_tree.TreeV4,
-	v6tree *bool_tree.TreeV6) error {
-	scanner := bufio.NewScanner(reader)
-
+	v6tree *bool_tree.TreeV6,
+	scanner *bufio.Scanner) error {
 	for scanner.Scan() {
 		text := scanner.Text()
 		text = fireholRegexpComment.ReplaceAllLiteralString(text, "")
@@ -271,7 +188,7 @@ func (f *Firehol) updateTrees(mutex sync.Locker,
 	}
 
 	if scanner.Err() != nil {
-		return fmt.Errorf("cannot parse a response: %w", scanner.Err())
+		return fmt.Errorf("cannot parse a file: %w", scanner.Err())
 	}
 
 	return nil
@@ -317,27 +234,36 @@ func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint,
 // when it is necessary.
 func NewFirehol(logger mtglib.Logger, network mtglib.Network,
 	downloadConcurrency uint,
-	remoteURLs []string,
+	urls []string,
 	localFiles []string) (*Firehol, error) {
-	for _, v := range remoteURLs {
-		parsed, err := url.Parse(v)
-		if err != nil {
-			return nil, fmt.Errorf("incorrect url %s: %w", v, err)
-		}
-
-		switch parsed.Scheme {
-		case "http", "https":
-		default:
-			return nil, fmt.Errorf("unsupported url %s", v)
-		}
-	}
+	blocklists := []files.File{}
 
 	for _, v := range localFiles {
-		if stat, err := os.Stat(v); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 {
-			return nil, fmt.Errorf("%s is not a readable file", v)
+		file, err := files.NewLocal(v)
+		if err != nil {
+			return nil, fmt.Errorf("cannot create a local file %s: %w", v, err)
 		}
+
+		blocklists = append(blocklists, file)
 	}
 
+	httpClient := network.MakeHTTPClient(nil)
+
+	for _, v := range urls {
+		file, err := files.NewHTTP(httpClient, v)
+		if err != nil {
+			return nil, fmt.Errorf("cannot create a HTTP file %s: %w", v, err)
+		}
+
+		blocklists = append(blocklists, file)
+	}
+
+	return NewFireholFromFiles(logger, downloadConcurrency, blocklists)
+}
+
+func NewFireholFromFiles(logger mtglib.Logger,
+	downloadConcurrency uint,
+	blocklists []files.File) (*Firehol, error) {
 	if downloadConcurrency == 0 {
 		downloadConcurrency = DefaultFireholDownloadConcurrency
 	}
@@ -349,11 +275,9 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network,
 		ctx:        ctx,
 		ctxCancel:  cancel,
 		logger:     logger.Named("firehol"),
-		httpClient: network.MakeHTTPClient(nil),
 		treeV4:     bool_tree.NewTreeV4(),
 		treeV6:     bool_tree.NewTreeV6(),
 		workerPool: workerPool,
-		remoteURLs: remoteURLs,
-		localFiles: localFiles,
+		blocklists: blocklists,
 	}, nil
 }

From 0ddaabb136e5375657c9cab5ae1845b0690f1b7b Mon Sep 17 00:00:00 2001
From: 9seconds <nineseconds@yandex.ru>
Date: Mon, 29 Nov 2021 17:02:53 +0300
Subject: [PATCH 5/5] Add whitelist support

---
 example.config.toml       | 21 +++++++++++++++++++++
 internal/cli/run_proxy.go | 24 ++++++++++++++++++------
 internal/config/config.go | 29 ++++++++++++++++++++---------
 internal/config/parse.go  |  6 ++++++
 mtglib/proxy.go           | 18 ++++++++++++++----
 mtglib/proxy_opts.go      |  5 +++++
 6 files changed, 84 insertions(+), 19 deletions(-)

diff --git a/example.config.toml b/example.config.toml
index 9a9c96b..b521d4e 100644
--- a/example.config.toml
+++ b/example.config.toml
@@ -174,6 +174,27 @@ urls = [
 # How often do we need to update a blocklist set.
 update-each = "24h"
 
+# Allowlist is an opposite to a blocklist. Only those IPs that are coming from
+# subnets defined in these lists are allowed. All others will be rejected.
+#
+# If this feature is disabled, then there won't be any check performed by this
+# validator. It is possible to combine both blocklist and whitelist.
+[defense.allowlist]
+# You can enable/disable this feature.
+enabled = false
+# This is a limiter for concurrency. In order to protect website
+# from overloading, we download files in this number of threads.
+download-concurrency = 2
+# A list of URLs in FireHOL format (https://iplists.firehol.org/)
+# You can provider links here (starts with https:// or http://) or
+# path to a local file, but in this case it should be absolute.
+urls = [
+    # "https://iplists.firehol.org/files/firehol_level1.netset",
+    # "/local.file"
+
+]
+update-each = "24h"
+
 # statsd statistics integration.
 [stats.statsd]
 # enabled/disabled
diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go
index cc5bed9..e8dde0c 100644
--- a/internal/cli/run_proxy.go
+++ b/internal/cli/run_proxy.go
@@ -86,15 +86,15 @@ func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache {
 	)
 }
 
-func makeIPBlocklist(conf *config.Config, logger mtglib.Logger, ntw mtglib.Network) (mtglib.IPBlocklist, error) {
-	if !conf.Defense.Blocklist.Enabled.Get(false) {
+func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Network) (mtglib.IPBlocklist, error) {
+	if !conf.Enabled.Get(false) {
 		return ipblocklist.NewNoop(), nil
 	}
 
 	remoteURLs := []string{}
 	localFiles := []string{}
 
-	for _, v := range conf.Defense.Blocklist.URLs {
+	for _, v := range conf.URLs {
 		if v.IsRemote() {
 			remoteURLs = append(remoteURLs, v.String())
 		} else {
@@ -104,7 +104,7 @@ func makeIPBlocklist(conf *config.Config, logger mtglib.Logger, ntw mtglib.Netwo
 
 	firehol, err := ipblocklist.NewFirehol(logger.Named("ipblockist"),
 		ntw,
-		conf.Defense.Blocklist.DownloadConcurrency.Get(1),
+		conf.DownloadConcurrency.Get(1),
 		remoteURLs,
 		localFiles)
 	if err != nil {
@@ -153,7 +153,7 @@ func makeEventStream(conf *config.Config, logger mtglib.Logger) (mtglib.EventStr
 	return events.NewNoopStream(), nil
 }
 
-func runProxy(conf *config.Config, version string) error {
+func runProxy(conf *config.Config, version string) error { // nolint: funlen
 	logger := makeLogger(conf)
 
 	logger.BindJSON("configuration", conf.String()).Debug("configuration")
@@ -163,11 +163,22 @@ func runProxy(conf *config.Config, version string) error {
 		return fmt.Errorf("cannot build network: %w", err)
 	}
 
-	blocklist, err := makeIPBlocklist(conf, logger, ntw)
+	blocklist, err := makeIPBlocklist(conf.Defense.Blocklist, logger, ntw)
 	if err != nil {
 		return fmt.Errorf("cannot build ip blocklist: %w", err)
 	}
 
+	var whitelist mtglib.IPBlocklist
+
+	if conf.Defense.Allowlist.Enabled.Get(false) {
+		whlist, err := makeIPBlocklist(conf.Defense.Allowlist, logger, ntw)
+		if err != nil {
+			return fmt.Errorf("cannot build ip blocklist: %w", err)
+		}
+
+		whitelist = whlist
+	}
+
 	eventStream, err := makeEventStream(conf, logger)
 	if err != nil {
 		return fmt.Errorf("cannot build event stream: %w", err)
@@ -178,6 +189,7 @@ func runProxy(conf *config.Config, version string) error {
 		Network:         ntw,
 		AntiReplayCache: makeAntiReplayCache(conf),
 		IPBlocklist:     blocklist,
+		IPWhitelist:     whitelist,
 		EventStream:     eventStream,
 
 		Secret:             conf.Secret,
diff --git a/internal/config/config.go b/internal/config/config.go
index c072dad..aa407c0 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -8,6 +8,18 @@ import (
 	"github.com/9seconds/mtg/v2/mtglib"
 )
 
+type Optional struct {
+	Enabled TypeBool `json:"enabled"`
+}
+
+type ListConfig struct {
+	Optional
+
+	DownloadConcurrency TypeConcurrency    `json:"downloadConcurrency"`
+	URLs                []TypeBlocklistURI `json:"urls"`
+	UpdateEach          TypeDuration       `json:"updateEach"`
+}
+
 type Config struct {
 	Debug                    TypeBool        `json:"debug"`
 	AllowFallbackOnUnknownDC TypeBool        `json:"allowFallbackOnUnknownDc"`
@@ -20,16 +32,13 @@ type Config struct {
 	Concurrency              TypeConcurrency `json:"concurrency"`
 	Defense                  struct {
 		AntiReplay struct {
-			Enabled   TypeBool      `json:"enabled"`
+			Optional
+
 			MaxSize   TypeBytes     `json:"maxSize"`
 			ErrorRate TypeErrorRate `json:"errorRate"`
 		} `json:"antiReplay"`
-		Blocklist struct {
-			Enabled             TypeBool           `json:"enabled"`
-			DownloadConcurrency TypeConcurrency    `json:"downloadConcurrency"`
-			URLs                []TypeBlocklistURI `json:"urls"`
-			UpdateEach          TypeDuration       `json:"updateEach"`
-		} `json:"blocklist"`
+		Blocklist ListConfig `json:"blocklist"`
+		Allowlist ListConfig `json:"allowlist"`
 	} `json:"defense"`
 	Network struct {
 		Timeout struct {
@@ -42,13 +51,15 @@ type Config struct {
 	} `json:"network"`
 	Stats struct {
 		StatsD struct {
-			Enabled      TypeBool            `json:"enabled"`
+			Optional
+
 			Address      TypeHostPort        `json:"address"`
 			MetricPrefix TypeMetricPrefix    `json:"metricPrefix"`
 			TagFormat    TypeStatsdTagFormat `json:"tagFormat"`
 		} `json:"statsd"`
 		Prometheus struct {
-			Enabled      TypeBool         `json:"enabled"`
+			Optional
+
 			BindTo       TypeHostPort     `json:"bindTo"`
 			HTTPPath     TypeHTTPPath     `json:"httpPath"`
 			MetricPrefix TypeMetricPrefix `json:"metricPrefix"`
diff --git a/internal/config/parse.go b/internal/config/parse.go
index a364712..90d6a78 100644
--- a/internal/config/parse.go
+++ b/internal/config/parse.go
@@ -30,6 +30,12 @@ type tomlConfig struct {
 			URLs                []string `toml:"urls" json:"urls,omitempty"`
 			UpdateEach          string   `toml:"update-each" json:"updateEach,omitempty"`
 		} `toml:"blocklist" json:"blocklist,omitempty"`
+		Allowlist struct {
+			Enabled             bool     `toml:"enabled" json:"enabled,omitempty"`
+			DownloadConcurrency uint     `toml:"download-concurrency" json:"downloadConcurrency,omitempty"`
+			URLs                []string `toml:"urls" json:"urls,omitempty"`
+			UpdateEach          string   `toml:"update-each" json:"updateEach,omitempty"`
+		} `toml:"allowlist" json:"allowlist,omitempty"`
 	} `toml:"defense" json:"defense,omitempty"`
 	Network struct {
 		Timeout struct {
diff --git a/mtglib/proxy.go b/mtglib/proxy.go
index 330d2f3..6910a63 100644
--- a/mtglib/proxy.go
+++ b/mtglib/proxy.go
@@ -33,7 +33,8 @@ type Proxy struct {
 	secret          Secret
 	network         Network
 	antiReplayCache AntiReplayCache
-	ipBlocklist     IPBlocklist
+	blocklist       IPBlocklist
+	whitelist       IPBlocklist
 	eventStream     EventStream
 	logger          Logger
 }
@@ -91,7 +92,7 @@ func (p *Proxy) ServeConn(conn net.Conn) {
 }
 
 // Serve starts a proxy on a given listener.
-func (p *Proxy) Serve(listener net.Listener) error {
+func (p *Proxy) Serve(listener net.Listener) error { // nolint: cyclop
 	p.streamWaitGroup.Add(1)
 	defer p.streamWaitGroup.Done()
 
@@ -109,7 +110,15 @@ func (p *Proxy) Serve(listener net.Listener) error {
 		ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP
 		logger := p.logger.BindStr("ip", ipAddr.String())
 
-		if p.ipBlocklist.Contains(ipAddr) {
+		if p.whitelist != nil && !p.whitelist.Contains(ipAddr) {
+			conn.Close()
+			logger.Info("ip was rejected by whitelist")
+			p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr))
+
+			continue
+		}
+
+		if p.blocklist.Contains(ipAddr) {
 			conn.Close()
 			logger.Info("ip was blacklisted")
 			p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr))
@@ -291,7 +300,8 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {
 		secret:                   opts.Secret,
 		network:                  opts.Network,
 		antiReplayCache:          opts.AntiReplayCache,
-		ipBlocklist:              opts.IPBlocklist,
+		blocklist:                opts.IPBlocklist,
+		whitelist:                opts.IPWhitelist,
 		eventStream:              opts.EventStream,
 		logger:                   opts.getLogger("proxy"),
 		domainFrontingPort:       opts.getDomainFrontingPort(),
diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go
index 8993de7..860e3ac 100644
--- a/mtglib/proxy_opts.go
+++ b/mtglib/proxy_opts.go
@@ -28,6 +28,11 @@ type ProxyOpts struct {
 	// This is a mandatory setting.
 	IPBlocklist IPBlocklist
 
+	// IPWhitelist defines a whitelist of IPs to allow to use proxy.
+	//
+	// This is an optional setting, ignored by default (no restrictions).
+	IPWhitelist IPBlocklist
+
 	// EventStream defines an instance of event stream.
 	//
 	// This ia a mandatory setting.