Merge remote-tracking branch 'origin/master' into stable

Revert "Consider test modules for update-deps"
This reverts commit 880dd04728.
2026-08-31 23:24:01 +03:00 · 2022-03-11 18:03:35 +03:00 · 2022-03-11 17:55:12 +03:00 · 2022-03-11 17:27:01 +03:00 · 2022-03-11 17:23:53 +03:00 · 2022-03-11 17:19:15 +03:00
43 changed files with 364 additions and 160 deletions
@@ -42,7 +42,6 @@ jobs:
    strategy:
      matrix:
        go_version:
-          - ~1.16
          - ^1.17
    steps:
      - name: Checkout
@@ -83,7 +82,7 @@ jobs:
      - name: Run linter
        uses: golangci/golangci-lint-action@v2
        with:
-          version: v1.43.0
+          version: v1.44.2

  docker:
    name: Docker
@@ -9,4 +9,4 @@ format = "colored-line-number"

 [linters]
 enable-all = true
-disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct"]
+disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"]
@@ -2,7 +2,7 @@ ROOT_DIR     := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 IMAGE_NAME   := mtg
 APP_NAME     := $(IMAGE_NAME)

-GOLANGCI_LINT_VERSION := v1.43.0
+GOLANGCI_LINT_VERSION := v1.44.2

 VERSION_GO         := $(shell go version)
 VERSION_DATE       := $(shell date -Ru)
@@ -1,6 +1,6 @@
 # mtg

-Highly-opionated (ex-bullshit-free) MTPROTO proxy for
+Highly-opinionated (ex-bullshit-free) MTPROTO proxy for
 [Telegram](https://telegram.org/).

 [![CI](https://github.com/9seconds/mtg/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/9seconds/mtg/actions/workflows/ci.yaml)
@@ -224,6 +224,16 @@ $ mtg generate-secret --hex google.com
 ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d
 ```

+equivalent commands with docker:
+
+```console
+$ docker run --rm nineseconds/mtg:2 generate-secret google.com
+7ibaERuTSGPH1RdztfYnN4tnb29nbGUuY29t
+
+$ docker run --rm nineseconds/mtg:2 generate-secret --hex google.com
+ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d
+```
+
 This secret is a keystone for a proxy and your password for a client.
 You need to keep it secured.

@@ -324,7 +334,7 @@ $ sudo systemctl start mtg
 or you can run a docker image

 ```console
-docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --restart=unless-stopped nineseconds/mtg:2
+docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --name mtg-proxy --restart=unless-stopped nineseconds/mtg:2
 ```

 where _443_ is a host port (a port you want to connect to from a
@@ -353,6 +363,12 @@ $ mtg access /etc/mtg.toml
 }
 ```

+or if you are using docker:
+
+```console
+$ docker exec mtg-proxy /mtg access /config.toml
+```
+
 ## Metrics

 Out of the box, mtg works with
@@ -367,6 +383,7 @@ Here goes a list of metrics with their types but without a prefix.
 | client_connections          | gauge   | `ip_family`                      | Count of processing client connections.                                                    |
 | telegram_connections        | gauge   | `telegram_ip`, `dc`              | Count of connections to Telegram servers.                                                  |
 | domain_fronting_connections | gauge   | `ip_family`                      | Count of connections to fronting domain.                                                   |
+| iplist_size                 | gauge   | `ip_list`                        | A size of either allowlist or blocklist in use.                                            |
 | telegram_traffic            | counter | `telegram_ip`, `dc`, `direction` | Count of bytes, transmitted to/from Telegram.                                              |
 | domain_fronting_traffic     | counter | `direction`                      | Count of bytes, transmitted to/from fronting domain.                                       |
 | domain_fronting             | counter | –                                | Count of domain fronting events.                                                           |
@@ -382,3 +399,4 @@ Tag meaning:
 | dc          |                            | A number of the Telegram DC for a connection. |
 | telegram_ip |                            | IP address of the Telegram server.            |
 | direction   | `to_client`, `from_client` | A direction of the traffic flow.              |
+| ip_list     | `allowlist`, `blocklist`   | A type of the IP list.                        |
@@ -102,6 +102,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob
 				observer.EventConcurrencyLimited(typedEvt)
 			case mtglib.EventReplayAttack:
 				observer.EventReplayAttack(typedEvt)
+			case mtglib.EventIPListSize:
+				observer.EventIPListSize(typedEvt)
 			}
 		}
 	}
@@ -204,6 +204,27 @@ func (suite *EventStreamTestSuite) TestEventReplayAttack() {
 	time.Sleep(100 * time.Millisecond)
 }

+func (suite *EventStreamTestSuite) TestEventIPListSize() {
+	evt := mtglib.NewEventIPListSize(10, true)
+
+	for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} {
+		v.
+			On("EventIPListSize", mock.Anything).
+			Once().
+			Run(func(args mock.Arguments) {
+				caught, ok := args.Get(0).(mtglib.EventIPListSize)
+
+				suite.True(ok)
+				suite.Equal(evt.Timestamp(), caught.Timestamp())
+				suite.Equal(evt.Size, caught.Size)
+				suite.Equal(evt.IsBlockList, caught.IsBlockList)
+			})
+	}
+
+	suite.stream.Send(suite.ctx, evt)
+	time.Sleep(100 * time.Millisecond)
+}
+
 func (suite *EventStreamTestSuite) TearDownTest() {
 	suite.stream.Shutdown()
 	suite.ctxCancel()
@@ -53,6 +53,9 @@ type Observer interface {
 	// EventReplayAttack reacts on incoming mtglib.EventReplayAttack event.
 	EventReplayAttack(mtglib.EventReplayAttack)

+	// EventIPListSize reacts on incoming mtglib.EventIPListSize
+	EventIPListSize(mtglib.EventIPListSize)
+
 	// Shutdown stop observer. Default event stream guarantees:
 	//   1. If shutdown is executed, it is executed only once
 	//   2. Observer won't receieve any new message after this
@@ -41,6 +41,10 @@ func (o *ObserverMock) EventReplayAttack(evt mtglib.EventReplayAttack) {
 	o.Called(evt)
 }

+func (o *ObserverMock) EventIPListSize(evt mtglib.EventIPListSize) {
+	o.Called(evt)
+}
+
 func (o *ObserverMock) Shutdown() {
 	o.Called()
 }
@@ -130,6 +130,21 @@ func (m multiObserver) EventReplayAttack(evt mtglib.EventReplayAttack) {
 	wg.Wait()
 }

+func (m multiObserver) EventIPListSize(evt mtglib.EventIPListSize) {
+	wg := &sync.WaitGroup{}
+	wg.Add(len(m.observers))
+
+	for _, v := range m.observers {
+		go func(obs Observer) {
+			defer wg.Done()
+
+			obs.EventIPListSize(evt)
+		}(v)
+	}
+
+	wg.Wait()
+}
+
 func (m multiObserver) Shutdown() {
 	for _, v := range m.observers {
 		v.Shutdown()
@@ -25,6 +25,7 @@ func (n noopObserver) EventFinish(_ mtglib.EventFinish)
 func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {}
 func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted)           {}
 func (n noopObserver) EventReplayAttack(_ mtglib.EventReplayAttack)             {}
+func (n noopObserver) EventIPListSize(_ mtglib.EventIPListSize)                 {}
 func (n noopObserver) Shutdown()                                                {}

 // NewNoopObserver creates an observer which discards each message.
@@ -27,6 +27,7 @@ func (suite *NoopTestSuite) SetupSuite() {
 		"concurrency-limited": mtglib.NewEventConcurrencyLimited(),
 		"ip-blacklisted":      mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")),
 		"replay-attack":       mtglib.NewEventReplayAttack("connID"),
+		"ip-list-size":        mtglib.NewEventIPListSize(10, true),
 	}
 	suite.ctx = context.Background()
 }
@@ -65,6 +66,8 @@ func (suite *NoopTestSuite) TestObserver() {
 				observer.EventIPBlocklisted(typedEvt)
 			case mtglib.EventReplayAttack:
 				observer.EventReplayAttack(typedEvt)
+			case mtglib.EventIPListSize:
+				observer.EventIPListSize(typedEvt)
 			}
 		})
 	}
@@ -170,7 +170,7 @@ download-concurrency = 2
 # You can provider links here (starts with https:// or http://) or
 # path to a local file, but in this case it should be absolute.
 urls = [
-    # "https://iplists.firehol.org/files/firehol_level1.netset",
+    "https://iplists.firehol.org/files/firehol_level1.netset",
    # "/local.file"
 ]
 # How often do we need to update a blocklist set.
@@ -4,33 +4,35 @@ go 1.17

 require (
 	github.com/OneOfOne/xxhash v1.2.8
-	github.com/alecthomas/kong v0.2.19
-	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a
+	github.com/alecthomas/kong v0.5.0
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gotd/td v0.34.0
 	github.com/jarcoal/httpmock v1.0.8
-	github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e
 	github.com/mccutchen/go-httpbin v1.1.1
-	github.com/panjf2000/ants/v2 v2.4.7
+	github.com/panjf2000/ants/v2 v2.4.8
 	github.com/pelletier/go-toml v1.9.4
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rs/zerolog v1.26.0
+	github.com/rs/zerolog v1.26.1
 	github.com/smira/go-statsd v1.3.2
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
-	golang.org/x/crypto v0.0.0-20211202192323-5770296d904e
+	golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70
 	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
-	golang.org/x/sys v0.0.0-20211124211545-fe61309f8881
+	golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
 	google.golang.org/protobuf v1.27.1 // indirect
 )

-require github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b
+require (
+	github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a
+	github.com/yl2chen/cidranger v1.0.2
+)

 require (
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -44,7 +46,7 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf // indirect
+	github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8 // indirect
 	github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
@@ -37,15 +37,17 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
-github.com/alecthomas/kong v0.2.19 h1:qBDfByO5XgWUXyNB4D6OOhGh5Z1eNOwWayDPQJFNWdc=
-github.com/alecthomas/kong v0.2.19/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ=
+github.com/alecthomas/kong v0.5.0 h1:u8Kdw+eeml93qtMZ04iei0CFYve/WPcA5IFh+9wSskE=
+github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc=
-github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
 github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -163,14 +165,13 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/k0kubun/pp v2.4.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
-github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e h1:1wAVuGu1c+lsdaOPQN+9xoP9+gaIMJV6H0ehGc+K5iA=
-github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -191,10 +192,11 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/panjf2000/ants/v2 v2.4.7 h1:MZnw2JRyTJxFwtaMtUJcwE618wKD04POWk2gwwP4E2M=
-github.com/panjf2000/ants/v2 v2.4.7/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
+github.com/panjf2000/ants/v2 v2.4.8 h1:JgTbolX6K6RreZ4+bfctI0Ifs+3mrE5BIHudQxUDQ9k=
+github.com/panjf2000/ants/v2 v2.4.8/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
@@ -209,8 +211,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -231,8 +234,8 @@ github.com/quasilyte/go-ruleguard/dsl v0.3.2/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQP
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
-github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
+github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc=
+github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc=
 github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -245,21 +248,23 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
 github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf h1:7PflaKRtU4np/epFxRXlFhlzLXZzKFrH5/I4so5Ove0=
 github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf/go.mod h1:CLUSJbazqETbaR+i0YAhXBICV9TrKH93pziccMhmhpM=
-github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b h1:6J/38A0Xmdnjacfie0Udams7OP/GdoExyTipKwuQWjY=
-github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b/go.mod h1:7NloQcrxaZYKURWph5HLxVDlIwMHJXCPkeWPtpftsIg=
+github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8 h1:iYc+JnXtzv6sdMx9Q7OTKkDAn7FhDPDogcjeSfEQcLY=
+github.com/txthinking/runnergroup v0.0.0-20220212043759-8da8edb7dae8/go.mod h1:CLUSJbazqETbaR+i0YAhXBICV9TrKH93pziccMhmhpM=
+github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a h1:BOqgJ4jku0LHPDoR51RD8Mxmo0LHxCzJT/M9MemYdHo=
+github.com/txthinking/socks5 v0.0.0-20220212043548-414499347d4a/go.mod h1:7NloQcrxaZYKURWph5HLxVDlIwMHJXCPkeWPtpftsIg=
 github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe h1:gMWxZxBFRAXqoGkwkYlPX2zvyyKNWJpxOxCrjqJkm5A=
 github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe/go.mod h1:WgqbSEmUYSjEV3B1qmee/PpP2NYEz4bL9/+mF1ma+s4=
 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg=
 github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM=
+github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -285,8 +290,9 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8=
-golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
+golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 h1:syTAU9FwmvzEoIYMqcPHOcVm4H3U5u90WsvuYgwpETU=
+golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -350,6 +356,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
@@ -411,8 +418,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 h1:TyHqChC80pFkXWraUUf6RuB5IqFdQieMLwwCJokV2pc=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1,6 +1,7 @@
 package cli

 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -85,7 +86,10 @@ func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache {
 	)
 }

-func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Network) (mtglib.IPBlocklist, error) {
+func makeIPBlocklist(conf config.ListConfig,
+	logger mtglib.Logger,
+	ntw mtglib.Network,
+	updateCallback ipblocklist.FireholUpdateCallback) (mtglib.IPBlocklist, error) {
 	if !conf.Enabled.Get(false) {
 		return ipblocklist.NewNoop(), nil
 	}
@@ -105,11 +109,14 @@ func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Ne
 		ntw,
 		conf.DownloadConcurrency.Get(1),
 		remoteURLs,
-		localFiles)
+		localFiles,
+		updateCallback)
 	if err != nil {
 		return nil, fmt.Errorf("incorrect parameters for firehol: %w", err)
 	}

+	go firehol.Run(conf.UpdateEach.Get(ipblocklist.DefaultFireholUpdateEach))
+
 	return firehol, nil
 }

@@ -157,12 +164,23 @@ func runProxy(conf *config.Config, version string) error { // nolint: funlen

 	logger.BindJSON("configuration", conf.String()).Debug("configuration")

+	eventStream, err := makeEventStream(conf, logger)
+	if err != nil {
+		return fmt.Errorf("cannot build event stream: %w", err)
+	}
+
 	ntw, err := makeNetwork(conf, version)
 	if err != nil {
 		return fmt.Errorf("cannot build network: %w", err)
 	}

-	blocklist, err := makeIPBlocklist(conf.Defense.Blocklist, logger, ntw)
+	blocklist, err := makeIPBlocklist(
+		conf.Defense.Blocklist,
+		logger.Named("blocklist"),
+		ntw,
+		func(ctx context.Context, size int) {
+			eventStream.Send(ctx, mtglib.NewEventIPListSize(size, true))
+		})
 	if err != nil {
 		return fmt.Errorf("cannot build ip blocklist: %w", err)
 	}
@@ -170,19 +188,20 @@ func runProxy(conf *config.Config, version string) error { // nolint: funlen
 	var whitelist mtglib.IPBlocklist

 	if conf.Defense.Allowlist.Enabled.Get(false) {
-		whlist, err := makeIPBlocklist(conf.Defense.Allowlist, logger, ntw)
+		whlist, err := makeIPBlocklist(
+			conf.Defense.Allowlist,
+			logger.Named("allowlist"),
+			ntw,
+			func(ctx context.Context, size int) {
+				eventStream.Send(ctx, mtglib.NewEventIPListSize(size, false))
+			})
 		if err != nil {
-			return fmt.Errorf("cannot build ip blocklist: %w", err)
+			return fmt.Errorf("cannot build ip allowlist: %w", err)
 		}

 		whitelist = whlist
 	}

-	eventStream, err := makeEventStream(conf, logger)
-	if err != nil {
-		return fmt.Errorf("cannot build event stream: %w", err)
-	}
-
 	opts := mtglib.ProxyOpts{
 		Logger:          logger,
 		Network:         ntw,
@@ -15,16 +15,16 @@ type MtglibNetworkMock struct {
 func (m *MtglibNetworkMock) Dial(network, address string) (essentials.Conn, error) {
 	args := m.Called(network, address)

-	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck
+	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert
 }

 func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) {
 	args := m.Called(ctx, network, address)

-	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck
+	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert
 }

 func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context,
 	network, address string) (essentials.Conn, error)) *http.Client {
-	return m.Called(dialFunc).Get(0).(*http.Client)
+	return m.Called(dialFunc).Get(0).(*http.Client) // nolint: forcetypeassert
 }
@@ -36,11 +36,11 @@ func (n *EssentialsConnMock) CloseWrite() error {
 }

 func (n *EssentialsConnMock) LocalAddr() net.Addr {
-	return n.Called().Get(0).(net.Addr)
+	return n.Called().Get(0).(net.Addr) // nolint: forcetypeassert
 }

 func (n *EssentialsConnMock) RemoteAddr() net.Addr {
-	return n.Called().Get(0).(net.Addr)
+	return n.Called().Get(0).(net.Addr) // nolint: forcetypeassert
 }

 func (n *EssentialsConnMock) SetDeadline(t time.Time) error {
@@ -12,17 +12,20 @@ import (

 	"github.com/9seconds/mtg/v2/ipblocklist/files"
 	"github.com/9seconds/mtg/v2/mtglib"
-	"github.com/kentik/patricia"
-	"github.com/kentik/patricia/bool_tree"
 	"github.com/panjf2000/ants/v2"
+	"github.com/yl2chen/cidranger"
 )

-const (
-	fireholIPv4DefaultCIDR = 32
-	fireholIPv6DefaultCIDR = 128
+var (
+	fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
+
+	fireholIPv4DefaultCIDR = net.CIDRMask(32, 32)   // nolint: gomnd
+	fireholIPv6DefaultCIDR = net.CIDRMask(128, 128) // nolint: gomnd
 )

-var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
+// FireholUpdateCallback defines a signature of the callback that has to be
+// execute when ip list is updated.
+type FireholUpdateCallback func(context.Context, int)

 // Firehol is IPBlocklist which uses lists from FireHOL:
 // https://iplists.firehol.org/
@@ -43,11 +46,12 @@ type Firehol struct {
 	logger      mtglib.Logger
 	updateMutex sync.RWMutex

+	updateCallback FireholUpdateCallback
+	ranger         cidranger.Ranger
+
 	blocklists []files.File

 	workerPool *ants.Pool
-	treeV4     *bool_tree.TreeV4
-	treeV6     *bool_tree.TreeV6
 }

 // Shutdown stop a background update process.
@@ -64,11 +68,12 @@ func (f *Firehol) Contains(ip net.IP) bool {
 	f.updateMutex.RLock()
 	defer f.updateMutex.RUnlock()

-	if ip4 := ip.To4(); ip4 != nil {
-		return f.containsIPv4(ip4)
+	ok, err := f.ranger.Contains(ip)
+	if err != nil {
+		f.logger.BindStr("ip", ip.String()).DebugError("Cannot check if ip is present", err)
 	}

-	return f.containsIPv6(ip.To16())
+	return ok && err == nil
 }

 // Run starts a background update process.
@@ -103,26 +108,6 @@ func (f *Firehol) Run(updateEach time.Duration) {
 	}
 }

-func (f *Firehol) containsIPv4(addr net.IP) bool {
-	ip := patricia.NewIPv4AddressFromBytes(addr, 32) // nolint: gomnd
-
-	if ok, _ := f.treeV4.FindDeepestTag(ip); ok {
-		return true
-	}
-
-	return false
-}
-
-func (f *Firehol) containsIPv6(addr net.IP) bool {
-	ip := patricia.NewIPv6Address(addr, 128) // nolint: gomnd
-
-	if ok, _ := f.treeV6.FindDeepestTag(ip); ok {
-		return true
-	}
-
-	return false
-}
-
 func (f *Firehol) update() {
 	ctx, cancel := context.WithCancel(f.ctx)
 	defer cancel()
@@ -130,9 +115,8 @@ func (f *Firehol) update() {
 	wg := &sync.WaitGroup{}
 	wg.Add(len(f.blocklists))

-	treeMutex := &sync.Mutex{}
-	v4tree := bool_tree.NewTreeV4()
-	v6tree := bool_tree.NewTreeV6()
+	mutex := &sync.Mutex{}
+	ranger := cidranger.NewPCTrieRanger()

 	for _, v := range f.blocklists {
 		go func(file files.File) {
@@ -149,7 +133,7 @@ func (f *Firehol) update() {

 			defer fileContent.Close()

-			if err := f.updateFromFile(treeMutex, v4tree, v6tree, bufio.NewScanner(fileContent)); err != nil {
+			if err := f.updateFromFile(mutex, ranger, bufio.NewScanner(fileContent)); err != nil {
 				logger.WarningError("update has failed", err)
 			}
 		}(v)
@@ -160,15 +144,17 @@ func (f *Firehol) update() {
 	f.updateMutex.Lock()
 	defer f.updateMutex.Unlock()

-	f.treeV4 = v4tree
-	f.treeV6 = v6tree
+	f.ranger = ranger

-	f.logger.Info("blocklist was updated")
+	if f.updateCallback != nil {
+		f.updateCallback(ctx, ranger.Len())
+	}
+
+	f.logger.Info("ip list was updated")
 }

 func (f *Firehol) updateFromFile(mutex sync.Locker,
-	v4tree *bool_tree.TreeV4,
-	v6tree *bool_tree.TreeV6,
+	ranger cidranger.Ranger,
 	scanner *bufio.Scanner) error {
 	for scanner.Scan() {
 		text := scanner.Text()
@@ -179,12 +165,18 @@ func (f *Firehol) updateFromFile(mutex sync.Locker,
 			continue
 		}

-		ip, cidr, err := f.updateParseLine(text)
+		ipnet, err := f.updateParseLine(text)
 		if err != nil {
 			return fmt.Errorf("cannot parse a line: %w", err)
 		}

-		f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree)
+		mutex.Lock()
+		err = ranger.Insert(cidranger.NewBasicRangerEntry(*ipnet))
+		mutex.Unlock()
+
+		if err != nil {
+			return fmt.Errorf("cannot insert %v into ranger: %w", ipnet, err)
+		}
 	}

 	if scanner.Err() != nil {
@@ -194,38 +186,26 @@ func (f *Firehol) updateFromFile(mutex sync.Locker,
 	return nil
 }

-func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) {
-	_, ipnet, err := net.ParseCIDR(text)
-	if err != nil {
-		ipaddr := net.ParseIP(text)
-		if ipaddr == nil {
-			return nil, 0, fmt.Errorf("incorrect ip address %s", text)
-		}
-
-		ip4 := ipaddr.To4()
-		if ip4 != nil {
-			return ip4, fireholIPv4DefaultCIDR, nil
-		}
-
-		return ipaddr.To16(), fireholIPv6DefaultCIDR, nil
+func (f *Firehol) updateParseLine(text string) (*net.IPNet, error) {
+	if _, ipnet, err := net.ParseCIDR(text); err == nil {
+		return ipnet, nil
 	}

-	ones, _ := ipnet.Mask.Size()
-
-	return ipnet.IP, uint(ones), nil
-}
-
-func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint,
-	mutex sync.Locker,
-	v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) {
-	mutex.Lock()
-	defer mutex.Unlock()
-
-	if ip.To4() != nil {
-		v4tree.Set(patricia.NewIPv4AddressFromBytes(ip, cidr), true)
-	} else {
-		v6tree.Set(patricia.NewIPv6Address(ip, cidr), true)
+	ipaddr := net.ParseIP(text)
+	if ipaddr == nil {
+		return nil, fmt.Errorf("incorrect ip address %s", text)
 	}
+
+	mask := fireholIPv4DefaultCIDR
+
+	if ipaddr.To4() == nil {
+		mask = fireholIPv6DefaultCIDR
+	}
+
+	return &net.IPNet{
+		IP:   ipaddr,
+		Mask: mask,
+	}, nil
 }

 // NewFirehol creates a new instance of FireHOL IP blocklist.
@@ -235,7 +215,8 @@ func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint,
 func NewFirehol(logger mtglib.Logger, network mtglib.Network,
 	downloadConcurrency uint,
 	urls []string,
-	localFiles []string) (*Firehol, error) {
+	localFiles []string,
+	updateCallback FireholUpdateCallback) (*Firehol, error) {
 	blocklists := []files.File{}

 	for _, v := range localFiles {
@@ -258,12 +239,13 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network,
 		blocklists = append(blocklists, file)
 	}

-	return NewFireholFromFiles(logger, downloadConcurrency, blocklists)
+	return NewFireholFromFiles(logger, downloadConcurrency, blocklists, updateCallback)
 }

 func NewFireholFromFiles(logger mtglib.Logger,
 	downloadConcurrency uint,
-	blocklists []files.File) (*Firehol, error) {
+	blocklists []files.File,
+	updateCallback FireholUpdateCallback) (*Firehol, error) {
 	if downloadConcurrency == 0 {
 		downloadConcurrency = DefaultFireholDownloadConcurrency
 	}
@@ -272,12 +254,12 @@ func NewFireholFromFiles(logger mtglib.Logger,
 	ctx, cancel := context.WithCancel(context.Background())

 	return &Firehol{
-		ctx:        ctx,
-		ctxCancel:  cancel,
-		logger:     logger.Named("firehol"),
-		treeV4:     bool_tree.NewTreeV4(),
-		treeV6:     bool_tree.NewTreeV6(),
-		workerPool: workerPool,
-		blocklists: blocklists,
+		ctx:            ctx,
+		ctxCancel:      cancel,
+		logger:         logger.Named("firehol"),
+		ranger:         cidranger.NewPCTrieRanger(),
+		workerPool:     workerPool,
+		blocklists:     blocklists,
+		updateCallback: updateCallback,
 	}, nil
 }
@@ -67,7 +67,8 @@ func (suite *FireholTestSuite) TearDownSuite() {
 func (suite *FireholTestSuite) TestLocalFail() {
 	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
 		suite.networkMock, 2,
-		nil, []string{filepath.Join("testdata", "broken_ipset.ipset")})
+		nil, []string{filepath.Join("testdata", "broken_ipset.ipset")},
+		nil)

 	suite.NoError(err)

@@ -85,7 +86,8 @@ func (suite *FireholTestSuite) TestLocalFail() {
 func (suite *FireholTestSuite) TestLocalOk() {
 	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
 		suite.networkMock, 2,
-		nil, []string{filepath.Join("testdata", "good_ipset.ipset")})
+		nil, []string{filepath.Join("testdata", "good_ipset.ipset")},
+		nil)

 	suite.NoError(err)

@@ -103,7 +105,7 @@ func (suite *FireholTestSuite) TestLocalOk() {
 func (suite *FireholTestSuite) TestRemoteFail() {
 	blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(),
 		suite.networkMock, 2,
-		[]string{"https://google.com"}, nil)
+		[]string{"https://google.com"}, nil, nil)

 	suite.NoError(err)

@@ -127,7 +129,7 @@ func (suite *FireholTestSuite) TestMixed() {
 			suite.httpServer.URL,
 		}, []string{
 			filepath.Join("testdata", "good_ipset.ipset"),
-		})
+		}, nil)

 	suite.NoError(err)

@@ -2,13 +2,16 @@ package ipblocklist

 import (
 	"net"
+	"time"

 	"github.com/9seconds/mtg/v2/mtglib"
 )

 type noop struct{}

-func (n noop) Contains(ip net.IP) bool { return false }
+func (n noop) Contains(ip net.IP) bool      { return false }
+func (n noop) Run(updateEach time.Duration) {}
+func (n noop) Shutdown()                    {}

 // NewNoop returns a dummy ipblocklist which allows all incoming
 // connections.
@@ -17,6 +17,13 @@ func (suite *NoopTestSuite) TestOp() {
 	suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10")))
 }

+func (suite *NoopTestSuite) TestRun() {
+	blocklist := ipblocklist.NewNoop()
+
+	blocklist.Run(0)
+	blocklist.Shutdown()
+}
+
 func TestNoop(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &NoopTestSuite{})
@@ -92,6 +92,15 @@ type EventReplayAttack struct {
 	eventBase
 }

+// EventIPListSize is emitted when mtg updates a contents of the ip lists:
+// allowlist or blocklist.
+type EventIPListSize struct {
+	eventBase
+
+	Size        int
+	IsBlockList bool
+}
+
 // NewEventStart creates a new EventStart event.
 func NewEventStart(streamID string, remoteIP net.IP) EventStart {
 	return EventStart{
@@ -176,3 +185,14 @@ func NewEventReplayAttack(streamID string) EventReplayAttack {
 		},
 	}
 }
+
+// NewEventIPListSize creates a new EventIPListSize event.
+func NewEventIPListSize(size int, isBlockList bool) EventIPListSize {
+	return EventIPListSize{
+		eventBase: eventBase{
+			timestamp: time.Now(),
+		},
+		Size:        size,
+		IsBlockList: isBlockList,
+	}
+}
@@ -69,6 +69,15 @@ func (suite *EventsTestSuite) TestEventReplayAttack() {
 	suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond)
 }

+func (suite *EventsTestSuite) TestEventIPListSize() {
+	evt := mtglib.NewEventIPListSize(10, false)
+
+	suite.Empty(evt.StreamID())
+	suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond)
+	suite.Equal(10, evt.Size)
+	suite.False(evt.IsBlockList)
+}
+
 func TestEvents(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &EventsTestSuite{})
@@ -176,6 +176,12 @@ type IPBlocklist interface {
 	// Contains checks if given IP address belongs to this blocklist If.
 	// it is, a connection is terminated                               .
 	Contains(net.IP) bool
+
+	// Run starts a background update procedure for a blocklist
+	Run(time.Duration)
+
+	// Shutdown stops a blocklist. It is assumed that none will access it after.
+	Shutdown()
 }

 // Event is a data structure which is populated during mtg request
@@ -12,7 +12,7 @@ var bytesBufferPool = sync.Pool{
 }

 func acquireBytesBuffer() *bytes.Buffer {
-	return bytesBufferPool.Get().(*bytes.Buffer)
+	return bytesBufferPool.Get().(*bytes.Buffer) // nolint: forcetypeassert
 }

 func releaseBytesBuffer(b *bytes.Buffer) {
@@ -11,7 +11,7 @@ var recordPool = sync.Pool{
 }

 func AcquireRecord() *Record {
-	return recordPool.Get().(*Record)
+	return recordPool.Get().(*Record) // nolint: forcetypeassert
 }

 func ReleaseRecord(r *Record) {
@@ -21,7 +21,7 @@ var (
 )

 func acquireSha256Hasher() hash.Hash {
-	return sha256HasherPool.Get().(hash.Hash)
+	return sha256HasherPool.Get().(hash.Hash) // nolint: forcetypeassert
 }

 func releaseSha256Hasher(h hash.Hash) {
@@ -30,7 +30,7 @@ func releaseSha256Hasher(h hash.Hash) {
 }

 func acquireBytesBuffer() *bytes.Buffer {
-	return bytesBufferPool.Get().(*bytes.Buffer)
+	return bytesBufferPool.Get().(*bytes.Buffer) // nolint: forcetypeassert
 }

 func releaseBytesBuffer(buf *bytes.Buffer) {
@@ -69,7 +69,7 @@ func (suite *ServerHandshakeTestSuite) TestSendToTelegram() {
 		Once().
 		Run(func(args mock.Arguments) {
 			message := make([]byte, len(messageToTelegram))
-			suite.decryptor.XORKeyStream(message, args.Get(0).([]byte))
+			suite.decryptor.XORKeyStream(message, args.Get(0).([]byte)) // nolint: forcetypeassert
 			suite.Equal(messageToTelegram, message)
 		})

@@ -89,7 +89,7 @@ func (suite *ServerHandshakeTestSuite) TestRecieveFromTelegram() {
 		Run(func(args mock.Arguments) {
 			message := make([]byte, len(messageFromTelegram))
 			suite.encryptor.XORKeyStream(message, messageFromTelegram)
-			copy(args.Get(0).([]byte), message)
+			copy(args.Get(0).([]byte), message) // nolint: forcetypeassert
 		})

 	n, err := suite.proxyConn.Read(buffer)
@@ -11,7 +11,7 @@ var copyBufferPool = sync.Pool{
 }

 func acquireCopyBuffer() *[]byte {
-	return copyBufferPool.Get().(*[]byte)
+	return copyBufferPool.Get().(*[]byte) // nolint: forcetypeassert
 }

 func releaseCopyBuffer(buf *[]byte) {
@@ -2,10 +2,13 @@ package telegram

 import (
 	"context"
+	"errors"

 	"github.com/9seconds/mtg/v2/essentials"
 )

+var errNoAddresses = errors.New("no addresses")
+
 type preferIP uint8

 const (
@@ -28,10 +28,9 @@ func (t Telegram) Dial(ctx context.Context, dc int) (essentials.Conn, error) {
 		addresses = append(t.pool.getV6(dc), t.pool.getV4(dc)...)
 	}

-	var (
-		conn essentials.Conn
-		err  error
-	)
+	var conn essentials.Conn
+
+	err := errNoAddresses

 	for _, v := range addresses {
 		conn, err = t.dialer.DialContext(ctx, v.network, v.address)
@@ -106,7 +106,7 @@ func (p *Proxy) Serve(listener net.Listener) error { // nolint: cyclop
 			}
 		}

-		ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP
+		ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP // nolint: forcetypeassert
 		logger := p.logger.BindStr("ip", ipAddr.String())

 		if p.whitelist != nil && !p.whitelist.Contains(ipAddr) {
@@ -144,6 +144,12 @@ func (p *Proxy) Shutdown() {
 	p.ctxCancel()
 	p.streamWaitGroup.Wait()
 	p.workerPool.Release()
+
+	if p.whitelist != nil {
+		p.whitelist.Shutdown()
+	}
+
+	p.blocklist.Shutdown()
 }

 func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool {
@@ -249,7 +255,10 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error {
 	}

 	p.eventStream.Send(ctx,
-		NewEventConnectedToDC(ctx.streamID, conn.RemoteAddr().(*net.TCPAddr).IP, ctx.dc))
+		NewEventConnectedToDC(ctx.streamID,
+			conn.RemoteAddr().(*net.TCPAddr).IP, // nolint: forcetypeassert
+			ctx.dc),
+	)

 	return nil
 }
@@ -310,7 +319,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {

 	pool, err := ants.NewPoolWithFunc(opts.getConcurrency(),
 		func(arg interface{}) {
-			proxy.ServeConn(arg.(essentials.Conn))
+			proxy.ServeConn(arg.(essentials.Conn)) // nolint: forcetypeassert
 		},
 		ants.WithLogger(opts.getLogger("ants")),
 		ants.WithNonblocking(true))
@@ -49,7 +49,7 @@ func (s *streamContext) Close() {
 }

 func (s *streamContext) ClientIP() net.IP {
-	return s.clientConn.RemoteAddr().(*net.TCPAddr).IP
+	return s.clientConn.RemoteAddr().(*net.TCPAddr).IP // nolint: forcetypeassert
 }

 func newStreamContext(ctx context.Context, logger Logger, clientConn essentials.Conn) *streamContext {
@@ -36,7 +36,7 @@ func (d *defaultDialer) DialContext(ctx context.Context, network, address string
 		return nil, fmt.Errorf("cannot set socket options: %w", err)
 	}

-	return conn.(essentials.Conn), nil
+	return conn.(essentials.Conn), nil // nolint: forcetypeassert
 }

 // NewDefaultDialer build a new dialer which dials bypassing proxies
@@ -14,11 +14,11 @@ type DialerMock struct {
 func (d *DialerMock) Dial(network, address string) (essentials.Conn, error) {
 	args := d.Called(network, address)

-	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck
+	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert
 }

 func (d *DialerMock) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) {
 	args := d.Called(ctx, network, address)

-	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck
+	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert
 }
@@ -22,13 +22,13 @@ type DialerMock struct {
 func (d *DialerMock) Dial(network, address string) (essentials.Conn, error) {
 	args := d.Called(network, address)

-	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck
+	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert
 }

 func (d *DialerMock) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) {
 	args := d.Called(ctx, network, address)

-	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck
+	return args.Get(0).(essentials.Conn), args.Error(1) // nolint: wrapcheck, forcetypeassert
 }

 type HTTPServerTestSuite struct {
@@ -10,13 +10,13 @@ import (
 //
 // bufferSize setting is deprecated and ignored.
 func SetClientSocketOptions(conn net.Conn, bufferSize int) error {
-	return setCommonSocketOptions(conn.(*net.TCPConn))
+	return setCommonSocketOptions(conn.(*net.TCPConn)) // nolint: forcetypeassert
 }

 // SetServerSocketOptions tunes a TCP socket that represents a connection to
 // remote server like Telegram or fronting domain (but not end user).
 func SetServerSocketOptions(conn net.Conn, bufferSize int) error {
-	return setCommonSocketOptions(conn.(*net.TCPConn))
+	return setCommonSocketOptions(conn.(*net.TCPConn)) // nolint: forcetypeassert
 }

 func setCommonSocketOptions(conn *net.TCPConn) error {
@@ -89,6 +89,13 @@ const (
 	//     Type: counter
 	MetricReplayAttacks = "replay_attacks"

+	// MetricIPListSize defines a metric for the size of the the ip list.
+	//
+	//     Type: gauge
+	//     Tags:
+	//       ip_list | 'allowlist' or 'blocklist'
+	MetricIPListSize = "iplist_size"
+
 	// TagIPFamily defines a name of the 'ip_family' tag and all values.
 	TagIPFamily = "ip_family"

@@ -114,4 +121,13 @@ const (
 	// TagDirectionFromClient defines that traffic is sent from a client to
 	// Telegram.
 	TagDirectionFromClient = "from_client"
+
+	// TagIPList defines a name of the 'ip_list' and all values.
+	TagIPList = "ip_list"
+
+	// TagIPListAllow defines a value of 'ip_list' of allowlist.
+	TagIPListAllow = "allowlist"
+
+	// TagIPListBlock defines a value of 'ip_list' of blocklist.
+	TagIPListBlock = "blocklist"
 )
@@ -11,7 +11,7 @@ var streamInfoPool = sync.Pool{
 }

 func acquireStreamInfo() *streamInfo {
-	return streamInfoPool.Get().(*streamInfo)
+	return streamInfoPool.Get().(*streamInfo) // nolint: forcetypeassert
 }

 func releaseStreamInfo(info *streamInfo) {
@@ -118,6 +118,15 @@ func (p prometheusProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) {
 	p.factory.metricReplayAttacks.Inc()
 }

+func (p prometheusProcessor) EventIPListSize(evt mtglib.EventIPListSize) {
+	tag := TagIPListBlock
+	if !evt.IsBlockList {
+		tag = TagIPListAllow
+	}
+
+	p.factory.metricIPListSize.WithLabelValues(tag).Set(float64(evt.Size))
+}
+
 func (p prometheusProcessor) Shutdown() {
 	for k, v := range p.streams {
 		releaseStreamInfo(v)
@@ -137,6 +146,7 @@ type PrometheusFactory struct {
 	metricClientConnections         *prometheus.GaugeVec
 	metricTelegramConnections       *prometheus.GaugeVec
 	metricDomainFrontingConnections *prometheus.GaugeVec
+	metricIPListSize                *prometheus.GaugeVec

 	metricTelegramTraffic       *prometheus.CounterVec
 	metricDomainFrontingTraffic *prometheus.CounterVec
@@ -197,6 +207,11 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint
 			Name:      MetricDomainFrontingConnections,
 			Help:      "A number of connections which talk to front domain.",
 		}, []string{TagIPFamily}),
+		metricIPListSize: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: metricPrefix,
+			Name:      MetricIPListSize,
+			Help:      "A size of the ip list (blocklist or allowlist)",
+		}, []string{TagIPList}),

 		metricTelegramTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Namespace: metricPrefix,
@@ -234,6 +249,7 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint
 	registry.MustRegister(factory.metricClientConnections)
 	registry.MustRegister(factory.metricTelegramConnections)
 	registry.MustRegister(factory.metricDomainFrontingConnections)
+	registry.MustRegister(factory.metricIPListSize)

 	registry.MustRegister(factory.metricTelegramTraffic)
 	registry.MustRegister(factory.metricDomainFrontingTraffic)
@@ -169,6 +169,18 @@ func (suite *PrometheusTestSuite) TestEventReplayAttack() {
 	suite.Contains(data, `mtg_replay_attacks 1`)
 }

+func (suite *PrometheusTestSuite) TestEventIPListSize() {
+	suite.prometheus.EventIPListSize(mtglib.NewEventIPListSize(10, false))
+	suite.prometheus.EventIPListSize(mtglib.NewEventIPListSize(3, true))
+
+	time.Sleep(100 * time.Millisecond)
+
+	data, err := suite.Get()
+	suite.NoError(err)
+	suite.Contains(data, `mtg_iplist_size{ip_list="allowlist"} 10`)
+	suite.Contains(data, `mtg_iplist_size{ip_list="blocklist"} 3`)
+}
+
 func TestPrometheus(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &PrometheusTestSuite{})
@@ -121,6 +121,15 @@ func (s statsdProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) {
 	s.client.Incr(MetricReplayAttacks, 1)
 }

+func (s statsdProcessor) EventIPListSize(evt mtglib.EventIPListSize) {
+	tag := TagIPListBlock
+	if !evt.IsBlockList {
+		tag = TagIPListAllow
+	}
+
+	s.client.Gauge(MetricIPListSize, int64(evt.Size), statsd.StringTag(TagIPList, tag))
+}
+
 func (s statsdProcessor) Shutdown() {
 	events := make([]mtglib.EventFinish, 0, len(s.streams))

@@ -196,6 +196,22 @@ func (suite *StatsdTestSuite) TestEventReplayAttack() {
 	suite.Equal("mtg.replay_attacks:1|c", suite.statsdServer.String())
 }

+func (suite *StatsdTestSuite) TestEventIPListSizeAllowlist() {
+	suite.statsd.EventIPListSize(mtglib.NewEventIPListSize(10, false))
+
+	time.Sleep(statsdSleepTime)
+	suite.Contains(suite.statsdServer.String(), "mtg.iplist_size:10|g")
+	suite.Contains(suite.statsdServer.String(), "allowlist")
+}
+
+func (suite *StatsdTestSuite) TestEventIPListSizeBlocklist() {
+	suite.statsd.EventIPListSize(mtglib.NewEventIPListSize(10, true))
+
+	time.Sleep(statsdSleepTime)
+	suite.Contains(suite.statsdServer.String(), "mtg.iplist_size:10|g")
+	suite.Contains(suite.statsdServer.String(), "blocklist")
+}
+
 func TestStatsd(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &StatsdTestSuite{})
Author	SHA1	Message	Date
9seconds	6d89f14c9b	Merge remote-tracking branch 'origin/master' into stable	2022-03-11 18:03:35 +03:00
9seconds	442e2da330	Revert "Consider test modules for update-deps" This reverts commit `880dd04728`.	2022-03-11 17:55:12 +03:00
9seconds	a9b3560b25	Remove obsolete Golang 1.16	2022-03-11 17:27:01 +03:00
9seconds	880dd04728	Consider test modules for update-deps	2022-03-11 17:23:53 +03:00
9seconds	8ea1aa3f5e	Update dependencies	2022-03-11 17:19:15 +03:00
Sergey ArkhipovandGitHub	4e5a83cfe9	Merge pull request #256 from 9seconds/golangcilint-1.44.2 Update golangci-lint to 1.44.2	2022-03-11 17:13:58 +03:00
9seconds	5282ca26f3	Update golangci-lint to 1.44.2	2022-03-11 17:08:33 +03:00
Sergey ArkhipovandGitHub	2077db1f1e	Merge pull request #255 from 9seconds/iplistsize Add iplist_size metric	2022-03-11 16:42:40 +03:00
9seconds	1a9bc80091	Unexport an error that is not required	2022-03-11 16:22:19 +03:00
9seconds	30170b9413	Add iplist_size metric	2022-03-11 16:20:11 +03:00
Sergey ArkhipovandGitHub	4687a7c899	Merge pull request #254 from 9seconds/cidranger Use cidrranger instead of patricia	2022-03-11 16:16:52 +03:00
9seconds	d467fba674	Use cidrranger instead of patricia	2022-03-11 16:12:45 +03:00
Sergey ArkhipovandGitHub	6e447b1d59	Merge pull request #253 from 9seconds/cannot-dial-err More correct error if no addresses are found	2022-03-11 11:33:36 +03:00
9seconds	b15a8ec4a5	More correct error if no addresses are found	2022-03-11 11:19:15 +03:00
Sergey ArkhipovandGitHub	6bef4df091	Merge pull request #252 from 9seconds/broken-whitelist Fix broken ip lists	2022-03-11 11:05:11 +03:00
9seconds	0ce0c668b9	Fix broken whitelists	2022-03-11 10:50:15 +03:00
Sergey ArkhipovandGitHub	01e201365b	Merge pull request #249 from themegabyte/master modified docker commands for ease of use	2022-03-09 09:49:07 +03:00
Shayan	0b52367a82	modified docker commands for ease of use	2022-03-07 13:52:57 +05:00
Sergey ArkhipovandGitHub	25c842daf1	Fix typo	2022-03-02 10:25:46 +03:00
9seconds	4c543aaea2	Merge remote-tracking branch 'origin/stable'	2021-12-03 09:36:47 +03:00