From d467fba674c615d8e660d1dedd4c2eca05a230bd Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 11 Mar 2022 13:28:47 +0300 Subject: [PATCH] Use cidrranger instead of patricia --- go.mod | 6 ++- go.sum | 5 +- ipblocklist/firehol.go | 114 +++++++++++++++-------------------------- 3 files changed, 48 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index ce52f2a..babf62c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/gotd/td v0.34.0 github.com/jarcoal/httpmock v1.0.8 - github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e github.com/mccutchen/go-httpbin v1.1.1 github.com/panjf2000/ants/v2 v2.4.7 github.com/pelletier/go-toml v1.9.4 @@ -30,7 +29,10 @@ require ( google.golang.org/protobuf v1.27.1 // indirect ) -require github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b +require ( + github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b + github.com/yl2chen/cidranger v1.0.2 +) require ( github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 6e70dd9..35ef06d 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.4.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e h1:1wAVuGu1c+lsdaOPQN+9xoP9+gaIMJV6H0ehGc+K5iA= -github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -245,7 +243,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -260,6 +257,8 @@ github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe h1:gMWxZxBFRAXqoGkwkY github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe/go.mod h1:WgqbSEmUYSjEV3B1qmee/PpP2NYEz4bL9/+mF1ma+s4= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg= github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 05d34cb..b2c3cce 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -12,17 +12,16 @@ import ( "github.com/9seconds/mtg/v2/ipblocklist/files" "github.com/9seconds/mtg/v2/mtglib" - "github.com/kentik/patricia" - "github.com/kentik/patricia/bool_tree" "github.com/panjf2000/ants/v2" + "github.com/yl2chen/cidranger" ) -const ( - fireholIPv4DefaultCIDR = 32 - fireholIPv6DefaultCIDR = 128 -) +var ( + fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) -var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) + fireholIPv4DefaultCIDR = net.CIDRMask(32, 32) // nolint: gomnd + fireholIPv6DefaultCIDR = net.CIDRMask(128, 128) // nolint: gomnd +) // Firehol is IPBlocklist which uses lists from FireHOL: // https://iplists.firehol.org/ @@ -43,11 +42,11 @@ type Firehol struct { logger mtglib.Logger updateMutex sync.RWMutex + ranger cidranger.Ranger + blocklists []files.File workerPool *ants.Pool - treeV4 *bool_tree.TreeV4 - treeV6 *bool_tree.TreeV6 } // Shutdown stop a background update process. @@ -64,11 +63,12 @@ func (f *Firehol) Contains(ip net.IP) bool { f.updateMutex.RLock() defer f.updateMutex.RUnlock() - if ip4 := ip.To4(); ip4 != nil { - return f.containsIPv4(ip4) + ok, err := f.ranger.Contains(ip) + if err != nil { + f.logger.BindStr("ip", ip.String()).DebugError("Cannot check if ip is present", err) } - return f.containsIPv6(ip.To16()) + return ok && err == nil } // Run starts a background update process. @@ -103,26 +103,6 @@ func (f *Firehol) Run(updateEach time.Duration) { } } -func (f *Firehol) containsIPv4(addr net.IP) bool { - ip := patricia.NewIPv4AddressFromBytes(addr, 32) // nolint: gomnd - - if ok, _ := f.treeV4.FindDeepestTag(ip); ok { - return true - } - - return false -} - -func (f *Firehol) containsIPv6(addr net.IP) bool { - ip := patricia.NewIPv6Address(addr, 128) // nolint: gomnd - - if ok, _ := f.treeV6.FindDeepestTag(ip); ok { - return true - } - - return false -} - func (f *Firehol) update() { ctx, cancel := context.WithCancel(f.ctx) defer cancel() @@ -131,8 +111,7 @@ func (f *Firehol) update() { wg.Add(len(f.blocklists)) treeMutex := &sync.Mutex{} - v4tree := bool_tree.NewTreeV4() - v6tree := bool_tree.NewTreeV6() + ranger := cidranger.NewPCTrieRanger() for _, v := range f.blocklists { go func(file files.File) { @@ -149,7 +128,7 @@ func (f *Firehol) update() { defer fileContent.Close() - if err := f.updateFromFile(treeMutex, v4tree, v6tree, bufio.NewScanner(fileContent)); err != nil { + if err := f.updateFromFile(treeMutex, ranger, bufio.NewScanner(fileContent)); err != nil { logger.WarningError("update has failed", err) } }(v) @@ -160,15 +139,13 @@ func (f *Firehol) update() { f.updateMutex.Lock() defer f.updateMutex.Unlock() - f.treeV4 = v4tree - f.treeV6 = v6tree + f.ranger = ranger f.logger.Info("ip list was updated") } func (f *Firehol) updateFromFile(mutex sync.Locker, - v4tree *bool_tree.TreeV4, - v6tree *bool_tree.TreeV6, + ranger cidranger.Ranger, scanner *bufio.Scanner) error { for scanner.Scan() { text := scanner.Text() @@ -179,12 +156,18 @@ func (f *Firehol) updateFromFile(mutex sync.Locker, continue } - ip, cidr, err := f.updateParseLine(text) + ipnet, err := f.updateParseLine(text) if err != nil { return fmt.Errorf("cannot parse a line: %w", err) } - f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree) + mutex.Lock() + err = ranger.Insert(cidranger.NewBasicRangerEntry(*ipnet)) + mutex.Unlock() + + if err != nil { + return fmt.Errorf("cannot insert %v into ranger: %w", ipnet, err) + } } if scanner.Err() != nil { @@ -194,38 +177,26 @@ func (f *Firehol) updateFromFile(mutex sync.Locker, return nil } -func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) { - _, ipnet, err := net.ParseCIDR(text) - if err != nil { - ipaddr := net.ParseIP(text) - if ipaddr == nil { - return nil, 0, fmt.Errorf("incorrect ip address %s", text) - } - - ip4 := ipaddr.To4() - if ip4 != nil { - return ip4, fireholIPv4DefaultCIDR, nil - } - - return ipaddr.To16(), fireholIPv6DefaultCIDR, nil +func (f *Firehol) updateParseLine(text string) (*net.IPNet, error) { + if _, ipnet, err := net.ParseCIDR(text); err == nil { + return ipnet, nil } - ones, _ := ipnet.Mask.Size() - - return ipnet.IP, uint(ones), nil -} - -func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, - mutex sync.Locker, - v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) { - mutex.Lock() - defer mutex.Unlock() - - if ip.To4() != nil { - v4tree.Set(patricia.NewIPv4AddressFromBytes(ip, cidr), true) - } else { - v6tree.Set(patricia.NewIPv6Address(ip, cidr), true) + ipaddr := net.ParseIP(text) + if ipaddr == nil { + return nil, fmt.Errorf("incorrect ip address %s", text) } + + mask := fireholIPv4DefaultCIDR + + if ipaddr.To4() == nil { + mask = fireholIPv6DefaultCIDR + } + + return &net.IPNet{ + IP: ipaddr, + Mask: mask, + }, nil } // NewFirehol creates a new instance of FireHOL IP blocklist. @@ -275,8 +246,7 @@ func NewFireholFromFiles(logger mtglib.Logger, ctx: ctx, ctxCancel: cancel, logger: logger.Named("firehol"), - treeV4: bool_tree.NewTreeV4(), - treeV6: bool_tree.NewTreeV6(), + ranger: cidranger.NewPCTrieRanger(), workerPool: workerPool, blocklists: blocklists, }, nil