ScuroNeko/mtg
1
0
Fork 0
mirror of https://github.com/ScuroNeko/mtg.git synced 2026-08-31 17:24:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #254 from 9seconds/cidranger

Browse Source 
Use cidrranger instead of patricia
This commit is contained in:
Sergey Arkhipov
2022-03-11 16:16:52 +03:00
committed by GitHub
parent 6e447b1d59 d467fba674
commit 4687a7c899
3 changed files with 48 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+4 -2
View File
@@ -12,7 +12,6 @@ require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/gotd/td v0.34.0
github.com/jarcoal/httpmock v1.0.8
github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e
github.com/mccutchen/go-httpbin v1.1.1
github.com/panjf2000/ants/v2 v2.4.7
github.com/pelletier/go-toml v1.9.4
@@ -30,7 +29,10 @@ require (
google.golang.org/protobuf v1.27.1 // indirect
)
require github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b
require (
github.com/txthinking/socks5 v0.0.0-20211121111206-e03c1217a50b
github.com/yl2chen/cidranger v1.0.2
)
require (
github.com/beorn7/perks v1.0.1 // indirect
go.sum
+2 -3
View File
@@ -169,8 +169,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/k0kubun/pp v2.4.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e h1:1wAVuGu1c+lsdaOPQN+9xoP9+gaIMJV6H0ehGc+K5iA=
github.com/kentik/patricia v0.0.0-20210909164817-21603333b70e/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -245,7 +243,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -260,6 +257,8 @@ github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe h1:gMWxZxBFRAXqoGkwkY
github.com/txthinking/x v0.0.0-20210326105829-476fab902fbe/go.mod h1:WgqbSEmUYSjEV3B1qmee/PpP2NYEz4bL9/+mF1ma+s4=
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43 h1:QEePdg0ty2r0t1+qwfZmQ4OOl/MB2UXIeJSpIZv56lg=
github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM=
github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
ipblocklist/firehol.go
+42 -72
View File
@@ -12,17 +12,16 @@ import (
"github.com/9seconds/mtg/v2/ipblocklist/files"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/kentik/patricia"
"github.com/kentik/patricia/bool_tree"
"github.com/panjf2000/ants/v2"
"github.com/yl2chen/cidranger"
)
const (
fireholIPv4DefaultCIDR = 32
fireholIPv6DefaultCIDR = 128
)
var (
fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`)
fireholIPv4DefaultCIDR = net.CIDRMask(32, 32) // nolint: gomnd
fireholIPv6DefaultCIDR = net.CIDRMask(128, 128) // nolint: gomnd
)
// Firehol is IPBlocklist which uses lists from FireHOL:
// https://iplists.firehol.org/
@@ -43,11 +42,11 @@ type Firehol struct {
logger mtglib.Logger
updateMutex sync.RWMutex
ranger cidranger.Ranger
blocklists []files.File
workerPool *ants.Pool
treeV4 *bool_tree.TreeV4
treeV6 *bool_tree.TreeV6
}
// Shutdown stop a background update process.
@@ -64,11 +63,12 @@ func (f *Firehol) Contains(ip net.IP) bool {
f.updateMutex.RLock()
defer f.updateMutex.RUnlock()
if ip4 := ip.To4(); ip4 != nil {
return f.containsIPv4(ip4)
ok, err := f.ranger.Contains(ip)
if err != nil {
f.logger.BindStr("ip", ip.String()).DebugError("Cannot check if ip is present", err)
}
return f.containsIPv6(ip.To16())
return ok && err == nil
}
// Run starts a background update process.
@@ -103,26 +103,6 @@ func (f *Firehol) Run(updateEach time.Duration) {
}
}
func (f *Firehol) containsIPv4(addr net.IP) bool {
ip := patricia.NewIPv4AddressFromBytes(addr, 32) // nolint: gomnd
if ok, _ := f.treeV4.FindDeepestTag(ip); ok {
return true
}
return false
}
func (f *Firehol) containsIPv6(addr net.IP) bool {
ip := patricia.NewIPv6Address(addr, 128) // nolint: gomnd
if ok, _ := f.treeV6.FindDeepestTag(ip); ok {
return true
}
return false
}
func (f *Firehol) update() {
ctx, cancel := context.WithCancel(f.ctx)
defer cancel()
@@ -131,8 +111,7 @@ func (f *Firehol) update() {
wg.Add(len(f.blocklists))
treeMutex := &sync.Mutex{}
v4tree := bool_tree.NewTreeV4()
v6tree := bool_tree.NewTreeV6()
ranger := cidranger.NewPCTrieRanger()
for _, v := range f.blocklists {
go func(file files.File) {
@@ -149,7 +128,7 @@ func (f *Firehol) update() {
defer fileContent.Close()
if err := f.updateFromFile(treeMutex, v4tree, v6tree, bufio.NewScanner(fileContent)); err != nil {
if err := f.updateFromFile(treeMutex, ranger, bufio.NewScanner(fileContent)); err != nil {
logger.WarningError("update has failed", err)
}
}(v)
@@ -160,15 +139,13 @@ func (f *Firehol) update() {
f.updateMutex.Lock()
defer f.updateMutex.Unlock()
f.treeV4 = v4tree
f.treeV6 = v6tree
f.ranger = ranger
f.logger.Info("ip list was updated")
}
func (f *Firehol) updateFromFile(mutex sync.Locker,
v4tree *bool_tree.TreeV4,
v6tree *bool_tree.TreeV6,
ranger cidranger.Ranger,
scanner *bufio.Scanner) error {
for scanner.Scan() {
text := scanner.Text()
@@ -179,12 +156,18 @@ func (f *Firehol) updateFromFile(mutex sync.Locker,
continue
}
ip, cidr, err := f.updateParseLine(text)
ipnet, err := f.updateParseLine(text)
if err != nil {
return fmt.Errorf("cannot parse a line: %w", err)
}
f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree)
mutex.Lock()
err = ranger.Insert(cidranger.NewBasicRangerEntry(*ipnet))
mutex.Unlock()
if err != nil {
return fmt.Errorf("cannot insert %v into ranger: %w", ipnet, err)
}
}
if scanner.Err() != nil {
@@ -194,38 +177,26 @@ func (f *Firehol) updateFromFile(mutex sync.Locker,
return nil
}
func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) {
_, ipnet, err := net.ParseCIDR(text)
if err != nil {
ipaddr := net.ParseIP(text)
if ipaddr == nil {
return nil, 0, fmt.Errorf("incorrect ip address %s", text)
}
ip4 := ipaddr.To4()
if ip4 != nil {
return ip4, fireholIPv4DefaultCIDR, nil
}
return ipaddr.To16(), fireholIPv6DefaultCIDR, nil
func (f *Firehol) updateParseLine(text string) (*net.IPNet, error) {
if _, ipnet, err := net.ParseCIDR(text); err == nil {
return ipnet, nil
}
ones, _ := ipnet.Mask.Size()
return ipnet.IP, uint(ones), nil
}
func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint,
mutex sync.Locker,
v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) {
mutex.Lock()
defer mutex.Unlock()
if ip.To4() != nil {
v4tree.Set(patricia.NewIPv4AddressFromBytes(ip, cidr), true)
} else {
v6tree.Set(patricia.NewIPv6Address(ip, cidr), true)
ipaddr := net.ParseIP(text)
if ipaddr == nil {
return nil, fmt.Errorf("incorrect ip address %s", text)
}
mask := fireholIPv4DefaultCIDR
if ipaddr.To4() == nil {
mask = fireholIPv6DefaultCIDR
}
return &net.IPNet{
IP: ipaddr,
Mask: mask,
}, nil
}
// NewFirehol creates a new instance of FireHOL IP blocklist.
@@ -275,8 +246,7 @@ func NewFireholFromFiles(logger mtglib.Logger,
ctx: ctx,
ctxCancel: cancel,
logger: logger.Named("firehol"),
treeV4: bool_tree.NewTreeV4(),
treeV6: bool_tree.NewTreeV6(),
ranger: cidranger.NewPCTrieRanger(),
workerPool: workerPool,
blocklists: blocklists,
}, nil