From 0274b3436a6d27cd90ad4ef1cd64148edcd9a9d2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 16:09:21 +0300 Subject: [PATCH] Add documentation for ipblocklist --- antireplay/stable_bloom_filter.go | 13 +++++- internal/cli/proxy.go | 2 +- ipblocklist/firehol.go | 73 +++++++++++++++++++++---------- ipblocklist/init.go | 14 +++++- ipblocklist/noop.go | 2 + mtglib/init.go | 6 +++ 6 files changed, 82 insertions(+), 28 deletions(-) diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index 4e075b7..61f9934 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -30,9 +30,18 @@ func (s *stableBloomFilter) SeenBefore(digest []byte) bool { // hardcore math which proves that if you choose this P correctly, you // can maintain the same error rate for a stream of elements. // -// byteSize is the number of bytes you want to give to a bloom filter . -// errorRate is desired false-positive error rate . +// byteSize is the number of bytes you want to give to a bloom filter. +// errorRate is desired false-positive error rate. If you want to use +// default values, please pass 0 for byteSize and <0 for errorRate. func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { + if byteSize == 0 { + byteSize = DefaultStableBloomFilterMaxSize + } + + if errorRate < 0 { + errorRate = DefaultStableBloomFilterErrorRate + } + sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd sf.SetHash(xxhash.New64()) diff --git a/internal/cli/proxy.go b/internal/cli/proxy.go index fed863a..713afcb 100644 --- a/internal/cli/proxy.go +++ b/internal/cli/proxy.go @@ -133,7 +133,7 @@ func (c *Proxy) setupIPBlocklist(opts *mtglib.ProxyOpts) error { return err // nolint: wrapcheck } - go firehol.Run(c.Config.Defense.Blocklist.UpdateEach.Value(ipblocklist.DefaultUpdateEach)) + go firehol.Run(c.Config.Defense.Blocklist.UpdateEach.Value(ipblocklist.DefaultFireholUpdateEach)) opts.IPBlocklist = firehol diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 20dd0d8..947134b 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -27,6 +27,19 @@ const ( var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) +// Firehol is IPBlocklist which uses lists from FireHOL: +// https://iplists.firehol.org/ +// +// It can use both local files and remote URLs. This is not necessary +// that blocklists should be taken from this website, we expect only +// compatible formats here. +// +// Example of the format: +// +// # this is a comment +// # to ignore +// 127.0.0.1 # you can specify an IP +// 10.0.0.0/8 # or cidr type Firehol struct { ctx context.Context ctxCancel context.CancelFunc @@ -44,6 +57,12 @@ type Firehol struct { treeV6 *bool_tree.TreeV6 } +// Shutdown stop a background update process. +func (f *Firehol) Shutdown() { + f.ctxCancel() +} + +// Contains is given IP list can be found in FireHOL blocklists. func (f *Firehol) Contains(ip net.IP) bool { if ip == nil { return true @@ -61,27 +80,15 @@ func (f *Firehol) Contains(ip net.IP) bool { return f.containsIPv6(ip.To16()) } -func (f *Firehol) containsIPv4(addr net.IP) bool { - ip := patricia.NewIPv4AddressFromBytes(addr, 32) - - if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { - return true - } - - return false -} - -func (f *Firehol) containsIPv6(addr net.IP) bool { - ip := patricia.NewIPv6Address(addr, 128) - - if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { - return true - } - - return false -} - +// Run starts a background update process. +// +// This is a blocking method so you probably want to run it in a +// goroutine. func (f *Firehol) Run(updateEach time.Duration) { + if updateEach == 0 { + updateEach = DefaultFireholUpdateEach + } + ticker := time.NewTicker(updateEach) defer func() { @@ -113,8 +120,24 @@ func (f *Firehol) Run(updateEach time.Duration) { } } -func (f *Firehol) Shutdown() { - f.ctxCancel() +func (f *Firehol) containsIPv4(addr net.IP) bool { + ip := patricia.NewIPv4AddressFromBytes(addr, 32) + + if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) containsIPv6(addr net.IP) bool { + ip := patricia.NewIPv6Address(addr, 128) + + if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false } func (f *Firehol) update() error { // nolint: funlen, cyclop @@ -302,6 +325,10 @@ func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, return nil } +// NewFirehol creates a new instance of FireHOL IP blocklist. +// +// This method does not start an update process so please execute Run +// when it is necessary. func NewFirehol(logger mtglib.Logger, network mtglib.Network, downloadConcurrency uint, remoteURLs []string, @@ -326,7 +353,7 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, } if downloadConcurrency == 0 { - downloadConcurrency = DefaultDownloadConcurrency + downloadConcurrency = DefaultFireholDownloadConcurrency } workerPool, _ := ants.NewPool(int(downloadConcurrency)) diff --git a/ipblocklist/init.go b/ipblocklist/init.go index cfa8df8..f5a06d3 100644 --- a/ipblocklist/init.go +++ b/ipblocklist/init.go @@ -1,8 +1,18 @@ +// Package ipblocklist contains default implementation of the +// IPBlocklist for mtg. +// +// Please check documentation for mtglib.IPBlocklist interface to get an +// idea of this abstraction. package ipblocklist import "time" const ( - DefaultDownloadConcurrency = 1 - DefaultUpdateEach = 12 * time.Hour + // DefaultFireholDownloadConcurrency defines a default max number of + // concurrent downloads of ip blocklists for Firehol. + DefaultFireholDownloadConcurrency = 1 + + // DefaultFireholUpdateEach defines a default time period when + // Firehol requests updates of the blocklists. + DefaultFireholUpdateEach = 6 * time.Hour ) diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go index dcbed19..2df31cb 100644 --- a/ipblocklist/noop.go +++ b/ipblocklist/noop.go @@ -10,6 +10,8 @@ type noop struct{} func (n noop) Contains(ip net.IP) bool { return false } +// NewNoop returns a dummy ipblocklist which allows all incoming +// connections. func NewNoop() mtglib.IPBlocklist { return noop{} } diff --git a/mtglib/init.go b/mtglib/init.go index 860e334..ca69ed5 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -69,6 +69,12 @@ type AntiReplayCache interface { SeenBefore(data []byte) bool } +// IPBlocklist filters requests based on IP address. +// +// If this filter has an IP address, then mtg closes a request without +// reading anything from a socket. It also does not give such request to +// a worker pool, so in worst cases you can expect that you invoke this +// object more frequent than defined proxy concurrency. type IPBlocklist interface { Contains(net.IP) bool }