From 87a720dd58ff0c1fccaac61ff46c4286e0022450 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 14:29:39 +0300 Subject: [PATCH] Do not use ristretto Memory usage is too big --- go.mod | 5 +- go.sum | 7 --- network/dns_resolver.go | 85 +++++++++++++++------------ network/dns_resolver_internal_test.go | 2 +- network/network.go | 2 +- 5 files changed, 50 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 9112aba..b9f9f50 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/d4l3k/messagediff v1.2.1 // indirect - github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/jarcoal/httpmock v1.0.8 github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 github.com/libp2p/go-reuseport v0.0.2 @@ -23,7 +22,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 - golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect + golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 ) diff --git a/go.sum b/go.sum index 12d7451..0cf083c 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -39,8 +38,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -58,10 +55,7 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -283,7 +277,6 @@ github.com/smira/go-statsd v1.3.2 h1:1EeuzxNZ/TD9apbTOFSM9nulqfcsQFmT4u1A2DREabI github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= diff --git a/network/dns_resolver.go b/network/dns_resolver.go index 3205b8f..23a4c37 100644 --- a/network/dns_resolver.go +++ b/network/dns_resolver.go @@ -2,27 +2,38 @@ package network import ( "net/http" + "sync" "time" doh "github.com/babolivier/go-doh-client" - "github.com/dgraph-io/ristretto" ) -const ( - dnsResolverSize = 1024 * 1024 // 1mb - dnsResolverKeepTime = 10 * time.Minute -) +const dnsResolverKeepTime = 10 * time.Minute -type dnsResolver struct { - resolver doh.Resolver - cache *ristretto.Cache +type dnsResolverCacheEntry struct { + ips []string + createdAt time.Time } -func (d dnsResolver) LookupA(hostname string) []string { - key := "\x00." + hostname +func (c dnsResolverCacheEntry) Ok() bool { + return time.Since(c.createdAt) < dnsResolverKeepTime +} - if value, ok := d.cache.Get(key); ok { - return value.([]string) +type dnsResolver struct { + resolver doh.Resolver + cache map[string]dnsResolverCacheEntry + cacheMutex sync.RWMutex +} + +func (d *dnsResolver) LookupA(hostname string) []string { + key := "\x00" + hostname + + d.cacheMutex.RLock() + entry, ok := d.cache[key] + d.cacheMutex.RUnlock() + + if ok && entry.Ok() { + return entry.ips } var ips []string @@ -32,17 +43,26 @@ func (d dnsResolver) LookupA(hostname string) []string { ips = append(ips, v.IP4) } - d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime) + d.cacheMutex.Lock() + d.cache[key] = dnsResolverCacheEntry{ + ips: ips, + createdAt: time.Now(), + } + d.cacheMutex.Unlock() } return ips } -func (d dnsResolver) LookupAAAA(hostname string) []string { - key := "\x01." + hostname +func (d *dnsResolver) LookupAAAA(hostname string) []string { + key := "\x01" + hostname - if value, ok := d.cache.Get(key); ok { - return value.([]string) + d.cacheMutex.RLock() + entry, ok := d.cache[key] + d.cacheMutex.RUnlock() + + if ok && entry.Ok() { + return entry.ips } var ips []string @@ -52,37 +72,24 @@ func (d dnsResolver) LookupAAAA(hostname string) []string { ips = append(ips, v.IP6) } - d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime) + d.cacheMutex.Lock() + d.cache[key] = dnsResolverCacheEntry{ + ips: ips, + createdAt: time.Now(), + } + d.cacheMutex.Unlock() } return ips } -func newDNSResolver(hostname string, httpClient *http.Client) dnsResolver { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 10 * dnsResolverSize, // nolint: gomnd // taken from official doc as a best practice value - MaxCost: dnsResolverSize, - BufferItems: 64, // nolint: gomnd // taken from official doc as a best practice value - Cost: func(value interface{}) int64 { - var cost int64 - - for _, v := range value.([]string) { - cost += int64(len([]byte(v))) - } - - return cost - }, - }) - if err != nil { - panic(err) - } - - return dnsResolver{ +func newDNSResolver(hostname string, httpClient *http.Client) *dnsResolver { + return &dnsResolver{ resolver: doh.Resolver{ Host: hostname, Class: doh.IN, HTTPClient: httpClient, }, - cache: cache, + cache: map[string]dnsResolverCacheEntry{}, } } diff --git a/network/dns_resolver_internal_test.go b/network/dns_resolver_internal_test.go index a5934e8..ba54c0c 100644 --- a/network/dns_resolver_internal_test.go +++ b/network/dns_resolver_internal_test.go @@ -12,7 +12,7 @@ import ( type DNSResolverTestSuite struct { suite.Suite - d dnsResolver + d *dnsResolver } func (suite *DNSResolverTestSuite) TestLookupA() { diff --git a/network/network.go b/network/network.go index c65c282..147290b 100644 --- a/network/network.go +++ b/network/network.go @@ -25,9 +25,9 @@ func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, erro type network struct { dialer Dialer - dns dnsResolver httpTimeout time.Duration userAgent string + dns *dnsResolver } func (n *network) Dial(protocol, address string) (net.Conn, error) {