From b53ead43720a7e44b7d43163a4938a113c8343b7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 16 Mar 2022 09:37:30 +0300 Subject: [PATCH 01/12] Build image on go 1.18 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 08003f7..b700b9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ############################################################################### # BUILD STAGE -FROM golang:1.17-alpine AS build +FROM golang:1.18-alpine AS build RUN set -x \ && apk --no-cache --update add \ From 63b425f24526880a7e53b85f3f1db025df0f7441 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 16 Mar 2022 09:37:52 +0300 Subject: [PATCH 02/12] Use Go 1.18 only for CI This is required due to fuzzing --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d813b55..900ecad 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: strategy: matrix: go_version: - - ^1.17 + - ^1.18 steps: - name: Checkout uses: actions/checkout@v2 From cbe5b8c94ecc944b78e11e32c48cae6e1c8e189b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 16 Mar 2022 09:39:58 +0300 Subject: [PATCH 03/12] Update go.mod to 1.18 --- go.mod | 2 +- go.sum | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a245cd2..3f9fb38 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/9seconds/mtg/v2 -go 1.17 +go 1.18 require ( github.com/OneOfOne/xxhash v1.2.8 diff --git a/go.sum b/go.sum index b4021d6..45b5193 100644 --- a/go.sum +++ b/go.sum @@ -416,7 +416,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= From 93755521805271544c0abdc53e7a10d22c777be6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 14:58:03 +0300 Subject: [PATCH 04/12] Add fuzz tests for obfuscated2 --- .../client_handshake_fuzz_internal_test.go | 32 ++++++++++ mtglib/internal/obfuscated2/init_test.go | 54 +++++++++++++++++ .../server_handshake_fuzz_internal_test.go | 30 ++++++++++ .../obfuscated2/server_handshake_fuzz_test.go | 58 +++++++++++++++++++ .../obfuscated2/server_handshake_test.go | 57 +++--------------- 5 files changed, 183 insertions(+), 48 deletions(-) create mode 100644 mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go create mode 100644 mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go create mode 100644 mtglib/internal/obfuscated2/server_handshake_fuzz_test.go diff --git a/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go b/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go new file mode 100644 index 0000000..1c9685d --- /dev/null +++ b/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go @@ -0,0 +1,32 @@ +package obfuscated2 + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +var FuzzClientHandshakeSecret = []byte{1,2,3} + +func FuzzClientHandshake(f *testing.F) { + f.Add([]byte{1,2,3}) + + f.Fuzz(func(t *testing.T, frame []byte) { + data := bytes.NewReader(frame) + + if _, _, _, err := ClientHandshake(FuzzClientHandshakeSecret, data); err != nil { + return + } + + handshake := clientHandhakeFrame{} + require.Len(t, frame, handshakeFrameLen) + + copy(handshake.data[:], frame) + + decryptor := handshake.decryptor(FuzzClientHandshakeSecret) + decryptor.XORKeyStream(handshake.data[:], handshake.data[:]) + + require.Equal(t, handshakeConnectionType, handshake.connectionType()) + }) +} diff --git a/mtglib/internal/obfuscated2/init_test.go b/mtglib/internal/obfuscated2/init_test.go index 001c84b..0cc5c17 100644 --- a/mtglib/internal/obfuscated2/init_test.go +++ b/mtglib/internal/obfuscated2/init_test.go @@ -1,12 +1,20 @@ package obfuscated2_test import ( + "bytes" + "crypto/aes" + "crypto/cipher" "encoding/base64" "encoding/json" "fmt" "os" "path/filepath" "strings" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/stretchr/testify/require" ) type snapshotBytes struct { @@ -50,6 +58,14 @@ type SnapshotTestSuite struct { snapshots map[string]*Obfuscated2Snapshot } +type ServerHandshakeTestData struct { + connMock *testlib.EssentialsConnMock + + proxyConn obfuscated2.Conn + encryptor cipher.Stream + decryptor cipher.Stream +} + func (suite *SnapshotTestSuite) IngestSnapshots(dirname, namePrefix string) error { suite.snapshots = map[string]*Obfuscated2Snapshot{} @@ -81,3 +97,41 @@ func (suite *SnapshotTestSuite) IngestSnapshots(dirname, namePrefix string) erro return nil } + +func NewServerHandshakeTestData(t *testing.T) ServerHandshakeTestData { + buf := &bytes.Buffer{} + connMock := &testlib.EssentialsConnMock{} + + handshakeEnc, handshakeDec, err := obfuscated2.ServerHandshake(buf) + require.NoError(t, err) + + serverEncrypted := buf.Bytes() + decBlock, _ := aes.NewCipher(serverEncrypted[8 : 8+32]) + decryptor := cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16]) + + serverDecrypted := make([]byte, len(serverEncrypted)) + decryptor.XORKeyStream(serverDecrypted, serverEncrypted) + + require.Equal(t, "3d3d3Q", + base64.RawStdEncoding.EncodeToString(serverDecrypted[8+32+16:8+32+16+4])) + + serverEncryptedReverted := make([]byte, len(serverEncrypted)) + + for i := 0; i < 32+16; i++ { + serverEncryptedReverted[8+i] = serverEncrypted[8+32+16-1-i] + } + + encBlock, _ := aes.NewCipher(serverEncryptedReverted[8 : 8+32]) + encryptor := cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16]) + + return ServerHandshakeTestData{ + connMock: connMock, + proxyConn: obfuscated2.Conn{ + Conn: connMock, + Encryptor: handshakeEnc, + Decryptor: handshakeDec, + }, + encryptor: encryptor, + decryptor: decryptor, + } +} diff --git a/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go b/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go new file mode 100644 index 0000000..ca9ad09 --- /dev/null +++ b/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go @@ -0,0 +1,30 @@ +package obfuscated2 + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/assert" +) + +func FuzzServerGenerateHandshakeFrame(f *testing.F) { + f.Fuzz(func(t *testing.T, arg int) { + frame := generateServerHanshakeFrame() + + assert.NotEqualValues(t, 0xef, frame.data[0]) + + firstBytes := binary.LittleEndian.Uint32(frame.data[:4]) + assert.NotEqualValues(t, 0x44414548, firstBytes) + assert.NotEqualValues(t, 0x54534f50, firstBytes) + assert.NotEqualValues(t, 0x20544547, firstBytes) + assert.NotEqualValues(t, 0x4954504f, firstBytes) + assert.NotEqualValues(t, 0xeeeeeeee, firstBytes) + + assert.NotEqualValues( + t, + 0, + frame.data[4] | frame.data[5] | frame.data[6] | frame.data[7]) + + assert.Equal(t, handshakeConnectionType, frame.connectionType()) + }) +} diff --git a/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go b/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go new file mode 100644 index 0000000..be1cea6 --- /dev/null +++ b/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go @@ -0,0 +1,58 @@ +package obfuscated2_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func FuzzServerSend(f *testing.F) { + f.Add([]byte{1,2,3,4,5}) + + f.Fuzz(func(t *testing.T, data []byte) { + handshakeData := NewServerHandshakeTestData(t) + + handshakeData.connMock. + On("Write", mock.Anything). + Return(len(data), nil). + Once(). + Run(func(args mock.Arguments) { + message := make([]byte, len(data)) + handshakeData.decryptor.XORKeyStream(message, args.Get(0).([]byte)) + assert.Equal(t, message, data) + }) + + n, err := handshakeData.proxyConn.Write(data) + + assert.EqualValues(t, len(data), n) + assert.NoError(t, err) + handshakeData.connMock.AssertExpectations(t) + }) +} + +func FuzzServerReceive(f *testing.F) { + f.Add([]byte{1,2,3,4,5}) + + f.Fuzz(func(t *testing.T, data []byte) { + handshakeData := NewServerHandshakeTestData(t) + buffer := make([]byte, len(data)) + + handshakeData.connMock. + On("Read", mock.Anything). + Return(len(data), nil). + Once(). + Run(func(args mock.Arguments) { + message := make([]byte, len(data)) + handshakeData.encryptor.XORKeyStream(message, data) + copy(args.Get(0).([]byte), message) + }) + + n, err := handshakeData.proxyConn.Read(buffer) + + assert.EqualValues(t, len(data), n) + assert.NoError(t, err) + assert.Equal(t, data, buffer) + handshakeData.connMock.AssertExpectations(t) + }) +} diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go index 5321c9a..09943ae 100644 --- a/mtglib/internal/obfuscated2/server_handshake_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -1,14 +1,8 @@ package obfuscated2_test import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "encoding/base64" "testing" - "github.com/9seconds/mtg/v2/internal/testlib" - "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -16,64 +10,31 @@ import ( type ServerHandshakeTestSuite struct { suite.Suite - connMock *testlib.EssentialsConnMock - proxyConn obfuscated2.Conn - encryptor cipher.Stream - decryptor cipher.Stream + data ServerHandshakeTestData } func (suite *ServerHandshakeTestSuite) SetupTest() { - buf := &bytes.Buffer{} - suite.connMock = &testlib.EssentialsConnMock{} - - encryptor, decryptor, err := obfuscated2.ServerHandshake(buf) - suite.NoError(err) - - suite.proxyConn = obfuscated2.Conn{ - Conn: suite.connMock, - Encryptor: encryptor, - Decryptor: decryptor, - } - - serverEncrypted := buf.Bytes() - - decBlock, _ := aes.NewCipher(serverEncrypted[8 : 8+32]) - suite.decryptor = cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16]) - - serverDecrypted := make([]byte, len(serverEncrypted)) - suite.decryptor.XORKeyStream(serverDecrypted, serverEncrypted) - - suite.Equal("3d3d3Q", - base64.RawStdEncoding.EncodeToString(serverDecrypted[8+32+16:8+32+16+4])) - - serverEncryptedReverted := make([]byte, len(serverEncrypted)) - - for i := 0; i < 32+16; i++ { - serverEncryptedReverted[8+i] = serverEncrypted[8+32+16-1-i] - } - - encBlock, _ := aes.NewCipher(serverEncryptedReverted[8 : 8+32]) - suite.encryptor = cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16]) + suite.data = NewServerHandshakeTestData(suite.T()) } func (suite *ServerHandshakeTestSuite) TearDownTest() { - suite.connMock.AssertExpectations(suite.T()) + suite.data.connMock.AssertExpectations(suite.T()) } func (suite *ServerHandshakeTestSuite) TestSendToTelegram() { messageToTelegram := []byte{10, 11, 12, 13, 14, 'a'} - suite.connMock. + suite.data.connMock. On("Write", mock.Anything). Return(len(messageToTelegram), nil). Once(). Run(func(args mock.Arguments) { message := make([]byte, len(messageToTelegram)) - suite.decryptor.XORKeyStream(message, args.Get(0).([]byte)) // nolint: forcetypeassert + suite.data.decryptor.XORKeyStream(message, args.Get(0).([]byte)) // nolint: forcetypeassert suite.Equal(messageToTelegram, message) }) - n, err := suite.proxyConn.Write(messageToTelegram) + n, err := suite.data.proxyConn.Write(messageToTelegram) suite.EqualValues(len(messageToTelegram), n) suite.NoError(err) } @@ -82,17 +43,17 @@ func (suite *ServerHandshakeTestSuite) TestRecieveFromTelegram() { messageFromTelegram := []byte{10, 11, 12, 13, 14, 'a'} buffer := make([]byte, len(messageFromTelegram)) - suite.connMock. + suite.data.connMock. On("Read", mock.Anything). Return(len(messageFromTelegram), nil). Once(). Run(func(args mock.Arguments) { message := make([]byte, len(messageFromTelegram)) - suite.encryptor.XORKeyStream(message, messageFromTelegram) + suite.data.encryptor.XORKeyStream(message, messageFromTelegram) copy(args.Get(0).([]byte), message) // nolint: forcetypeassert }) - n, err := suite.proxyConn.Read(buffer) + n, err := suite.data.proxyConn.Read(buffer) suite.EqualValues(len(messageFromTelegram), n) suite.NoError(err) suite.Equal(messageFromTelegram, buffer) From e0850869ba3e709f76924a10edbc63d168f875b1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 14:58:08 +0300 Subject: [PATCH 05/12] Linting --- internal/cli/run_proxy.go | 3 ++- internal/testlib/mtglib_network_mock.go | 3 ++- ipblocklist/firehol.go | 9 ++++++--- .../client_handshake_fuzz_internal_test.go | 4 ++-- mtglib/internal/obfuscated2/init_test.go | 2 +- .../server_handshake_fuzz_internal_test.go | 2 +- .../obfuscated2/server_handshake_fuzz_test.go | 4 ++-- network/circuit_breaker.go | 15 ++++++++++----- network/network.go | 9 ++++++--- stats/statsd.go | 3 ++- 10 files changed, 34 insertions(+), 20 deletions(-) diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index ae06a80..01ec512 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -89,7 +89,8 @@ func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache { func makeIPBlocklist(conf config.ListConfig, logger mtglib.Logger, ntw mtglib.Network, - updateCallback ipblocklist.FireholUpdateCallback) (mtglib.IPBlocklist, error) { + updateCallback ipblocklist.FireholUpdateCallback, +) (mtglib.IPBlocklist, error) { if !conf.Enabled.Get(false) { return ipblocklist.NewNoop(), nil } diff --git a/internal/testlib/mtglib_network_mock.go b/internal/testlib/mtglib_network_mock.go index a6d08bf..7eb21a4 100644 --- a/internal/testlib/mtglib_network_mock.go +++ b/internal/testlib/mtglib_network_mock.go @@ -25,6 +25,7 @@ func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address st } func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, - network, address string) (essentials.Conn, error)) *http.Client { + network, address string) (essentials.Conn, error), +) *http.Client { return m.Called(dialFunc).Get(0).(*http.Client) // nolint: forcetypeassert } diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index d6fa3e8..efe57f0 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -155,7 +155,8 @@ func (f *Firehol) update() { func (f *Firehol) updateFromFile(mutex sync.Locker, ranger cidranger.Ranger, - scanner *bufio.Scanner) error { + scanner *bufio.Scanner, +) error { for scanner.Scan() { text := scanner.Text() text = fireholRegexpComment.ReplaceAllLiteralString(text, "") @@ -216,7 +217,8 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, downloadConcurrency uint, urls []string, localFiles []string, - updateCallback FireholUpdateCallback) (*Firehol, error) { + updateCallback FireholUpdateCallback, +) (*Firehol, error) { blocklists := []files.File{} for _, v := range localFiles { @@ -245,7 +247,8 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, func NewFireholFromFiles(logger mtglib.Logger, downloadConcurrency uint, blocklists []files.File, - updateCallback FireholUpdateCallback) (*Firehol, error) { + updateCallback FireholUpdateCallback, +) (*Firehol, error) { if downloadConcurrency == 0 { downloadConcurrency = DefaultFireholDownloadConcurrency } diff --git a/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go b/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go index 1c9685d..01a8a1e 100644 --- a/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go +++ b/mtglib/internal/obfuscated2/client_handshake_fuzz_internal_test.go @@ -7,10 +7,10 @@ import ( "github.com/stretchr/testify/require" ) -var FuzzClientHandshakeSecret = []byte{1,2,3} +var FuzzClientHandshakeSecret = []byte{1, 2, 3} func FuzzClientHandshake(f *testing.F) { - f.Add([]byte{1,2,3}) + f.Add([]byte{1, 2, 3}) f.Fuzz(func(t *testing.T, frame []byte) { data := bytes.NewReader(frame) diff --git a/mtglib/internal/obfuscated2/init_test.go b/mtglib/internal/obfuscated2/init_test.go index 0cc5c17..50d031c 100644 --- a/mtglib/internal/obfuscated2/init_test.go +++ b/mtglib/internal/obfuscated2/init_test.go @@ -127,7 +127,7 @@ func NewServerHandshakeTestData(t *testing.T) ServerHandshakeTestData { return ServerHandshakeTestData{ connMock: connMock, proxyConn: obfuscated2.Conn{ - Conn: connMock, + Conn: connMock, Encryptor: handshakeEnc, Decryptor: handshakeDec, }, diff --git a/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go b/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go index ca9ad09..125fd9c 100644 --- a/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_fuzz_internal_test.go @@ -23,7 +23,7 @@ func FuzzServerGenerateHandshakeFrame(f *testing.F) { assert.NotEqualValues( t, 0, - frame.data[4] | frame.data[5] | frame.data[6] | frame.data[7]) + frame.data[4]|frame.data[5]|frame.data[6]|frame.data[7]) assert.Equal(t, handshakeConnectionType, frame.connectionType()) }) diff --git a/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go b/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go index be1cea6..fb5df81 100644 --- a/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go @@ -8,7 +8,7 @@ import ( ) func FuzzServerSend(f *testing.F) { - f.Add([]byte{1,2,3,4,5}) + f.Add([]byte{1, 2, 3, 4, 5}) f.Fuzz(func(t *testing.T, data []byte) { handshakeData := NewServerHandshakeTestData(t) @@ -32,7 +32,7 @@ func FuzzServerSend(f *testing.F) { } func FuzzServerReceive(f *testing.F) { - f.Add([]byte{1,2,3,4,5}) + f.Add([]byte{1, 2, 3, 4, 5}) f.Fuzz(func(t *testing.T, data []byte) { handshakeData := NewServerHandshakeTestData(t) diff --git a/network/circuit_breaker.go b/network/circuit_breaker.go index 91e1317..e5f2702 100644 --- a/network/circuit_breaker.go +++ b/network/circuit_breaker.go @@ -36,7 +36,8 @@ func (c *circuitBreakerDialer) Dial(network, address string) (essentials.Conn, e } func (c *circuitBreakerDialer) DialContext(ctx context.Context, - network, address string) (essentials.Conn, error) { + network, address string, +) (essentials.Conn, error) { switch atomic.LoadUint32(&c.state) { case circuitBreakerStateClosed: return c.doClosed(ctx, network, address) @@ -48,7 +49,8 @@ func (c *circuitBreakerDialer) DialContext(ctx context.Context, } func (c *circuitBreakerDialer) doClosed(ctx context.Context, - network, address string) (essentials.Conn, error) { + network, address string, +) (essentials.Conn, error) { conn, err := c.Dialer.DialContext(ctx, network, address) select { @@ -80,7 +82,8 @@ func (c *circuitBreakerDialer) doClosed(ctx context.Context, } func (c *circuitBreakerDialer) doHalfOpened(ctx context.Context, - network, address string) (essentials.Conn, error) { + network, address string, +) (essentials.Conn, error) { if !atomic.CompareAndSwapUint32(&c.halfOpenAttempts, 0, 1) { return nil, ErrCircuitBreakerOpened } @@ -174,14 +177,16 @@ func (c *circuitBreakerDialer) stopTimer(timerRef **time.Timer) { } func (c *circuitBreakerDialer) ensureTimer(timerRef **time.Timer, - timeout time.Duration, callback func()) { + timeout time.Duration, callback func(), +) { if *timerRef == nil { *timerRef = time.AfterFunc(timeout, callback) } } func newCircuitBreakerDialer(baseDialer Dialer, - openThreshold uint32, halfOpenTimeout, resetFailuresTimeout time.Duration) Dialer { + openThreshold uint32, halfOpenTimeout, resetFailuresTimeout time.Duration, +) Dialer { cb := &circuitBreakerDialer{ Dialer: baseDialer, stateMutexChan: make(chan bool, 1), diff --git a/network/network.go b/network/network.go index e42508f..6aecd6a 100644 --- a/network/network.go +++ b/network/network.go @@ -61,7 +61,8 @@ func (n *network) DialContext(ctx context.Context, protocol, address string) (es } func (n *network) MakeHTTPClient(dialFunc func(ctx context.Context, - network, address string) (essentials.Conn, error)) *http.Client { + network, address string) (essentials.Conn, error), +) *http.Client { if dialFunc == nil { dialFunc = n.DialContext } @@ -123,7 +124,8 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { // It brings simple DNS cache and DNS-Over-HTTPS when necessary. func NewNetwork(dialer Dialer, userAgent, dohHostname string, - httpTimeout time.Duration) (mtglib.Network, error) { + httpTimeout time.Duration, +) (mtglib.Network, error) { switch { case httpTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", httpTimeout) @@ -146,7 +148,8 @@ func NewNetwork(dialer Dialer, func makeHTTPClient(userAgent string, timeout time.Duration, - dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client { + dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error), +) *http.Client { return &http.Client{ Timeout: timeout, Transport: networkHTTPTransport{ diff --git a/stats/statsd.go b/stats/statsd.go index bd3f6fb..a19f561 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -171,7 +171,8 @@ func (s StatsdFactory) Make() events.Observer { // // Valid tagFormats are 'datadog', 'influxdb' and 'graphite'. func NewStatsd(address string, log logger.StdLikeLogger, - metricPrefix, tagFormat string) (StatsdFactory, error) { + metricPrefix, tagFormat string, +) (StatsdFactory, error) { options := []statsd.Option{ statsd.MetricPrefix(metricPrefix), statsd.Logger(log), From d8b14dc765d4fdf14eae97a790e1843089eb5466 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 15:42:58 +0300 Subject: [PATCH 06/12] Add fuzzing for client hello --- .../faketls/client_hello_fuzz_test.go | 21 +++++++++++++++++++ mtglib/internal/faketls/init.go | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 mtglib/internal/faketls/client_hello_fuzz_test.go diff --git a/mtglib/internal/faketls/client_hello_fuzz_test.go b/mtglib/internal/faketls/client_hello_fuzz_test.go new file mode 100644 index 0000000..84db20d --- /dev/null +++ b/mtglib/internal/faketls/client_hello_fuzz_test.go @@ -0,0 +1,21 @@ +package faketls_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/stretchr/testify/require" +) + +var FuzzClientHelloSecret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} + +func FuzzClientHello(f *testing.F) { + f.Add([]byte{1, 2, 3}) + + f.Fuzz(func(t *testing.T, frame []byte) { + _, err := faketls.ParseClientHello(FuzzClientHelloSecret, frame) + + // a probability of having != err is almost negligible + require.Error(t, err) + }) +} diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go index 3a305a5..b514b21 100644 --- a/mtglib/internal/faketls/init.go +++ b/mtglib/internal/faketls/init.go @@ -19,7 +19,7 @@ const ( // ClientHelloMinLen is a minimal possible length of // ClientHello record. - ClientHelloMinLen = 4 + ClientHelloMinLen = 6 // WelcomePacketRandomOffset is an offset of random in ServerHello // packet (including record envelope). From 75357b3e3c40cb81eaca26400fe5f056fbcaec9f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 16:45:12 +0300 Subject: [PATCH 07/12] Add fuzz to makefile --- Makefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Makefile b/Makefile index 354cdec..33de934 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ VERSION_DATE := $(shell date -Ru) VERSION_TAG := $(shell git describe --tags --always) COMMON_BUILD_FLAGS := -trimpath -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'" +FUZZ_FLAGS := -fuzztime=120s + GOBIN := $(ROOT_DIR)/.bin GOTOOL := env "GOBIN=$(GOBIN)" "PATH=$(ROOT_DIR)/.bin:$(PATH)" @@ -96,3 +98,26 @@ install-tools-goreleaser: .bin .PHONY: update-deps update-deps: @go get -u && go mod tidy -go=1.17 + +.PHONY: fuzz +fuzz: fuzz-ClientHello fuzz-ServerGenerateHandshakeFrame fuzz-ClientHandshake fuzz-ServerReceive fuzz-ServerSend + +.PHONY: fuzz-ClientHello +fuzz-ClientHello: + @go test -fuzz=FuzzClientHello $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/faketls" + +.PHONY: fuzz-ServerGenerateHandshakeFrame +fuzz-ServerGenerateHandshakeFrame: + @go test -fuzz=FuzzServerGenerateHandshakeFrame $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2" + +.PHONY: fuzz-ClientHandshake +fuzz-ClientHandshake: + @go test -fuzz=FuzzClientHandshake $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2" + +.PHONY: fuzz-ServerReceive +fuzz-ServerReceive: + @go test -fuzz=FuzzServerReceive $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2" + +.PHONY: fuzz-ServerSend +fuzz-ServerSend: + @go test -fuzz=FuzzServerSend $(FUZZ_FLAGS) "$(ROOT_DIR)/mtglib/internal/obfuscated2" From 2b8c7825caed9bbed2b2bb92b8876fe38e3cd07a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 16:52:24 +0300 Subject: [PATCH 08/12] Run fuzzing in GitHub actions --- .github/workflows/ci.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 900ecad..7521d3e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -69,6 +69,38 @@ jobs: with: file: ./coverage.txt + fuzz: + name: Fuzzing + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Setup Go + uses: actions/setup-go@v2 + with: + go-version: ^1.18 + + - name: Cache fuzz results + uses: actions/cache@v2 + with: + path: ~/.cache/go-build/fuzz + key: ${{ runner.os }}-go-${{ hashFiles('**/*_fuzz_test.go', '**/*_fuzz_internal_test.go') }} + restore-keys: ${{ runner.os }}-go- + + - name: Cache dependencies + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Run fuzzing + run: make -j4 fuzz + lint: name: Lint runs-on: ubuntu-latest From 02ad052c496672df651563cf501763dea4e8eff1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 17:25:14 +0300 Subject: [PATCH 09/12] Update update-deps make command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 33de934..1df17f4 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ install-tools-goreleaser: .bin .PHONY: update-deps update-deps: - @go get -u && go mod tidy -go=1.17 + @go get -u && go mod tidy -go=1.18 .PHONY: fuzz fuzz: fuzz-ClientHello fuzz-ServerGenerateHandshakeFrame fuzz-ClientHandshake fuzz-ServerReceive fuzz-ServerSend From 5df1f594ccfa3cf659d85f221b89c1ce50e60b61 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 18 Mar 2022 17:52:04 +0300 Subject: [PATCH 10/12] Use embedded buildinfo --- Makefile | 6 ++---- main.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 1df17f4..cefbd34 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,8 @@ APP_NAME := $(IMAGE_NAME) GOLANGCI_LINT_VERSION := v1.44.2 -VERSION_GO := $(shell go version) -VERSION_DATE := $(shell date -Ru) -VERSION_TAG := $(shell git describe --tags --always) -COMMON_BUILD_FLAGS := -trimpath -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'" +VERSION := $(shell git describe --exact-match HEAD 2>/dev/null || git describe --tags --always) +COMMON_BUILD_FLAGS := -trimpath -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION)'" FUZZ_FLAGS := -fuzztime=120s diff --git a/main.go b/main.go index 019278b..dd795c7 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,10 @@ package main import ( + "fmt" "math/rand" + "runtime/debug" + "strconv" "time" "github.com/9seconds/mtg/v2/internal/cli" @@ -26,6 +29,32 @@ func main() { panic(err) } + if buildInfo, ok := debug.ReadBuildInfo(); ok { + vcsCommit := "" + vcsDate := time.Now() + vcsDirty := "" + + for _, setting := range buildInfo.Settings { + switch setting.Key { + case "vcs.time": + vcsDate, _ = time.Parse(time.RFC3339, setting.Value) + case "vcs.revision": + vcsCommit = setting.Value + case "vcs.modified": + if isDirty, _ := strconv.ParseBool(setting.Value); isDirty { + vcsDirty = " [dirty]" + } + } + } + + version = fmt.Sprintf("%s (%s: %s on %s%s)", + version, + buildInfo.GoVersion, + vcsDate.Format(time.RFC3339), + vcsCommit, + vcsDirty) + } + cli := &cli.CLI{} ctx := kong.Parse(cli, kong.Vars{ "version": version, From 1575b82688d7d4390dc162633bd9ccbec49d3897 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 19 Mar 2022 13:46:57 +0300 Subject: [PATCH 11/12] Update golangci-lint --- .github/workflows/ci.yaml | 11 ++++++++--- .golangci.toml | 2 +- Makefile | 2 +- .../obfuscated2/server_handshake_fuzz_test.go | 4 ++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7521d3e..27fa54e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -111,10 +111,15 @@ jobs: with: submodules: recursive - - name: Run linter - uses: golangci/golangci-lint-action@v2 + - name: Setup Go + uses: actions/setup-go@v2 with: - version: v1.44.2 + go-version: ^1.18 + + - name: Run linter + uses: golangci/golangci-lint-action@v3 + with: + version: v1.45.0 docker: name: Docker diff --git a/.golangci.toml b/.golangci.toml index 353b1d9..a595c75 100644 --- a/.golangci.toml +++ b/.golangci.toml @@ -9,4 +9,4 @@ format = "colored-line-number" [linters] enable-all = true -disable = ["ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"] +disable = ["thelper", "ireturn", "varnamelen", "gochecknoglobals", "gas", "goerr113", "exhaustivestruct", "containedctx"] diff --git a/Makefile b/Makefile index cefbd34..7533f8e 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) IMAGE_NAME := mtg APP_NAME := $(IMAGE_NAME) -GOLANGCI_LINT_VERSION := v1.44.2 +GOLANGCI_LINT_VERSION := v1.45.0 VERSION := $(shell git describe --exact-match HEAD 2>/dev/null || git describe --tags --always) COMMON_BUILD_FLAGS := -trimpath -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION)'" diff --git a/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go b/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go index fb5df81..d129f37 100644 --- a/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_fuzz_test.go @@ -19,7 +19,7 @@ func FuzzServerSend(f *testing.F) { Once(). Run(func(args mock.Arguments) { message := make([]byte, len(data)) - handshakeData.decryptor.XORKeyStream(message, args.Get(0).([]byte)) + handshakeData.decryptor.XORKeyStream(message, args.Get(0).([]byte)) // nolint: forcetypeassert assert.Equal(t, message, data) }) @@ -45,7 +45,7 @@ func FuzzServerReceive(f *testing.F) { Run(func(args mock.Arguments) { message := make([]byte, len(data)) handshakeData.encryptor.XORKeyStream(message, data) - copy(args.Get(0).([]byte), message) + copy(args.Get(0).([]byte), message) // nolint: forcetypeassert }) n, err := handshakeData.proxyConn.Read(buffer) From d7e554540dfc9e50e3b458eed83dab26c81b37ad Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 19 Mar 2022 13:47:10 +0300 Subject: [PATCH 12/12] Use recommended way of running golangci-lint install --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7533f8e..9a4c7a0 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ install-tools: install-tools-lint install-tools-godoc install-tools-gofumpt inst .PHONY: install-tools-lint install-tools-lint: .bin - @curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh \ + @curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ | bash -s -- -b "$(GOBIN)" "$(GOLANGCI_LINT_VERSION)" .PHONY: install-tools-godoc