From 036b10be67790523b6c312d4bbbaa9053f78fe61 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Mar 2026 17:14:52 +0100 Subject: [PATCH 01/42] Include example config into distribution bundle --- .goreleaser.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.goreleaser.yml b/.goreleaser.yml index 37e5606..4fb638e 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -54,6 +54,7 @@ archives: - LICENSE - README.md - SECURITY.md + - example.config.toml gomod: proxy: true From c886ffdd81dd42c74bf62091cc2eb3423f734cac Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 10 Mar 2026 00:25:47 +0100 Subject: [PATCH 02/42] Add TrustTLS method to networkHTTPTransport --- network/v2/http.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/network/v2/http.go b/network/v2/http.go index 58e6a5e..0504352 100644 --- a/network/v2/http.go +++ b/network/v2/http.go @@ -1,6 +1,9 @@ package network -import "net/http" +import ( + "crypto/tls" + "net/http" +) type networkHTTPTransport struct { userAgent string @@ -12,3 +15,11 @@ func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, erro return n.next.RoundTrip(req) //nolint: wrapcheck } + +func (n networkHTTPTransport) TrustTLS() { + tr := n.next.(*http.Transport) + if tr.TLSClientConfig == nil { + tr.TLSClientConfig = &tls.Config{} + } + tr.TLSClientConfig.InsecureSkipVerify = true +} From 1182b9ef6fc2f9f2548d23e6054bbc8e0928e3cc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 10 Mar 2026 22:19:41 +0100 Subject: [PATCH 03/42] Add doppel and tls packages --- mtglib/internal/doppel/clock.go | 35 ++++ mtglib/internal/doppel/clock_test.go | 80 ++++++++ mtglib/internal/doppel/conn.go | 95 +++++++++ mtglib/internal/doppel/conn_test.go | 163 +++++++++++++++ mtglib/internal/doppel/ganger.go | 173 ++++++++++++++++ mtglib/internal/doppel/ganger_test.go | 107 ++++++++++ mtglib/internal/doppel/init.go | 38 ++++ mtglib/internal/doppel/init_test.go | 104 ++++++++++ mtglib/internal/doppel/logger.go | 6 + mtglib/internal/doppel/scout.go | 104 ++++++++++ mtglib/internal/doppel/scout_conn.go | 57 +++++ .../internal/doppel/scout_conn_collected.go | 29 +++ .../doppel/scout_conn_collected_test.go | 42 ++++ mtglib/internal/doppel/scout_test.go | 39 ++++ mtglib/internal/doppel/stats.go | 150 ++++++++++++++ mtglib/internal/doppel/stats_test.go | 194 ++++++++++++++++++ mtglib/internal/tls/conn.go | 88 ++++++++ mtglib/internal/tls/conn_test.go | 160 +++++++++++++++ mtglib/internal/tls/init_test.go | 30 +++ mtglib/internal/tls/utils.go | 48 +++++ mtglib/internal/tls/utils_test.go | 125 +++++++++++ 21 files changed, 1867 insertions(+) create mode 100644 mtglib/internal/doppel/clock.go create mode 100644 mtglib/internal/doppel/clock_test.go create mode 100644 mtglib/internal/doppel/conn.go create mode 100644 mtglib/internal/doppel/conn_test.go create mode 100644 mtglib/internal/doppel/ganger.go create mode 100644 mtglib/internal/doppel/ganger_test.go create mode 100644 mtglib/internal/doppel/init.go create mode 100644 mtglib/internal/doppel/init_test.go create mode 100644 mtglib/internal/doppel/logger.go create mode 100644 mtglib/internal/doppel/scout.go create mode 100644 mtglib/internal/doppel/scout_conn.go create mode 100644 mtglib/internal/doppel/scout_conn_collected.go create mode 100644 mtglib/internal/doppel/scout_conn_collected_test.go create mode 100644 mtglib/internal/doppel/scout_test.go create mode 100644 mtglib/internal/doppel/stats.go create mode 100644 mtglib/internal/doppel/stats_test.go create mode 100644 mtglib/internal/tls/conn.go create mode 100644 mtglib/internal/tls/conn_test.go create mode 100644 mtglib/internal/tls/init_test.go create mode 100644 mtglib/internal/tls/utils.go create mode 100644 mtglib/internal/tls/utils_test.go diff --git a/mtglib/internal/doppel/clock.go b/mtglib/internal/doppel/clock.go new file mode 100644 index 0000000..8b29078 --- /dev/null +++ b/mtglib/internal/doppel/clock.go @@ -0,0 +1,35 @@ +package doppel + +import ( + "context" + "time" +) + +type Clock struct { + stats *Stats + tick chan struct{} +} + +func (c Clock) Start(ctx context.Context) { + tickTock := time.NewTimer(c.stats.Delay()) + defer func() { + tickTock.Stop() + select { + case <-tickTock.C: + default: + } + }() + + for { + select { + case <-ctx.Done(): + return + case <-tickTock.C: + select { + case <-ctx.Done(): + case c.tick <- struct{}{}: + } + tickTock.Reset(c.stats.Delay()) + } + } +} diff --git a/mtglib/internal/doppel/clock_test.go b/mtglib/internal/doppel/clock_test.go new file mode 100644 index 0000000..37fbb62 --- /dev/null +++ b/mtglib/internal/doppel/clock_test.go @@ -0,0 +1,80 @@ +package doppel + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type ClockTestSuite struct { + suite.Suite + + clock Clock + wg sync.WaitGroup + ctx context.Context + ctxCancel context.CancelFunc +} + +func (suite *ClockTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + + suite.ctx = ctx + suite.ctxCancel = cancel + suite.clock = Clock{ + stats: &Stats{ + k: StatsDefaultK, + lambda: StatsDefaultLambda, + }, + tick: make(chan struct{}), + } + + suite.wg.Go(func() { + suite.clock.Start(suite.ctx) + }) +} + +func (suite *ClockTestSuite) TearDownTest() { + suite.ctxCancel() + suite.wg.Wait() +} + +func (suite *ClockTestSuite) TestTicks() { + received := 0 + + for range 3 { + select { + case <-suite.clock.tick: + received++ + case <-time.After(2 * time.Second): + suite.Fail("timed out waiting for tick") + } + } + + suite.Equal(3, received) +} + +func (suite *ClockTestSuite) TestStopsOnCancel() { + select { + case <-suite.clock.tick: + case <-time.After(2 * time.Second): + suite.Fail("timed out waiting for first tick") + } + + suite.ctxCancel() + + time.Sleep(50 * time.Millisecond) + + select { + case <-suite.clock.tick: + suite.Fail("received tick after cancel") + default: + } +} + +func TestClock(t *testing.T) { + t.Parallel() + suite.Run(t, &ClockTestSuite{}) +} diff --git a/mtglib/internal/doppel/conn.go b/mtglib/internal/doppel/conn.go new file mode 100644 index 0000000..a2543cd --- /dev/null +++ b/mtglib/internal/doppel/conn.go @@ -0,0 +1,95 @@ +package doppel + +import ( + "bytes" + "context" + "sync" + + "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" +) + +type Conn struct { + essentials.Conn + + p *connPayload +} + +type connPayload struct { + ctx context.Context + ctxCancel context.CancelCauseFunc + clock Clock + wg sync.WaitGroup + writeLock sync.Mutex + writeStream bytes.Buffer +} + +func (c Conn) Write(p []byte) (int, error) { + c.p.writeLock.Lock() + c.p.writeStream.Write(p) + c.p.writeLock.Unlock() + + return len(p), context.Cause(c.p.ctx) +} + +func (c Conn) Start() { + c.p.wg.Go(func() { + c.start() + }) +} + +func (c Conn) start() { + buf := [tls.MaxRecordSize]byte{} + + for { + select { + case <-c.p.ctx.Done(): + return + case <-c.p.clock.tick: + } + + c.p.writeLock.Lock() + n, err := c.p.writeStream.Read(buf[:c.p.clock.stats.Size()]) + c.p.writeLock.Unlock() + + if n == 0 || err != nil { + continue + } + + if err := tls.WriteRecord(c.Conn, buf[:n]); err != nil { + c.p.ctxCancel(err) + return + } + } +} + +func (c Conn) Stop() { + c.p.ctxCancel(nil) + c.p.wg.Wait() +} + +func NewConn(ctx context.Context, conn essentials.Conn, stats *Stats) Conn { + ctx, cancel := context.WithCancelCause(ctx) + rv := Conn{ + Conn: conn, + p: &connPayload{ + ctx: ctx, + ctxCancel: cancel, + clock: Clock{ + stats: stats, + tick: make(chan struct{}), + }, + }, + } + + rv.p.writeStream.Grow(tls.DefaultBufferSize) + + rv.p.wg.Go(func() { + rv.p.clock.Start(ctx) + }) + rv.p.wg.Go(func() { + rv.start() + }) + + return rv +} diff --git a/mtglib/internal/doppel/conn_test.go b/mtglib/internal/doppel/conn_test.go new file mode 100644 index 0000000..e774bce --- /dev/null +++ b/mtglib/internal/doppel/conn_test.go @@ -0,0 +1,163 @@ +package doppel + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "sync" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnMock struct { + testlib.EssentialsConnMock + + mu sync.Mutex + writeBuffer bytes.Buffer +} + +func (m *ConnMock) Write(p []byte) (int, error) { + args := m.Called(p) + if err := args.Error(1); err != nil { + return args.Int(0), err + } + + m.mu.Lock() + defer m.mu.Unlock() + + return m.writeBuffer.Write(p) +} + +func (m *ConnMock) Written() []byte { + m.mu.Lock() + defer m.mu.Unlock() + + return bytes.Clone(m.writeBuffer.Bytes()) +} + +type ConnTestSuite struct { + suite.Suite + + connMock *ConnMock + ctx context.Context + ctxCancel context.CancelFunc +} + +func (suite *ConnTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + suite.ctx = ctx + suite.ctxCancel = cancel + suite.connMock = &ConnMock{} +} + +func (suite *ConnTestSuite) TearDownTest() { + suite.ctxCancel() + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnTestSuite) makeConn() Conn { + return NewConn(suite.ctx, suite.connMock, &Stats{ + k: 2.0, + lambda: 0.01, + }) +} + +func (suite *ConnTestSuite) TestWriteBuffersData() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + defer c.Stop() + + n, err := c.Write([]byte{1, 2, 3}) + suite.NoError(err) + suite.Equal(3, n) +} + +func (suite *ConnTestSuite) TestWriteOutputsTLSRecords() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + + payload := []byte("hello doppelganger") + _, err := c.Write(payload) + suite.NoError(err) + + suite.Eventually(func() bool { + return len(suite.connMock.Written()) > 0 + }, 2*time.Second, time.Millisecond) + + c.Stop() + + assembled := &bytes.Buffer{} + reader := bytes.NewReader(suite.connMock.Written()) + + for { + header := make([]byte, tls.SizeHeader) + if _, err := io.ReadFull(reader, header); err != nil { + break + } + + suite.Equal(byte(tls.TypeApplicationData), header[0]) + suite.Equal(tls.TLSVersion[:], header[tls.SizeRecordType:tls.SizeRecordType+tls.SizeVersion]) + + length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:]) + suite.Greater(length, uint16(0)) + + rec := make([]byte, length) + _, err := io.ReadFull(reader, rec) + suite.NoError(err) + + assembled.Write(rec) + } + + suite.Equal(payload, assembled.Bytes()) +} + +func (suite *ConnTestSuite) TestWriteReturnsErrorAfterStop() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + c.Stop() + + time.Sleep(10 * time.Millisecond) + + _, err := c.Write([]byte{1}) + suite.Error(err) +} + +func (suite *ConnTestSuite) TestStopOnUnderlyingWriteError() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, errors.New("connection reset")). + Maybe() + + c := suite.makeConn() + + _, _ = c.Write([]byte("data")) + + suite.Eventually(func() bool { + _, err := c.Write([]byte{1}) + return err != nil + }, 2*time.Second, time.Millisecond) +} + +func TestConn(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTestSuite{}) +} diff --git a/mtglib/internal/doppel/ganger.go b/mtglib/internal/doppel/ganger.go new file mode 100644 index 0000000..0aa735c --- /dev/null +++ b/mtglib/internal/doppel/ganger.go @@ -0,0 +1,173 @@ +package doppel + +import ( + "context" + "sync" + "time" + + "github.com/9seconds/mtg/v2/essentials" +) + +const ( + DoppelGangerMaxDurations = 4096 + DoppelGangerScoutMissionEach = 30 * time.Minute + DoppelGangerScoutRepeats = 10 +) + +type gangerConnRequest struct { + ret chan Conn + payload essentials.Conn +} + +type Ganger struct { + ctx context.Context + ctxCancel context.CancelFunc + logger Logger + wg sync.WaitGroup + + scout Scout + scoutMissionEach time.Duration + scoutMissionRepeats int + + stats *Stats + durations []time.Duration + + connRequests chan gangerConnRequest +} + +func (g *Ganger) Shutdown() { + g.ctxCancel() + g.wg.Wait() +} + +func (g *Ganger) Run() { + g.wg.Go(func() { + g.run() + }) +} + +func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) { + req := gangerConnRequest{ + ret: make(chan Conn), + payload: conn, + } + defer close(req.ret) + + select { + case <-g.ctx.Done(): + return Conn{}, context.Cause(g.ctx) + case g.connRequests <- req: + } + + select { + case <-g.ctx.Done(): + return Conn{}, context.Cause(g.ctx) + case conn := <-req.ret: + return conn, nil + } +} + +func (g *Ganger) run() { + scoutTicker := time.NewTicker(g.scoutMissionEach) + defer func() { + scoutTicker.Stop() + + select { + case <-scoutTicker.C: + default: + } + }() + + scoutCollectedChan := make(chan []time.Duration) + currentScoutCollectedChan := scoutCollectedChan + + updatedStatsChan := make(chan *Stats) + + g.wg.Go(func() { + g.runScoutMission(scoutCollectedChan) + }) + + for { + select { + case <-g.ctx.Done(): + return + case durations := <-currentScoutCollectedChan: + g.durations = append(g.durations, durations...) + if len(g.durations) > DoppelGangerMaxDurations { + g.durations = g.durations[len(g.durations)-DoppelGangerMaxDurations:] + } + + currentScoutCollectedChan = nil + g.wg.Go(func() { + select { + case <-g.ctx.Done(): + case updatedStatsChan <- NewStats(durations): + } + }) + case stats := <-updatedStatsChan: + g.stats = stats + currentScoutCollectedChan = scoutCollectedChan + case <-scoutTicker.C: + g.wg.Go(func() { + g.runScoutMission(scoutCollectedChan) + }) + case req := <-g.connRequests: + select { + case <-g.ctx.Done(): + case req.ret <- NewConn(g.ctx, req.payload, g.stats): + } + } + } +} + +func (g *Ganger) runScoutMission(rvChan chan<- []time.Duration) { + durations := []time.Duration{} + + for range g.scoutMissionRepeats { + learned, err := g.scout.Learn(g.ctx) + if err != nil { + g.logger.WarningError("cannot learn", err) + continue + } + durations = append(durations, learned...) + } + + select { + case <-g.ctx.Done(): + return + case rvChan <- durations: + } +} + +func NewGanger( + ctx context.Context, + network Network, + logger Logger, + scoutEach time.Duration, + scoutRepeats int, + urls []string, +) *Ganger { + ctx, cancel := context.WithCancel(ctx) + + if scoutEach == 0 { + scoutEach = DoppelGangerScoutMissionEach + } + + if scoutRepeats == 0 { + scoutRepeats = DoppelGangerScoutRepeats + } + + return &Ganger{ + ctx: ctx, + ctxCancel: cancel, + logger: logger, + scoutMissionEach: scoutEach, + scoutMissionRepeats: scoutRepeats, + stats: &Stats{ + k: StatsDefaultK, + lambda: StatsDefaultLambda, + }, + scout: NewScout(network, urls), + connRequests: make(chan gangerConnRequest), + } +} diff --git a/mtglib/internal/doppel/ganger_test.go b/mtglib/internal/doppel/ganger_test.go new file mode 100644 index 0000000..0343eb4 --- /dev/null +++ b/mtglib/internal/doppel/ganger_test.go @@ -0,0 +1,107 @@ +package doppel + +import ( + "bytes" + "sync" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type GangerTestSuite struct { + TLSServerTestSuite + + log *LoggerMock + g *Ganger +} + +func (suite *GangerTestSuite) SetupTest() { + suite.TLSServerTestSuite.SetupTest() + + suite.log = &LoggerMock{} + suite.log. + On("Info", mock.AnythingOfType("string")). + Maybe() + suite.log. + On("WarningError", mock.AnythingOfType("string"), mock.Anything). + Maybe() + + suite.g = NewGanger(suite.ctx, suite.network, suite.log, time.Hour, 1, suite.urls) + suite.g.Run() +} + +func (suite *GangerTestSuite) TearDownTest() { + suite.g.Shutdown() + + suite.log.AssertExpectations(suite.T()) + suite.TLSServerTestSuite.TearDownTest() +} + +func (suite *GangerTestSuite) TestNewConnAfterShutdown() { + suite.g.Shutdown() + connMock := &testlib.EssentialsConnMock{} + + _, err := suite.g.NewConn(connMock) + suite.Error(err) +} + +func (suite *GangerTestSuite) TestNewConnWhileRunning() { + connMock := &testlib.EssentialsConnMock{} + connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + connMock.On("Close"). + Return(nil). + Maybe() + + conn, err := suite.g.NewConn(connMock) + suite.NoError(err) + + conn.Stop() +} + +func (suite *GangerTestSuite) TestNewConnWriteProducesTLSRecords() { + var ( + mu sync.Mutex + buf bytes.Buffer + ) + + connMock := &testlib.EssentialsConnMock{} + connMock.On("Write", mock.AnythingOfType("[]uint8")). + Run(func(args mock.Arguments) { + mu.Lock() + buf.Write(args.Get(0).([]byte)) + mu.Unlock() + }). + Return(0, nil). + Maybe() + connMock.On("Close"). + Return(nil). + Maybe() + + conn, err := suite.g.NewConn(connMock) + suite.NoError(err) + + payload := bytes.Repeat([]byte("x"), 512) + _, err = conn.Write(payload) + suite.NoError(err) + + time.Sleep(500 * time.Millisecond) + conn.Stop() + + mu.Lock() + written := buf.Bytes() + mu.Unlock() + + suite.NotEmpty(written) +} + +func TestGanger(t *testing.T) { + t.Parallel() + + suite.Run(t, &GangerTestSuite{}) +} diff --git a/mtglib/internal/doppel/init.go b/mtglib/internal/doppel/init.go new file mode 100644 index 0000000..9baa193 --- /dev/null +++ b/mtglib/internal/doppel/init.go @@ -0,0 +1,38 @@ +package doppel + +import ( + "context" + "net/http" + "time" + + "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" +) + +const ( + // Please see Stats description + // https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/ + // https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch + TLSRecordSizeStart = 1369 + TLSRecordSizeAccel = 4229 + TLSRecordSizeMax = 16384 - tls.SizeHeader + + TLSCounterAccelAfter = 40 + TLSCounterMaxAfter = TLSCounterAccelAfter + 20 + + TLSRecordSizeResetAfter = time.Second +) + +// copypasted from mtglib +type Network interface { + // Dial establishes context-free TCP connections. + Dial(network, address string) (essentials.Conn, error) + + // DialContext dials using a context. This is a preferrable way of + // establishing TCP connections. + DialContext(ctx context.Context, network, address string) (essentials.Conn, error) + + // MakeHTTPClient build an HTTP client with given dial function. If nothing is + // provided, then DialContext of this interface is going to be used. + MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client +} diff --git a/mtglib/internal/doppel/init_test.go b/mtglib/internal/doppel/init_test.go new file mode 100644 index 0000000..e8eec5e --- /dev/null +++ b/mtglib/internal/doppel/init_test.go @@ -0,0 +1,104 @@ +package doppel + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/http/httptest" + "time" + + "github.com/9seconds/mtg/v2/essentials" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type SimpleNetwork struct { +} + +func (s SimpleNetwork) Dial(network, address string) (essentials.Conn, error) { + return s.DialContext(context.Background(), network, address) +} + +func (s SimpleNetwork) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) { + d := &net.Dialer{} + + conn, err := d.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + + return conn.(*net.TCPConn), nil +} + +func (s SimpleNetwork) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client { + if dialFunc == nil { + dialFunc = s.DialContext + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, //nolint: gosec + }, + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return dialFunc(ctx, network, address) + }, + }, + } +} + +type TLSServerTestSuite struct { + suite.Suite + + tlsServer *httptest.Server + ctx context.Context + ctxCancel context.CancelFunc + network SimpleNetwork + urls []string +} + +func (suite *TLSServerTestSuite) SetupSuite() { + suite.tlsServer = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Add("Hello", "how long") + + if _, err := w.Write([]byte{1, 2, 3}); err != nil { + panic(err) + } + + time.Sleep(5 * time.Millisecond) + + if _, err := w.Write([]byte{1, 2, 3}); err != nil { + panic(err) + } + })) + suite.urls = []string{suite.tlsServer.URL} +} + +func (suite *TLSServerTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + suite.ctx = ctx + suite.ctxCancel = cancel +} + +func (suite *TLSServerTestSuite) TearDownTest() { + suite.ctxCancel() + suite.tlsServer.CloseClientConnections() +} + +func (suite *TLSServerTestSuite) TearDownSuite() { + suite.tlsServer.Close() +} + +type LoggerMock struct { + mock.Mock +} + +func (l *LoggerMock) Info(msg string) { + l.Called(msg) +} + +func (l *LoggerMock) WarningError(msg string, err error) { + l.Called(msg, err) +} diff --git a/mtglib/internal/doppel/logger.go b/mtglib/internal/doppel/logger.go new file mode 100644 index 0000000..b91fc7e --- /dev/null +++ b/mtglib/internal/doppel/logger.go @@ -0,0 +1,6 @@ +package doppel + +type Logger interface { + Info(msg string) + WarningError(msg string, err error) +} diff --git a/mtglib/internal/doppel/scout.go b/mtglib/internal/doppel/scout.go new file mode 100644 index 0000000..88a478b --- /dev/null +++ b/mtglib/internal/doppel/scout.go @@ -0,0 +1,104 @@ +package doppel + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" +) + +type Scout struct { + network Network + urls []string +} + +func (s Scout) Learn(ctx context.Context) ([]time.Duration, error) { + var durations []time.Duration + + for _, url := range s.urls { + learned, err := s.learn(ctx, url) + if err != nil { + return nil, err + } + + durations = append(durations, learned...) + } + + return durations, nil +} + +func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) { + client, results := s.makeClient() + + if !strings.HasPrefix(url, "https://") { + return nil, fmt.Errorf("url %s must be https", url) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := client.Do(req) + if resp != nil { + io.Copy(io.Discard, resp.Body) //nolint: errcheck + resp.Body.Close() //nolint: errcheck + client.CloseIdleConnections() + } + + if err != nil || len(results.data) == 0 { + return nil, err + } + + durations := []time.Duration{} + lastTimestamp := time.Time{} + + for i, v := range results.data { + if v.recordType != tls.TypeApplicationData { + continue + } + + if lastTimestamp.IsZero() { + if i > 0 { + lastTimestamp = results.data[i-1].timestamp + } else { + lastTimestamp = v.timestamp + } + } + + durations = append(durations, v.timestamp.Sub(lastTimestamp)) + lastTimestamp = v.timestamp + } + + return durations, nil +} + +func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) { + collected := NewScoutConnCollected() + client := s.network.MakeHTTPClient(func( + ctx context.Context, + network string, + address string, + ) (essentials.Conn, error) { + conn, err := s.network.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + + return NewScoutConn(conn, collected), nil + }) + + return client, collected +} + +func NewScout(network Network, urls []string) Scout { + return Scout{ + network: network, + urls: urls, + } +} diff --git a/mtglib/internal/doppel/scout_conn.go b/mtglib/internal/doppel/scout_conn.go new file mode 100644 index 0000000..0aaa0b5 --- /dev/null +++ b/mtglib/internal/doppel/scout_conn.go @@ -0,0 +1,57 @@ +package doppel + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" +) + +type ScoutConn struct { + tls.Conn + + results *ScoutConnCollected + rawBuf *bytes.Buffer +} + +func (s ScoutConn) Read(p []byte) (int, error) { + buf := &bytes.Buffer{} + + for { + if n, err := s.rawBuf.Read(p); err == nil { + return n, nil + } + + s.rawBuf.Reset() + + recordType, length, err := tls.ReadRecord(s.Conn, buf) + if err != nil { + return 0, err + } + + s.results.Add(recordType) + s.rawBuf.Write([]byte{recordType}) + s.rawBuf.Write(tls.TLSVersion[:]) + + if err := binary.Write(s.rawBuf, binary.BigEndian, uint16(length)); err != nil { + return 0, err + } + + if _, err := io.Copy(s.rawBuf, buf); err != nil { + return 0, err + } + } +} + +func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) ScoutConn { + rawBuf := &bytes.Buffer{} + rawBuf.Grow(tls.MaxRecordSize) + + return ScoutConn{ + Conn: tls.New(conn, false, false), + results: results, + rawBuf: rawBuf, + } +} diff --git a/mtglib/internal/doppel/scout_conn_collected.go b/mtglib/internal/doppel/scout_conn_collected.go new file mode 100644 index 0000000..daf98cb --- /dev/null +++ b/mtglib/internal/doppel/scout_conn_collected.go @@ -0,0 +1,29 @@ +package doppel + +import "time" + +const ( + ScoutConnCollectedPreallocSize = 100 +) + +type ScoutConnResult struct { + timestamp time.Time + recordType byte +} + +type ScoutConnCollected struct { + data []ScoutConnResult +} + +func (s *ScoutConnCollected) Add(record byte) { + s.data = append(s.data, ScoutConnResult{ + timestamp: time.Now(), + recordType: record, + }) +} + +func NewScoutConnCollected() *ScoutConnCollected { + return &ScoutConnCollected{ + data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize), + } +} diff --git a/mtglib/internal/doppel/scout_conn_collected_test.go b/mtglib/internal/doppel/scout_conn_collected_test.go new file mode 100644 index 0000000..df4dbdd --- /dev/null +++ b/mtglib/internal/doppel/scout_conn_collected_test.go @@ -0,0 +1,42 @@ +package doppel + +import ( + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib/internal/tls" + "github.com/stretchr/testify/suite" +) + +type ScoutConnCollectedTestSuite struct { + suite.Suite +} + +func (suite *ScoutConnCollectedTestSuite) TestAddSingle() { + collected := NewScoutConnCollected() + collected.Add(tls.TypeApplicationData) + + suite.Len(collected.data, 1) + suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType) +} + +func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() { + collected := NewScoutConnCollected() + + collected.Add(tls.TypeApplicationData) + + time.Sleep(time.Microsecond) + collected.Add(tls.TypeApplicationData) + + time.Sleep(time.Microsecond) + collected.Add(tls.TypeApplicationData) + + for i := 1; i < len(collected.data); i++ { + suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp)) + } +} + +func TestScoutConnCollected(t *testing.T) { + t.Parallel() + suite.Run(t, &ScoutConnCollectedTestSuite{}) +} diff --git a/mtglib/internal/doppel/scout_test.go b/mtglib/internal/doppel/scout_test.go new file mode 100644 index 0000000..d9fe850 --- /dev/null +++ b/mtglib/internal/doppel/scout_test.go @@ -0,0 +1,39 @@ +package doppel + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type ScoutTestSuite struct { + TLSServerTestSuite + + scout Scout +} + +func (suite *ScoutTestSuite) SetupSuite() { + suite.TLSServerTestSuite.SetupSuite() + + suite.scout = Scout{ + network: suite.network, + urls: suite.urls, + } +} + +func (suite *ScoutTestSuite) TestCollectResults() { + durations, err := suite.scout.Learn(suite.ctx) + suite.NoError(err) + suite.Less(3, len(durations)) +} + +func (suite *ScoutTestSuite) TestCollectNothing() { + suite.ctxCancel() + + _, err := suite.scout.Learn(suite.ctx) + suite.Error(err) +} + +func TestScout(t *testing.T) { + suite.Run(t, &ScoutTestSuite{}) +} diff --git a/mtglib/internal/doppel/stats.go b/mtglib/internal/doppel/stats.go new file mode 100644 index 0000000..a6cbc9b --- /dev/null +++ b/mtglib/internal/doppel/stats.go @@ -0,0 +1,150 @@ +package doppel + +import ( + "math" + "math/rand/v2" + "time" +) + +const ( + StatsBisectTimes = 70 + StatsLowK = 0.01 + StatsHighK = 10.0 + + StatsDefaultK = 0.6 + StatsDefaultLambda = 0.002 +) + +// Stats is responsible for generating values that are distributed according +// to some statistical distribution. +// +// It follows several ideas: +// 1. Based on nginx and Cloudflare behaviour, even if server is eager +// to send a lot, they all start with small TLS packets that are +// approximately MTU-sized. After +// 2. After ~40 TLS records, server considers TCP session as somewhat solid +// and reliable and ramps up to 4096. +// 3. After ~20 TLS records more it jumps to the max 16384 bytes and keep +// this size as long as it can +// 4. If there is no any byte within a connection for a longer time period, +// this counter resets. +// +// This is called Dynamic TLS Record Sizing +// - https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/ +// - https://community.f5.com/kb/technicalarticles/boosting-tls-performance-with-dynamic-record-sizing-on-big-ip/280798 +// - https://www.igvita.com/2013/10/24/optimizing-tls-record-size-and-buffering-latency/ +// +// And this optimized for the very first byte, so web browsers could start to +// render as early as possible, showing user some preliminary results, optimizing +// for perceived latency. +// +// Since this is very typical for the website, we also aim for that. +// +// Another important idea is how delays between TLS packets are distributed. +// In case of sending huge heavy content with max sized record, delays have +// lognormal distribution. But a nature of a typical website shows that +// it eagers to deliver as fast as it can in a few very first records and +// could possibly slow down later. +// +// This is perfectly described by Weibull distribution: +// - https://en.wikipedia.org/wiki/Weibull_distribution +// - https://ieeexplore.ieee.org/document/6662948 +// - https://www.researchgate.net/publication/224621285_Traffic_modelling_and_cost_optimization_for_transmitting_traffic_messages_over_a_hybrid_broadcast_and_cellular_network +// - https://ir.uitm.edu.my/id/eprint/105386/1/105386.pdf +// +// In other word, a combination of Dynamic TLS Record Sizing hints us for +// Weibull distribution. +type Stats struct { + sizeLastRequested time.Time + sizeCounter int + + // https://en.wikipedia.org/wiki/Shape_parameter + k float64 + // https://en.wikipedia.org/wiki/Scale_parameter + lambda float64 +} + +func (d *Stats) Delay() time.Duration { + // u ∈ (0, 1], avoids ln(0) + u := 1.0 - rand.Float64() + + // X = λ·(-ln U)^(1/k) + generated := d.lambda * math.Pow(-math.Log(u), 1.0/d.k) + + // generated is in milliseconds + return time.Duration(generated * float64(time.Millisecond)) +} + +func (d *Stats) Size() int { + if time.Since(d.sizeLastRequested) > TLSRecordSizeResetAfter { + d.sizeCounter = 0 + } + + d.sizeLastRequested = time.Now() + d.sizeCounter++ + + switch { + case d.sizeCounter <= TLSCounterAccelAfter: + return TLSRecordSizeStart + case d.sizeCounter <= TLSCounterMaxAfter: + return TLSRecordSizeAccel + } + + return TLSRecordSizeMax +} + +func NewStats(durations []time.Duration) *Stats { + n := float64(len(durations)) + + // in milliseconds + durFloats := make([]float64, len(durations)) + for i, v := range durations { + durFloats[i] = float64(v.Microseconds()) / 1000.0 + } + + // The bisection solves the standard Weibull MLE equation for shape + // parameter k. There is no any good formula for doing that so we + // approximate it by several bisections. The number of operations + // is statically defined by a constant. + + sumLog := 0.0 + for _, v := range durFloats { + sumLog += math.Log(v) + } + + lowK := StatsLowK + highK := StatsHighK + + for range StatsBisectTimes { + midK := (lowK + highK) / 2.0 + sumXK := 0.0 + sumXKLog := 0.0 + + for _, v := range durFloats { + xk := math.Pow(v, midK) + sumXK += xk + sumXKLog += xk * math.Log(v) + } + + if (1.0/midK)+(sumLog/n)-(sumXKLog/sumXK) > 0 { + lowK = midK + } else { + highK = midK + } + } + + k := (lowK + highK) / 2 + + sumXK := 0.0 + for _, v := range durFloats { + sumXK += math.Pow(v, k) + } + + // λ = (Σxᵢᵏ / n)^(1/k) + lambda := math.Pow(sumXK/n, 1.0/k) + + return &Stats{ + k: k, + lambda: lambda, + } +} diff --git a/mtglib/internal/doppel/stats_test.go b/mtglib/internal/doppel/stats_test.go new file mode 100644 index 0000000..1e45a1f --- /dev/null +++ b/mtglib/internal/doppel/stats_test.go @@ -0,0 +1,194 @@ +package doppel + +import ( + "math" + "math/rand/v2" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type StatsTestSuite struct { + suite.Suite +} + +func (suite *StatsTestSuite) GenWeibull(k, lambda float64, n int, seed uint64) []time.Duration { + rng := rand.New(rand.NewPCG(seed, 0)) + samples := make([]time.Duration, n) + + for i := range samples { + u := 1.0 - rng.Float64() + ms := lambda * math.Pow(-math.Log(u), 1.0/k) + d := time.Duration(ms * float64(time.Millisecond)) + + if d < time.Microsecond { + time.Sleep(time.Microsecond) + d = time.Microsecond + } + + samples[i] = d + } + + return samples +} + +func (suite *StatsTestSuite) TestNewStatsRecoverParameters() { + knownK := 1.5 + knownLambda := 100.0 + + samples := suite.GenWeibull(knownK, knownLambda, 5000, 42) + stats := NewStats(samples) + + suite.InDelta(knownK, stats.k, 0.1) + suite.InDelta(knownLambda, stats.lambda, 5.0) +} + +func (suite *StatsTestSuite) TestNewStatsExponentialCase() { + // When k=1, Weibull reduces to exponential distribution. + knownK := 1.0 + knownLambda := 50.0 + + samples := suite.GenWeibull(knownK, knownLambda, 5000, 123) + stats := NewStats(samples) + + suite.InDelta(knownK, stats.k, 0.1) + suite.InDelta(knownLambda, stats.lambda, 5.0) +} + +func (suite *StatsTestSuite) TestNewStatsSmallK() { + // k < 1 produces a heavy-tailed distribution typical for network delays. + // Lambda must be large enough so samples stay above microsecond precision + // after time.Duration round-trip. + knownK := 0.6 + knownLambda := 100.0 + + samples := suite.GenWeibull(knownK, knownLambda, 10000, 99) + stats := NewStats(samples) + + suite.InDelta(knownK, stats.k, 0.05) + suite.InDelta(knownLambda, stats.lambda, 5.0) +} + +func (suite *StatsTestSuite) TestNewStatsLargeK() { + // k > 1: light tail, concentrated around the mode. + knownK := 5.0 + knownLambda := 200.0 + + samples := suite.GenWeibull(knownK, knownLambda, 5000, 77) + stats := NewStats(samples) + + suite.InDelta(knownK, stats.k, 0.3) + suite.InDelta(knownLambda, stats.lambda, 5.0) +} + +func (suite *StatsTestSuite) TestDelayNonNegative() { + stats := &Stats{ + k: 1.5, + lambda: 100.0, + } + + for range 200 { + dur := stats.Delay() + suite.GreaterOrEqual(dur, time.Duration(0)) + } +} + +func (suite *StatsTestSuite) TestDelayDistributionMean() { + // Weibull mean = λ · Γ(1 + 1/k) + k := 2.0 + lambda := 50.0 + stats := &Stats{k: k, lambda: lambda} + + n := 50000 + sum := 0.0 + + for range n { + dur := stats.Delay() + sum += float64(dur) / float64(time.Millisecond) + } + + sampleMean := sum / float64(n) + expectedMean := lambda * math.Gamma(1.0+1.0/k) + + suite.InDelta(expectedMean, sampleMean, expectedMean*0.05) +} + +func (suite *StatsTestSuite) TestNewStatsRoundTrip() { + // Estimate parameters from data, then verify that Delay samples + // from the fitted distribution have approximately the same mean. + knownK := 1.2 + knownLambda := 80.0 + + samples := suite.GenWeibull(knownK, knownLambda, 5000, 555) + stats := NewStats(samples) + + n := 50000 + sum := 0.0 + + for range n { + dur := stats.Delay() + sum += float64(dur) / float64(time.Millisecond) + } + + sampleMean := sum / float64(n) + expectedMean := knownLambda * math.Gamma(1.0+1.0/knownK) + + suite.InDelta(expectedMean, sampleMean, expectedMean*0.05) +} + +func (suite *StatsTestSuite) TestSizeStartPhase() { + stats := &Stats{k: 1.0, lambda: 1.0} + + for range TLSCounterAccelAfter { + size := stats.Size() + suite.Equal(TLSRecordSizeStart, size) + } +} + +func (suite *StatsTestSuite) TestSizeAccelPhase() { + stats := &Stats{k: 1.0, lambda: 1.0} + + for range TLSCounterAccelAfter { + stats.Size() + } + + for range TLSCounterMaxAfter - TLSCounterAccelAfter { + size := stats.Size() + suite.Equal(TLSRecordSizeAccel, size) + } +} + +func (suite *StatsTestSuite) TestSizeMaxPhase() { + stats := &Stats{k: 1.0, lambda: 1.0} + + for range TLSCounterMaxAfter { + stats.Size() + } + + for range 20 { + size := stats.Size() + suite.Equal(TLSRecordSizeMax, size) + } +} + +func (suite *StatsTestSuite) TestSizeResetsAfterInactivity() { + stats := &Stats{k: 1.0, lambda: 1.0} + + // Advance past start phase. + for range TLSCounterMaxAfter { + stats.Size() + } + + suite.Equal(TLSRecordSizeMax, stats.Size()) + + // Simulate inactivity by backdating sizeLastRequested. + stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond) + + suite.Equal(TLSRecordSizeStart, stats.Size()) +} + +func TestStats(t *testing.T) { + t.Parallel() + suite.Run(t, &StatsTestSuite{}) +} diff --git a/mtglib/internal/tls/conn.go b/mtglib/internal/tls/conn.go new file mode 100644 index 0000000..86c5f59 --- /dev/null +++ b/mtglib/internal/tls/conn.go @@ -0,0 +1,88 @@ +package tls + +import ( + "bufio" + "bytes" + + "github.com/9seconds/mtg/v2/essentials" +) + +const ( + SizeRecordType = 1 + SizeVersion = 2 + SizeSize = 2 + SizeHeader = SizeRecordType + SizeVersion + SizeSize + + MaxRecordSize = 16384 + MaxRecordPayloadSize = MaxRecordSize - SizeHeader + DefaultBufferSize = 4096 + + TypeChangeCipherSpec = 0x14 + TypeHandshake = 0x16 + TypeApplicationData = 0x17 +) + +var ( + // TLS 1.2 is used for both TLS 1.2 and 1.3 + TLSVersion = [SizeVersion]byte{3, 3} +) + +// Conn presents an established TLS 1.3 connection, after handshake +type Conn struct { + essentials.Conn + + p *connPayload +} + +type connPayload struct { + readBuf bytes.Buffer + writeBuf bytes.Buffer + connBuffered *bufio.Reader + read bool + write bool +} + +func (c Conn) Write(p []byte) (int, error) { + if !c.p.write { + return c.Conn.Write(p) + } + + return len(p), WriteRecord(c.Conn, p) +} + +func (c Conn) Read(p []byte) (int, error) { + if !c.p.read { + return c.Conn.Read(p) + } + + for { + if n, err := c.p.readBuf.Read(p); err == nil { + return n, nil + } + + recordType, _, err := ReadRecord(c.p.connBuffered, &c.p.readBuf) + if err != nil { + return 0, err + } + + if recordType != TypeApplicationData { + c.p.readBuf.Reset() + } + } +} + +func New(conn essentials.Conn, read, write bool) Conn { + newConn := Conn{ + Conn: conn, + p: &connPayload{ + connBuffered: bufio.NewReaderSize(conn, DefaultBufferSize), + read: read, + write: write, + }, + } + + newConn.p.readBuf.Grow(DefaultBufferSize) + newConn.p.writeBuf.Grow(DefaultBufferSize) + + return newConn +} diff --git a/mtglib/internal/tls/conn_test.go b/mtglib/internal/tls/conn_test.go new file mode 100644 index 0000000..3669688 --- /dev/null +++ b/mtglib/internal/tls/conn_test.go @@ -0,0 +1,160 @@ +package tls + +import ( + "io" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnTestSuite struct { + suite.Suite + + connMock *testlib.EssentialsConnMock +} + +func (suite *ConnTestSuite) SetupTest() { + suite.connMock = &testlib.EssentialsConnMock{} +} + +func (suite *ConnTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnTestSuite) feedRead(raw []byte) { + suite.connMock. + On("Read", mock.AnythingOfType("[]uint8")). + Run(func(args mock.Arguments) { + copy(args.Get(0).([]byte), raw) + }). + Return(len(raw), nil). + Once() + suite.connMock. + On("Read", mock.AnythingOfType("[]uint8")). + Return(0, io.EOF). + Maybe() +} + +func (suite *ConnTestSuite) TestReadTLSEnabled() { + payload := []byte("hello world") + suite.feedRead(MakeTLSRecord(0x17, payload)) + + conn := New(suite.connMock, true, false) + + buf := make([]byte, 128) + n, err := conn.Read(buf) + + suite.NoError(err) + suite.Equal(payload, buf[:n]) +} + +func (suite *ConnTestSuite) TestReadTLSSkipsNonApplicationData() { + raw := append( + MakeTLSRecord(0x14, []byte{1}), + MakeTLSRecord(0x17, []byte("real data"))..., + ) + suite.feedRead(raw) + + conn := New(suite.connMock, true, false) + + buf := make([]byte, 128) + n, err := conn.Read(buf) + + suite.NoError(err) + suite.Equal([]byte("real data"), buf[:n]) +} + +func (suite *ConnTestSuite) TestReadTLSMultipleRecords() { + raw := append( + MakeTLSRecord(0x17, []byte("first")), + MakeTLSRecord(0x17, []byte("second"))..., + ) + suite.feedRead(raw) + + conn := New(suite.connMock, true, false) + buf := make([]byte, 128) + + n, err := conn.Read(buf) + suite.NoError(err) + suite.Equal([]byte("first"), buf[:n]) + + n, err = conn.Read(buf) + suite.NoError(err) + suite.Equal([]byte("second"), buf[:n]) +} + +func (suite *ConnTestSuite) TestReadTLSSmallBuffer() { + payload := []byte("hello world, this is a longer payload") + suite.feedRead(MakeTLSRecord(0x17, payload)) + + conn := New(suite.connMock, true, false) + + small := make([]byte, 5) + n, err := conn.Read(small) + suite.NoError(err) + suite.Equal(payload[:5], small[:n]) + + rest := make([]byte, 128) + n, err = conn.Read(rest) + suite.NoError(err) + suite.Equal(payload[5:], rest[:n]) +} + +func (suite *ConnTestSuite) TestReadPassthrough() { + data := []byte("raw bytes") + + suite.connMock. + On("Read", mock.AnythingOfType("[]uint8")). + Run(func(args mock.Arguments) { + copy(args.Get(0).([]byte), data) + }). + Return(len(data), nil). + Once() + + conn := New(suite.connMock, false, false) + + buf := make([]byte, 128) + n, err := conn.Read(buf) + + suite.NoError(err) + suite.Equal(data, buf[:n]) +} + +func (suite *ConnTestSuite) TestWritePassthrough() { + data := []byte("outgoing data") + + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(len(data), nil). + Once() + + conn := New(suite.connMock, false, false) + + n, err := conn.Write(data) + + suite.NoError(err) + suite.Equal(len(data), n) +} + +func (suite *ConnTestSuite) TestWriteTLSEnabled() { + data := []byte("outgoing data") + + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(len(data), nil). + Once() + + conn := New(suite.connMock, false, true) + + n, err := conn.Write(data) + + suite.NoError(err) + suite.Equal(len(data), n) +} + +func TestConn(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTestSuite{}) +} diff --git a/mtglib/internal/tls/init_test.go b/mtglib/internal/tls/init_test.go new file mode 100644 index 0000000..65eb28d --- /dev/null +++ b/mtglib/internal/tls/init_test.go @@ -0,0 +1,30 @@ +package tls + +import ( + "encoding/binary" + + "github.com/stretchr/testify/mock" +) + +type WriterMock struct { + mock.Mock +} + +func (m *WriterMock) Write(p []byte) (int, error) { + args := m.Called(p) + return args.Int(0), args.Error(1) +} + +// makeTLSRecord builds a raw TLS record from hardcoded offsets: +// type(1) + version(2, {3,3}) + length(2, big-endian) + payload. +func MakeTLSRecord(recordType byte, payload []byte) []byte { + buf := make([]byte, 5+len(payload)) + + buf[0] = recordType + buf[1] = 3 + buf[2] = 3 + binary.BigEndian.PutUint16(buf[3:5], uint16(len(payload))) + copy(buf[5:], payload) + + return buf +} diff --git a/mtglib/internal/tls/utils.go b/mtglib/internal/tls/utils.go new file mode 100644 index 0000000..978e048 --- /dev/null +++ b/mtglib/internal/tls/utils.go @@ -0,0 +1,48 @@ +package tls + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" +) + +func ReadRecord(r io.Reader, w io.Writer) (byte, int64, error) { + buf := [SizeHeader]byte{} + + if _, err := io.ReadFull(r, buf[:]); err != nil { + return 0, 0, err + } + + pVer := buf[SizeRecordType:] + pLen := pVer[SizeVersion:] + + if !bytes.Equal(TLSVersion[:], pVer[:SizeVersion]) { + return 0, 0, fmt.Errorf("incorrect tls version %v", pVer) + } + + length := int64(binary.BigEndian.Uint16(pLen[:SizeSize])) + _, err := io.CopyN(w, r, length) + + return buf[0], length, err +} + +func WriteRecord(w io.Writer, payload []byte) error { + buf := [MaxRecordSize]byte{} + buf[0] = TypeApplicationData + + bufV := buf[SizeRecordType:] + copy(bufV[:SizeVersion], TLSVersion[:]) + + bufS := bufV[SizeVersion:] + binary.BigEndian.PutUint16(bufS[:SizeSize], uint16(len(payload))) + + bufP := buf[SizeHeader:] + if n := copy(bufP, payload); n != len(payload) { + return fmt.Errorf("copied %d bytes of payload instead of %d", n, len(payload)) + } + + _, err := w.Write(buf[:SizeHeader+len(payload)]) + + return err +} diff --git a/mtglib/internal/tls/utils_test.go b/mtglib/internal/tls/utils_test.go new file mode 100644 index 0000000..9ddbfa8 --- /dev/null +++ b/mtglib/internal/tls/utils_test.go @@ -0,0 +1,125 @@ +package tls + +import ( + "bytes" + "encoding/binary" + "errors" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type UtilsTestSuite struct { + suite.Suite + + dst *bytes.Buffer +} + +func (suite *UtilsTestSuite) SetupTest() { + suite.dst = &bytes.Buffer{} +} + +func (suite *UtilsTestSuite) TestReadRecord() { + payload := []byte("hello world") + raw := MakeTLSRecord(0x17, payload) + + recordType, length, err := ReadRecord(bytes.NewReader(raw), suite.dst) + + suite.NoError(err) + suite.Equal(byte(0x17), recordType) + suite.Equal(int64(len(payload)), length) + suite.Equal(payload, suite.dst.Bytes()) +} + +func (suite *UtilsTestSuite) TestReadRecordChangeCipherSpec() { + payload := []byte{1} + raw := MakeTLSRecord(0x14, payload) + + recordType, length, err := ReadRecord(bytes.NewReader(raw), suite.dst) + + suite.NoError(err) + suite.Equal(byte(0x14), recordType) + suite.Equal(int64(1), length) +} + +func (suite *UtilsTestSuite) TestReadRecordRejectsWrongVersion() { + record := []byte{0x17, 3, 1, 0, 5, 0, 0, 0, 0, 0} + + _, _, err := ReadRecord(bytes.NewReader(record), suite.dst) + suite.ErrorContains(err, "incorrect tls version") +} + +func (suite *UtilsTestSuite) TestReadRecordEmptyReader() { + _, _, err := ReadRecord(bytes.NewReader(nil), suite.dst) + suite.Error(err) +} + +func (suite *UtilsTestSuite) TestReadRecordTruncatedHeader() { + _, _, err := ReadRecord(bytes.NewReader([]byte{0x17, 3}), suite.dst) + suite.Error(err) +} + +func (suite *UtilsTestSuite) TestReadRecordTruncatedPayload() { + raw := MakeTLSRecord(0x17, []byte("full payload")) + truncated := raw[:5+3] + + _, _, err := ReadRecord(bytes.NewReader(truncated), suite.dst) + suite.Error(err) +} + +func (suite *UtilsTestSuite) TestWriteRecord() { + payload := []byte("hello world") + + err := WriteRecord(suite.dst, payload) + suite.NoError(err) + + written := suite.dst.Bytes() + suite.Equal(byte(0x17), written[0]) + suite.Equal([]byte{3, 3}, written[1:3]) + + length := binary.BigEndian.Uint16(written[3:5]) + suite.Equal(uint16(len(payload)), length) + suite.Equal(payload, written[5:]) +} + +func (suite *UtilsTestSuite) TestWriteRecordRoundTrip() { + payload := []byte("round trip test") + + var wire bytes.Buffer + + err := WriteRecord(&wire, payload) + suite.NoError(err) + + var recovered bytes.Buffer + + recordType, length, err := ReadRecord(&wire, &recovered) + + suite.NoError(err) + suite.Equal(byte(0x17), recordType) + suite.Equal(int64(len(payload)), length) + suite.Equal(payload, recovered.Bytes()) +} + +func (suite *UtilsTestSuite) TestWriteRecordPropagatesError() { + m := &WriterMock{} + m. + On("Write", mock.AnythingOfType("[]uint8")). + Once(). + Return(0, errors.New("dist full")) + + err := WriteRecord(m, []byte("data")) + suite.Error(err) + + m.AssertExpectations(suite.T()) +} + +func (suite *UtilsTestSuite) TestWriteRecordPayloadTooLarge() { + err := WriteRecord(suite.dst, make([]byte, MaxRecordPayloadSize+1)) + suite.Error(err) +} + +func TestUtils(t *testing.T) { + t.Parallel() + suite.Run(t, &UtilsTestSuite{}) +} From 59557059df8c5746965c314c2c8aa5ecfb3a1c27 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 11 Mar 2026 20:15:01 +0100 Subject: [PATCH 04/42] ReadClientHello function --- .mise.toml | 2 +- mtglib/internal/tls/fake/client_side.go | 305 ++++++++++++++ .../tls/fake/client_side_fuzz_test.go | 49 +++ .../tls/fake/client_side_snapshot_test.go | 143 +++++++ mtglib/internal/tls/fake/client_side_test.go | 395 ++++++++++++++++++ mtglib/internal/tls/fake/init.go | 16 + .../client-hello-bad-fa2e46cdb33e2a1b.json | 8 + .../client-hello-ok-19dfe38384b9884b.json | 8 + .../client-hello-ok-48f8a72a56f3174a.json | 8 + .../client-hello-ok-651054256093c6cd.json | 8 + .../client-hello-ok-79d01ef18a9d2621.json | 8 + .../client-hello-ok-7a5569f05b118145.json | 8 + 12 files changed, 957 insertions(+), 1 deletion(-) create mode 100644 mtglib/internal/tls/fake/client_side.go create mode 100644 mtglib/internal/tls/fake/client_side_fuzz_test.go create mode 100644 mtglib/internal/tls/fake/client_side_snapshot_test.go create mode 100644 mtglib/internal/tls/fake/client_side_test.go create mode 100644 mtglib/internal/tls/fake/init.go create mode 100644 mtglib/internal/tls/fake/testdata/client-hello-bad-fa2e46cdb33e2a1b.json create mode 100644 mtglib/internal/tls/fake/testdata/client-hello-ok-19dfe38384b9884b.json create mode 100644 mtglib/internal/tls/fake/testdata/client-hello-ok-48f8a72a56f3174a.json create mode 100644 mtglib/internal/tls/fake/testdata/client-hello-ok-651054256093c6cd.json create mode 100644 mtglib/internal/tls/fake/testdata/client-hello-ok-79d01ef18a9d2621.json create mode 100644 mtglib/internal/tls/fake/testdata/client-hello-ok-7a5569f05b118145.json diff --git a/.mise.toml b/.mise.toml index f95845b..c5bdf73 100644 --- a/.mise.toml +++ b/.mise.toml @@ -48,7 +48,7 @@ depends = [ [tasks."test:fuzz:client-hello"] description = "Run fuzzy test for ClientHello" -run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzClientHello ./mtglib/internal/faketls" +run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzReadClientHello ./mtglib/internal/tls/fake" [tasks."test:fuzz:client-handshake"] description = "Run fuzzy test for ClientHandshake" diff --git a/mtglib/internal/tls/fake/client_side.go b/mtglib/internal/tls/fake/client_side.go new file mode 100644 index 0000000..dc41397 --- /dev/null +++ b/mtglib/internal/tls/fake/client_side.go @@ -0,0 +1,305 @@ +package fake + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/binary" + "fmt" + "io" + "net" + "slices" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" +) + +const ( + TypeHandshakeClient = 0x01 + + RandomLen = 32 + + // record_type(1) + version(2) + size(2) + handshake_type(1) + uint24_length(3) + client_version(2) + clientRandomOffset = 1 + 2 + 2 + 1 + 3 + 2 + sniDNSNamesListType = 0 +) + +var ( + emptyRandom = [RandomLen]byte{} + extTypeSNI = [2]byte{} +) + +type ClientHello struct { + Random [RandomLen]byte + SessionID []byte + CipherSuite uint16 +} + +func ReadClientHello(conn net.Conn, secret mtglib.Secret, tolerateTimeSkewness time.Duration) (*ClientHello, error) { + if err := conn.SetReadDeadline(time.Now().Add(ClientHelloReadTimeout)); err != nil { + return nil, fmt.Errorf("cannot set read deadline: %w", err) + } + defer conn.SetReadDeadline(resetDeadline) //nolint: errcheck + + // This is how FakeTLS is organized: + // 1. We create sha256 HMAC with a given secret + // 2. We dump there a whole TLS frame except of the fact that random + // is filled with all zeroes + // 3. Digest is computed. This digest should be XORed with + // original client random + // 4. New digest should be all 0 except of last 4 bytes + // 5. Last 4 bytes are little endian uint32 of UNIX timestamp when + // this message was created. + handshakeCopyBuf := &bytes.Buffer{} + reader := io.TeeReader(conn, handshakeCopyBuf) + + reader, err := parseTLSHeader(reader) + if err != nil { + return nil, fmt.Errorf("cannot parse tls header: %w", err) + } + + reader, err = parseHandshakeHeader(reader) + if err != nil { + return nil, fmt.Errorf("cannot parse handshake header: %w", err) + } + + hello, err := parseHandshake(reader) + if err != nil { + return nil, fmt.Errorf("cannot parse handshake: %w", err) + } + + sniHostnames, err := parseSNI(reader) + if err != nil { + return nil, fmt.Errorf("cannot parse SNI: %w", err) + } + + if !slices.Contains(sniHostnames, secret.Host) { + return nil, fmt.Errorf("cannot find %s in %v", secret.Host, sniHostnames) + } + + digest := hmac.New(sha256.New, secret.Key[:]) + // we write a copy of the handshake with client random all nullified. + digest.Write(handshakeCopyBuf.Next(clientRandomOffset)) + handshakeCopyBuf.Next(RandomLen) + digest.Write(emptyRandom[:]) + digest.Write(handshakeCopyBuf.Bytes()) + + computed := digest.Sum(nil) + + for i := range RandomLen { + computed[i] ^= hello.Random[i] + } + + if subtle.ConstantTimeCompare(emptyRandom[:RandomLen-4], computed[:RandomLen-4]) != 1 { + return nil, ErrBadDigest + } + + timestamp := int64(binary.LittleEndian.Uint32(computed[RandomLen-4:])) + createdAt := time.Unix(timestamp, 0) + + if tdiff := time.Since(createdAt).Abs(); tdiff > tolerateTimeSkewness { + return nil, fmt.Errorf("timestamp %q is too old %s", createdAt, tdiff) + } + + return hello, nil +} + +func parseTLSHeader(r io.Reader) (io.Reader, error) { + // record_type(1) + version(2) + size(2) + // 16 - type is 0x16 (handshake record) + // 03 01 - protocol version is "3,1" (also known as TLS 1.0) + // 00 f8 - 0xF8 (248) bytes of handshake message follows + header := [1 + 2 + 2]byte{} + + if _, err := io.ReadFull(r, header[:]); err != nil { + return nil, fmt.Errorf("cannot read record header: %w", err) + } + + if header[0] != tls.TypeHandshake { + return nil, fmt.Errorf("unexpected record type %#x", header[0]) + } + + if header[1] != 3 || header[2] != 1 { + return nil, fmt.Errorf("unexpected protocol version %#x %#x", header[1], header[2]) + } + + length := int64(binary.BigEndian.Uint16(header[3:])) + buf := &bytes.Buffer{} + + _, err := io.CopyN(buf, r, length) + + return buf, err +} + +func parseHandshakeHeader(r io.Reader) (io.Reader, error) { + // type(1) + size(3 / uint24) + // 01 - handshake message type 0x01 (client hello) + // 00 00 f4 - 0xF4 (244) bytes of client hello data follows + header := [1 + 3]byte{} + + if _, err := io.ReadFull(r, header[:]); err != nil { + return nil, fmt.Errorf("cannot read handshake header: %w", err) + } + + if header[0] != TypeHandshakeClient { + return nil, fmt.Errorf("incorrect handshake type: %#x", header[0]) + } + + // unfortunately there is not uint24 in golang, so we just reust header + header[0] = 0 + + length := int64(binary.BigEndian.Uint32(header[:])) + buf := &bytes.Buffer{} + + _, err := io.CopyN(buf, r, length) + + return buf, err +} + +func parseHandshake(r io.Reader) (*ClientHello, error) { + // A protocol version of "3,3" (meaning TLS 1.2) is given. + header := [2]byte{} + + if _, err := io.ReadFull(r, header[:]); err != nil { + return nil, fmt.Errorf("cannot read client version: %w", err) + } + + hello := &ClientHello{} + + if _, err := io.ReadFull(r, hello.Random[:]); err != nil { + return nil, fmt.Errorf("cannot read client random: %w", err) + } + + if _, err := io.ReadFull(r, header[:1]); err != nil { + return nil, fmt.Errorf("cannot read session ID length: %w", err) + } + + hello.SessionID = make([]byte, int(header[0])) + + if _, err := io.ReadFull(r, hello.SessionID); err != nil { + return nil, fmt.Errorf("cannot read session id: %w", err) + } + + if _, err := io.ReadFull(r, header[:]); err != nil { + return nil, fmt.Errorf("cannot read cipher suite length: %w", err) + } + + cipherSuiteLen := int64(binary.BigEndian.Uint16(header[:])) + + // we do not care about picking up any cipher. we pick the first one, + // so it is always should be present. + if _, err := io.ReadFull(r, header[:]); err != nil { + return nil, fmt.Errorf("cannot read first cipher suite: %w", err) + } + + hello.CipherSuite = binary.BigEndian.Uint16(header[:]) + + if _, err := io.CopyN(io.Discard, r, cipherSuiteLen-2); err != nil { + return nil, fmt.Errorf("cannot skip remaining cipher suites: %w", err) + } + + if _, err := io.ReadFull(r, header[:1]); err != nil { + return nil, fmt.Errorf("cannot read compression methods length: %w", err) + } + + if _, err := io.CopyN(io.Discard, r, int64(header[0])); err != nil { + return nil, fmt.Errorf("cannot skip compression methods: %w", err) + } + + return hello, nil +} + +func parseSNI(r io.Reader) ([]string, error) { + header := [2]byte{} + + if _, err := io.ReadFull(r, header[:]); err != nil { + return nil, fmt.Errorf("cannot read length of TLS extensions: %w", err) + } + + extensionsLength := int64(binary.BigEndian.Uint16(header[:])) + buf := &bytes.Buffer{} + buf.Grow(int(extensionsLength)) + + if _, err := io.CopyN(buf, r, extensionsLength); err != nil { + return nil, fmt.Errorf("cannot read extensions: %w", err) + } + + for buf.Len() > 0 { + // 00 00 - assigned value for extension "server name" + // 00 18 - 0x18 (24) bytes of "server name" extension data follows + // 00 16 - 0x16 (22) bytes of first (and only) list entry follows + // 00 - list entry is type 0x00 "DNS hostname" + // 00 13 - 0x13 (19) bytes of hostname follows + // 65 78 61 ... 6e 65 74 - "example.ulfheim.net" + + // 00 00 - assigned value for extension "server name" + extTypeB := buf.Next(2) + if len(extTypeB) != 2 { + return nil, fmt.Errorf("cannot read extension type: %v", extTypeB) + } + + // 00 18 - 0x18 (24) bytes of "server name" extension data follows + lengthB := buf.Next(2) + if len(lengthB) != 2 { + return nil, fmt.Errorf("cannot read extension %v length: %v", extTypeB, lengthB) + } + length := int(binary.BigEndian.Uint16(lengthB)) + + extDataB := buf.Next(length) + if len(extDataB) != length { + return nil, fmt.Errorf("cannot read extension %v data: len %d != %d", extTypeB, length, len(extDataB)) + } + + if !bytes.Equal(extTypeB, extTypeSNI[:]) { + continue + } + + buf.Reset() + buf.Write(extDataB) + + // 00 16 - 0x16 (22) bytes of first (and only) list entry follows + lengthB = buf.Next(2) + if len(lengthB) != 2 { + return nil, fmt.Errorf("cannot read the length of the SNI record: %v", lengthB) + } + + length = int(binary.BigEndian.Uint16(lengthB)) + if length == 0 { + return nil, nil + } + + listType, err := buf.ReadByte() + if err != nil { + return nil, fmt.Errorf("cannot read SNI list type: %w", err) + } + + // 00 - list entry is type 0x00 "DNS hostname" + if listType != sniDNSNamesListType { + return nil, fmt.Errorf("incorrect SNI list type %#x", listType) + } + + names := []string{} + + for buf.Len() > 0 { + // 00 13 - 0x13 (19) bytes of hostname follows + lengthB = buf.Next(2) + if len(lengthB) != 2 { + return nil, fmt.Errorf("incorrect length of the hostname: %v", lengthB) + } + length = int(binary.BigEndian.Uint16(lengthB)) + + name := buf.Next(length) + if len(name) != length { + return nil, fmt.Errorf("incorrect length of SNI hostname: len %d != %d", length, len(name)) + } + + names = append(names, string(name)) + } + + return names, nil + } + + return nil, nil +} diff --git a/mtglib/internal/tls/fake/client_side_fuzz_test.go b/mtglib/internal/tls/fake/client_side_fuzz_test.go new file mode 100644 index 0000000..109873b --- /dev/null +++ b/mtglib/internal/tls/fake/client_side_fuzz_test.go @@ -0,0 +1,49 @@ +package fake_test + +import ( + "bytes" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls/fake" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +type connMock struct { + testlib.EssentialsConnMock + + readBuf *bytes.Buffer +} + +func (f *connMock) Read(p []byte) (int, error) { + return f.readBuf.Read(p) +} + +func FuzzReadClientHello(f *testing.F) { + seed := [248]byte{} + + secret, err := mtglib.ParseSecret( + "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d", + ) + require.NoError(f, err) + + f.Add(seed[:]) + + f.Fuzz(func(t *testing.T, value []byte) { + r := &connMock{ + readBuf: bytes.NewBuffer(value), + } + r. + On("SetReadDeadline", mock.AnythingOfType("time.Time")). + Twice(). + Return(nil) + + _, err := fake.ReadClientHello(r, secret, time.Hour) + assert.Error(t, err) + }) + +} diff --git a/mtglib/internal/tls/fake/client_side_snapshot_test.go b/mtglib/internal/tls/fake/client_side_snapshot_test.go new file mode 100644 index 0000000..fc60ce7 --- /dev/null +++ b/mtglib/internal/tls/fake/client_side_snapshot_test.go @@ -0,0 +1,143 @@ +package fake_test + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls/fake" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type clientHelloSnapshot struct { + Time int `json:"time"` + Random string `json:"random"` + SessionID string `json:"sessionId"` + Host string `json:"host"` + CipherSuite int `json:"cipherSuite"` + Full string `json:"full"` +} + +func (c clientHelloSnapshot) GetRandom() []byte { + data, _ := base64.StdEncoding.DecodeString(c.Random) + + return data +} + +func (c clientHelloSnapshot) GetSessionID() []byte { + data, _ := base64.StdEncoding.DecodeString(c.SessionID) + + return data +} + +func (c clientHelloSnapshot) GetCipherSuite() uint16 { + return uint16(c.CipherSuite) +} + +func (c clientHelloSnapshot) GetFull() []byte { + data, _ := base64.StdEncoding.DecodeString(c.Full) + + return data +} + +type ParseClientHelloSnapshotTestSuite struct { + suite.Suite + + secret mtglib.Secret +} + +func (suite *ParseClientHelloSnapshotTestSuite) SetupSuite() { + parsed, err := mtglib.ParseSecret( + "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d", + ) + require.NoError(suite.T(), err) + + suite.secret = parsed +} + +func (suite *ParseClientHelloSnapshotTestSuite) makeConn(data []byte) *parseClientHelloConnMock { + readBuf := &bytes.Buffer{} + readBuf.Write(data) + + connMock := &parseClientHelloConnMock{ + readBuf: readBuf, + } + + connMock. + On("SetReadDeadline", mock.AnythingOfType("time.Time")). + Twice(). + Return(nil) + + return connMock +} + +func (suite *ParseClientHelloSnapshotTestSuite) TestSnapshotOk() { + files, err := os.ReadDir("testdata") + require.NoError(suite.T(), err) + + for _, v := range files { + if !strings.HasPrefix(v.Name(), "client-hello-ok") { + continue + } + + path := filepath.Join("testdata", v.Name()) + + suite.T().Run(v.Name(), func(t *testing.T) { + fileData, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &clientHelloSnapshot{} + assert.NoError(t, json.Unmarshal(fileData, snapshot)) + + connMock := suite.makeConn(snapshot.GetFull()) + defer connMock.AssertExpectations(t) + + hello, err := fake.ReadClientHello(connMock, suite.secret, TolerateTime) + require.NoError(t, err) + + assert.Equal(t, snapshot.GetRandom(), hello.Random[:]) + assert.Equal(t, snapshot.GetSessionID(), hello.SessionID) + assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite) + }) + } +} + +func (suite *ParseClientHelloSnapshotTestSuite) TestSnapshotBad() { + files, err := os.ReadDir("testdata") + require.NoError(suite.T(), err) + + for _, v := range files { + if !strings.HasPrefix(v.Name(), "client-hello-bad") { + continue + } + + path := filepath.Join("testdata", v.Name()) + + suite.T().Run(v.Name(), func(t *testing.T) { + fileData, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &clientHelloSnapshot{} + assert.NoError(t, json.Unmarshal(fileData, snapshot)) + + connMock := suite.makeConn(snapshot.GetFull()) + defer connMock.AssertExpectations(t) + + _, err = fake.ReadClientHello(connMock, suite.secret, TolerateTime) + assert.ErrorIs(t, err, fake.ErrBadDigest) + }) + } +} + +func TestParseClientHelloSnapshot(t *testing.T) { + t.Parallel() + suite.Run(t, &ParseClientHelloSnapshotTestSuite{}) +} diff --git a/mtglib/internal/tls/fake/client_side_test.go b/mtglib/internal/tls/fake/client_side_test.go new file mode 100644 index 0000000..edf5901 --- /dev/null +++ b/mtglib/internal/tls/fake/client_side_test.go @@ -0,0 +1,395 @@ +package fake_test + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" + "github.com/9seconds/mtg/v2/mtglib/internal/tls/fake" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + TolerateTime = 365 * 30 * 24 * time.Hour +) + +type parseClientHelloConnMock struct { + testlib.EssentialsConnMock + + readBuf *bytes.Buffer +} + +func (m *parseClientHelloConnMock) Read(p []byte) (int, error) { + return m.readBuf.Read(p) +} + +type ParseClientHelloTestSuite struct { + suite.Suite + + secret mtglib.Secret + readBuf *bytes.Buffer + connMock *parseClientHelloConnMock +} + +func (suite *ParseClientHelloTestSuite) SetupSuite() { + parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d") + require.NoError(suite.T(), err) + + suite.secret = parsed +} + +func (suite *ParseClientHelloTestSuite) SetupTest() { + suite.readBuf = &bytes.Buffer{} + suite.connMock = &parseClientHelloConnMock{ + readBuf: suite.readBuf, + } + + suite.connMock. + On("SetReadDeadline", mock.AnythingOfType("time.Time")). + Twice(). + Return(nil) +} + +func (suite *ParseClientHelloTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +type ParseClientHello_TLSHeaderTestSuite struct { + ParseClientHelloTestSuite +} + +func (suite *ParseClientHello_TLSHeaderTestSuite) TestEmpty() { + suite.connMock.ExpectedCalls = []*mock.Call{} + suite.connMock. + On("SetReadDeadline", mock.AnythingOfType("time.Time")). + Once(). + Return(errors.New("fail")) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "fail") +} + +func (suite *ParseClientHello_TLSHeaderTestSuite) TestNothing() { + suite.connMock.ExpectedCalls = []*mock.Call{} + suite.connMock. + On("SetReadDeadline", mock.AnythingOfType("time.Time")). + Twice(). + Return(nil) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorIs(err, io.EOF) +} + +func (suite *ParseClientHello_TLSHeaderTestSuite) TestUnknownRecord() { + suite.readBuf.Write([]byte{ + 10, + 3, 3, + 0, 0, + }) + suite.readBuf.WriteByte(10) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "unexpected record type 0xa") +} + +func (suite *ParseClientHello_TLSHeaderTestSuite) TestUnknownProtocolVersion() { + suite.readBuf.Write([]byte{ + tls.TypeHandshake, + 3, 3, + 0, 0, + }) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "unexpected protocol version") +} + +func (suite *ParseClientHello_TLSHeaderTestSuite) TestCannotReadRestOfRecord() { + suite.readBuf.Write([]byte{ + tls.TypeHandshake, + 3, 1, + 0, 10, + }) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorIs(err, io.EOF) +} + +type ParseClientHelloHandshakeTestSuite struct { + ParseClientHelloTestSuite +} + +func (suite *ParseClientHelloHandshakeTestSuite) SetupTest() { + suite.ParseClientHelloTestSuite.SetupTest() + + suite.readBuf.Write([]byte{ + tls.TypeHandshake, + 3, 1, + 0, + }) +} + +func (suite *ParseClientHelloHandshakeTestSuite) TestCannotReadHeader() { + suite.readBuf.Write([]byte{ + 1, + 10, + }) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read handshake header") +} + +func (suite *ParseClientHelloHandshakeTestSuite) TestIncorrectHandshakeType() { + suite.readBuf.Write([]byte{ + 4, + 10, 0, 0, 0, + }) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "incorrect handshake type") +} + +func (suite *ParseClientHelloHandshakeTestSuite) TestCannotReadHandshake() { + suite.readBuf.Write([]byte{ + 4 + 3, + 10, 0, 0, 0, + }) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorIs(err, io.EOF) +} + +type ParseClientHelloHandshakeBodyTestSuite struct { + ParseClientHelloTestSuite +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) SetupTest() { + suite.ParseClientHelloTestSuite.SetupTest() + + suite.readBuf.Write([]byte{ + tls.TypeHandshake, + 3, 1, + 0, + }) +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) writeBody(body []byte) { + suite.readBuf.WriteByte(byte(4 + len(body))) + suite.readBuf.Write([]byte{ + fake.TypeHandshakeClient, + 0, 0, byte(len(body)), + }) + suite.readBuf.Write(body) +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadVersion() { + suite.writeBody(nil) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read client version") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadRandom() { + suite.writeBody([]byte{3, 3}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read client random") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadSessionIDLength() { + body := make([]byte, 2+fake.RandomLen) + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read session ID length") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadSessionID() { + body := make([]byte, 2+fake.RandomLen+1) + body[2+fake.RandomLen] = 32 + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read session id") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadCipherSuiteLength() { + body := make([]byte, 2+fake.RandomLen+1) + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read cipher suite length") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadFirstCipherSuite() { + body := make([]byte, 2+fake.RandomLen+1+2) + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read first cipher suite") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotSkipRemainingCipherSuites() { + body := make([]byte, 2+fake.RandomLen+1+2+2) + binary.BigEndian.PutUint16(body[2+fake.RandomLen+1:], 4) + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot skip remaining cipher suites") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadCompressionMethodsLength() { + body := make([]byte, 2+fake.RandomLen+1+2+2) + binary.BigEndian.PutUint16(body[2+fake.RandomLen+1:], 2) + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read compression methods length") +} + +func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotSkipCompressionMethods() { + body := make([]byte, 2+fake.RandomLen+1+2+2+1) + binary.BigEndian.PutUint16(body[2+fake.RandomLen+1:], 2) + body[2+fake.RandomLen+1+2+2] = 1 + + suite.writeBody(body) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot skip compression methods") +} + +type ParseClientHelloSNITestSuite struct { + ParseClientHelloTestSuite +} + +func (suite *ParseClientHelloSNITestSuite) SetupTest() { + suite.ParseClientHelloTestSuite.SetupTest() + + suite.readBuf.Write([]byte{ + tls.TypeHandshake, + 3, 1, + 0, + }) +} + +func (suite *ParseClientHelloSNITestSuite) writeExtensions(extensions []byte) { + handshakeBodyLen := 41 + len(extensions) + + suite.readBuf.WriteByte(byte(4 + handshakeBodyLen)) + suite.readBuf.Write([]byte{ + fake.TypeHandshakeClient, + 0, 0, byte(handshakeBodyLen), + }) + + // version(2) + random(32) + sessionIDLen(1) + cipherSuiteLen(2) + + // cipherSuite(2) + compressionLen(1) + compression(1) = 41 + body := make([]byte, 41) + binary.BigEndian.PutUint16(body[35:], 2) + body[39] = 1 + + suite.readBuf.Write(body) + suite.readBuf.Write(extensions) +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionsLength() { + suite.writeExtensions(nil) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read length of TLS extensions") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensions() { + suite.writeExtensions([]byte{0, 10}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read extensions") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionType() { + suite.writeExtensions([]byte{0, 1, 0xAB}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read extension type") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionLength() { + suite.writeExtensions([]byte{0, 2, 0xFF, 0xFF}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "length:") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionData() { + suite.writeExtensions([]byte{0, 4, 0xFF, 0xFF, 0, 5}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "data: len") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadSNIRecordLength() { + suite.writeExtensions([]byte{0, 5, 0, 0, 0, 1, 0xAB}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read the length of the SNI record") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadSNIListType() { + suite.writeExtensions([]byte{0, 6, 0, 0, 0, 2, 0, 1}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "cannot read SNI list type") +} + +func (suite *ParseClientHelloSNITestSuite) TestIncorrectSNIListType() { + suite.writeExtensions([]byte{0, 7, 0, 0, 0, 3, 0, 1, 5}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "incorrect SNI list type") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadHostnameLength() { + suite.writeExtensions([]byte{0, 8, 0, 0, 0, 4, 0, 2, 0, 0xAB}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "incorrect length of the hostname") +} + +func (suite *ParseClientHelloSNITestSuite) TestCannotReadHostname() { + suite.writeExtensions([]byte{0, 9, 0, 0, 0, 5, 0, 3, 0, 0, 5}) + + _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + suite.ErrorContains(err, "incorrect length of SNI hostname") +} + +func TestParseClientHelloTLSHeader(t *testing.T) { + t.Parallel() + suite.Run(t, &ParseClientHello_TLSHeaderTestSuite{}) +} + +func TestParseClientHelloHandshake(t *testing.T) { + t.Parallel() + suite.Run(t, &ParseClientHelloHandshakeTestSuite{}) +} + +func TestParseClientHelloHandshakeBody(t *testing.T) { + t.Parallel() + suite.Run(t, &ParseClientHelloHandshakeBodyTestSuite{}) +} + +func TestParseClientHelloSNI(t *testing.T) { + t.Parallel() + suite.Run(t, &ParseClientHelloSNITestSuite{}) +} diff --git a/mtglib/internal/tls/fake/init.go b/mtglib/internal/tls/fake/init.go new file mode 100644 index 0000000..77bbe6b --- /dev/null +++ b/mtglib/internal/tls/fake/init.go @@ -0,0 +1,16 @@ +package fake + +import ( + "errors" + "time" +) + +const ( + ClientHelloReadTimeout = 5 * time.Second +) + +var ( + resetDeadline time.Time + + ErrBadDigest = errors.New("incorrect client random") +) diff --git a/mtglib/internal/tls/fake/testdata/client-hello-bad-fa2e46cdb33e2a1b.json b/mtglib/internal/tls/fake/testdata/client-hello-bad-fa2e46cdb33e2a1b.json new file mode 100644 index 0000000..3bacba3 --- /dev/null +++ b/mtglib/internal/tls/fake/testdata/client-hello-bad-fa2e46cdb33e2a1b.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", + "sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", + "host": "storage.googleapis.com", + "cipherSuite": 4867, + "full": "FgMBAgABAAH8AwNe8I9zdoBsduFEu/SRSbLoF89k4a+y6x53n8c2wpcQ+yBK3YFna4cwWfcHa2sPWN922mOgk46DokF4uEVzIIAKrgA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUA0AAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACAH/ugvH0kSUgAuwslL3UfZA3JTUfSiwrAhR6VWd2wvIgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +} diff --git a/mtglib/internal/tls/fake/testdata/client-hello-ok-19dfe38384b9884b.json b/mtglib/internal/tls/fake/testdata/client-hello-ok-19dfe38384b9884b.json new file mode 100644 index 0000000..e7a2ad5 --- /dev/null +++ b/mtglib/internal/tls/fake/testdata/client-hello-ok-19dfe38384b9884b.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", + "sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", + "host": "storage.googleapis.com", + "cipherSuite": 4867, + "full": "FgMBAgABAAH8AwNe8I9zdoBsduFEu/SRSbLoF89k4a+y6x53n8c2wpcQ+yBK3YFna4cwWfcHa2sPWN922mOgk46DokF4uEVzIIwKrgA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACAH/ugvH0kSUgAuwslL3UfZA3JTUfSiwrAhR6VWd2wvIgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +} diff --git a/mtglib/internal/tls/fake/testdata/client-hello-ok-48f8a72a56f3174a.json b/mtglib/internal/tls/fake/testdata/client-hello-ok-48f8a72a56f3174a.json new file mode 100644 index 0000000..e24429e --- /dev/null +++ b/mtglib/internal/tls/fake/testdata/client-hello-ok-48f8a72a56f3174a.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=", + "sessionId": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=", + "host": "storage.googleapis.com", + "cipherSuite": 4867, + "full": "FgMBAgABAAH8AwOhgS7feOX7NBtQoy1BtXU4cHSBcgzbLZqr2JjRBfXaiyAUaoDdkVitKWP//GXuVqaY2frgr38wraZDcclQa8/7yAA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACBroKhykU/xB3hgIVH2mRoKv3umjYAuPQ/mcj02dvdRYwAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +} diff --git a/mtglib/internal/tls/fake/testdata/client-hello-ok-651054256093c6cd.json b/mtglib/internal/tls/fake/testdata/client-hello-ok-651054256093c6cd.json new file mode 100644 index 0000000..517bc1b --- /dev/null +++ b/mtglib/internal/tls/fake/testdata/client-hello-ok-651054256093c6cd.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=", + "sessionId": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=", + "host": "storage.googleapis.com", + "cipherSuite": 4867, + "full": "FgMBAgABAAH8AwPlXmxKmuT+0UiDL7HUF4o0aEuUWSp+CAumA3sZg4DWuiCPGvh3o9c8OT4vDHdZSn3C+PxMaU7ER2tEnTJ3In34fwA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACCu6UBqpR0p5VgzQX6m7qif+HosGk7LM4objEUgpygWTgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +} diff --git a/mtglib/internal/tls/fake/testdata/client-hello-ok-79d01ef18a9d2621.json b/mtglib/internal/tls/fake/testdata/client-hello-ok-79d01ef18a9d2621.json new file mode 100644 index 0000000..b8b0d63 --- /dev/null +++ b/mtglib/internal/tls/fake/testdata/client-hello-ok-79d01ef18a9d2621.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=", + "sessionId": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=", + "host": "storage.googleapis.com", + "cipherSuite": 4867, + "full": "FgMBAgABAAH8AwPzGWOU6GQOWRp8QXm+7d7Wvd9a+HwBfnjzALeEtnezNCDTS68NgqcXJkUrJ/ceUvBYYI7J4ew8WJTlRn4WzmkMBQA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACD/0/vXjQ20rOPIPAF/32Y7LX4WNE8A8dM1D1bEc4qlXgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +} diff --git a/mtglib/internal/tls/fake/testdata/client-hello-ok-7a5569f05b118145.json b/mtglib/internal/tls/fake/testdata/client-hello-ok-7a5569f05b118145.json new file mode 100644 index 0000000..944d586 --- /dev/null +++ b/mtglib/internal/tls/fake/testdata/client-hello-ok-7a5569f05b118145.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=", + "sessionId": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=", + "host": "storage.googleapis.com", + "cipherSuite": 4867, + "full": "FgMBAgABAAH8AwPONrcwtnxYZJ+xBG09Xvn61jqBsrfMo+LVLLtIEFp+DiCo+63bIypdr3MYsgj/zvZJDd3hN+qjS6gNvu2mRF8GcAA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACBe+ItECoBgnzE4t2VyxSGV0jheXSD+z37LZCt3yto8SAAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" +} From 37f8d18be510b42483032087d6b884d0925c543b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 11 Mar 2026 23:20:26 +0100 Subject: [PATCH 05/42] Add server side of things --- mtglib/internal/tls/fake/bytes_pool.go | 23 +++ mtglib/internal/tls/fake/client_side.go | 6 +- mtglib/internal/tls/fake/server_side.go | 141 +++++++++++++++++++ mtglib/internal/tls/fake/server_side_test.go | 131 +++++++++++++++++ 4 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 mtglib/internal/tls/fake/bytes_pool.go create mode 100644 mtglib/internal/tls/fake/server_side.go create mode 100644 mtglib/internal/tls/fake/server_side_test.go diff --git a/mtglib/internal/tls/fake/bytes_pool.go b/mtglib/internal/tls/fake/bytes_pool.go new file mode 100644 index 0000000..2eaca15 --- /dev/null +++ b/mtglib/internal/tls/fake/bytes_pool.go @@ -0,0 +1,23 @@ +package fake + +import ( + "bytes" + "sync" +) + +var ( + bytesPool = sync.Pool{ + New: func() any { + return &bytes.Buffer{} + }, + } +) + +func acquireBuffer() *bytes.Buffer { + return bytesPool.Get().(*bytes.Buffer) +} + +func releaseBuffer(b *bytes.Buffer) { + b.Reset() + bytesPool.Put(b) +} diff --git a/mtglib/internal/tls/fake/client_side.go b/mtglib/internal/tls/fake/client_side.go index dc41397..8d89b5a 100644 --- a/mtglib/internal/tls/fake/client_side.go +++ b/mtglib/internal/tls/fake/client_side.go @@ -20,9 +20,9 @@ const ( TypeHandshakeClient = 0x01 RandomLen = 32 - // record_type(1) + version(2) + size(2) + handshake_type(1) + uint24_length(3) + client_version(2) - clientRandomOffset = 1 + 2 + 2 + 1 + 3 + 2 + RandomOffset = 1 + 2 + 2 + 1 + 3 + 2 + sniDNSNamesListType = 0 ) @@ -81,7 +81,7 @@ func ReadClientHello(conn net.Conn, secret mtglib.Secret, tolerateTimeSkewness t digest := hmac.New(sha256.New, secret.Key[:]) // we write a copy of the handshake with client random all nullified. - digest.Write(handshakeCopyBuf.Next(clientRandomOffset)) + digest.Write(handshakeCopyBuf.Next(RandomOffset)) handshakeCopyBuf.Next(RandomLen) digest.Write(emptyRandom[:]) digest.Write(handshakeCopyBuf.Bytes()) diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go new file mode 100644 index 0000000..8f920c5 --- /dev/null +++ b/mtglib/internal/tls/fake/server_side.go @@ -0,0 +1,141 @@ +package fake + +import ( + "bytes" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "io" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" + "golang.org/x/crypto/curve25519" +) + +const ( + TypeHandshakeServer = 0x02 + + ChangeCipherValue = 0x01 + + EllipticCurveLen = 32 +) + +var ( + serverHelloSuffix = []byte{ + 0x00, // no compression + 0x00, 0x2e, // 46 bytes of data + 0x00, 0x2b, // Extension - Supported Versions + 0x00, 0x02, // 2 bytes are following + 0x03, 0x04, // TLS 1.3 + 0x00, 0x33, // Extension - Key Share + 0x00, 0x24, // 36 bytes + 0x00, 0x1d, // x25519 curve + 0x00, 0x20, // 32 bytes of key + } +) + +func SendServerHello(w io.Writer, secret mtglib.Secret, clientHello *ClientHello) ([]byte, error) { + buf := &bytes.Buffer{} + buf.Grow(tls.MaxRecordSize) + + generateServerHello(buf, clientHello) + generateChangeCipherValue(buf) + + noise := &bytes.Buffer{} + generateNoise(noise) + + packet := buf.Bytes() + digest := hmac.New(sha256.New, secret.Key[:]) + + digest.Write(clientHello.Random[:]) + digest.Write(packet) + digest.Write(noise.Bytes()) + copy(packet[RandomOffset:], digest.Sum(nil)) + + _, err := w.Write(packet) + + return noise.Bytes(), err +} + +func generateServerHello(buf *bytes.Buffer, hello *ClientHello) { + payload := acquireBuffer() + defer releaseBuffer(payload) + + generateServerHelloPayload(payload, hello) + + // 16 - type is 0x16 (handshake record) + // 03 03 - legacy protocol version of "3,3" (TLS 1.2) + // 00 7a - 0x7A (122) bytes of handshake message follows + + // 16 - type is 0x16 (handshake record) + buf.WriteByte(tls.TypeHandshake) + // 03 03 - legacy protocol version of "3,3" (TLS 1.2) + buf.Write(tls.TLSVersion[:]) + // 00 7a - 0x7A (122) bytes of handshake message follows + binary.Write(buf, binary.BigEndian, uint16(payload.Len())) //nolint: errcheck + + payload.WriteTo(buf) //nolint: errcheck +} + +func generateServerHelloPayload(buf *bytes.Buffer, hello *ClientHello) { + data := [4]byte{} + + payload := acquireBuffer() + defer releaseBuffer(payload) + + generateServerHelloHandshakePayload(payload, hello) + + // 02 - handshake message type 0x02 (server hello) + // 00 00 76 - 0x76 (118) bytes of server hello data follows + buf.WriteByte(TypeHandshakeServer) + // 00 00 76 - 0x76 (118) bytes of server hello data follows + binary.BigEndian.PutUint32(data[:], uint32(payload.Len())) + buf.Write(data[1:]) + + payload.WriteTo(buf) //nolint: errcheck +} + +func generateServerHelloHandshakePayload(buf *bytes.Buffer, hello *ClientHello) { + // The unusual version number ("3,3" representing TLS 1.2) is due to + // TLS 1.0 being a minor revision of the SSL 3.0 protocol. Therefore + // TLS 1.0 is represented by "3,1", TLS 1.1 is "3,2", and so on. + buf.Write(tls.TLSVersion[:]) + + buf.Write(emptyRandom[:]) + + // 20 - 0x20 (32) bytes of session ID follow + // e0 e1 ... fe ff - session ID copied from Client Hello + buf.WriteByte(byte(len(hello.SessionID))) + buf.Write(hello.SessionID) + + binary.Write(buf, binary.BigEndian, hello.CipherSuite) //nolint: errcheck + + buf.Write(serverHelloSuffix) + + scalar := [EllipticCurveLen]byte{} + + if _, err := rand.Read(scalar[:]); err != nil { + panic(err) + } + + curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) + buf.Write(curve) +} + +func generateChangeCipherValue(buf *bytes.Buffer) { + buf.WriteByte(tls.TypeChangeCipherSpec) + buf.Write(tls.TLSVersion[:]) + binary.Write(buf, binary.BigEndian, uint16(1)) //nolint: errcheck + buf.WriteByte(ChangeCipherValue) +} + +func generateNoise(buf *bytes.Buffer) { + data := [1369]byte{} + + if _, err := rand.Read(data[:]); err != nil { + panic(err) + } + + tls.WriteRecord(buf, data[:]) //nolint: errcheck +} diff --git a/mtglib/internal/tls/fake/server_side_test.go b/mtglib/internal/tls/fake/server_side_test.go new file mode 100644 index 0000000..05508ba --- /dev/null +++ b/mtglib/internal/tls/fake/server_side_test.go @@ -0,0 +1,131 @@ +package fake_test + +import ( + "bytes" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "testing" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" + "github.com/9seconds/mtg/v2/mtglib/internal/tls/fake" + "github.com/stretchr/testify/suite" +) + +type SendServerHelloTestSuite struct { + suite.Suite + + hello *fake.ClientHello + buf *bytes.Buffer + secret mtglib.Secret +} + +func (suite *SendServerHelloTestSuite) SetupTest() { + suite.hello = &fake.ClientHello{ + CipherSuite: 4867, + SessionID: make([]byte, 32), + } + + _, err := rand.Read(suite.hello.SessionID) + suite.NoError(err) + + _, err = rand.Read(suite.hello.Random[:]) + suite.NoError(err) + + suite.buf = &bytes.Buffer{} + suite.secret = mtglib.GenerateSecret("google.com") +} + +func (suite *SendServerHelloTestSuite) TestRecordStructure() { + noise, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + suite.NoError(err) + + var rec bytes.Buffer + + recordType, _, err := tls.ReadRecord(suite.buf, &rec) + suite.NoError(err) + suite.Equal(byte(tls.TypeHandshake), recordType) + + rec.Reset() + + recordType, _, err = tls.ReadRecord(suite.buf, &rec) + suite.NoError(err) + suite.Equal(byte(tls.TypeChangeCipherSpec), recordType) + + suite.Empty(suite.buf.Bytes()) + + noiseBuf := bytes.NewReader(noise) + rec.Reset() + + recordType, _, err = tls.ReadRecord(noiseBuf, &rec) + suite.NoError(err) + suite.Equal(byte(tls.TypeApplicationData), recordType) + suite.Zero(noiseBuf.Len()) +} + +func (suite *SendServerHelloTestSuite) TestHMAC() { + noise, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + suite.NoError(err) + + packet := make([]byte, suite.buf.Len()) + copy(packet, suite.buf.Bytes()) + + random := make([]byte, fake.RandomLen) + copy(random, packet[fake.RandomOffset:]) + copy(packet[fake.RandomOffset:], make([]byte, fake.RandomLen)) + + mac := hmac.New(sha256.New, suite.secret.Key[:]) + mac.Write(suite.hello.Random[:]) + mac.Write(packet) + mac.Write(noise) + + suite.Equal(random, mac.Sum(nil)) +} + +func (suite *SendServerHelloTestSuite) TestHandshakePayload() { + _, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + suite.NoError(err) + + packet := suite.buf.Bytes() + + // TLS record header: type(1) + version(2) + length(2) + suite.Equal(byte(tls.TypeHandshake), packet[0]) + suite.Equal([]byte{3, 3}, packet[1:3]) + + // Handshake header: type(1) + uint24_length(3) + suite.Equal(byte(fake.TypeHandshakeServer), packet[5]) + + // ServerHello version + suite.Equal([]byte{3, 3}, packet[9:11]) + + // Session ID + sessionIDOffset := fake.RandomOffset + fake.RandomLen + suite.Equal(byte(len(suite.hello.SessionID)), packet[sessionIDOffset]) + suite.Equal(suite.hello.SessionID, packet[sessionIDOffset+1:sessionIDOffset+1+len(suite.hello.SessionID)]) +} + +func (suite *SendServerHelloTestSuite) TestChangeCipherSpec() { + _, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + suite.NoError(err) + + // Skip first record + var rec bytes.Buffer + + _, _, err = tls.ReadRecord(suite.buf, &rec) + suite.NoError(err) + + // Read ChangeCipherSpec record + rec.Reset() + + recordType, length, err := tls.ReadRecord(suite.buf, &rec) + suite.NoError(err) + suite.Equal(byte(tls.TypeChangeCipherSpec), recordType) + suite.Equal(int64(1), length) + suite.Equal([]byte{fake.ChangeCipherValue}, rec.Bytes()) +} + +func TestSendServerHello(t *testing.T) { + t.Parallel() + suite.Run(t, &SendServerHelloTestSuite{}) +} From bb49c6a55d30ef11eb9e6cf2a84c52e602a55aa8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 00:04:37 +0100 Subject: [PATCH 06/42] Integrate new fake package and doppel into proxy --- mtglib/internal/tls/fake/client_side.go | 14 +++-- .../tls/fake/client_side_fuzz_test.go | 2 +- .../tls/fake/client_side_snapshot_test.go | 14 ++++- mtglib/internal/tls/fake/client_side_test.go | 54 +++++++++---------- mtglib/internal/tls/fake/server_side.go | 5 +- mtglib/internal/tls/fake/server_side_test.go | 8 +-- mtglib/proxy.go | 51 +++++++----------- network/v2/http.go | 13 +---- 8 files changed, 76 insertions(+), 85 deletions(-) diff --git a/mtglib/internal/tls/fake/client_side.go b/mtglib/internal/tls/fake/client_side.go index 8d89b5a..3b7e5a0 100644 --- a/mtglib/internal/tls/fake/client_side.go +++ b/mtglib/internal/tls/fake/client_side.go @@ -12,7 +12,6 @@ import ( "slices" "time" - "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/mtglib/internal/tls" ) @@ -37,7 +36,12 @@ type ClientHello struct { CipherSuite uint16 } -func ReadClientHello(conn net.Conn, secret mtglib.Secret, tolerateTimeSkewness time.Duration) (*ClientHello, error) { +func ReadClientHello( + conn net.Conn, + secret []byte, + hostname string, + tolerateTimeSkewness time.Duration, +) (*ClientHello, error) { if err := conn.SetReadDeadline(time.Now().Add(ClientHelloReadTimeout)); err != nil { return nil, fmt.Errorf("cannot set read deadline: %w", err) } @@ -75,11 +79,11 @@ func ReadClientHello(conn net.Conn, secret mtglib.Secret, tolerateTimeSkewness t return nil, fmt.Errorf("cannot parse SNI: %w", err) } - if !slices.Contains(sniHostnames, secret.Host) { - return nil, fmt.Errorf("cannot find %s in %v", secret.Host, sniHostnames) + if !slices.Contains(sniHostnames, hostname) { + return nil, fmt.Errorf("cannot find %s in %v", hostname, sniHostnames) } - digest := hmac.New(sha256.New, secret.Key[:]) + digest := hmac.New(sha256.New, secret) // we write a copy of the handshake with client random all nullified. digest.Write(handshakeCopyBuf.Next(RandomOffset)) handshakeCopyBuf.Next(RandomLen) diff --git a/mtglib/internal/tls/fake/client_side_fuzz_test.go b/mtglib/internal/tls/fake/client_side_fuzz_test.go index 109873b..28f3337 100644 --- a/mtglib/internal/tls/fake/client_side_fuzz_test.go +++ b/mtglib/internal/tls/fake/client_side_fuzz_test.go @@ -42,7 +42,7 @@ func FuzzReadClientHello(f *testing.F) { Twice(). Return(nil) - _, err := fake.ReadClientHello(r, secret, time.Hour) + _, err := fake.ReadClientHello(r, secret.Key[:], secret.Host, time.Hour) assert.Error(t, err) }) diff --git a/mtglib/internal/tls/fake/client_side_snapshot_test.go b/mtglib/internal/tls/fake/client_side_snapshot_test.go index fc60ce7..e4e0480 100644 --- a/mtglib/internal/tls/fake/client_side_snapshot_test.go +++ b/mtglib/internal/tls/fake/client_side_snapshot_test.go @@ -100,7 +100,12 @@ func (suite *ParseClientHelloSnapshotTestSuite) TestSnapshotOk() { connMock := suite.makeConn(snapshot.GetFull()) defer connMock.AssertExpectations(t) - hello, err := fake.ReadClientHello(connMock, suite.secret, TolerateTime) + hello, err := fake.ReadClientHello( + connMock, + suite.secret.Key[:], + suite.secret.Host, + TolerateTime, + ) require.NoError(t, err) assert.Equal(t, snapshot.GetRandom(), hello.Random[:]) @@ -131,7 +136,12 @@ func (suite *ParseClientHelloSnapshotTestSuite) TestSnapshotBad() { connMock := suite.makeConn(snapshot.GetFull()) defer connMock.AssertExpectations(t) - _, err = fake.ReadClientHello(connMock, suite.secret, TolerateTime) + _, err = fake.ReadClientHello( + connMock, + suite.secret.Key[:], + suite.secret.Host, + TolerateTime, + ) assert.ErrorIs(t, err, fake.ErrBadDigest) }) } diff --git a/mtglib/internal/tls/fake/client_side_test.go b/mtglib/internal/tls/fake/client_side_test.go index edf5901..2e66b6c 100644 --- a/mtglib/internal/tls/fake/client_side_test.go +++ b/mtglib/internal/tls/fake/client_side_test.go @@ -73,7 +73,7 @@ func (suite *ParseClientHello_TLSHeaderTestSuite) TestEmpty() { Once(). Return(errors.New("fail")) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "fail") } @@ -84,7 +84,7 @@ func (suite *ParseClientHello_TLSHeaderTestSuite) TestNothing() { Twice(). Return(nil) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorIs(err, io.EOF) } @@ -96,7 +96,7 @@ func (suite *ParseClientHello_TLSHeaderTestSuite) TestUnknownRecord() { }) suite.readBuf.WriteByte(10) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "unexpected record type 0xa") } @@ -107,7 +107,7 @@ func (suite *ParseClientHello_TLSHeaderTestSuite) TestUnknownProtocolVersion() { 0, 0, }) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "unexpected protocol version") } @@ -118,7 +118,7 @@ func (suite *ParseClientHello_TLSHeaderTestSuite) TestCannotReadRestOfRecord() { 0, 10, }) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorIs(err, io.EOF) } @@ -142,7 +142,7 @@ func (suite *ParseClientHelloHandshakeTestSuite) TestCannotReadHeader() { 10, }) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read handshake header") } @@ -152,7 +152,7 @@ func (suite *ParseClientHelloHandshakeTestSuite) TestIncorrectHandshakeType() { 10, 0, 0, 0, }) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "incorrect handshake type") } @@ -162,7 +162,7 @@ func (suite *ParseClientHelloHandshakeTestSuite) TestCannotReadHandshake() { 10, 0, 0, 0, }) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorIs(err, io.EOF) } @@ -192,14 +192,14 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) writeBody(body []byte) { func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadVersion() { suite.writeBody(nil) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read client version") } func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadRandom() { suite.writeBody([]byte{3, 3}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read client random") } @@ -208,7 +208,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadSessionIDLeng suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read session ID length") } @@ -218,7 +218,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadSessionID() { suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read session id") } @@ -227,7 +227,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadCipherSuiteLe suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read cipher suite length") } @@ -236,7 +236,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadFirstCipherSu suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read first cipher suite") } @@ -246,7 +246,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotSkipRemainingCiph suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot skip remaining cipher suites") } @@ -256,7 +256,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadCompressionMe suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read compression methods length") } @@ -267,7 +267,7 @@ func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotSkipCompressionMe suite.writeBody(body) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot skip compression methods") } @@ -307,70 +307,70 @@ func (suite *ParseClientHelloSNITestSuite) writeExtensions(extensions []byte) { func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionsLength() { suite.writeExtensions(nil) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read length of TLS extensions") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensions() { suite.writeExtensions([]byte{0, 10}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read extensions") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionType() { suite.writeExtensions([]byte{0, 1, 0xAB}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read extension type") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionLength() { suite.writeExtensions([]byte{0, 2, 0xFF, 0xFF}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "length:") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionData() { suite.writeExtensions([]byte{0, 4, 0xFF, 0xFF, 0, 5}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "data: len") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadSNIRecordLength() { suite.writeExtensions([]byte{0, 5, 0, 0, 0, 1, 0xAB}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read the length of the SNI record") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadSNIListType() { suite.writeExtensions([]byte{0, 6, 0, 0, 0, 2, 0, 1}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "cannot read SNI list type") } func (suite *ParseClientHelloSNITestSuite) TestIncorrectSNIListType() { suite.writeExtensions([]byte{0, 7, 0, 0, 0, 3, 0, 1, 5}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "incorrect SNI list type") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadHostnameLength() { suite.writeExtensions([]byte{0, 8, 0, 0, 0, 4, 0, 2, 0, 0xAB}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "incorrect length of the hostname") } func (suite *ParseClientHelloSNITestSuite) TestCannotReadHostname() { suite.writeExtensions([]byte{0, 9, 0, 0, 0, 5, 0, 3, 0, 0, 5}) - _, err := fake.ReadClientHello(suite.connMock, suite.secret, TolerateTime) + _, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime) suite.ErrorContains(err, "incorrect length of SNI hostname") } diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go index 8f920c5..27ae0d6 100644 --- a/mtglib/internal/tls/fake/server_side.go +++ b/mtglib/internal/tls/fake/server_side.go @@ -8,7 +8,6 @@ import ( "encoding/binary" "io" - "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/mtglib/internal/tls" "golang.org/x/crypto/curve25519" ) @@ -35,7 +34,7 @@ var ( } ) -func SendServerHello(w io.Writer, secret mtglib.Secret, clientHello *ClientHello) ([]byte, error) { +func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) ([]byte, error) { buf := &bytes.Buffer{} buf.Grow(tls.MaxRecordSize) @@ -46,7 +45,7 @@ func SendServerHello(w io.Writer, secret mtglib.Secret, clientHello *ClientHello generateNoise(noise) packet := buf.Bytes() - digest := hmac.New(sha256.New, secret.Key[:]) + digest := hmac.New(sha256.New, secret) digest.Write(clientHello.Random[:]) digest.Write(packet) diff --git a/mtglib/internal/tls/fake/server_side_test.go b/mtglib/internal/tls/fake/server_side_test.go index 05508ba..37d4246 100644 --- a/mtglib/internal/tls/fake/server_side_test.go +++ b/mtglib/internal/tls/fake/server_side_test.go @@ -38,7 +38,7 @@ func (suite *SendServerHelloTestSuite) SetupTest() { } func (suite *SendServerHelloTestSuite) TestRecordStructure() { - noise, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + noise, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) var rec bytes.Buffer @@ -65,7 +65,7 @@ func (suite *SendServerHelloTestSuite) TestRecordStructure() { } func (suite *SendServerHelloTestSuite) TestHMAC() { - noise, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + noise, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) packet := make([]byte, suite.buf.Len()) @@ -84,7 +84,7 @@ func (suite *SendServerHelloTestSuite) TestHMAC() { } func (suite *SendServerHelloTestSuite) TestHandshakePayload() { - _, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + _, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) packet := suite.buf.Bytes() @@ -106,7 +106,7 @@ func (suite *SendServerHelloTestSuite) TestHandshakePayload() { } func (suite *SendServerHelloTestSuite) TestChangeCipherSpec() { - _, err := fake.SendServerHello(suite.buf, suite.secret, suite.hello) + _, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) // Skip first record diff --git a/mtglib/proxy.go b/mtglib/proxy.go index ce740a1..9445752 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -11,10 +11,11 @@ import ( "github.com/9seconds/mtg/v2/essentials" "github.com/9seconds/mtg/v2/mtglib/internal/dc" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/9seconds/mtg/v2/mtglib/internal/doppel" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscation" "github.com/9seconds/mtg/v2/mtglib/internal/relay" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" + "github.com/9seconds/mtg/v2/mtglib/internal/tls/fake" "github.com/panjf2000/ants/v2" ) @@ -32,6 +33,7 @@ type Proxy struct { workerPool *ants.PoolWithFunc telegram *dc.Telegram configUpdater *dc.PublicConfigUpdater + doppelGanger *doppel.Ganger clientObfuscatror obfuscation.Obfuscator secret Secret @@ -161,52 +163,39 @@ func (p *Proxy) Shutdown() { } func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - rewind := newConnRewind(ctx.clientConn) - if err := rec.Read(rewind); err != nil { + clientHello, err := fake.ReadClientHello( + rewind, + p.secret.Key[:], + p.secret.Host, + p.tolerateTimeSkewness, + ) + if err != nil { p.logger.InfoError("cannot read client hello", err) p.doDomainFronting(ctx, rewind) - return false } - hello, err := faketls.ParseClientHello(p.secret.Key[:], rec.Payload.Bytes()) - if err != nil { - p.logger.InfoError("cannot parse client hello", err) - p.doDomainFronting(ctx, rewind) - - return false - } - - if err := hello.Valid(p.secret.Host, p.tolerateTimeSkewness); err != nil { - p.logger. - BindStr("hostname", hello.Host). - BindStr("hello-time", hello.Time.String()). - InfoError("invalid faketls client hello", err) - p.doDomainFronting(ctx, rewind) - - return false - } - - if p.antiReplayCache.SeenBefore(hello.SessionID) { + if p.antiReplayCache.SeenBefore(clientHello.SessionID) { p.logger.Warning("replay attack has been detected!") p.eventStream.Send(p.ctx, NewEventReplayAttack(ctx.streamID)) p.doDomainFronting(ctx, rewind) - return false } - if err := faketls.SendWelcomePacket(rewind, p.secret.Key[:], hello); err != nil { + _, err = fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello) + if err != nil { p.logger.InfoError("cannot send welcome packet", err) - return false } - ctx.clientConn = &faketls.Conn{ - Conn: ctx.clientConn, + ctx.clientConn = tls.New(ctx.clientConn, true, true) + + ctx.clientConn, err = p.doppelGanger.NewConn(ctx.clientConn) + if err != nil { + p.logger.WarningError("cannot create connection", err) + return false } return true diff --git a/network/v2/http.go b/network/v2/http.go index 0504352..58e6a5e 100644 --- a/network/v2/http.go +++ b/network/v2/http.go @@ -1,9 +1,6 @@ package network -import ( - "crypto/tls" - "net/http" -) +import "net/http" type networkHTTPTransport struct { userAgent string @@ -15,11 +12,3 @@ func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, erro return n.next.RoundTrip(req) //nolint: wrapcheck } - -func (n networkHTTPTransport) TrustTLS() { - tr := n.next.(*http.Transport) - if tr.TLSClientConfig == nil { - tr.TLSClientConfig = &tls.Config{} - } - tr.TLSClientConfig.InsecureSkipVerify = true -} From 5557393b389a08476b8708a352907a9b7a1565e8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 15:03:01 +0100 Subject: [PATCH 07/42] Add config section --- example.config.toml | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/example.config.toml b/example.config.toml index ce248dd..2815873 100644 --- a/example.config.toml +++ b/example.config.toml @@ -206,6 +206,67 @@ tcp = "5s" http = "10s" idle = "1m" +# mtg has to mimic real websites. It does not mean domain fronting, it also +# means that traffic characteristics should be similar to real world traffic. +# websites and applications behave differently, their traffic patterns are also +# different. Applications do bursts of RPC-style messages (or JSON communication, +# does not really matter), while websites pump heavy content in HTTP2 streams +# +# It means that statistically there is a different between traffic shape: +# TLS packet sizes are different, delays between packets are also different. +# In order to avoid censorship detection based on these patterns, there is a +# mtg subsystem called "Doppelganger" that aims to mimic website statistics +# as close as it could. +# +# It does that by 2 ideas: +# 1. Delays between TLS packets are not constant. There are many factors +# that come in play. Application should generate some response, it could +# send some headers first and stream content with chunked encoding. So +# some first packets could come as soon as possible, with some delays +# after first ones. Such phenomenon is described by different statistic +# distribution. There are 2 distribution that describe it: lognormal +# distribution and Weibul distribution. Lognormal is all about steady streams +# of heavy content like a video. Weibul is great about short bursts like +# user who requested a static page an a couple of images. +# +# mtg tries to adapt Weibul distribution. It comes with some sensible +# defaults that were taken from ok.ru. But when you use domain fronting, +# it always make sense to take statistics from that website. You can specify +# some urls here. mtg will crawl them from time to time, accumulate time +# series and approximates parameters for Weibul. +# 2. TLS record sizes are not random. +# https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/ +# https://aws.github.io/s2n-tls/usage-guide/ch08-record-sizes.html +# +# The idea is that huge TLS records could negatively affect performance. +# You cannot simply decrypt a part of the packet, you need to wait it +# whole, and huge packets could involve several RTTs if you do not use +# any specific software that treat TLS in a very special way. So +# servers start with small packets, usually around MTU, and ramp up +# later. This optimizes a time-to-first byte so web browsers start to +# render early. +# +# mtg uses the same technique as was introduced by Cloudflare in their +# patches to nginx 10 years ago: +# https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch +[defense.doppelganger] +# This is a list of URLs that would be crawled by mtg to approximate delay +# statistics. They MUST be HTTPS urls. +# +# You can come to the website and collect different URLs, with light and +# heavy content. We recommend to search for CDNs. +urls = [ + # "https://st-ok.cdn-vk.ru/res/react/vendor/clsx-2.1.1-amd.js" +] +# A collection is done in raids. Each raid makes this number of requests to +# each URL in this list. Do not use a huge number, 10 is probably ok. +repeats-per-raid = 10 +# This is a duration between each raid. It makes no sense to have a small number +# here as you would start to make a noticeable activity. Usually traffic patterns +# do not change a lot, so do not expect different results if you request +# each 10 minutes. +raid-each = "6h" + # Some countries do active probing on Telegram connections. This technique # allows to protect from such effort. # From 25ad776b6fad5158bf741f2bc3a79925c7b07148 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 15:03:16 +0100 Subject: [PATCH 08/42] Propagate doppelganger to config --- internal/config/config.go | 9 ++- internal/config/parse.go | 5 ++ internal/config/type_https_url.go | 53 +++++++++++++ internal/config/type_https_url_test.go | 100 +++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 internal/config/type_https_url.go create mode 100644 internal/config/type_https_url_test.go diff --git a/internal/config/config.go b/internal/config/config.go index e3d8bd2..91c93b0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,8 +47,13 @@ type Config struct { MaxSize TypeBytes `json:"maxSize"` ErrorRate TypeErrorRate `json:"errorRate"` } `json:"antiReplay"` - Blocklist ListConfig `json:"blocklist"` - Allowlist ListConfig `json:"allowlist"` + Blocklist ListConfig `json:"blocklist"` + Allowlist ListConfig `json:"allowlist"` + Doppelganger struct { + URLs []TypeHttpsURL `json:"urls"` + Repeats TypeConcurrency `json:"repeats_per_raid"` + UpdateEach TypeDuration `json:"raid_each"` + } `json:"doppelganger"` } `json:"defense"` Network struct { Timeout struct { diff --git a/internal/config/parse.go b/internal/config/parse.go index 1186769..06183c7 100644 --- a/internal/config/parse.go +++ b/internal/config/parse.go @@ -44,6 +44,11 @@ type tomlConfig struct { URLs []string `toml:"urls" json:"urls,omitempty"` UpdateEach string `toml:"update-each" json:"updateEach,omitempty"` } `toml:"allowlist" json:"allowlist,omitempty"` + Doppelganger struct { + URLs []string `toml:"urls" json:"urls,omitempty"` + Repeats uint `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"` + UpdateEach string `toml:"raid-each" json:"raid_each,omitempty"` + } `toml:"doppelganger" json:"doppelganger,omitempty"` } `toml:"defense" json:"defense,omitempty"` Network struct { Timeout struct { diff --git a/internal/config/type_https_url.go b/internal/config/type_https_url.go new file mode 100644 index 0000000..6509064 --- /dev/null +++ b/internal/config/type_https_url.go @@ -0,0 +1,53 @@ +package config + +import ( + "fmt" + "net/url" +) + +type TypeHttpsURL struct { + Value *url.URL +} + +func (t *TypeHttpsURL) Set(value string) error { + parsedURL, err := url.Parse(value) + if err != nil { + return fmt.Errorf("value is not correct URL (%s): %w", value, err) + } + + if parsedURL.Host == "" { + return fmt.Errorf("url has to have a schema: %s", value) + } + + if parsedURL.Scheme != "https" { + return fmt.Errorf("unsupported schema: %s", parsedURL.Scheme) + } + + t.Value = parsedURL + + return nil +} + +func (t *TypeHttpsURL) Get(defaultValue *url.URL) *url.URL { + if t.Value == nil { + return defaultValue + } + + return t.Value +} + +func (t *TypeHttpsURL) UnmarshalText(data []byte) error { + return t.Set(string(data)) +} + +func (t TypeHttpsURL) MarshalText() ([]byte, error) { + return []byte(t.String()), nil +} + +func (t TypeHttpsURL) String() string { + if t.Value == nil { + return "" + } + + return t.Value.String() +} diff --git a/internal/config/type_https_url_test.go b/internal/config/type_https_url_test.go new file mode 100644 index 0000000..6c7db83 --- /dev/null +++ b/internal/config/type_https_url_test.go @@ -0,0 +1,100 @@ +package config_test + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeHttpsURLTestStruct struct { + Value config.TypeHttpsURL `json:"value"` +} + +type HttpsURLTestSuite struct { + suite.Suite +} + +func (suite *HttpsURLTestSuite) TestUnmarshalFail() { + testData := []string{ + "", + "https://", + "://lala", + "/path", + "http://example.com", + "socks5://example.com", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeHttpsURLTestStruct{})) + }) + } +} + +func (suite *HttpsURLTestSuite) TestUnmarshalOk() { + testData := map[string]string{ + "https://example.com": "https://example.com", + "https://example.com:8443": "https://example.com:8443", + "https://example.com/path?q=1": "https://example.com/path?q=1", + "https://user:pass@example.com": "https://user:pass@example.com", + } + + for k, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": k, + }) + suite.NoError(err) + + suite.T().Run(k, func(t *testing.T) { + testStruct := &typeHttpsURLTestStruct{} + assert.NoError(t, json.Unmarshal(data, testStruct)) + + parsed, _ := url.Parse(value) + + assert.Equal(t, parsed.Scheme, testStruct.Value.Get(nil).Scheme) + assert.Equal(t, parsed.Host, testStruct.Value.Get(nil).Host) + assert.Equal(t, parsed.RawQuery, testStruct.Value.Get(nil).RawQuery) + assert.Equal(t, parsed.Path, testStruct.Value.Get(nil).Path) + }) + } +} + +func (suite *HttpsURLTestSuite) TestMarshalOk() { + parsed, _ := url.Parse("https://example.com/path?q=1") + testStruct := &typeHttpsURLTestStruct{ + Value: config.TypeHttpsURL{ + Value: parsed, + }, + } + + encodedJSON, err := json.Marshal(testStruct) + suite.NoError(err) + suite.JSONEq(`{"value": "https://example.com/path?q=1"}`, + string(encodedJSON)) +} + +func (suite *HttpsURLTestSuite) TestGet() { + emptyURL := &url.URL{} + + value := config.TypeHttpsURL{} + suite.Equal(emptyURL, value.Get(emptyURL)) + + value.Value = &url.URL{} + suite.Equal(value.Value, value.Get(emptyURL)) +} + +func TestTypeHttpsURL(t *testing.T) { + t.Parallel() + suite.Run(t, &HttpsURLTestSuite{}) +} From 75392941dafe75336b8974e34c3c9aea2bc178d6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 15:11:57 +0100 Subject: [PATCH 09/42] Propagate settings to Proxy --- internal/cli/run_proxy.go | 9 +++++++++ mtglib/init.go | 7 +++++++ mtglib/proxy.go | 11 +++++++++++ mtglib/proxy_opts.go | 15 +++++++++++++++ 4 files changed, 42 insertions(+) diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index b48fcbc..7774953 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -239,6 +239,11 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen return fmt.Errorf("cannot build ip allowlist: %w", err) } + doppelGangerURLs := make([]string, len(conf.Defense.Doppelganger.URLs)) + for i, v := range conf.Defense.Doppelganger.URLs { + doppelGangerURLs[i] = v.String() + } + opts := mtglib.ProxyOpts{ Logger: logger, Network: ntw, @@ -256,6 +261,10 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen AllowFallbackOnUnknownDC: conf.AllowFallbackOnUnknownDC.Get(false), TolerateTimeSkewness: conf.TolerateTimeSkewness.Value, + + DoppelGangerURLs: doppelGangerURLs, + DoppelGangerPerRaid: conf.Defense.Doppelganger.Repeats.Get(mtglib.DoppelGangerPerRaid), + DoppelGangerEach: conf.Defense.Doppelganger.UpdateEach.Get(mtglib.DoppelGangerEach), } proxy, err := mtglib.NewProxy(opts) diff --git a/mtglib/init.go b/mtglib/init.go index 81dee7a..5b2d0bf 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -99,6 +99,13 @@ const ( // reads from Telegram after which connection will be terminated. This is // required to abort stale connections. TCPRelayReadTimeout = 20 * time.Second + + // DoppelGangerPerRaid defines a number of requests to each URL + // per raid. + DoppelGangerPerRaid = 10 + + // DoppelGangerEach defines a time period between each crawl attempt. + DoppelGangerEach = 6 * time.Hour ) // Network defines a knowledge how to work with a network. It may sound fun but diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 9445752..9ac8412 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -157,6 +157,7 @@ func (p *Proxy) Shutdown() { p.streamWaitGroup.Wait() p.workerPool.Release() p.configUpdater.Wait() + p.doppelGanger.Shutdown() p.allowlist.Shutdown() p.blocklist.Shutdown() @@ -327,6 +328,14 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { tolerateTimeSkewness: opts.getTolerateTimeSkewness(), allowFallbackOnUnknownDC: opts.AllowFallbackOnUnknownDC, telegram: tg, + doppelGanger: doppel.NewGanger( + ctx, + opts.Network, + logger.Named("doppelganger"), + opts.DoppelGangerEach, + int(opts.DoppelGangerPerRaid), + opts.DoppelGangerURLs, + ), configUpdater: dc.NewPublicConfigUpdater( tg, updatersLogger.Named("public-config"), @@ -338,6 +347,8 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { domainFrontingProxyProtocol: opts.DomainFrontingProxyProtocol, } + proxy.doppelGanger.Run() + if opts.AutoUpdate { proxy.configUpdater.Run(ctx, dc.PublicConfigUpdateURLv4, "tcp4") proxy.configUpdater.Run(ctx, dc.PublicConfigUpdateURLv6, "tcp6") diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index 88af9ea..0f8600e 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -142,6 +142,21 @@ type ProxyOpts struct { // // OBSOLETE and DEPRECATED. Ignored. DCOverrides map[int][]string + + // DoppelGangerURLs is a list of URLs that should be crawled by + // mtg to calculate parameters for statistical distribution of a + // traffic for fronting domains. If nothing is given, then predefined + // statistics is going to be used. + DoppelGangerURLs []string + + // DoppelGangerPerRaid defines how many time each URL from + // DoppelGangerURLs list should be crawled per raid. We recommend to + // have this number ~10. + DoppelGangerPerRaid uint + + // DoppelGangerEach defines a time period between each raid. We recommend + // to use hours here. + DoppelGangerEach time.Duration } func (p ProxyOpts) valid() error { From d43d6692d7f26d8384eafeff58d1b844d02bd5ee Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 15:23:08 +0100 Subject: [PATCH 10/42] Align naming about scout raids --- mtglib/internal/doppel/ganger.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mtglib/internal/doppel/ganger.go b/mtglib/internal/doppel/ganger.go index 0aa735c..835ab12 100644 --- a/mtglib/internal/doppel/ganger.go +++ b/mtglib/internal/doppel/ganger.go @@ -10,7 +10,7 @@ import ( const ( DoppelGangerMaxDurations = 4096 - DoppelGangerScoutMissionEach = 30 * time.Minute + DoppelGangerScoutRaidEach = 30 * time.Minute DoppelGangerScoutRepeats = 10 ) @@ -26,8 +26,8 @@ type Ganger struct { wg sync.WaitGroup scout Scout - scoutMissionEach time.Duration - scoutMissionRepeats int + scoutRaidEach time.Duration + scoutRaidRepeats int stats *Stats durations []time.Duration @@ -68,7 +68,7 @@ func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) { } func (g *Ganger) run() { - scoutTicker := time.NewTicker(g.scoutMissionEach) + scoutTicker := time.NewTicker(g.scoutRaidEach) defer func() { scoutTicker.Stop() @@ -84,7 +84,7 @@ func (g *Ganger) run() { updatedStatsChan := make(chan *Stats) g.wg.Go(func() { - g.runScoutMission(scoutCollectedChan) + g.runScoutRaid(scoutCollectedChan) }) for { @@ -109,7 +109,7 @@ func (g *Ganger) run() { currentScoutCollectedChan = scoutCollectedChan case <-scoutTicker.C: g.wg.Go(func() { - g.runScoutMission(scoutCollectedChan) + g.runScoutRaid(scoutCollectedChan) }) case req := <-g.connRequests: select { @@ -120,10 +120,10 @@ func (g *Ganger) run() { } } -func (g *Ganger) runScoutMission(rvChan chan<- []time.Duration) { +func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) { durations := []time.Duration{} - for range g.scoutMissionRepeats { + for range g.scoutRaidRepeats { learned, err := g.scout.Learn(g.ctx) if err != nil { g.logger.WarningError("cannot learn", err) @@ -150,7 +150,7 @@ func NewGanger( ctx, cancel := context.WithCancel(ctx) if scoutEach == 0 { - scoutEach = DoppelGangerScoutMissionEach + scoutEach = DoppelGangerScoutRaidEach } if scoutRepeats == 0 { @@ -161,8 +161,8 @@ func NewGanger( ctx: ctx, ctxCancel: cancel, logger: logger, - scoutMissionEach: scoutEach, - scoutMissionRepeats: scoutRepeats, + scoutRaidEach: scoutEach, + scoutRaidRepeats: scoutRepeats, stats: &Stats{ k: StatsDefaultK, lambda: StatsDefaultLambda, From 7a58c74cfe7c3a9961c8e2afd7944f06463173f6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 15:40:11 +0100 Subject: [PATCH 11/42] Correctly process doppel conns --- mtglib/internal/doppel/ganger.go | 19 ++++++++------- mtglib/proxy.go | 41 +++++++++++++++++++------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/mtglib/internal/doppel/ganger.go b/mtglib/internal/doppel/ganger.go index 835ab12..4d6b355 100644 --- a/mtglib/internal/doppel/ganger.go +++ b/mtglib/internal/doppel/ganger.go @@ -9,13 +9,13 @@ import ( ) const ( - DoppelGangerMaxDurations = 4096 + DoppelGangerMaxDurations = 4096 DoppelGangerScoutRaidEach = 30 * time.Minute - DoppelGangerScoutRepeats = 10 + DoppelGangerScoutRepeats = 10 ) type gangerConnRequest struct { - ret chan Conn + ret chan<- Conn payload essentials.Conn } @@ -25,7 +25,7 @@ type Ganger struct { logger Logger wg sync.WaitGroup - scout Scout + scout Scout scoutRaidEach time.Duration scoutRaidRepeats int @@ -47,8 +47,9 @@ func (g *Ganger) Run() { } func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) { + rvChan := make(chan Conn) req := gangerConnRequest{ - ret: make(chan Conn), + ret: rvChan, payload: conn, } defer close(req.ret) @@ -62,7 +63,7 @@ func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) { select { case <-g.ctx.Done(): return Conn{}, context.Cause(g.ctx) - case conn := <-req.ret: + case conn := <-rvChan: return conn, nil } } @@ -158,9 +159,9 @@ func NewGanger( } return &Ganger{ - ctx: ctx, - ctxCancel: cancel, - logger: logger, + ctx: ctx, + ctxCancel: cancel, + logger: logger, scoutRaidEach: scoutEach, scoutRaidRepeats: scoutRepeats, stats: &Stats{ diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 9ac8412..40a2bbf 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -78,19 +78,32 @@ func (p *Proxy) ServeConn(conn essentials.Conn) { ctx.logger.Info("Stream has been finished") }() - if !p.doFakeTLSHandshake(ctx) { + noise, ok := p.doFakeTLSHandshake(ctx) + if !ok { return } - if err := p.doObfuscatedHandshake(ctx); err != nil { - p.logger.InfoError("obfuscated handshake is failed", err) + clientConn, err := p.doppelGanger.NewConn(ctx.clientConn) + if err != nil { + ctx.logger.InfoError("cannot wrap into doppelganger connection", err) + return + } + defer clientConn.Stop() + if _, err := clientConn.Write(noise); err != nil { + ctx.logger.InfoError("cannot send the first packet", err) + return + } + + ctx.clientConn = clientConn + + if err := p.doObfuscatedHandshake(ctx); err != nil { + ctx.logger.InfoError("obfuscated handshake is failed", err) return } if err := p.doTelegramCall(ctx); err != nil { - p.logger.WarningError("cannot dial to telegram", err) - + ctx.logger.WarningError("cannot dial to telegram", err) return } @@ -163,7 +176,7 @@ func (p *Proxy) Shutdown() { p.blocklist.Shutdown() } -func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) ([]byte, bool) { rewind := newConnRewind(ctx.clientConn) clientHello, err := fake.ReadClientHello( @@ -175,31 +188,25 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { if err != nil { p.logger.InfoError("cannot read client hello", err) p.doDomainFronting(ctx, rewind) - return false + return nil, false } if p.antiReplayCache.SeenBefore(clientHello.SessionID) { p.logger.Warning("replay attack has been detected!") p.eventStream.Send(p.ctx, NewEventReplayAttack(ctx.streamID)) p.doDomainFronting(ctx, rewind) - return false + return nil, false } - _, err = fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello) + noise, err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello) if err != nil { p.logger.InfoError("cannot send welcome packet", err) - return false + return nil, false } ctx.clientConn = tls.New(ctx.clientConn, true, true) - ctx.clientConn, err = p.doppelGanger.NewConn(ctx.clientConn) - if err != nil { - p.logger.WarningError("cannot create connection", err) - return false - } - - return true + return noise, true } func (p *Proxy) doObfuscatedHandshake(ctx *streamContext) error { From 83ff4ee266cbc387e4646aef30985fdad1d6735f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 16:09:31 +0100 Subject: [PATCH 12/42] Align scout defaults with config defaults --- mtglib/internal/doppel/ganger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtglib/internal/doppel/ganger.go b/mtglib/internal/doppel/ganger.go index 4d6b355..666baa8 100644 --- a/mtglib/internal/doppel/ganger.go +++ b/mtglib/internal/doppel/ganger.go @@ -10,7 +10,7 @@ import ( const ( DoppelGangerMaxDurations = 4096 - DoppelGangerScoutRaidEach = 30 * time.Minute + DoppelGangerScoutRaidEach = 6 * time.Hour DoppelGangerScoutRepeats = 10 ) From 1c0e8472478861af3daaa2bd42ea90cd2dd4043c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 18:28:24 +0100 Subject: [PATCH 13/42] Set statistics measured from real life --- mtglib/internal/doppel/stats.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mtglib/internal/doppel/stats.go b/mtglib/internal/doppel/stats.go index a6cbc9b..6ed7c9f 100644 --- a/mtglib/internal/doppel/stats.go +++ b/mtglib/internal/doppel/stats.go @@ -11,8 +11,12 @@ const ( StatsLowK = 0.01 StatsHighK = 10.0 - StatsDefaultK = 0.6 - StatsDefaultLambda = 0.002 + // do not calculate statistics if we have < than this number of durations + MinDurationsToCalculate = 100 + + // these values are taken from ok-cdn. measured from moscow site. + StatsDefaultK = 1.2908978568647322 + StatsDefaultLambda = 1.4258402622793287 ) // Stats is responsible for generating values that are distributed according From 7aa01dcebe0eaf0ac1edc9627a5757dbbe078606 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 18:28:40 +0100 Subject: [PATCH 14/42] Limit min number of collected stats --- mtglib/internal/doppel/ganger.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mtglib/internal/doppel/ganger.go b/mtglib/internal/doppel/ganger.go index 666baa8..ee4a3eb 100644 --- a/mtglib/internal/doppel/ganger.go +++ b/mtglib/internal/doppel/ganger.go @@ -94,10 +94,15 @@ func (g *Ganger) run() { return case durations := <-currentScoutCollectedChan: g.durations = append(g.durations, durations...) + if len(g.durations) > DoppelGangerMaxDurations { g.durations = g.durations[len(g.durations)-DoppelGangerMaxDurations:] } + if len(g.durations) < MinDurationsToCalculate { + continue + } + currentScoutCollectedChan = nil g.wg.Go(func() { select { From 0bfc1ef2d4feedb0110824bbe8f1ca43064693be Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 18:59:18 +0100 Subject: [PATCH 15/42] Fix double TLS wrapping for noise --- mtglib/internal/tls/fake/server_side.go | 2 +- mtglib/internal/tls/fake/server_side_test.go | 17 +++++++++-------- mtglib/proxy.go | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go index 27ae0d6..14e0cde 100644 --- a/mtglib/internal/tls/fake/server_side.go +++ b/mtglib/internal/tls/fake/server_side.go @@ -54,7 +54,7 @@ func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) ([]by _, err := w.Write(packet) - return noise.Bytes(), err + return noise.Bytes()[tls.SizeHeader:], err } func generateServerHello(buf *bytes.Buffer, hello *ClientHello) { diff --git a/mtglib/internal/tls/fake/server_side_test.go b/mtglib/internal/tls/fake/server_side_test.go index 37d4246..0b4ee67 100644 --- a/mtglib/internal/tls/fake/server_side_test.go +++ b/mtglib/internal/tls/fake/server_side_test.go @@ -55,13 +55,8 @@ func (suite *SendServerHelloTestSuite) TestRecordStructure() { suite.Empty(suite.buf.Bytes()) - noiseBuf := bytes.NewReader(noise) - rec.Reset() - - recordType, _, err = tls.ReadRecord(noiseBuf, &rec) - suite.NoError(err) - suite.Equal(byte(tls.TypeApplicationData), recordType) - suite.Zero(noiseBuf.Len()) + // noise is raw payload without TLS record header + suite.Len(noise, 1369) } func (suite *SendServerHelloTestSuite) TestHMAC() { @@ -78,7 +73,13 @@ func (suite *SendServerHelloTestSuite) TestHMAC() { mac := hmac.New(sha256.New, suite.secret.Key[:]) mac.Write(suite.hello.Random[:]) mac.Write(packet) - mac.Write(noise) + + // HMAC is computed over the full noise TLS record (with header), + // but SendServerHello returns noise without the header, + // so we reconstruct the full record. + var fullNoise bytes.Buffer + tls.WriteRecord(&fullNoise, noise) //nolint: errcheck + mac.Write(fullNoise.Bytes()) suite.Equal(random, mac.Sum(nil)) } diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 40a2bbf..995f877 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -204,7 +204,7 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) ([]byte, bool) { return nil, false } - ctx.clientConn = tls.New(ctx.clientConn, true, true) + ctx.clientConn = tls.New(ctx.clientConn, true, false) return noise, true } From 00403e3a948d1fc43aeb91b4d3bc9453f5df9c37 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 19:11:39 +0100 Subject: [PATCH 16/42] Remove old faketls package --- mtglib/internal/faketls/client_hello.go | 134 ------------ .../faketls/client_hello_fuzz_test.go | 21 -- mtglib/internal/faketls/client_hello_test.go | 191 ------------------ mtglib/internal/faketls/conn.go | 72 ------- mtglib/internal/faketls/conn_test.go | 153 -------------- mtglib/internal/faketls/init.go | 59 ------ mtglib/internal/faketls/record/init.go | 84 -------- mtglib/internal/faketls/record/init_test.go | 79 -------- mtglib/internal/faketls/record/pools.go | 20 -- mtglib/internal/faketls/record/record.go | 86 -------- mtglib/internal/faketls/record/record_test.go | 110 ---------- .../record/testdata/05eb6b71f87b6802.json | 6 - .../record/testdata/4eef4abc15b206b6.json | 6 - .../record/testdata/736f358216afe91f.json | 6 - .../record/testdata/8405d94222bd0b6a.json | 6 - .../record/testdata/9036f76e517f0cd1.json | 6 - .../record/testdata/9244766a0fe4a02a.json | 6 - .../record/testdata/9255c73d3de76e7b.json | 6 - .../record/testdata/aeb65b9924315cf8.json | 6 - .../record/testdata/b0acd44296056b54.json | 6 - .../record/testdata/c0545a13fd9a3fa3.json | 6 - .../record/testdata/f083f4501668b759.json | 6 - .../record/testdata/f5696bcdffd11706.json | 6 - .../client-hello-bad-fa2e46cdb33e2a1b.json | 8 - .../client-hello-ok-19dfe38384b9884b.json | 8 - .../client-hello-ok-48f8a72a56f3174a.json | 8 - .../client-hello-ok-651054256093c6cd.json | 8 - .../client-hello-ok-79d01ef18a9d2621.json | 8 - .../client-hello-ok-7a5569f05b118145.json | 8 - mtglib/internal/faketls/welcome.go | 91 --------- mtglib/internal/faketls/welcome_test.go | 82 -------- 31 files changed, 1302 deletions(-) delete mode 100644 mtglib/internal/faketls/client_hello.go delete mode 100644 mtglib/internal/faketls/client_hello_fuzz_test.go delete mode 100644 mtglib/internal/faketls/client_hello_test.go delete mode 100644 mtglib/internal/faketls/conn.go delete mode 100644 mtglib/internal/faketls/conn_test.go delete mode 100644 mtglib/internal/faketls/init.go delete mode 100644 mtglib/internal/faketls/record/init.go delete mode 100644 mtglib/internal/faketls/record/init_test.go delete mode 100644 mtglib/internal/faketls/record/pools.go delete mode 100644 mtglib/internal/faketls/record/record.go delete mode 100644 mtglib/internal/faketls/record/record_test.go delete mode 100644 mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json delete mode 100644 mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json delete mode 100644 mtglib/internal/faketls/record/testdata/736f358216afe91f.json delete mode 100644 mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json delete mode 100644 mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json delete mode 100644 mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json delete mode 100644 mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json delete mode 100644 mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json delete mode 100644 mtglib/internal/faketls/record/testdata/b0acd44296056b54.json delete mode 100644 mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json delete mode 100644 mtglib/internal/faketls/record/testdata/f083f4501668b759.json delete mode 100644 mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json delete mode 100644 mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json delete mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json delete mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json delete mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json delete mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json delete mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json delete mode 100644 mtglib/internal/faketls/welcome.go delete mode 100644 mtglib/internal/faketls/welcome_test.go diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go deleted file mode 100644 index 0e5e5b4..0000000 --- a/mtglib/internal/faketls/client_hello.go +++ /dev/null @@ -1,134 +0,0 @@ -package faketls - -import ( - "crypto/hmac" - "crypto/sha256" - "crypto/subtle" - "encoding/binary" - "fmt" - "time" - - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" -) - -type ClientHello struct { - Time time.Time - Random [RandomLen]byte - SessionID []byte - Host string - CipherSuite uint16 -} - -func (c ClientHello) Valid(hostname string, tolerateTimeSkewness time.Duration) error { - if c.Host != "" && c.Host != hostname { - return fmt.Errorf("incorrect hostname %s", hostname) - } - - now := time.Now() - - timeDiff := now.Sub(c.Time) - if timeDiff < 0 { - timeDiff = -timeDiff - } - - if timeDiff > tolerateTimeSkewness { - return fmt.Errorf("incorrect timestamp. got=%d, now=%d, diff=%s", - c.Time.Unix(), now.Unix(), timeDiff.String()) - } - - return nil -} - -func ParseClientHello(secret, handshake []byte) (ClientHello, error) { - hello := ClientHello{} - - if len(handshake) < ClientHelloMinLen { - return hello, fmt.Errorf("lengh of handshake is too small: %d", len(handshake)) - } - - if handshake[0] != HandshakeTypeClient { - return hello, fmt.Errorf("unknown handshake type %#x", handshake[0]) - } - - handshakeSizeBytes := [4]byte{0, handshake[1], handshake[2], handshake[3]} - handshakeLength := binary.BigEndian.Uint32(handshakeSizeBytes[:]) - - if len(handshake)-4 != int(handshakeLength) { - return hello, - fmt.Errorf("incorrect handshake size. manifested=%d, real=%d", - handshakeLength, len(handshake)-4) - } - - copy(hello.Random[:], handshake[ClientHelloRandomOffset:]) - copy(handshake[ClientHelloRandomOffset:], clientHelloEmptyRandom) - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - rec.Type = record.TypeHandshake - rec.Version = record.Version10 - rec.Payload.Write(handshake) - - // mac is calculated for the whole record, not only - // for the payload part - mac := hmac.New(sha256.New, secret) - rec.Dump(mac) //nolint: errcheck - - computedRandom := mac.Sum(nil) - - for i := range RandomLen { - computedRandom[i] ^= hello.Random[i] - } - - if subtle.ConstantTimeCompare(clientHelloEmptyRandom[:RandomLen-4], computedRandom[:RandomLen-4]) != 1 { - return hello, ErrBadDigest - } - - timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:])) - hello.Time = time.Unix(timestamp, 0) - - parseSessionID(&hello, handshake) - parseCipherSuite(&hello, handshake) - parseSNI(&hello, handshake) - - return hello, nil -} - -func parseSessionID(hello *ClientHello, handshake []byte) { - hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) - copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) -} - -func parseCipherSuite(hello *ClientHello, handshake []byte) { - cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3 - hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2]) -} - -func parseSNI(hello *ClientHello, handshake []byte) { - cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 1 - handshake = handshake[cipherSuiteOffset:] - - cipherSuiteLength := binary.BigEndian.Uint16(handshake[:2]) - handshake = handshake[2+cipherSuiteLength:] - - compressionMethodsLength := int(handshake[0]) - handshake = handshake[1+compressionMethodsLength:] - - extensionsLength := binary.BigEndian.Uint16(handshake[:2]) - handshake = handshake[2 : 2+extensionsLength] - - for len(handshake) > 0 { - if binary.BigEndian.Uint16(handshake[:2]) != ExtensionSNI { - extensionsLength := binary.BigEndian.Uint16(handshake[2:4]) - handshake = handshake[4+extensionsLength:] - - continue - } - - hostnameLength := binary.BigEndian.Uint16(handshake[7:9]) - handshake = handshake[9:] - hello.Host = string(handshake[:int(hostnameLength)]) - - return - } -} diff --git a/mtglib/internal/faketls/client_hello_fuzz_test.go b/mtglib/internal/faketls/client_hello_fuzz_test.go deleted file mode 100644 index 84db20d..0000000 --- a/mtglib/internal/faketls/client_hello_fuzz_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package faketls_test - -import ( - "testing" - - "github.com/9seconds/mtg/v2/mtglib/internal/faketls" - "github.com/stretchr/testify/require" -) - -var FuzzClientHelloSecret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} - -func FuzzClientHello(f *testing.F) { - f.Add([]byte{1, 2, 3}) - - f.Fuzz(func(t *testing.T, frame []byte) { - _, err := faketls.ParseClientHello(FuzzClientHelloSecret, frame) - - // a probability of having != err is almost negligible - require.Error(t, err) - }) -} diff --git a/mtglib/internal/faketls/client_hello_test.go b/mtglib/internal/faketls/client_hello_test.go deleted file mode 100644 index 7c9446e..0000000 --- a/mtglib/internal/faketls/client_hello_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package faketls_test - -import ( - "encoding/base64" - "encoding/json" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/9seconds/mtg/v2/mtglib" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" -) - -type ClientHelloSnapshot struct { - Time int `json:"time"` - Random string `json:"random"` - SessionID string `json:"sessionId"` - Host string `json:"host"` - CipherSuite int `json:"cipherSuite"` - Full string `json:"full"` -} - -func (c ClientHelloSnapshot) GetTime() time.Time { - return time.Unix(int64(c.Time), 0) -} - -func (c ClientHelloSnapshot) GetRandom() []byte { - data, _ := base64.StdEncoding.DecodeString(c.Random) - - return data -} - -func (c ClientHelloSnapshot) GetSessionID() []byte { - data, _ := base64.StdEncoding.DecodeString(c.SessionID) - - return data -} - -func (c ClientHelloSnapshot) GetHost() string { - return c.Host -} - -func (c ClientHelloSnapshot) GetCipherSuite() uint16 { - return uint16(c.CipherSuite) -} - -func (c ClientHelloSnapshot) GetFull() []byte { - data, _ := base64.StdEncoding.DecodeString(c.Full) - - return data -} - -type ClientHelloTestSuite struct { - suite.Suite - - secret mtglib.Secret -} - -func (suite *ClientHelloTestSuite) SetupSuite() { - parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d") - if err != nil { - panic(err) - } - - suite.secret = parsed -} - -func (suite *ClientHelloTestSuite) TestEmptyHandshake() { - _, err := faketls.ParseClientHello(suite.secret.Key[:], nil) - suite.Error(err) -} - -func (suite *ClientHelloTestSuite) TestIncorrectHandshakeType() { - data := make([]byte, 1024) - data[0] = 0x02 - - _, err := faketls.ParseClientHello(suite.secret.Key[:], data) - suite.Error(err) -} - -func (suite *ClientHelloTestSuite) TestIncorrectLength() { - data := make([]byte, 1024) - data[0] = 0x01 - data[1] = 0xff - data[2] = 0xff - - _, err := faketls.ParseClientHello(suite.secret.Key[:], data) - suite.Error(err) -} - -func (suite *ClientHelloTestSuite) TestSnapshotOk() { - files, err := os.ReadDir("testdata") - suite.NoError(err) - - testData := []string{} - - for _, v := range files { - if strings.HasPrefix(v.Name(), "client-hello-ok") { - testData = append(testData, v.Name()) - } - } - - for _, name := range testData { - path := filepath.Join("testdata", name) - - suite.T().Run(name, func(t *testing.T) { - fileData, err := os.ReadFile(path) - assert.NoError(t, err) - - snapshot := &ClientHelloSnapshot{} - assert.NoError(t, json.Unmarshal(fileData, snapshot)) - - hello, err := faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull()) - assert.NoError(t, err) - assert.WithinDuration(t, snapshot.GetTime(), hello.Time, time.Second) - assert.Equal(t, snapshot.GetRandom(), hello.Random[:]) - assert.Equal(t, snapshot.GetSessionID(), hello.SessionID) - assert.Equal(t, snapshot.GetHost(), hello.Host) - assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite) - }) - } -} - -func (suite *ClientHelloTestSuite) TestSnapshotBad() { - files, err := os.ReadDir("testdata") - suite.NoError(err) - - testData := []string{} - - for _, v := range files { - if strings.HasPrefix(v.Name(), "client-hello-bad") { - testData = append(testData, v.Name()) - } - } - - for _, name := range testData { - path := filepath.Join("testdata", name) - - suite.T().Run(name, func(t *testing.T) { - fileData, err := os.ReadFile(path) - assert.NoError(t, err) - - snapshot := &ClientHelloSnapshot{} - assert.NoError(t, json.Unmarshal(fileData, snapshot)) - - _, err = faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull()) - assert.Error(t, err) - }) - } -} - -func (suite *ClientHelloTestSuite) TestValidateHostname() { - hello := faketls.ClientHello{ - Time: time.Now(), - } - suite.NoError(hello.Valid("hostname", time.Second)) - - hello.Host = "hostname" - suite.Error(hello.Valid("hostname2", time.Second)) - suite.NoError(hello.Valid("hostname", time.Second)) -} - -func (suite *ClientHelloTestSuite) TestValidateTime() { - testData := []time.Duration{ - -2 * time.Second, - 2 * time.Second, - } - - for _, v := range testData { - value := v - - suite.T().Run(value.String(), func(t *testing.T) { - hello := faketls.ClientHello{ - Host: "hostname", - Time: time.Now().Add(value), - } - suite.Error(hello.Valid("hostname", 500*time.Millisecond)) - suite.Error(hello.Valid("hostname", time.Second)) - suite.NoError(hello.Valid("hostname", 3*time.Second)) - }) - } -} - -func TestClientHello(t *testing.T) { - t.Parallel() - suite.Run(t, &ClientHelloTestSuite{}) -} diff --git a/mtglib/internal/faketls/conn.go b/mtglib/internal/faketls/conn.go deleted file mode 100644 index 2d04978..0000000 --- a/mtglib/internal/faketls/conn.go +++ /dev/null @@ -1,72 +0,0 @@ -package faketls - -import ( - "bytes" - "fmt" - "math/rand/v2" - - "github.com/9seconds/mtg/v2/essentials" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" -) - -type Conn struct { - essentials.Conn - - readBuffer bytes.Buffer -} - -func (c *Conn) Read(p []byte) (int, error) { - if n, _ := c.readBuffer.Read(p); n > 0 { - return n, nil - } - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - for { - if err := rec.Read(c.Conn); err != nil { - return 0, err //nolint: wrapcheck - } - - switch rec.Type { //nolint: exhaustive - case record.TypeApplicationData: - rec.Payload.WriteTo(&c.readBuffer) //nolint: errcheck - - return c.readBuffer.Read(p) //nolint: wrapcheck - case record.TypeChangeCipherSpec: - default: - return 0, fmt.Errorf("unsupported record type %v", rec.Type) - } - } -} - -func (c *Conn) Write(p []byte) (int, error) { - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - rec.Type = record.TypeApplicationData - rec.Version = record.Version12 - - written := 0 - - for len(p) > 0 { - chunkSize := rand.IntN(record.TLSMaxRecordSize) - if chunkSize > len(p) || chunkSize == 0 { - chunkSize = len(p) - } - - rec.Payload.Reset() - rec.Payload.Write(p[:chunkSize]) - - err := rec.Dump(c.Conn) - written += chunkSize - - if err != nil { - return written, err - } - - p = p[chunkSize:] - } - - return written, nil -} diff --git a/mtglib/internal/faketls/conn_test.go b/mtglib/internal/faketls/conn_test.go deleted file mode 100644 index 29a8c0d..0000000 --- a/mtglib/internal/faketls/conn_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package faketls_test - -import ( - "bytes" - "crypto/rand" - "errors" - "io" - "testing" - - "github.com/9seconds/mtg/v2/internal/testlib" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" -) - -type ConnMock struct { - testlib.EssentialsConnMock - - readBuffer bytes.Buffer - writeBuffer bytes.Buffer -} - -func (m *ConnMock) Read(p []byte) (int, error) { - m.Called(p) - - return m.readBuffer.Read(p) //nolint: wrapcheck -} - -func (m *ConnMock) Write(p []byte) (int, error) { - m.Called(p) - - return m.writeBuffer.Write(p) //nolint: wrapcheck -} - -type ConnTestSuite struct { - suite.Suite - - connMock *ConnMock - c *faketls.Conn -} - -func (suite *ConnTestSuite) SetupTest() { - suite.connMock = &ConnMock{} - suite.c = &faketls.Conn{ - Conn: suite.connMock, - } -} - -func (suite *ConnTestSuite) TearDownTest() { - suite.connMock.AssertExpectations(suite.T()) -} - -func (suite *ConnTestSuite) TestRead() { - suite.connMock.On("Read", mock.Anything).Return(0, nil) - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - rec.Type = record.TypeChangeCipherSpec - rec.Version = record.Version12 - - rec.Payload.WriteByte(0x01) - rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck - rec.Reset() - - rec.Type = record.TypeApplicationData - rec.Version = record.Version12 - - rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck - - resultBuffer := &bytes.Buffer{} - buf := make([]byte, 2) - - for { - n, err := suite.c.Read(buf) - if errors.Is(err, io.EOF) { - break - } - - resultBuffer.Write(buf[:n]) - } - - suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, resultBuffer.Bytes()) -} - -func (suite *ConnTestSuite) TestReadUnexpected() { - suite.connMock.On("Read", mock.Anything).Return(0, nil) - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - rec.Type = record.TypeChangeCipherSpec - rec.Version = record.Version12 - - rec.Payload.WriteByte(0x01) - rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck - rec.Reset() - - rec.Type = record.TypeHandshake - rec.Version = record.Version12 - - rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) - rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck - - buf := make([]byte, 2) - - for { - _, err := suite.c.Read(buf) - - switch { - case err == nil: - case errors.Is(err, io.EOF): - suite.FailNow("unexpected to finish") - default: - return - } - } -} - -func (suite *ConnTestSuite) TestWrite() { - suite.connMock.On("Write", mock.Anything).Return(0, nil) - - dataToRec := make([]byte, record.TLSMaxRecordSize*2) - rand.Read(dataToRec) //nolint: staticcheck, errcheck - - n, err := suite.c.Write(dataToRec) - suite.NoError(err) - suite.Equal(len(dataToRec), n) - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - buf := &bytes.Buffer{} - - for { - if err := rec.Read(&suite.connMock.writeBuffer); err != nil { - break - } - - suite.Equal(record.TypeApplicationData, rec.Type) - suite.Equal(record.Version12, rec.Version) - rec.Payload.WriteTo(buf) //nolint: errcheck - } - - suite.Equal(dataToRec, buf.Bytes()) -} - -func TestConn(t *testing.T) { - t.Parallel() - suite.Run(t, &ConnTestSuite{}) -} diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go deleted file mode 100644 index b514b21..0000000 --- a/mtglib/internal/faketls/init.go +++ /dev/null @@ -1,59 +0,0 @@ -package faketls - -import ( - "bytes" - "errors" -) - -const ( - // RandomLen defines a size of the random digest in TLS Hellos. - RandomLen = 32 - - // ClientHelloRandomOffset is an offset in ClientHello record where - // random digest is started. - ClientHelloRandomOffset = 6 - - // ClientHelloSessionIDOffset is an offset in ClientHello record where - // SessionID is started. - ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen - - // ClientHelloMinLen is a minimal possible length of - // ClientHello record. - ClientHelloMinLen = 6 - - // WelcomePacketRandomOffset is an offset of random in ServerHello - // packet (including record envelope). - WelcomePacketRandomOffset = 11 - - // HandshakeTypeClient is a value representing a client handshake. - HandshakeTypeClient = 0x01 - - // HandshakeTypeServer is a value representing a server handshake. - HandshakeTypeServer = 0x02 - - // ChangeCipherValue is a value representing a change cipher - // specification record. - ChangeCipherValue = 0x01 - - // ExtensionSNI is a value for TLS extension 'SNI'. - ExtensionSNI = 0x00 -) - -var ( - // ErrBadDigest is returned if given TLS Client Hello mismatches with a - // derived one. - ErrBadDigest = errors.New("bad digest") - - serverHelloSuffix = []byte{ - 0x00, // no compression - 0x00, 0x2e, // 46 bytes of data - 0x00, 0x2b, // Extension - Supported Versions - 0x00, 0x02, // 2 bytes are following - 0x03, 0x04, // TLS 1.3 - 0x00, 0x33, // Extension - Key Share - 0x00, 0x24, // 36 bytes - 0x00, 0x1d, // x25519 curve - 0x00, 0x20, // 32 bytes of key - } - clientHelloEmptyRandom = bytes.Repeat([]byte{0}, RandomLen) -) diff --git a/mtglib/internal/faketls/record/init.go b/mtglib/internal/faketls/record/init.go deleted file mode 100644 index 9195113..0000000 --- a/mtglib/internal/faketls/record/init.go +++ /dev/null @@ -1,84 +0,0 @@ -package record - -import "fmt" - -const TLSMaxRecordSize = 65535 // max uint16 - -type Type uint8 - -const ( - // TypeChangeCipherSpec defines a byte value of the TLS record when a - // peer wants to change a specifications of the chosen cipher. - TypeChangeCipherSpec Type = 0x14 - - // TypeHandshake defines a byte value of the TLS record when a peer - // initiates a new TLS connection and wants to make a handshake - // ceremony. - TypeHandshake Type = 0x16 - - // TypeApplicationData defines a byte value of the TLS record when a - // peer sends an user data, not a control frames. - TypeApplicationData Type = 0x17 -) - -func (t Type) String() string { - switch t { - case TypeChangeCipherSpec: - return "changeCipher(0x14)" - case TypeHandshake: - return "handshake(0x16)" - case TypeApplicationData: - return "applicationData(0x17)" - } - - return fmt.Sprintf("unknown(%#x)", byte(t)) -} - -func (t Type) Valid() error { - switch t { - case TypeChangeCipherSpec, TypeHandshake, TypeApplicationData: - return nil - } - - return fmt.Errorf("unknown type %#x", byte(t)) -} - -type Version uint16 - -const ( - // Version10 defines a TLS1.0. - Version10 Version = 769 // 0x03 0x01 - - // Version11 defines a TLS1.1. - Version11 Version = 770 // 0x03 0x02 - - // Version12 defines a TLS1.2. - Version12 Version = 771 // 0x03 0x03 - - // Version13 defines a TLS1.3. - Version13 Version = 772 // 0x03 0x04 -) - -func (v Version) String() string { - switch v { - case Version10: - return "tls1.0" - case Version11: - return "tls1.1" - case Version12: - return "tls1.2" - case Version13: - return "tls1.3" - } - - return fmt.Sprintf("tls?(%d)", uint16(v)) -} - -func (v Version) Valid() error { - switch v { - case Version10, Version11, Version12, Version13: - return nil - } - - return fmt.Errorf("unknown version %d", uint16(v)) -} diff --git a/mtglib/internal/faketls/record/init_test.go b/mtglib/internal/faketls/record/init_test.go deleted file mode 100644 index 7922777..0000000 --- a/mtglib/internal/faketls/record/init_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package record_test - -import ( - "testing" - - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" - "github.com/stretchr/testify/suite" -) - -type TypeTestSuite struct { - suite.Suite -} - -func (suite *TypeTestSuite) TestChangeCipherSpec() { - suite.Contains(record.TypeChangeCipherSpec.String(), "changeCipher") - suite.Contains(record.TypeChangeCipherSpec.String(), "0x14") - suite.NoError(record.TypeChangeCipherSpec.Valid()) -} - -func (suite *TypeTestSuite) TestHandshake() { - suite.Contains(record.TypeHandshake.String(), "handshake") - suite.Contains(record.TypeHandshake.String(), "0x16") - suite.NoError(record.TypeHandshake.Valid()) -} - -func (suite *TypeTestSuite) TestApplicationData() { - suite.Contains(record.TypeApplicationData.String(), "applicationData") - suite.Contains(record.TypeApplicationData.String(), "0x17") - suite.NoError(record.TypeApplicationData.Valid()) -} - -func (suite *TypeTestSuite) TestUnknown() { - value := record.Type(0x20) - - suite.Contains(value.String(), "unknown") - suite.Contains(value.String(), "0x20") - suite.Error(value.Valid()) -} - -type VersionTestSuite struct { - suite.Suite -} - -func (suite *VersionTestSuite) Test10() { - suite.Equal("tls1.0", record.Version10.String()) - suite.NoError(record.Version10.Valid()) -} - -func (suite *VersionTestSuite) Test11() { - suite.Equal("tls1.1", record.Version11.String()) - suite.NoError(record.Version11.Valid()) -} - -func (suite *VersionTestSuite) Test12() { - suite.Equal("tls1.2", record.Version12.String()) - suite.NoError(record.Version12.Valid()) -} - -func (suite *VersionTestSuite) Test13() { - suite.Equal("tls1.3", record.Version13.String()) - suite.NoError(record.Version13.Valid()) -} - -func (suite *VersionTestSuite) TestUnknown() { - value := record.Version(900) - - suite.Equal("tls?(900)", value.String()) - suite.Error(value.Valid()) -} - -func TestType(t *testing.T) { - t.Parallel() - suite.Run(t, &TypeTestSuite{}) -} - -func TestVersion(t *testing.T) { - t.Parallel() - suite.Run(t, &VersionTestSuite{}) -} diff --git a/mtglib/internal/faketls/record/pools.go b/mtglib/internal/faketls/record/pools.go deleted file mode 100644 index 4f758b7..0000000 --- a/mtglib/internal/faketls/record/pools.go +++ /dev/null @@ -1,20 +0,0 @@ -package record - -import ( - "sync" -) - -var recordPool = sync.Pool{ - New: func() any { - return &Record{} - }, -} - -func AcquireRecord() *Record { - return recordPool.Get().(*Record) //nolint: forcetypeassert -} - -func ReleaseRecord(r *Record) { - r.Reset() - recordPool.Put(r) -} diff --git a/mtglib/internal/faketls/record/record.go b/mtglib/internal/faketls/record/record.go deleted file mode 100644 index 2c8b23e..0000000 --- a/mtglib/internal/faketls/record/record.go +++ /dev/null @@ -1,86 +0,0 @@ -package record - -import ( - "bytes" - "encoding/base64" - "encoding/binary" - "fmt" - "io" -) - -type Record struct { - Type Type - Version Version - Payload bytes.Buffer -} - -func (r *Record) String() string { - return fmt.Sprintf("", - r.Type, - r.Version, - base64.StdEncoding.EncodeToString(r.Payload.Bytes())) -} - -func (r *Record) Reset() { - r.Payload.Reset() -} - -func (r *Record) Read(reader io.Reader) error { - r.Reset() - - buf := [2]byte{} - - if _, err := io.ReadFull(reader, buf[:1]); err != nil { - return fmt.Errorf("cannot read type: %w", err) - } - - r.Type = Type(buf[0]) - if err := r.Type.Valid(); err != nil { - return fmt.Errorf("invalid type: %w", err) - } - - if _, err := io.ReadFull(reader, buf[:]); err != nil { - return fmt.Errorf("cannot read version: %w", err) - } - - r.Version = Version(binary.BigEndian.Uint16(buf[:])) - if err := r.Version.Valid(); err != nil { - return fmt.Errorf("invalid version: %w", err) - } - - if _, err := io.ReadFull(reader, buf[:]); err != nil { - return fmt.Errorf("cannot read payload length: %w", err) - } - - length := int64(binary.BigEndian.Uint16(buf[:])) - if _, err := io.CopyN(&r.Payload, reader, length); err != nil { - return fmt.Errorf("cannot read payload: %w", err) - } - - return nil -} - -func (r *Record) Dump(writer io.Writer) error { - buf := [2]byte{byte(r.Type), 0} - if _, err := writer.Write(buf[:1]); err != nil { - return fmt.Errorf("cannot dump record type: %w", err) - } - - binary.BigEndian.PutUint16(buf[:], uint16(r.Version)) - - if _, err := writer.Write(buf[:]); err != nil { - return fmt.Errorf("cannot dump version: %w", err) - } - - binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len())) - - if _, err := writer.Write(buf[:]); err != nil { - return fmt.Errorf("cannot dump payload length: %w", err) - } - - if _, err := writer.Write(r.Payload.Bytes()); err != nil { - return fmt.Errorf("cannot dump record: %w", err) - } - - return nil -} diff --git a/mtglib/internal/faketls/record/record_test.go b/mtglib/internal/faketls/record/record_test.go deleted file mode 100644 index e29b5ba..0000000 --- a/mtglib/internal/faketls/record/record_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package record_test - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" -) - -type RecordTestSnapshot struct { - Type int `json:"type"` - Version int `json:"version"` - Payload string `json:"payload"` - Record string `json:"record"` -} - -func (r RecordTestSnapshot) RecordBytes() []byte { - data, _ := base64.StdEncoding.DecodeString(r.Record) - - return data -} - -func (r RecordTestSnapshot) PayloadBytes() []byte { - data, _ := base64.StdEncoding.DecodeString(r.Payload) - - return data -} - -type RecordTestSuite struct { - suite.Suite - - r *record.Record - buf *bytes.Buffer -} - -func (suite *RecordTestSuite) SetupTest() { - suite.r = record.AcquireRecord() - suite.buf = &bytes.Buffer{} -} - -func (suite *RecordTestSuite) TearDownTest() { - record.ReleaseRecord(suite.r) - suite.buf.Reset() -} - -func (suite *RecordTestSuite) TestIdempotent() { - suite.r.Type = record.TypeApplicationData - suite.r.Version = record.Version13 - - suite.r.Payload.Write([]byte{1, 2, 3}) - suite.NoError(suite.r.Dump(suite.buf)) - - suite.r.Reset() - suite.NoError(suite.r.Read(suite.buf)) - - suite.Equal(0, suite.buf.Len()) - suite.Equal(record.TypeApplicationData, suite.r.Type) - suite.Equal(record.Version13, suite.r.Version) - suite.Equal([]byte{1, 2, 3}, suite.r.Payload.Bytes()) -} - -func (suite *RecordTestSuite) TestString() { - _ = suite.r.String() -} - -func (suite *RecordTestSuite) TestSnapshot() { - files, err := os.ReadDir("testdata") - suite.NoError(err) - - testData := map[string]string{} - - for _, f := range files { - testData[f.Name()] = filepath.Join("testdata", f.Name()) - } - - for name, pathV := range testData { - path := pathV - - suite.T().Run(name, func(t *testing.T) { - data, err := os.ReadFile(path) - assert.NoError(t, err) - - snapshot := &RecordTestSnapshot{} - assert.NoError(t, json.Unmarshal(data, snapshot)) - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - assert.NoError(t, rec.Read(bytes.NewReader(snapshot.RecordBytes()))) - assert.Equal(t, snapshot.Type, int(rec.Type)) - assert.Equal(t, snapshot.Version, int(rec.Version)) - assert.Equal(t, snapshot.PayloadBytes(), rec.Payload.Bytes()) - - buf := &bytes.Buffer{} - assert.NoError(t, rec.Dump(buf)) - assert.Equal(t, snapshot.RecordBytes(), buf.Bytes()) - }) - } -} - -func TestRecord(t *testing.T) { - t.Parallel() - suite.Run(t, &RecordTestSuite{}) -} diff --git a/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json b/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json deleted file mode 100644 index 6cac5ae..0000000 --- a/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 20, - "version": 772, - "payload": "sxS+0oAyk+NBv0LLVtQOp9WSx4CweyUZPz01tQ0o4oyp8aaBl6/kMFvLq3q52KE8lCiKejLw2NxVBUkE+4izCf2gLx9qfr81opWnqJTChWzcDijvttbq9cmtDFNL+odKsS3v1/TfYEFtPsoRPrJRmOHRAnqnf49Y5Q==", - "record": "FAMEAHmzFL7SgDKT40G/QstW1A6n1ZLHgLB7JRk/PTW1DSjijKnxpoGXr+QwW8urernYoTyUKIp6MvDY3FUFSQT7iLMJ/aAvH2p+vzWilaeolMKFbNwOKO+21ur1ya0MU0v6h0qxLe/X9N9gQW0+yhE+slGY4dECeqd/j1jl" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json b/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json deleted file mode 100644 index 875d64f..0000000 --- a/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 22, - "version": 772, - "payload": "waNH223htyxCBKAb6hm0u/SK/9mhI8Ck91nfWob7QMOaIREogrDYREJH4Djcp47XrpAlEaUIDiCvoFLVJ/LK1nYs4swzfHSSl/+Aj1eqPA63XqPa8EG4FAbf0DwjwXxV9qVIhvP9b2TafKbzr4Yb5GCygzFRb/zawA==", - "record": "FgMEAHnBo0fbbeG3LEIEoBvqGbS79Ir/2aEjwKT3Wd9ahvtAw5ohESiCsNhEQkfgONynjteukCURpQgOIK+gUtUn8srWdizizDN8dJKX/4CPV6o8Drdeo9rwQbgUBt/QPCPBfFX2pUiG8/1vZNp8pvOvhhvkYLKDMVFv/NrA" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/736f358216afe91f.json b/mtglib/internal/faketls/record/testdata/736f358216afe91f.json deleted file mode 100644 index 5b322d4..0000000 --- a/mtglib/internal/faketls/record/testdata/736f358216afe91f.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 23, - "version": 769, - "payload": "jmJ0o1E5+ehAHHYAbCo4AMV03X7RSivYl250s06nD9CO44fyjaoGELz0N7IeCg1jFKcRVSCRmYYmiIY9wydn2fXOJhKif8B0BlM3qhbethYgyP+l1S8hyyETpIiOtiiiOnAJwl1D1j9OryFiJFSdRRXReIMZ4CPqPg==", - "record": "FwMBAHmOYnSjUTn56EAcdgBsKjgAxXTdftFKK9iXbnSzTqcP0I7jh/KNqgYQvPQ3sh4KDWMUpxFVIJGZhiaIhj3DJ2fZ9c4mEqJ/wHQGUzeqFt62FiDI/6XVLyHLIROkiI62KKI6cAnCXUPWP06vIWIkVJ1FFdF4gxngI+o+" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json b/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json deleted file mode 100644 index 16a4ba7..0000000 --- a/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 22, - "version": 769, - "payload": "hBnpBnNUdlqe/rKXa7Judcz79u7AkUgSGOycn8EqvbkZpVxnI31rNOvAsPZqG+GF7DWJ3R7H2ETmFmrpnyyng32MjSs1jptmV1oAs63zTADD7sVipgid9AJHwfl4CrC3FIQr43IPMYd29JPOl5bqu/SfrgI16PBiJw==", - "record": "FgMBAHmEGekGc1R2Wp7+spdrsm51zPv27sCRSBIY7JyfwSq9uRmlXGcjfWs068Cw9mob4YXsNYndHsfYROYWaumfLKeDfYyNKzWOm2ZXWgCzrfNMAMPuxWKmCJ30AkfB+XgKsLcUhCvjcg8xh3b0k86Xluq79J+uAjXo8GIn" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json b/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json deleted file mode 100644 index 5d60155..0000000 --- a/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 23, - "version": 770, - "payload": "Vm/C+DO56czlbtR915aHzsugSyDtp8CtojF9w1jKY0efyyfcLrNuhNg/pZm3gQ7v2BBbL1UJ97v/RIjST+5gRIfg3bBN1BE9hkf+N2AYY2lHLi0yeInHB0zFWPeHscsDopDFadIi5KtC8HvbEMuK+kK8POVk5tN9UQ==", - "record": "FwMCAHlWb8L4M7npzOVu1H3XlofOy6BLIO2nwK2iMX3DWMpjR5/LJ9wus26E2D+lmbeBDu/YEFsvVQn3u/9EiNJP7mBEh+DdsE3UET2GR/43YBhjaUcuLTJ4iccHTMVY94exywOikMVp0iLkq0Lwe9sQy4r6Qrw85WTm031R" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json b/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json deleted file mode 100644 index 459986b..0000000 --- a/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 22, - "version": 770, - "payload": "ajPzpsgk4gwm2stRQKbllvKRLdI7vmyaj1uxEJ/kKoQnQSPumdDNKD618U2Cq6PVd0/b+9YtH67Uzx1QxtpKuby5fUXqw06WUuDAQsmjq7F26EkE5FND6rQUjUPC+e1U0dF4TQzOUSS4IAkFQPAaVehUVTRxVWa/0g==", - "record": "FgMCAHlqM/OmyCTiDCbay1FApuWW8pEt0ju+bJqPW7EQn+QqhCdBI+6Z0M0oPrXxTYKro9V3T9v71i0frtTPHVDG2kq5vLl9RerDTpZS4MBCyaOrsXboSQTkU0PqtBSNQ8L57VTR0XhNDM5RJLggCQVA8BpV6FRVNHFVZr/S" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json b/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json deleted file mode 100644 index e49ea0a..0000000 --- a/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 20, - "version": 771, - "payload": "d1Hiv1NYVgEDR9mtJyv9j8mg3dWqfUpeKfOsL+jzSDfVIxeDiJZFLDT50TjNW44/yEOVEX/Y/pk+wnc7E8aCEiwGwAvB+Insw1UCJ2ejt689VWLo2u4klGVKTHuOpUvdGVTc7Lo4FAt91KQSPLYB5iqxomjEv5e3Vg==", - "record": "FAMDAHl3UeK/U1hWAQNH2a0nK/2PyaDd1ap9Sl4p86wv6PNIN9UjF4OIlkUsNPnROM1bjj/IQ5URf9j+mT7CdzsTxoISLAbAC8H4iezDVQInZ6O3rz1VYuja7iSUZUpMe46lS90ZVNzsujgUC33UpBI8tgHmKrGiaMS/l7dW" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json b/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json deleted file mode 100644 index 7eac0da..0000000 --- a/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 23, - "version": 771, - "payload": "wbdU1CbrzuAJDsh6CFjGyE+AFArJj/Wmsa2wtDyW0kRuE2vUO8gg+nXkg0kkoz0WnvQEOdaswfJIaVrloD78yoyeQVfBB+VUP/63vqn60v5ccaQEn0jLdxgLjiTAxKDQDxCTMRoLnFE2ZZf28zw+HfqpIxiOZs8LhQ==", - "record": "FwMDAHnBt1TUJuvO4AkOyHoIWMbIT4AUCsmP9aaxrbC0PJbSRG4Ta9Q7yCD6deSDSSSjPRae9AQ51qzB8khpWuWgPvzKjJ5BV8EH5VQ//re+qfrS/lxxpASfSMt3GAuOJMDEoNAPEJMxGgucUTZll/bzPD4d+qkjGI5mzwuF" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json b/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json deleted file mode 100644 index 3fffda7..0000000 --- a/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 23, - "version": 772, - "payload": "qqnBMb1Af3zZt4DPHpVRuIiON9ODGJUNFicFjranORh67L/HI4D6HnHyycZFUSBOw2FjMBF6UialY8snOYaRKrQmQzuUNg1Ztq7yAZ+Lgj3TBarR6OMlYhEAY0Px9Xv1UuJ0YcvQx33gdM1skJ5HBR3yZvEKNJV1LA==", - "record": "FwMEAHmqqcExvUB/fNm3gM8elVG4iI4304MYlQ0WJwWOtqc5GHrsv8cjgPoecfLJxkVRIE7DYWMwEXpSJqVjyyc5hpEqtCZDO5Q2DVm2rvIBn4uCPdMFqtHo4yViEQBjQ/H1e/VS4nRhy9DHfeB0zWyQnkcFHfJm8Qo0lXUs" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json b/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json deleted file mode 100644 index aa11362..0000000 --- a/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 20, - "version": 769, - "payload": "NEe735TuQFp7bWpFQhASas/e1XaySvus0ovXmkfCbFq334MyFHq2eDMadziXsfu/GfBjoYggvk0LgYUeoAkBNKR0dfSovjSndaqmIUonoWl+6sZObiGZkRIMwuY2q4Eaw4/iuDu/pZhjRW/iAIH+YH7cyk/1tgdJDg==", - "record": "FAMBAHk0R7vflO5AWnttakVCEBJqz97VdrJK+6zSi9eaR8JsWrffgzIUerZ4Mxp3OJex+78Z8GOhiCC+TQuBhR6gCQE0pHR19Ki+NKd1qqYhSiehaX7qxk5uIZmREgzC5jargRrDj+K4O7+lmGNFb+IAgf5gftzKT/W2B0kO" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/f083f4501668b759.json b/mtglib/internal/faketls/record/testdata/f083f4501668b759.json deleted file mode 100644 index 2518193..0000000 --- a/mtglib/internal/faketls/record/testdata/f083f4501668b759.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 22, - "version": 771, - "payload": "wrXjZrPm3OSyzO0klv6/G+z2PDloR/colS/RlWwQE31Vb2xm8YkEchDDKwlc/KPLD73qMoz3MQOQLtSLc8LhVYp+l7L9jz49yTaVKtBI5UuGbo09snsKxFCgCyYUBETKabATBQtiaEu/D8dmF4Yk/2ww4sEb8DwKLQ==", - "record": "FgMDAHnCteNms+bc5LLM7SSW/r8b7PY8OWhH9yiVL9GVbBATfVVvbGbxiQRyEMMrCVz8o8sPveoyjPcxA5Au1ItzwuFVin6Xsv2PPj3JNpUq0EjlS4ZujT2yewrEUKALJhQERMppsBMFC2JoS78Px2YXhiT/bDDiwRvwPAot" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json b/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json deleted file mode 100644 index 055a7a6..0000000 --- a/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "type": 20, - "version": 770, - "payload": "OU5s8Sa11hpXWEarWzFlX55IZt3Eo+F4AMbQ/2RwB4rfHS/JNl8n63OR4oYs9QXw3RfCrYJuU9n6Xn+I/+7ZzAgZ0PbLSXW1PrLtttdfmhTErK90b49YEWdY9na4g++NMkKykwgXvY1hNxZIHX/qawEWJgxXUR3DdQ==", - "record": "FAMCAHk5TmzxJrXWGldYRqtbMWVfnkhm3cSj4XgAxtD/ZHAHit8dL8k2Xyfrc5Hihiz1BfDdF8Ktgm5T2fpef4j/7tnMCBnQ9stJdbU+su2211+aFMSsr3Rvj1gRZ1j2driD740yQrKTCBe9jWE3Fkgdf+prARYmDFdRHcN1" -} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json b/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json deleted file mode 100644 index 68bfd38..0000000 --- a/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1617181365, - "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", - "sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", - "host": "storage.googleapis.com", - "cipherSuite": 4867, - "full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCACq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFANAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json b/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json deleted file mode 100644 index 097df60..0000000 --- a/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1617181365, - "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", - "sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", - "host": "storage.googleapis.com", - "cipherSuite": 4867, - "full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json b/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json deleted file mode 100644 index f4fe585..0000000 --- a/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1617181352, - "random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=", - "sessionId": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=", - "host": "storage.googleapis.com", - "cipherSuite": 4867, - "full": "AQAB/AMDoYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12osgFGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8gANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAga6CocpFP8Qd4YCFR9pkaCr97po2ALj0P5nI9Nnb3UWMALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json b/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json deleted file mode 100644 index 432f170..0000000 --- a/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1617181352, - "random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=", - "sessionId": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=", - "host": "storage.googleapis.com", - "cipherSuite": 4867, - "full": "AQAB/AMD5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1rogjxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgrulAaqUdKeVYM0F+pu6on/h6LBpOyzOKG4xFIKcoFk4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json b/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json deleted file mode 100644 index 0faafd8..0000000 --- a/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1617181365, - "random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=", - "sessionId": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=", - "host": "storage.googleapis.com", - "cipherSuite": 4867, - "full": "AQAB/AMD8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQg00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAUANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAg/9P7140NtKzjyDwBf99mOy1+FjRPAPHTNQ9WxHOKpV4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json b/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json deleted file mode 100644 index 7bd1122..0000000 --- a/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "time": 1617181352, - "random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=", - "sessionId": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=", - "host": "storage.googleapis.com", - "cipherSuite": 4867, - "full": "AQAB/AMDzja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4gqPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnAANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgXviLRAqAYJ8xOLdlcsUhldI4Xl0g/s9+y2Qrd8raPEgALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -} diff --git a/mtglib/internal/faketls/welcome.go b/mtglib/internal/faketls/welcome.go deleted file mode 100644 index e9fa85b..0000000 --- a/mtglib/internal/faketls/welcome.go +++ /dev/null @@ -1,91 +0,0 @@ -package faketls - -import ( - "bytes" - "crypto/hmac" - "crypto/rand" - "crypto/sha256" - "encoding/binary" - "io" - mrand "math/rand/v2" - - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" - "golang.org/x/crypto/curve25519" -) - -func SendWelcomePacket(writer io.Writer, secret []byte, clientHello ClientHello) error { - buf := &bytes.Buffer{} - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - rec.Type = record.TypeHandshake - rec.Version = record.Version12 - - generateServerHello(&rec.Payload, clientHello) - rec.Dump(buf) //nolint: errcheck - rec.Reset() - - rec.Type = record.TypeChangeCipherSpec - rec.Version = record.Version12 - rec.Payload.WriteByte(ChangeCipherValue) - - rec.Dump(buf) //nolint: errcheck - rec.Reset() - - rec.Type = record.TypeApplicationData - rec.Version = record.Version12 - - if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.IntN(3092))); err != nil { - panic(err) - } - - rec.Dump(buf) //nolint: errcheck - - packet := buf.Bytes() - mac := hmac.New(sha256.New, secret) - - mac.Write(clientHello.Random[:]) - mac.Write(packet) - - copy(packet[WelcomePacketRandomOffset:], mac.Sum(nil)) - - if _, err := writer.Write(packet); err != nil { - return err //nolint: wrapcheck - } - - return nil -} - -func generateServerHello(writer io.Writer, clientHello ClientHello) { - bodyBuf := &bytes.Buffer{} - - sliceBuf := [2]byte{} - digest := [RandomLen]byte{} - - binary.BigEndian.PutUint16(sliceBuf[:], uint16(record.Version12)) - bodyBuf.Write(sliceBuf[:]) - bodyBuf.Write(digest[:]) - bodyBuf.WriteByte(byte(len(clientHello.SessionID))) - bodyBuf.Write(clientHello.SessionID) - - binary.BigEndian.PutUint16(sliceBuf[:], clientHello.CipherSuite) - bodyBuf.Write(sliceBuf[:]) - bodyBuf.Write(serverHelloSuffix) - - scalar := [32]byte{} - - if _, err := rand.Read(scalar[:]); err != nil { - panic(err) - } - - curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) - bodyBuf.Write(curve) - - header := [4]byte{0, 0, 0, 0} - binary.BigEndian.PutUint32(header[:], uint32(bodyBuf.Len())) - header[0] = HandshakeTypeServer - - writer.Write(header[:]) //nolint: errcheck - bodyBuf.WriteTo(writer) //nolint: errcheck -} diff --git a/mtglib/internal/faketls/welcome_test.go b/mtglib/internal/faketls/welcome_test.go deleted file mode 100644 index e722007..0000000 --- a/mtglib/internal/faketls/welcome_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package faketls_test - -import ( - "bytes" - "crypto/hmac" - "crypto/rand" - "crypto/sha256" - "testing" - "time" - - "github.com/9seconds/mtg/v2/mtglib" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" - "github.com/stretchr/testify/suite" -) - -type WelcomeTestSuite struct { - suite.Suite - - h *faketls.ClientHello - buf *bytes.Buffer - secret mtglib.Secret -} - -func (suite *WelcomeTestSuite) SetupTest() { - suite.h = &faketls.ClientHello{ - Time: time.Now(), - Host: "google.com", - CipherSuite: 4867, - SessionID: make([]byte, 32), - } - - _, err := rand.Read(suite.h.SessionID) //nolint: staticcheck - suite.NoError(err) - - _, err = rand.Read(suite.h.Random[:]) //nolint: staticcheck - suite.NoError(err) - - suite.buf = &bytes.Buffer{} - - suite.secret = mtglib.GenerateSecret("google.com") -} - -func (suite *WelcomeTestSuite) TestOk() { - suite.NoError(faketls.SendWelcomePacket(suite.buf, suite.secret.Key[:], *suite.h)) - - welcomePacket := []byte{} - welcomePacket = append(welcomePacket, suite.buf.Bytes()...) - - rec := record.AcquireRecord() - defer record.ReleaseRecord(rec) - - suite.NoError(rec.Read(suite.buf)) - suite.Equal(record.TypeHandshake, rec.Type) - suite.Equal(record.Version12, rec.Version) - - suite.NoError(rec.Read(suite.buf)) - suite.Equal(record.TypeChangeCipherSpec, rec.Type) - suite.Equal(record.Version12, rec.Version) - - suite.NoError(rec.Read(suite.buf)) - suite.Equal(record.TypeApplicationData, rec.Type) - suite.Equal(record.Version12, rec.Version) - suite.Empty(suite.buf.Bytes()) - - random := make([]byte, 32) - copy(random, welcomePacket[11:]) - - empty := make([]byte, 32) - copy(welcomePacket[11:], empty) - - mac := hmac.New(sha256.New, suite.secret.Key[:]) - mac.Write(suite.h.Random[:]) - mac.Write(welcomePacket) - - suite.Equal(random, mac.Sum(nil)) -} - -func TestWelcome(t *testing.T) { - t.Parallel() - suite.Run(t, &WelcomeTestSuite{}) -} From ffd3ab03ccfc70e080853cedb7615c95c9360b29 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 19:13:33 +0100 Subject: [PATCH 17/42] Formatting with gofumpt --- internal/config/type_https_url_test.go | 8 +++---- mtglib/internal/doppel/init_test.go | 3 +-- mtglib/internal/tls/conn.go | 6 ++--- mtglib/internal/tls/fake/bytes_pool.go | 12 ++++------ .../tls/fake/client_side_fuzz_test.go | 1 - mtglib/internal/tls/fake/server_side.go | 24 +++++++++---------- 6 files changed, 23 insertions(+), 31 deletions(-) diff --git a/internal/config/type_https_url_test.go b/internal/config/type_https_url_test.go index 6c7db83..79a39fe 100644 --- a/internal/config/type_https_url_test.go +++ b/internal/config/type_https_url_test.go @@ -42,10 +42,10 @@ func (suite *HttpsURLTestSuite) TestUnmarshalFail() { func (suite *HttpsURLTestSuite) TestUnmarshalOk() { testData := map[string]string{ - "https://example.com": "https://example.com", - "https://example.com:8443": "https://example.com:8443", - "https://example.com/path?q=1": "https://example.com/path?q=1", - "https://user:pass@example.com": "https://user:pass@example.com", + "https://example.com": "https://example.com", + "https://example.com:8443": "https://example.com:8443", + "https://example.com/path?q=1": "https://example.com/path?q=1", + "https://user:pass@example.com": "https://user:pass@example.com", } for k, v := range testData { diff --git a/mtglib/internal/doppel/init_test.go b/mtglib/internal/doppel/init_test.go index e8eec5e..8723959 100644 --- a/mtglib/internal/doppel/init_test.go +++ b/mtglib/internal/doppel/init_test.go @@ -13,8 +13,7 @@ import ( "github.com/stretchr/testify/suite" ) -type SimpleNetwork struct { -} +type SimpleNetwork struct{} func (s SimpleNetwork) Dial(network, address string) (essentials.Conn, error) { return s.DialContext(context.Background(), network, address) diff --git a/mtglib/internal/tls/conn.go b/mtglib/internal/tls/conn.go index 86c5f59..e889310 100644 --- a/mtglib/internal/tls/conn.go +++ b/mtglib/internal/tls/conn.go @@ -22,10 +22,8 @@ const ( TypeApplicationData = 0x17 ) -var ( - // TLS 1.2 is used for both TLS 1.2 and 1.3 - TLSVersion = [SizeVersion]byte{3, 3} -) +// TLS 1.2 is used for both TLS 1.2 and 1.3 +var TLSVersion = [SizeVersion]byte{3, 3} // Conn presents an established TLS 1.3 connection, after handshake type Conn struct { diff --git a/mtglib/internal/tls/fake/bytes_pool.go b/mtglib/internal/tls/fake/bytes_pool.go index 2eaca15..533ed24 100644 --- a/mtglib/internal/tls/fake/bytes_pool.go +++ b/mtglib/internal/tls/fake/bytes_pool.go @@ -5,13 +5,11 @@ import ( "sync" ) -var ( - bytesPool = sync.Pool{ - New: func() any { - return &bytes.Buffer{} - }, - } -) +var bytesPool = sync.Pool{ + New: func() any { + return &bytes.Buffer{} + }, +} func acquireBuffer() *bytes.Buffer { return bytesPool.Get().(*bytes.Buffer) diff --git a/mtglib/internal/tls/fake/client_side_fuzz_test.go b/mtglib/internal/tls/fake/client_side_fuzz_test.go index 28f3337..6de5c82 100644 --- a/mtglib/internal/tls/fake/client_side_fuzz_test.go +++ b/mtglib/internal/tls/fake/client_side_fuzz_test.go @@ -45,5 +45,4 @@ func FuzzReadClientHello(f *testing.F) { _, err := fake.ReadClientHello(r, secret.Key[:], secret.Host, time.Hour) assert.Error(t, err) }) - } diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go index 14e0cde..3a0d43a 100644 --- a/mtglib/internal/tls/fake/server_side.go +++ b/mtglib/internal/tls/fake/server_side.go @@ -20,19 +20,17 @@ const ( EllipticCurveLen = 32 ) -var ( - serverHelloSuffix = []byte{ - 0x00, // no compression - 0x00, 0x2e, // 46 bytes of data - 0x00, 0x2b, // Extension - Supported Versions - 0x00, 0x02, // 2 bytes are following - 0x03, 0x04, // TLS 1.3 - 0x00, 0x33, // Extension - Key Share - 0x00, 0x24, // 36 bytes - 0x00, 0x1d, // x25519 curve - 0x00, 0x20, // 32 bytes of key - } -) +var serverHelloSuffix = []byte{ + 0x00, // no compression + 0x00, 0x2e, // 46 bytes of data + 0x00, 0x2b, // Extension - Supported Versions + 0x00, 0x02, // 2 bytes are following + 0x03, 0x04, // TLS 1.3 + 0x00, 0x33, // Extension - Key Share + 0x00, 0x24, // 36 bytes + 0x00, 0x1d, // x25519 curve + 0x00, 0x20, // 32 bytes of key +} func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) ([]byte, error) { buf := &bytes.Buffer{} From 94e4179fb724eabf530a799d7968cea911458052 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 19:25:53 +0100 Subject: [PATCH 18/42] Add NativeDialer method to mtglib.Network --- internal/cli/run_proxy.go | 2 +- mtglib/init.go | 6 +++++- network/network.go | 4 ++++ network/v2/init.go | 9 --------- network/v2/multi_network.go | 7 ++++--- network/v2/network.go | 3 ++- network/v2/proxy_network.go | 5 +++-- 7 files changed, 19 insertions(+), 17 deletions(-) diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index 7774953..e0dd9bd 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -52,7 +52,7 @@ func makeNetwork(conf *config.Config, version string) (mtglib.Network, error) { conf.Network.Timeout.Idle.Get(0), ) - proxyDialers := make([]network.Network, len(conf.Network.Proxies)) + proxyDialers := make([]mtglib.Network, len(conf.Network.Proxies)) for idx, v := range conf.Network.Proxies { value, err := network.NewProxyNetwork(base, v.Get(nil)) if err != nil { diff --git a/mtglib/init.go b/mtglib/init.go index 5b2d0bf..3d7f630 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -124,13 +124,17 @@ type Network interface { // Dial establishes context-free TCP connections. Dial(network, address string) (essentials.Conn, error) - // DialContext dials using a context. This is a preferrable way of + // DialContext dials using a context. This is a preferable way of // establishing TCP connections. DialContext(ctx context.Context, network, address string) (essentials.Conn, error) // MakeHTTPClient build an HTTP client with given dial function. If nothing is // provided, then DialContext of this interface is going to be used. MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client + + // NativeDialer returns a configured instance of native dialer that + // skips proxy connections or any other irrelevant settings. + NativeDialer() *net.Dialer } // AntiReplayCache is an interface that is used to detect replay attacks based diff --git a/network/network.go b/network/network.go index 7a7b332..c2d4e84 100644 --- a/network/network.go +++ b/network/network.go @@ -60,6 +60,10 @@ func (n *network) DialContext(ctx context.Context, protocol, address string) (es return nil, fmt.Errorf("cannot dial to %s:%s: %w", protocol, address, err) } +func (n *network) NativeDialer() *net.Dialer { + return &net.Dialer{} +} + func (n *network) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error), ) *http.Client { diff --git a/network/v2/init.go b/network/v2/init.go index 7cc4302..bfdfa5c 100644 --- a/network/v2/init.go +++ b/network/v2/init.go @@ -11,10 +11,7 @@ package network import ( "errors" - "net" "time" - - "github.com/9seconds/mtg/v2/mtglib" ) const ( @@ -37,9 +34,3 @@ const ( ) var ErrCannotDial = errors.New("cannot dial to any address") - -type Network interface { - mtglib.Network - - NativeDialer() *net.Dialer -} diff --git a/network/v2/multi_network.go b/network/v2/multi_network.go index 45e501c..07f4bd4 100644 --- a/network/v2/multi_network.go +++ b/network/v2/multi_network.go @@ -8,10 +8,11 @@ import ( "net/http" "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib" ) type multiNetwork struct { - networks []Network + networks []mtglib.Network } func (m multiNetwork) Dial(network, address string) (essentials.Conn, error) { @@ -22,7 +23,7 @@ func (m multiNetwork) DialContext(ctx context.Context, network, address string) networks := m.networks if len(networks) > 1 { - networks = make([]Network, len(m.networks)) + networks = make([]mtglib.Network, len(m.networks)) copy(networks, m.networks) rand.Shuffle(len(m.networks), func(i, j int) { @@ -59,7 +60,7 @@ func (m multiNetwork) MakeHTTPClient( return m.networks[0].MakeHTTPClient(dialFunc) } -func Join(networks ...Network) (Network, error) { +func Join(networks ...mtglib.Network) (mtglib.Network, error) { if len(networks) == 0 { return nil, errors.New("cannot join no networks") } diff --git a/network/v2/network.go b/network/v2/network.go index 9921f8b..f3b0405 100644 --- a/network/v2/network.go +++ b/network/v2/network.go @@ -8,6 +8,7 @@ import ( "time" "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib" ) type network struct { @@ -70,7 +71,7 @@ func New( tcpTimeout, httpTimeout, idleTimeout time.Duration, -) Network { +) mtglib.Network { if dnsResolver == nil { dnsResolver = net.DefaultResolver } diff --git a/network/v2/proxy_network.go b/network/v2/proxy_network.go index c27162d..58d5469 100644 --- a/network/v2/proxy_network.go +++ b/network/v2/proxy_network.go @@ -6,11 +6,12 @@ import ( "net/url" "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib" "golang.org/x/net/proxy" ) type proxyNetwork struct { - Network + mtglib.Network client proxy.ContextDialer } @@ -23,7 +24,7 @@ func (p proxyNetwork) DialContext(ctx context.Context, network, address string) return essentials.WrapNetConn(conn), nil } -func NewProxyNetwork(base Network, proxyURL *url.URL) (*proxyNetwork, error) { +func NewProxyNetwork(base mtglib.Network, proxyURL *url.URL) (*proxyNetwork, error) { socks, err := proxy.FromURL(proxyURL, base.NativeDialer()) if err != nil { return nil, fmt.Errorf("cannot build proxy dialer: %w", err) From 62cba24071b2cab359902ba07eee6d435c54e81e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 20:51:02 +0100 Subject: [PATCH 19/42] Use native dialer to communicate with fronting domain --- internal/testlib/mtglib_network_mock.go | 5 +++++ mtglib/proxy.go | 5 ++++- network/v2/base_network_test.go | 3 ++- network/v2/socks_proxy_test.go | 5 +++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/testlib/mtglib_network_mock.go b/internal/testlib/mtglib_network_mock.go index 21ae79c..9dce30d 100644 --- a/internal/testlib/mtglib_network_mock.go +++ b/internal/testlib/mtglib_network_mock.go @@ -2,6 +2,7 @@ package testlib import ( "context" + "net" "net/http" "github.com/9seconds/mtg/v2/essentials" @@ -24,6 +25,10 @@ func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address st return args.Get(0).(essentials.Conn), args.Error(1) //nolint: wrapcheck, forcetypeassert } +func (m *MtglibNetworkMock) NativeDialer() *net.Dialer { + return m.Called().Get(0).(*net.Dialer) +} + func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error), ) *http.Client { diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 995f877..aac79de 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -279,13 +279,16 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { p.eventStream.Send(p.ctx, NewEventDomainFronting(ctx.streamID)) conn.Rewind() - frontConn, err := p.network.DialContext(ctx, "tcp", p.DomainFrontingAddress()) + nativeDialer := p.network.NativeDialer() + fConn, err := nativeDialer.DialContext(ctx, "tcp", p.DomainFrontingAddress()) if err != nil { p.logger.WarningError("cannot dial to the fronting domain", err) return } + frontConn := essentials.WrapNetConn(fConn) + if p.domainFrontingProxyProtocol { frontConn = newConnProxyProtocol(ctx.clientConn, frontConn) } diff --git a/network/v2/base_network_test.go b/network/v2/base_network_test.go index f9729e4..f2d7a10 100644 --- a/network/v2/base_network_test.go +++ b/network/v2/base_network_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/network/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" @@ -12,7 +13,7 @@ import ( type BaseNetworkTestSuite struct { EchoServerTestSuite - net network.Network + net mtglib.Network } func (suite *BaseNetworkTestSuite) SetupSuite() { diff --git a/network/v2/socks_proxy_test.go b/network/v2/socks_proxy_test.go index cc02dda..d6d41ae 100644 --- a/network/v2/socks_proxy_test.go +++ b/network/v2/socks_proxy_test.go @@ -6,6 +6,7 @@ import ( "sync" "testing" + "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/network/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,7 +18,7 @@ type SocksProxyTestSuite struct { EchoServerTestSuite wg sync.WaitGroup - baseNetwork network.Network + baseNetwork mtglib.Network noAuthURL *url.URL authURL *url.URL @@ -85,7 +86,7 @@ func (suite *SocksProxyTestSuite) TestRead() { for name, proxies := range testData { suite.T().Run(name, func(t *testing.T) { - proxyNetworks := []network.Network{} + proxyNetworks := []mtglib.Network{} for _, u := range proxies { value, err := network.NewProxyNetwork(suite.baseNetwork, u) From a85348d6bedfa177ee02f4cd6e83584edae20ba3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 21:43:04 +0100 Subject: [PATCH 20/42] Use curl as user agent --- internal/cli/run_proxy.go | 2 +- network/v2/init.go | 3 +++ network/v2/network.go | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index e0dd9bd..1ecb270 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -46,7 +46,7 @@ func makeNetwork(conf *config.Config, version string) (mtglib.Network, error) { base := network.New( resolver, - "mtg/"+version, + "", conf.Network.Timeout.TCP.Get(0), conf.Network.Timeout.HTTP.Get(0), conf.Network.Timeout.Idle.Get(0), diff --git a/network/v2/init.go b/network/v2/init.go index bfdfa5c..14c6144 100644 --- a/network/v2/init.go +++ b/network/v2/init.go @@ -28,6 +28,9 @@ const ( // probes. DefaultTCPKeepAlivePeriod = 10 * time.Second + // User Agent to use in HTTP client. + UserAgent = "curl/8.5.0" + // tcpLingerTimeout defines a number of seconds to wait for sending // unacknowledged data. tcpLingerTimeout = 1 diff --git a/network/v2/network.go b/network/v2/network.go index f3b0405..3f961ed 100644 --- a/network/v2/network.go +++ b/network/v2/network.go @@ -76,6 +76,10 @@ func New( dnsResolver = net.DefaultResolver } + if userAgent == "" { + userAgent = UserAgent + } + return &network{ Dialer: net.Dialer{ Timeout: tcpTimeout, From cc63e762e372fe38d5f6ccdbeaf8116ecadf236d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 21:47:03 +0100 Subject: [PATCH 21/42] Use native dialer for scout --- mtglib/internal/doppel/init.go | 7 ++++++- mtglib/internal/doppel/init_test.go | 4 ++++ mtglib/internal/doppel/scout.go | 5 +++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mtglib/internal/doppel/init.go b/mtglib/internal/doppel/init.go index 9baa193..15bcfd8 100644 --- a/mtglib/internal/doppel/init.go +++ b/mtglib/internal/doppel/init.go @@ -2,6 +2,7 @@ package doppel import ( "context" + "net" "net/http" "time" @@ -28,11 +29,15 @@ type Network interface { // Dial establishes context-free TCP connections. Dial(network, address string) (essentials.Conn, error) - // DialContext dials using a context. This is a preferrable way of + // DialContext dials using a context. This is a preferable way of // establishing TCP connections. DialContext(ctx context.Context, network, address string) (essentials.Conn, error) // MakeHTTPClient build an HTTP client with given dial function. If nothing is // provided, then DialContext of this interface is going to be used. MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client + + // NativeDialer returns a configured instance of native dialer that + // skips proxy connections or any other irrelevant settings. + NativeDialer() *net.Dialer } diff --git a/mtglib/internal/doppel/init_test.go b/mtglib/internal/doppel/init_test.go index 8723959..7f8fc98 100644 --- a/mtglib/internal/doppel/init_test.go +++ b/mtglib/internal/doppel/init_test.go @@ -30,6 +30,10 @@ func (s SimpleNetwork) DialContext(ctx context.Context, network, address string) return conn.(*net.TCPConn), nil } +func (s SimpleNetwork) NativeDialer() *net.Dialer { + return &net.Dialer{} +} + func (s SimpleNetwork) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client { if dialFunc == nil { dialFunc = s.DialContext diff --git a/mtglib/internal/doppel/scout.go b/mtglib/internal/doppel/scout.go index 88a478b..e6650c8 100644 --- a/mtglib/internal/doppel/scout.go +++ b/mtglib/internal/doppel/scout.go @@ -79,18 +79,19 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) { } func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) { + dialer := s.network.NativeDialer() collected := NewScoutConnCollected() client := s.network.MakeHTTPClient(func( ctx context.Context, network string, address string, ) (essentials.Conn, error) { - conn, err := s.network.DialContext(ctx, network, address) + conn, err := dialer.DialContext(ctx, network, address) if err != nil { return nil, err } - return NewScoutConn(conn, collected), nil + return NewScoutConn(essentials.WrapNetConn(conn), collected), nil }) return client, collected From 991346621cd9d64dffd0cd50f92f6aed4e8c5668 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 22:13:00 +0100 Subject: [PATCH 22/42] Speedup docker builds by using cache --- .github/workflows/ci.yaml | 14 +++----------- Dockerfile | 6 +++++- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d85c508..262be17 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -123,14 +123,6 @@ jobs: - name: Setup BuildX uses: docker/setup-buildx-action@v3 - - name: Setup cache - uses: actions/cache@v5 - with: - path: /tmp/buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to DockerHub if: github.event_name != 'pull_request' uses: docker/login-action@v3 @@ -147,7 +139,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: pull: true context: . @@ -155,5 +147,5 @@ jobs: push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=local,src=/tmp/buildx-cache - cache-to: type=local,dest=/tmp/buildx-cache + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 43d996b..9824196 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,13 @@ RUN set -x \ ca-certificates \ git -COPY . /app +COPY go.mod go.sum /app/ WORKDIR /app +RUN go mod download + +COPY . /app + RUN set -x \ && version="$(git describe --exact-match HEAD 2>/dev/null || git describe --tags --always)" \ && go build \ From 3db1be068795ac559814234f2f67cb72529be983 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 22:13:17 +0100 Subject: [PATCH 23/42] Use cache to speed up jobs --- .github/workflows/ci.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 262be17..0161e01 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -48,6 +48,16 @@ jobs: - uses: jdx/mise-action@v3 name: Install mise + - name: Cache Go modules and build + uses: actions/cache@v5 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run tests run: mise tasks run covtest @@ -69,6 +79,16 @@ jobs: - uses: jdx/mise-action@v3 name: Install mise + - name: Cache Go modules and build + uses: actions/cache@v5 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run fuzzing run: mise tasks run 'test:fuzz:*' @@ -86,6 +106,16 @@ jobs: - uses: jdx/mise-action@v3 name: Install mise + - name: Cache Go modules and build + uses: actions/cache@v5 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run linter run: mise tasks run lint From d4822989ecb5c23784cdac728add586996fa1884 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 22:22:00 +0100 Subject: [PATCH 24/42] Add count number to covtest --- .mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mise.toml b/.mise.toml index c5bdf73..18e8245 100644 --- a/.mise.toml +++ b/.mise.toml @@ -37,7 +37,7 @@ run = "go test -v ./..." [tasks.covtest] description = "Run tests with code coverage" -run = "go test -coverprofile=coverage.txt -covermode=atomic -parallel 2 -race -v ./..." +run = "go test -coverprofile=coverage.txt -covermode=atomic -count=2 -race -v ./..." [tasks.test-all] description = "Run all tests" From 287a794772110b7d795d5239c2d68268cd9e867b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 22:22:09 +0100 Subject: [PATCH 25/42] Always run tests with race detector --- .mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mise.toml b/.mise.toml index 18e8245..e49496c 100644 --- a/.mise.toml +++ b/.mise.toml @@ -33,7 +33,7 @@ run = "govulncheck ./..." [tasks.test] description = "Run tests" -run = "go test -v ./..." +run = "go test -v -race ./..." [tasks.covtest] description = "Run tests with code coverage" From 4dca1d2b071a3b33ff068220840647743dd525de Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 22:46:19 +0100 Subject: [PATCH 26/42] Add cache mount for apk downloads --- Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9824196..fdb4b56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,12 @@ FROM golang:1.26-alpine AS build ENV CGO_ENABLED=0 -RUN set -x \ - && apk --no-cache --update add \ - bash \ - ca-certificates \ - git +RUN --mount=type=cache,target=/var/cache/apk \ + set -x \ + && apk --update add \ + bash \ + ca-certificates \ + git COPY go.mod go.sum /app/ WORKDIR /app From ad945c77a91fbae7420d28d6accd90f36c00cf50 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 12 Mar 2026 23:01:26 +0100 Subject: [PATCH 27/42] Enable caching for govulncheck --- .github/workflows/govulncheck.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index a1d0be7..c0338d3 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -35,6 +35,7 @@ jobs: uses: actions/setup-go@v6 with: go-version-file: go.mod + cache: true - name: Check for vulnerabilities run: | From 8df162988243a0bfed7d3084e45a359c728bdecb Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 08:42:15 +0100 Subject: [PATCH 28/42] Access command should use native dialer instead of network one --- internal/cli/access.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/cli/access.go b/internal/cli/access.go index 4657822..550babc 100644 --- a/internal/cli/access.go +++ b/internal/cli/access.go @@ -101,8 +101,13 @@ func (a *Access) Run(cli *CLI, version string) error { } func (a *Access) getIP(ntw mtglib.Network, protocol string) net.IP { + dialer := ntw.NativeDialer() client := ntw.MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error) { - return ntw.DialContext(ctx, protocol, address) //nolint: wrapcheck + conn, err := dialer.DialContext(ctx, protocol, address) + if err != nil { + return nil, err + } + return essentials.WrapNetConn(conn), err }) req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) //nolint: noctx From ea71fe81b2226185a96f7c1b754c32c04bf8c197 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 08:59:59 +0100 Subject: [PATCH 29/42] Update stats defaults --- mtglib/internal/doppel/stats.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mtglib/internal/doppel/stats.go b/mtglib/internal/doppel/stats.go index 6ed7c9f..8fb0c8d 100644 --- a/mtglib/internal/doppel/stats.go +++ b/mtglib/internal/doppel/stats.go @@ -14,9 +14,9 @@ const ( // do not calculate statistics if we have < than this number of durations MinDurationsToCalculate = 100 - // these values are taken from ok-cdn. measured from moscow site. - StatsDefaultK = 1.2908978568647322 - StatsDefaultLambda = 1.4258402622793287 + // these values are taken from ok.ru. measured from moscow site. + StatsDefaultK = 0.37846373895785335 + StatsDefaultLambda = 1.73177086015485 ) // Stats is responsible for generating values that are distributed according From 21d752235674e25b2746f762325c0ae63048ff04 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 11:04:01 +0100 Subject: [PATCH 30/42] Make DRS optional --- example.config.toml | 63 +++++++++++++-------------- internal/cli/run_proxy.go | 1 + internal/config/config.go | 1 + internal/config/parse.go | 1 + mtglib/internal/doppel/ganger.go | 7 ++- mtglib/internal/doppel/ganger_test.go | 2 +- mtglib/internal/doppel/init.go | 4 +- mtglib/internal/doppel/stats.go | 17 ++++++-- mtglib/internal/doppel/stats_test.go | 49 ++++++++++++++++----- mtglib/proxy.go | 1 + mtglib/proxy_opts.go | 3 ++ 11 files changed, 98 insertions(+), 51 deletions(-) diff --git a/example.config.toml b/example.config.toml index 2815873..c0ca32c 100644 --- a/example.config.toml +++ b/example.config.toml @@ -213,42 +213,20 @@ idle = "1m" # does not really matter), while websites pump heavy content in HTTP2 streams # # It means that statistically there is a different between traffic shape: -# TLS packet sizes are different, delays between packets are also different. +# delays between packets are also different. # In order to avoid censorship detection based on these patterns, there is a # mtg subsystem called "Doppelganger" that aims to mimic website statistics # as close as it could. # -# It does that by 2 ideas: -# 1. Delays between TLS packets are not constant. There are many factors -# that come in play. Application should generate some response, it could -# send some headers first and stream content with chunked encoding. So -# some first packets could come as soon as possible, with some delays -# after first ones. Such phenomenon is described by different statistic -# distribution. There are 2 distribution that describe it: lognormal -# distribution and Weibul distribution. Lognormal is all about steady streams -# of heavy content like a video. Weibul is great about short bursts like -# user who requested a static page an a couple of images. -# -# mtg tries to adapt Weibul distribution. It comes with some sensible -# defaults that were taken from ok.ru. But when you use domain fronting, -# it always make sense to take statistics from that website. You can specify -# some urls here. mtg will crawl them from time to time, accumulate time -# series and approximates parameters for Weibul. -# 2. TLS record sizes are not random. -# https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/ -# https://aws.github.io/s2n-tls/usage-guide/ch08-record-sizes.html -# -# The idea is that huge TLS records could negatively affect performance. -# You cannot simply decrypt a part of the packet, you need to wait it -# whole, and huge packets could involve several RTTs if you do not use -# any specific software that treat TLS in a very special way. So -# servers start with small packets, usually around MTU, and ramp up -# later. This optimizes a time-to-first byte so web browsers start to -# render early. -# -# mtg uses the same technique as was introduced by Cloudflare in their -# patches to nginx 10 years ago: -# https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch +# Delays between TLS packets are not constant. There are many factors +# that come in play. Application should generate some response, it could +# send some headers first and stream content with chunked encoding. So +# some first packets could come as soon as possible, with some delays +# after first ones. Such phenomenon is described by different statistic +# distribution. There are 2 distribution that describe it: lognormal +# distribution and Weibul distribution. Lognormal is all about steady streams +# of heavy content like a video. Weibul is great about short bursts like +# user who requested a static page an a couple of images. [defense.doppelganger] # This is a list of URLs that would be crawled by mtg to approximate delay # statistics. They MUST be HTTPS urls. @@ -266,6 +244,27 @@ repeats-per-raid = 10 # do not change a lot, so do not expect different results if you request # each 10 minutes. raid-each = "6h" +# This enables dynamic tls record sizing. +# +# Some modern stacks and platforms start to use the technique that is called +# DRS. They start with small TLS packets and ramp up eventually. First packets +# are usually about MTU size, after that we get 4k and eventually max size. +# This is done with a good intention: to minimize a time to the first byte, +# so application could start doing something with the data right after first +# RTT. +# +# Apparently, about 90% of application do not employ this technique, they use +# max size always: nginx, apache, java stuff. But Golang tools, angie and +# some specific patches activate this technique. +# +# In order to mimic a real website we need to know something about software +# it uses. Usually nobody cares: openssl does 16384, Python does it, nginx +# does it. So this setting is disabled by default. +# +# https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/ +# https://aws.github.io/s2n-tls/usage-guide/ch08-record-sizes.html +# https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch +drs = false # Some countries do active probing on Telegram connections. This technique # allows to protect from such effort. diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index 1ecb270..9453eda 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -265,6 +265,7 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen DoppelGangerURLs: doppelGangerURLs, DoppelGangerPerRaid: conf.Defense.Doppelganger.Repeats.Get(mtglib.DoppelGangerPerRaid), DoppelGangerEach: conf.Defense.Doppelganger.UpdateEach.Get(mtglib.DoppelGangerEach), + DoppelGangerDRS: conf.Defense.Doppelganger.DRS.Get(false), } proxy, err := mtglib.NewProxy(opts) diff --git a/internal/config/config.go b/internal/config/config.go index 91c93b0..9ced832 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -53,6 +53,7 @@ type Config struct { URLs []TypeHttpsURL `json:"urls"` Repeats TypeConcurrency `json:"repeats_per_raid"` UpdateEach TypeDuration `json:"raid_each"` + DRS TypeBool `json:"drs"` } `json:"doppelganger"` } `json:"defense"` Network struct { diff --git a/internal/config/parse.go b/internal/config/parse.go index 06183c7..81a50f3 100644 --- a/internal/config/parse.go +++ b/internal/config/parse.go @@ -48,6 +48,7 @@ type tomlConfig struct { URLs []string `toml:"urls" json:"urls,omitempty"` Repeats uint `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"` UpdateEach string `toml:"raid-each" json:"raid_each,omitempty"` + DRS bool `toml:"drs" json:"drs,omitempty"` } `toml:"doppelganger" json:"doppelganger,omitempty"` } `toml:"defense" json:"defense,omitempty"` Network struct { diff --git a/mtglib/internal/doppel/ganger.go b/mtglib/internal/doppel/ganger.go index ee4a3eb..6621761 100644 --- a/mtglib/internal/doppel/ganger.go +++ b/mtglib/internal/doppel/ganger.go @@ -29,6 +29,8 @@ type Ganger struct { scoutRaidEach time.Duration scoutRaidRepeats int + drs bool + stats *Stats durations []time.Duration @@ -107,7 +109,7 @@ func (g *Ganger) run() { g.wg.Go(func() { select { case <-g.ctx.Done(): - case updatedStatsChan <- NewStats(durations): + case updatedStatsChan <- NewStats(durations, g.drs): } }) case stats := <-updatedStatsChan: @@ -152,6 +154,7 @@ func NewGanger( scoutEach time.Duration, scoutRepeats int, urls []string, + drs bool, ) *Ganger { ctx, cancel := context.WithCancel(ctx) @@ -169,9 +172,11 @@ func NewGanger( logger: logger, scoutRaidEach: scoutEach, scoutRaidRepeats: scoutRepeats, + drs: drs, stats: &Stats{ k: StatsDefaultK, lambda: StatsDefaultLambda, + drs: drs, }, scout: NewScout(network, urls), connRequests: make(chan gangerConnRequest), diff --git a/mtglib/internal/doppel/ganger_test.go b/mtglib/internal/doppel/ganger_test.go index 0343eb4..412eb8a 100644 --- a/mtglib/internal/doppel/ganger_test.go +++ b/mtglib/internal/doppel/ganger_test.go @@ -29,7 +29,7 @@ func (suite *GangerTestSuite) SetupTest() { On("WarningError", mock.AnythingOfType("string"), mock.Anything). Maybe() - suite.g = NewGanger(suite.ctx, suite.network, suite.log, time.Hour, 1, suite.urls) + suite.g = NewGanger(suite.ctx, suite.network, suite.log, time.Hour, 1, suite.urls, true) suite.g.Run() } diff --git a/mtglib/internal/doppel/init.go b/mtglib/internal/doppel/init.go index 15bcfd8..eb74a88 100644 --- a/mtglib/internal/doppel/init.go +++ b/mtglib/internal/doppel/init.go @@ -14,8 +14,8 @@ const ( // Please see Stats description // https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/ // https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch - TLSRecordSizeStart = 1369 - TLSRecordSizeAccel = 4229 + TLSRecordSizeStart = 1450 + TLSRecordSizeAccel = 4096 TLSRecordSizeMax = 16384 - tls.SizeHeader TLSCounterAccelAfter = 40 diff --git a/mtglib/internal/doppel/stats.go b/mtglib/internal/doppel/stats.go index 8fb0c8d..eb73b61 100644 --- a/mtglib/internal/doppel/stats.go +++ b/mtglib/internal/doppel/stats.go @@ -17,6 +17,9 @@ const ( // these values are taken from ok.ru. measured from moscow site. StatsDefaultK = 0.37846373895785335 StatsDefaultLambda = 1.73177086015485 + + // how many bytes should we drift + DRSNoise = 100 ) // Stats is responsible for generating values that are distributed according @@ -66,6 +69,9 @@ type Stats struct { k float64 // https://en.wikipedia.org/wiki/Scale_parameter lambda float64 + + // Dynamic Record Sizing + drs bool } func (d *Stats) Delay() time.Duration { @@ -84,20 +90,24 @@ func (d *Stats) Size() int { d.sizeCounter = 0 } + if !d.drs { + return TLSRecordSizeMax + } + d.sizeLastRequested = time.Now() d.sizeCounter++ switch { case d.sizeCounter <= TLSCounterAccelAfter: - return TLSRecordSizeStart + return TLSRecordSizeStart - rand.IntN(DRSNoise) case d.sizeCounter <= TLSCounterMaxAfter: - return TLSRecordSizeAccel + return TLSRecordSizeAccel - rand.IntN(DRSNoise) } return TLSRecordSizeMax } -func NewStats(durations []time.Duration) *Stats { +func NewStats(durations []time.Duration, drs bool) *Stats { n := float64(len(durations)) // in milliseconds @@ -150,5 +160,6 @@ func NewStats(durations []time.Duration) *Stats { return &Stats{ k: k, lambda: lambda, + drs: drs, } } diff --git a/mtglib/internal/doppel/stats_test.go b/mtglib/internal/doppel/stats_test.go index 1e45a1f..94240d1 100644 --- a/mtglib/internal/doppel/stats_test.go +++ b/mtglib/internal/doppel/stats_test.go @@ -38,7 +38,7 @@ func (suite *StatsTestSuite) TestNewStatsRecoverParameters() { knownLambda := 100.0 samples := suite.GenWeibull(knownK, knownLambda, 5000, 42) - stats := NewStats(samples) + stats := NewStats(samples, true) suite.InDelta(knownK, stats.k, 0.1) suite.InDelta(knownLambda, stats.lambda, 5.0) @@ -50,7 +50,7 @@ func (suite *StatsTestSuite) TestNewStatsExponentialCase() { knownLambda := 50.0 samples := suite.GenWeibull(knownK, knownLambda, 5000, 123) - stats := NewStats(samples) + stats := NewStats(samples, true) suite.InDelta(knownK, stats.k, 0.1) suite.InDelta(knownLambda, stats.lambda, 5.0) @@ -64,7 +64,7 @@ func (suite *StatsTestSuite) TestNewStatsSmallK() { knownLambda := 100.0 samples := suite.GenWeibull(knownK, knownLambda, 10000, 99) - stats := NewStats(samples) + stats := NewStats(samples, true) suite.InDelta(knownK, stats.k, 0.05) suite.InDelta(knownLambda, stats.lambda, 5.0) @@ -76,7 +76,7 @@ func (suite *StatsTestSuite) TestNewStatsLargeK() { knownLambda := 200.0 samples := suite.GenWeibull(knownK, knownLambda, 5000, 77) - stats := NewStats(samples) + stats := NewStats(samples, true) suite.InDelta(knownK, stats.k, 0.3) suite.InDelta(knownLambda, stats.lambda, 5.0) @@ -121,7 +121,7 @@ func (suite *StatsTestSuite) TestNewStatsRoundTrip() { knownLambda := 80.0 samples := suite.GenWeibull(knownK, knownLambda, 5000, 555) - stats := NewStats(samples) + stats := NewStats(samples, true) n := 50000 sum := 0.0 @@ -138,16 +138,17 @@ func (suite *StatsTestSuite) TestNewStatsRoundTrip() { } func (suite *StatsTestSuite) TestSizeStartPhase() { - stats := &Stats{k: 1.0, lambda: 1.0} + stats := &Stats{k: 1.0, lambda: 1.0, drs: true} for range TLSCounterAccelAfter { size := stats.Size() - suite.Equal(TLSRecordSizeStart, size) + suite.GreaterOrEqual(size, TLSRecordSizeStart-DRSNoise) + suite.LessOrEqual(size, TLSRecordSizeStart) } } func (suite *StatsTestSuite) TestSizeAccelPhase() { - stats := &Stats{k: 1.0, lambda: 1.0} + stats := &Stats{k: 1.0, lambda: 1.0, drs: true} for range TLSCounterAccelAfter { stats.Size() @@ -155,12 +156,13 @@ func (suite *StatsTestSuite) TestSizeAccelPhase() { for range TLSCounterMaxAfter - TLSCounterAccelAfter { size := stats.Size() - suite.Equal(TLSRecordSizeAccel, size) + suite.GreaterOrEqual(size, TLSRecordSizeAccel-DRSNoise) + suite.LessOrEqual(size, TLSRecordSizeAccel) } } func (suite *StatsTestSuite) TestSizeMaxPhase() { - stats := &Stats{k: 1.0, lambda: 1.0} + stats := &Stats{k: 1.0, lambda: 1.0, drs: true} for range TLSCounterMaxAfter { stats.Size() @@ -173,7 +175,7 @@ func (suite *StatsTestSuite) TestSizeMaxPhase() { } func (suite *StatsTestSuite) TestSizeResetsAfterInactivity() { - stats := &Stats{k: 1.0, lambda: 1.0} + stats := &Stats{k: 1.0, lambda: 1.0, drs: true} // Advance past start phase. for range TLSCounterMaxAfter { @@ -185,7 +187,30 @@ func (suite *StatsTestSuite) TestSizeResetsAfterInactivity() { // Simulate inactivity by backdating sizeLastRequested. stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond) - suite.Equal(TLSRecordSizeStart, stats.Size()) + size := stats.Size() + suite.GreaterOrEqual(size, TLSRecordSizeStart-DRSNoise) + suite.LessOrEqual(size, TLSRecordSizeStart) +} + +func (suite *StatsTestSuite) TestSizeNoDRSAlwaysMax() { + stats := &Stats{k: 1.0, lambda: 1.0, drs: false} + + for range TLSCounterMaxAfter + 20 { + suite.Equal(TLSRecordSizeMax, stats.Size()) + } +} + +func (suite *StatsTestSuite) TestSizeNoDRSIgnoresCounter() { + stats := &Stats{k: 1.0, lambda: 1.0, drs: false} + + // Even after many calls, always returns max. + for range 200 { + suite.Equal(TLSRecordSizeMax, stats.Size()) + } + + // Inactivity has no effect either. + stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond) + suite.Equal(TLSRecordSizeMax, stats.Size()) } func TestStats(t *testing.T) { diff --git a/mtglib/proxy.go b/mtglib/proxy.go index aac79de..da1b636 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -345,6 +345,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { opts.DoppelGangerEach, int(opts.DoppelGangerPerRaid), opts.DoppelGangerURLs, + opts.DoppelGangerDRS, ), configUpdater: dc.NewPublicConfigUpdater( tg, diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index 0f8600e..cea9cad 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -157,6 +157,9 @@ type ProxyOpts struct { // DoppelGangerEach defines a time period between each raid. We recommend // to use hours here. DoppelGangerEach time.Duration + + // DoppelGangerDRS defines if TLS Dynamic Record Sizing is active. + DoppelGangerDRS bool } func (p ProxyOpts) valid() error { From 33c0fa9bf725acf57b7af51b5f7996a665251063 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 11:30:52 +0100 Subject: [PATCH 31/42] Add SyncWrite method to doppel.Conn --- mtglib/internal/doppel/conn.go | 55 +++++++++--- mtglib/internal/doppel/conn_test.go | 130 ++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 10 deletions(-) diff --git a/mtglib/internal/doppel/conn.go b/mtglib/internal/doppel/conn.go index a2543cd..7e8ed30 100644 --- a/mtglib/internal/doppel/conn.go +++ b/mtglib/internal/doppel/conn.go @@ -16,18 +16,48 @@ type Conn struct { } type connPayload struct { - ctx context.Context - ctxCancel context.CancelCauseFunc - clock Clock - wg sync.WaitGroup - writeLock sync.Mutex - writeStream bytes.Buffer + ctx context.Context + ctxCancel context.CancelCauseFunc + clock Clock + wg sync.WaitGroup + syncWriteLock sync.RWMutex + writeStream bytes.Buffer + writeCond *sync.Cond } func (c Conn) Write(p []byte) (int, error) { - c.p.writeLock.Lock() + c.p.syncWriteLock.RLock() + defer c.p.syncWriteLock.RUnlock() + + c.p.writeCond.L.Lock() c.p.writeStream.Write(p) - c.p.writeLock.Unlock() + c.p.writeCond.L.Unlock() + + return len(p), context.Cause(c.p.ctx) +} + +func (c Conn) SyncWrite(p []byte) (int, error) { + c.p.syncWriteLock.Lock() + defer c.p.syncWriteLock.Unlock() + + c.p.writeCond.L.Lock() + // wait until buffer is exhausted + for c.p.writeStream.Len() != 0 && context.Cause(c.p.ctx) == nil { + c.p.writeCond.Wait() + } + c.p.writeStream.Write(p) + c.p.writeCond.L.Unlock() + + if err := context.Cause(c.p.ctx); err != nil { + return len(p), err + } + + c.p.writeCond.L.Lock() + // wait until data will be sent + for c.p.writeStream.Len() != 0 && context.Cause(c.p.ctx) == nil { + c.p.writeCond.Wait() + } + c.p.writeCond.L.Unlock() return len(p), context.Cause(c.p.ctx) } @@ -39,6 +69,8 @@ func (c Conn) Start() { } func (c Conn) start() { + defer c.p.writeCond.Broadcast() + buf := [tls.MaxRecordSize]byte{} for { @@ -48,9 +80,9 @@ func (c Conn) start() { case <-c.p.clock.tick: } - c.p.writeLock.Lock() + c.p.writeCond.L.Lock() n, err := c.p.writeStream.Read(buf[:c.p.clock.stats.Size()]) - c.p.writeLock.Unlock() + c.p.writeCond.L.Unlock() if n == 0 || err != nil { continue @@ -60,6 +92,8 @@ func (c Conn) start() { c.p.ctxCancel(err) return } + + c.p.writeCond.Signal() } } @@ -75,6 +109,7 @@ func NewConn(ctx context.Context, conn essentials.Conn, stats *Stats) Conn { p: &connPayload{ ctx: ctx, ctxCancel: cancel, + writeCond: sync.NewCond(&sync.Mutex{}), clock: Clock{ stats: stats, tick: make(chan struct{}), diff --git a/mtglib/internal/doppel/conn_test.go b/mtglib/internal/doppel/conn_test.go index e774bce..f8adb91 100644 --- a/mtglib/internal/doppel/conn_test.go +++ b/mtglib/internal/doppel/conn_test.go @@ -157,6 +157,136 @@ func (suite *ConnTestSuite) TestStopOnUnderlyingWriteError() { }, 2*time.Second, time.Millisecond) } +func (suite *ConnTestSuite) TestSyncWriteDataSent() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + defer c.Stop() + + payload := []byte("sync hello") + n, err := c.SyncWrite(payload) + suite.NoError(err) + suite.Equal(len(payload), n) + + // SyncWrite returns only after data is flushed to the wire. + assembled := &bytes.Buffer{} + reader := bytes.NewReader(suite.connMock.Written()) + + for { + header := make([]byte, tls.SizeHeader) + if _, err := io.ReadFull(reader, header); err != nil { + break + } + + suite.Equal(byte(tls.TypeApplicationData), header[0]) + + length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:]) + rec := make([]byte, length) + _, err := io.ReadFull(reader, rec) + suite.NoError(err) + + assembled.Write(rec) + } + + suite.Equal(payload, assembled.Bytes()) +} + +func (suite *ConnTestSuite) TestSyncWriteDrainsBufferFirst() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + defer c.Stop() + + // Buffer some data via async Write. + _, err := c.Write([]byte("first")) + suite.NoError(err) + + // SyncWrite must drain "first" before sending "second". + n, err := c.SyncWrite([]byte("second")) + suite.NoError(err) + suite.Equal(6, n) + + // All data should be on the wire now. + assembled := &bytes.Buffer{} + reader := bytes.NewReader(suite.connMock.Written()) + + for { + header := make([]byte, tls.SizeHeader) + if _, err := io.ReadFull(reader, header); err != nil { + break + } + + length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:]) + rec := make([]byte, length) + _, err := io.ReadFull(reader, rec) + suite.NoError(err) + + assembled.Write(rec) + } + + suite.Equal([]byte("firstsecond"), assembled.Bytes()) +} + +func (suite *ConnTestSuite) TestSyncWriteBlocksAsyncWrite() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + defer c.Stop() + + // Start SyncWrite — it holds exclusive lock. + syncDone := make(chan struct{}) + + go func() { + defer close(syncDone) + c.SyncWrite([]byte("exclusive")) + }() + + // Give SyncWrite time to acquire the lock. + time.Sleep(10 * time.Millisecond) + + // Async Write should block until SyncWrite completes. + writeDone := make(chan struct{}) + + go func() { + defer close(writeDone) + c.Write([]byte("blocked")) + }() + + // SyncWrite should finish first. + <-syncDone + + select { + case <-writeDone: + // Write completed after SyncWrite — correct. + case <-time.After(2 * time.Second): + suite.Fail("async Write did not unblock after SyncWrite completed") + } +} + +func (suite *ConnTestSuite) TestSyncWriteReturnsErrorAfterStop() { + suite.connMock. + On("Write", mock.AnythingOfType("[]uint8")). + Return(0, nil). + Maybe() + + c := suite.makeConn() + c.Stop() + + time.Sleep(10 * time.Millisecond) + + _, err := c.SyncWrite([]byte("too late")) + suite.Error(err) +} + func TestConn(t *testing.T) { t.Parallel() suite.Run(t, &ConnTestSuite{}) From 4138cc649408df36b94b0093bfc9f4500718313e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 11:31:28 +0100 Subject: [PATCH 32/42] Use SyncWrite call to send a packet to proxy --- mtglib/internal/doppel/conn_test.go | 4 ++-- mtglib/proxy.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mtglib/internal/doppel/conn_test.go b/mtglib/internal/doppel/conn_test.go index f8adb91..eec1f6f 100644 --- a/mtglib/internal/doppel/conn_test.go +++ b/mtglib/internal/doppel/conn_test.go @@ -247,7 +247,7 @@ func (suite *ConnTestSuite) TestSyncWriteBlocksAsyncWrite() { go func() { defer close(syncDone) - c.SyncWrite([]byte("exclusive")) + c.SyncWrite([]byte("exclusive")) //nolint: errcheck }() // Give SyncWrite time to acquire the lock. @@ -258,7 +258,7 @@ func (suite *ConnTestSuite) TestSyncWriteBlocksAsyncWrite() { go func() { defer close(writeDone) - c.Write([]byte("blocked")) + c.Write([]byte("blocked")) //nolint: errcheck }() // SyncWrite should finish first. diff --git a/mtglib/proxy.go b/mtglib/proxy.go index da1b636..4afcdd8 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -90,7 +90,7 @@ func (p *Proxy) ServeConn(conn essentials.Conn) { } defer clientConn.Stop() - if _, err := clientConn.Write(noise); err != nil { + if _, err := clientConn.SyncWrite(noise); err != nil { ctx.logger.InfoError("cannot send the first packet", err) return } From 45b0964afd560ba96a8dbf6473af04887cf27795 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 16:20:18 +0100 Subject: [PATCH 33/42] Fix android ping --- mtglib/internal/tls/fake/server_side.go | 9 +++---- mtglib/internal/tls/fake/server_side_test.go | 25 +++++++++----------- mtglib/proxy.go | 21 ++++++---------- 3 files changed, 21 insertions(+), 34 deletions(-) diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go index 3a0d43a..4c2fe3a 100644 --- a/mtglib/internal/tls/fake/server_side.go +++ b/mtglib/internal/tls/fake/server_side.go @@ -32,27 +32,24 @@ var serverHelloSuffix = []byte{ 0x00, 0x20, // 32 bytes of key } -func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) ([]byte, error) { +func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) error { buf := &bytes.Buffer{} buf.Grow(tls.MaxRecordSize) generateServerHello(buf, clientHello) generateChangeCipherValue(buf) - - noise := &bytes.Buffer{} - generateNoise(noise) + generateNoise(buf) packet := buf.Bytes() digest := hmac.New(sha256.New, secret) digest.Write(clientHello.Random[:]) digest.Write(packet) - digest.Write(noise.Bytes()) copy(packet[RandomOffset:], digest.Sum(nil)) _, err := w.Write(packet) - return noise.Bytes()[tls.SizeHeader:], err + return err } func generateServerHello(buf *bytes.Buffer, hello *ClientHello) { diff --git a/mtglib/internal/tls/fake/server_side_test.go b/mtglib/internal/tls/fake/server_side_test.go index 0b4ee67..abde893 100644 --- a/mtglib/internal/tls/fake/server_side_test.go +++ b/mtglib/internal/tls/fake/server_side_test.go @@ -38,7 +38,7 @@ func (suite *SendServerHelloTestSuite) SetupTest() { } func (suite *SendServerHelloTestSuite) TestRecordStructure() { - noise, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) + err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) var rec bytes.Buffer @@ -53,14 +53,18 @@ func (suite *SendServerHelloTestSuite) TestRecordStructure() { suite.NoError(err) suite.Equal(byte(tls.TypeChangeCipherSpec), recordType) - suite.Empty(suite.buf.Bytes()) + rec.Reset() - // noise is raw payload without TLS record header - suite.Len(noise, 1369) + recordType, length, err := tls.ReadRecord(suite.buf, &rec) + suite.NoError(err) + suite.Equal(byte(tls.TypeApplicationData), recordType) + suite.Equal(int64(1369), length) + + suite.Empty(suite.buf.Bytes()) } func (suite *SendServerHelloTestSuite) TestHMAC() { - noise, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) + err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) packet := make([]byte, suite.buf.Len()) @@ -74,18 +78,11 @@ func (suite *SendServerHelloTestSuite) TestHMAC() { mac.Write(suite.hello.Random[:]) mac.Write(packet) - // HMAC is computed over the full noise TLS record (with header), - // but SendServerHello returns noise without the header, - // so we reconstruct the full record. - var fullNoise bytes.Buffer - tls.WriteRecord(&fullNoise, noise) //nolint: errcheck - mac.Write(fullNoise.Bytes()) - suite.Equal(random, mac.Sum(nil)) } func (suite *SendServerHelloTestSuite) TestHandshakePayload() { - _, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) + err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) packet := suite.buf.Bytes() @@ -107,7 +104,7 @@ func (suite *SendServerHelloTestSuite) TestHandshakePayload() { } func (suite *SendServerHelloTestSuite) TestChangeCipherSpec() { - _, err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) + err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello) suite.NoError(err) // Skip first record diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 4afcdd8..7eb31a6 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -78,8 +78,7 @@ func (p *Proxy) ServeConn(conn essentials.Conn) { ctx.logger.Info("Stream has been finished") }() - noise, ok := p.doFakeTLSHandshake(ctx) - if !ok { + if !p.doFakeTLSHandshake(ctx) { return } @@ -90,11 +89,6 @@ func (p *Proxy) ServeConn(conn essentials.Conn) { } defer clientConn.Stop() - if _, err := clientConn.SyncWrite(noise); err != nil { - ctx.logger.InfoError("cannot send the first packet", err) - return - } - ctx.clientConn = clientConn if err := p.doObfuscatedHandshake(ctx); err != nil { @@ -176,7 +170,7 @@ func (p *Proxy) Shutdown() { p.blocklist.Shutdown() } -func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) ([]byte, bool) { +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { rewind := newConnRewind(ctx.clientConn) clientHello, err := fake.ReadClientHello( @@ -188,25 +182,24 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) ([]byte, bool) { if err != nil { p.logger.InfoError("cannot read client hello", err) p.doDomainFronting(ctx, rewind) - return nil, false + return false } if p.antiReplayCache.SeenBefore(clientHello.SessionID) { p.logger.Warning("replay attack has been detected!") p.eventStream.Send(p.ctx, NewEventReplayAttack(ctx.streamID)) p.doDomainFronting(ctx, rewind) - return nil, false + return false } - noise, err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello) - if err != nil { + if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello); err != nil { p.logger.InfoError("cannot send welcome packet", err) - return nil, false + return false } ctx.clientConn = tls.New(ctx.clientConn, true, false) - return noise, true + return true } func (p *Proxy) doObfuscatedHandshake(ctx *streamContext) error { From b35c212d65e21d7528cc34cc762938e3d6e3c421 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 16:39:16 +0100 Subject: [PATCH 34/42] Have a noise of variable size --- mtglib/internal/tls/fake/server_side.go | 9 ++++++--- mtglib/internal/tls/fake/server_side_test.go | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go index 4c2fe3a..90b736d 100644 --- a/mtglib/internal/tls/fake/server_side.go +++ b/mtglib/internal/tls/fake/server_side.go @@ -7,15 +7,17 @@ import ( "crypto/sha256" "encoding/binary" "io" + "math" + rnd "math/rand/v2" + "github.com/9seconds/mtg/v2/mtglib/internal/doppel" "github.com/9seconds/mtg/v2/mtglib/internal/tls" "golang.org/x/crypto/curve25519" ) const ( TypeHandshakeServer = 0x02 - - ChangeCipherValue = 0x01 + ChangeCipherValue = 0x01 EllipticCurveLen = 32 ) @@ -125,7 +127,8 @@ func generateChangeCipherValue(buf *bytes.Buffer) { } func generateNoise(buf *bytes.Buffer) { - data := [1369]byte{} + minSize := int(math.Round(0.75 * float64(doppel.TLSRecordSizeMax))) + data := make([]byte, minSize+rnd.IntN(doppel.TLSRecordSizeMax-minSize)) if _, err := rand.Read(data[:]); err != nil { panic(err) diff --git a/mtglib/internal/tls/fake/server_side_test.go b/mtglib/internal/tls/fake/server_side_test.go index abde893..c4b54a6 100644 --- a/mtglib/internal/tls/fake/server_side_test.go +++ b/mtglib/internal/tls/fake/server_side_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/doppel" "github.com/9seconds/mtg/v2/mtglib/internal/tls" "github.com/9seconds/mtg/v2/mtglib/internal/tls/fake" "github.com/stretchr/testify/suite" @@ -58,7 +59,7 @@ func (suite *SendServerHelloTestSuite) TestRecordStructure() { recordType, length, err := tls.ReadRecord(suite.buf, &rec) suite.NoError(err) suite.Equal(byte(tls.TypeApplicationData), recordType) - suite.Equal(int64(1369), length) + suite.Greater(length, int64(doppel.TLSRecordSizeStart)) suite.Empty(suite.buf.Bytes()) } From 7adfc0352ef46609f842202107f59c3544dab877 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 13 Mar 2026 16:39:16 +0100 Subject: [PATCH 35/42] Have a noise of variable size --- mtglib/internal/tls/fake/server_side.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mtglib/internal/tls/fake/server_side.go b/mtglib/internal/tls/fake/server_side.go index 90b736d..e3a0139 100644 --- a/mtglib/internal/tls/fake/server_side.go +++ b/mtglib/internal/tls/fake/server_side.go @@ -7,10 +7,8 @@ import ( "crypto/sha256" "encoding/binary" "io" - "math" rnd "math/rand/v2" - "github.com/9seconds/mtg/v2/mtglib/internal/doppel" "github.com/9seconds/mtg/v2/mtglib/internal/tls" "golang.org/x/crypto/curve25519" ) @@ -127,8 +125,7 @@ func generateChangeCipherValue(buf *bytes.Buffer) { } func generateNoise(buf *bytes.Buffer) { - minSize := int(math.Round(0.75 * float64(doppel.TLSRecordSizeMax))) - data := make([]byte, minSize+rnd.IntN(doppel.TLSRecordSizeMax-minSize)) + data := make([]byte, int64(1024+rnd.IntN(3092))) if _, err := rand.Read(data[:]); err != nil { panic(err) From e675baa86057e681313b2cc86962da6094ee2b39 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 14 Mar 2026 09:42:28 +0100 Subject: [PATCH 36/42] Add docs about doppelganger --- README.md | 76 +++++++++++++++++++++++++++++++++ mtglib/internal/doppel/stats.go | 5 +++ 2 files changed, 81 insertions(+) diff --git a/README.md b/README.md index 82f1eee..26aa65b 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,33 @@ goal: to give a possibility to connect to Telegram in a restricted, censored environment. But it does it slightly differently in details that probably matter. +* **Domain fronting** + +For years mtg supports domain fronting. This technique means that it fallbacks +to accessing a real website in case if request fails. It could fail by many +reasons: anti-replay protection, accidental access to the webserver or +stale request. Anyway, if mtg rejects this request, it does not break a +connection. It connects to the websites and replicates everything that client +has sent, and simply proxies it back as is. Users will see a response from +the real website, _byte-to-byte identical_ to the response of the real netloc. + +* **Doppelganger** + +mtg also is a doppelganger of the website it fronts. Sure, with domain fronting +users will see replies of the real website in case if something will go wrong. +But what about such cases when _everything is fine_? + +In that case mtg mimics TLS connection statistical characteristics as close as +possible. Different application have different statistics of their patterns. +Big CDN steadily pumping the data, small websites burst with short easily +compressiable chunks of traffic. + +mtg artificially emulates those delays to be statistically indistinguishable +from the real website even if it covers connection of the very specific app. +It also follows 2 most common patterns of traffic chunking, so censors +will have to put more resources to find out that we have Telegram here +but not a hookah webshop served by nginx. + * **Resource-efficient** It has to be resource-efficient. It does not mean that you will see @@ -398,6 +425,55 @@ or if you are using docker: $ docker exec mtg-proxy /mtg access /config.toml ``` +## Doppelganger + +mtg can mimic real websites, please take a look at relevant section in example +config file. + +mtg comes with some very good precollected statistics coming from +[ok.ru](https://ok.ru/). It does not mean that you have to cover yourself +by pretending that mtg is _ok.ru_. **Do not do that: ok.ru comes from very specific +ASNs, but not from VPS providers you are going to use.** What I want to say +is that defaults are very good enough to use as is because ok.ru for public +pages has a very generic profile of TLS packets delay. + +But for better results it is recommended to teach mtg about the website you +will use as a domain front. In order to do that, you need to specify URLs +from this website. Just go to it, open WebDeveloper console and pick up +random URLs. For better results they have to be **from the same domain name +you are going to use as a disguise** but serve light and heavy content: pages, +images etc. Do not use many, 2-3 will probably work. + +mtg will crawl these pages periodically, accumulating statistics and +using it as you go. + +```toml +[defense.doppelganger] +urls = [ + "https://lalala.com/index.html", + "https://lalala.com/contacts.html", +] +``` + +This is not very necessary. Keep in mind these rules: + +1. If you are not sure what is this all about, do nothing. Defaults are good. +2. All URLs must be HTTPS +3. All URLs should be from the same domain name (but this is not a rule) +4. Do not use a lot of pages. Use _different_ pages. mtg will start using this + statistics when it will accumulate enough anyway. +5. These URLs should be directly accessible from mtg without proxies whatsoever +6. Do not create huge raids. mtg will repeatedly crawl in raids, making N repeats. + Do not use high N, you do not want to be noticeable. +7. It makes no sense to have small delay between raids. Usually webservers + do not update their TLS settings each hour. +8. If you have some specific knowledge if webserver is using + [TLS Dynamic Record Sizing](https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/), you + can use a very specific setting. This are Cloudflare, Go standard webservers, + [caddy](https://caddyserver.com/) and [H2O](https://h2o.examp1e.net/). If so, + you can enable `drs` setting. +9. **If you are not sure, touch nothing!** + ## Metrics Out of the box, mtg works with diff --git a/mtglib/internal/doppel/stats.go b/mtglib/internal/doppel/stats.go index eb73b61..18371b6 100644 --- a/mtglib/internal/doppel/stats.go +++ b/mtglib/internal/doppel/stats.go @@ -61,6 +61,11 @@ const ( // // In other word, a combination of Dynamic TLS Record Sizing hints us for // Weibull distribution. +// +// But we also have to keep in mind that DRS is not well spread yet. In most cases +// users still rely on OpenSSL or webserver defaults. OpenSSL chunks with +// biggest packet sizes, nginx relies on static setting that is 16k by default. +// Thus, dynamic sizing has to be present but we cannot oblige users to use that. type Stats struct { sizeLastRequested time.Time sizeCounter int From 879ebd132daf52472d75811323adcb9442cb55ac Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 14 Mar 2026 09:44:07 +0100 Subject: [PATCH 37/42] Formatting of the README --- README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 26aa65b..09ce48a 100644 --- a/README.md +++ b/README.md @@ -40,30 +40,30 @@ that probably matter. * **Domain fronting** -For years mtg supports domain fronting. This technique means that it fallbacks -to accessing a real website in case if request fails. It could fail by many -reasons: anti-replay protection, accidental access to the webserver or -stale request. Anyway, if mtg rejects this request, it does not break a -connection. It connects to the websites and replicates everything that client -has sent, and simply proxies it back as is. Users will see a response from -the real website, _byte-to-byte identical_ to the response of the real netloc. + For years mtg supports domain fronting. This technique means that it fallbacks + to accessing a real website in case if request fails. It could fail by many + reasons: anti-replay protection, accidental access to the webserver or + stale request. Anyway, if mtg rejects this request, it does not break a + connection. It connects to the websites and replicates everything that client + has sent, and simply proxies it back as is. Users will see a response from + the real website, _byte-to-byte identical_ to the response of the real netloc. * **Doppelganger** -mtg also is a doppelganger of the website it fronts. Sure, with domain fronting -users will see replies of the real website in case if something will go wrong. -But what about such cases when _everything is fine_? + mtg also is a doppelganger of the website it fronts. Sure, with domain fronting + users will see replies of the real website in case if something will go wrong. + But what about such cases when _everything is fine_? -In that case mtg mimics TLS connection statistical characteristics as close as -possible. Different application have different statistics of their patterns. -Big CDN steadily pumping the data, small websites burst with short easily -compressiable chunks of traffic. + In that case mtg mimics TLS connection statistical characteristics as close as + possible. Different application have different statistics of their patterns. + Big CDN steadily pumping the data, small websites burst with short easily + compressiable chunks of traffic. -mtg artificially emulates those delays to be statistically indistinguishable -from the real website even if it covers connection of the very specific app. -It also follows 2 most common patterns of traffic chunking, so censors -will have to put more resources to find out that we have Telegram here -but not a hookah webshop served by nginx. + mtg artificially emulates those delays to be statistically indistinguishable + from the real website even if it covers connection of the very specific app. + It also follows 2 most common patterns of traffic chunking, so censors + will have to put more resources to find out that we have Telegram here + but not a hookah webshop served by nginx. * **Resource-efficient** From bb90bcc127451f835c580d93f0a0ca9934a418c8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 14 Mar 2026 14:06:48 +0100 Subject: [PATCH 38/42] Decrease relay buffer size Even if it makes sense to have a huge buffers, we do artificial delays now. In that case we could achieve the same results with a lower buffer. If not, then we won't send a packet bigger that this value --- mtglib/internal/relay/init.go | 4 ---- mtglib/internal/relay/relay.go | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mtglib/internal/relay/init.go b/mtglib/internal/relay/init.go index 8bcc66c..b096b1e 100644 --- a/mtglib/internal/relay/init.go +++ b/mtglib/internal/relay/init.go @@ -1,9 +1,5 @@ package relay -const ( - copyBufferSize = 64 * 1024 -) - type Logger interface { Printf(msg string, args ...any) } diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go index 756a52d..986b3e2 100644 --- a/mtglib/internal/relay/relay.go +++ b/mtglib/internal/relay/relay.go @@ -6,6 +6,7 @@ import ( "io" "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib/internal/tls" ) func Relay(ctx context.Context, log Logger, telegramConn, clientConn essentials.Conn) { @@ -35,7 +36,7 @@ func Relay(ctx context.Context, log Logger, telegramConn, clientConn essentials. } func pump(log Logger, src, dst essentials.Conn, direction string) { - var buf [copyBufferSize]byte + var buf [tls.MaxRecordPayloadSize]byte defer src.CloseRead() //nolint: errcheck defer dst.CloseWrite() //nolint: errcheck From 57529904e35390eb7813672c72a0ecac5d752e0a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 15 Mar 2026 21:50:43 +0100 Subject: [PATCH 39/42] Add document about best practices --- .goreleaser.yml | 1 + BEST_PRACTICES.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 BEST_PRACTICES.md diff --git a/.goreleaser.yml b/.goreleaser.yml index 4fb638e..87e4f73 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -54,6 +54,7 @@ archives: - LICENSE - README.md - SECURITY.md + - BEST_PRACTICES.md - example.config.toml gomod: diff --git a/BEST_PRACTICES.md b/BEST_PRACTICES.md new file mode 100644 index 0000000..3ea3274 --- /dev/null +++ b/BEST_PRACTICES.md @@ -0,0 +1,50 @@ +# Best practices + +This is unfortunate, but since 2018 many things were changed. Most of them +became way worse. Previous iterations of censorship systems were very dumb, +DPI were primitive and filtered very obvious things. Nowadays they are +way more intelligent and it is very naive to treat them frivolously. + +In 2026 is not enough to pretend that your mtg installation is a Microsoft +website that sits in Amsterdam Digital Ocean location. Now your installation +has to be a website that is mtg in disguise. Yes, it requires a bit more effort +but this effort is probably less than rotating proxies each other day. + +mtproto traffic, even with FakeTLS, has its specifics that are probably +very well known by DPI systems. These specifics are not something unique but +could mark an IP address as suspicious. Now let's think: + +1. You have a proxy in Amsterdam Digital Ocean that tells it is microsoft.com + how hard could it be to find out that this is probably fake? 1 or probably 2 + DNS queries for `microsoft.com`? In case of some CDN, there are ECS-powered + resolvers that are very capable to return results from POV of some subnets. + If censor sees no relevant results, will they be afraid to block IP? +2. You have a proxy in Amsterdam Digital Ocean that tells it is a website from + the same public subnet. But not the same. Would it be hard to make these DNS + queries and ban IP? + +The correct way of having this proxy is following: + +1. Register a domain name +2. Get some VPS, probably in your domestic location +3. Set that domain name from a step 1 to IP address of that VPS +4. Generate a couple of HTML pages by LLMs or even copy them from elsewhere +5. Set some webserver and issue TLS certificates with Let's Encrypt or any other + name +6. Set mtg before this webserver. +7. Use sing-box or anything like that to provide local socks5 interface and + have VPNized uplinks +8. Set up mtg to use socks5 from a 7 step. + +In that case you will get a match of DNS and SNI in requests. As a side effect, +your proxy will work with XTLS and its friends: XTLS in sniff mode ignores +IP address a client wants to connect to. Instead, it reads SNI and connect +to resolved address: a clever idea if user does not have a trustworthy DNS +set up. + +Yes, this is much longer that usual technique, and requires more effort. But +this is could probably be very well automated to some reasonable extent. + +Unfortunately, this is a best practice right now. + +_March 2026._ From c036558df4cc83f1baca3fd0e0788bde563b7397 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 15 Mar 2026 21:58:04 +0100 Subject: [PATCH 40/42] Update docs --- BEST_PRACTICES.md | 5 +++++ README.md | 2 ++ 2 files changed, 7 insertions(+) diff --git a/BEST_PRACTICES.md b/BEST_PRACTICES.md index 3ea3274..6557b43 100644 --- a/BEST_PRACTICES.md +++ b/BEST_PRACTICES.md @@ -47,4 +47,9 @@ this is could probably be very well automated to some reasonable extent. Unfortunately, this is a best practice right now. +Do not also forget about other implementation, like +[telemt](https://github.com/telemt/telemt). Try everything. Use VPNs. It does +not really matter which project you are going to use as long it helps you to +stay connected. + _March 2026._ diff --git a/README.md b/README.md index 09ce48a..106a02b 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,8 @@ that probably matter. software (written in Golang) with a minimum effort + you can replace some parts with those you want. +Please also to read about [best practices](https://github.com/9seconds/mtg/blob/master/BEST_PRACTICES.md). + ### Version 2 If you use version 1.x before, you are probably noticed some major From 94ac9d6ffad09363a781152f72353796d2bed384 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 15 Mar 2026 22:00:08 +0100 Subject: [PATCH 41/42] Update tooling --- mise.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/mise.lock b/mise.lock index bfa377d..f04d0d2 100644 --- a/mise.lock +++ b/mise.lock @@ -1,11 +1,11 @@ [[tools.go]] -version = "1.26.0" +version = "1.26.1" backend = "core:go" -"platforms.linux-arm64" = { checksum = "sha256:bd03b743eb6eb4193ea3c3fd3956546bf0e3ca5b7076c8226334afe6b75704cd", url = "https://dl.google.com/go/go1.26.0.linux-arm64.tar.gz"} -"platforms.linux-x64" = { checksum = "sha256:aac1b08a0fb0c4e0a7c1555beb7b59180b05dfc5a3d62e40e9de90cd42f88235", url = "https://dl.google.com/go/go1.26.0.linux-amd64.tar.gz"} -"platforms.macos-arm64" = { checksum = "sha256:b1640525dfe68f066d56f200bef7bf4dce955a1a893bd061de6754c211431023", url = "https://dl.google.com/go/go1.26.0.darwin-arm64.tar.gz"} -"platforms.macos-x64" = { checksum = "sha256:1ca28b7703cbea05a65b2a1d92d6b308610ef92f8824578a0874f2e60c9d5a22", url = "https://dl.google.com/go/go1.26.0.darwin-amd64.tar.gz"} -"platforms.windows-x64" = { checksum = "sha256:9bbe0fc64236b2b51f6255c05c4232532b8ecc0e6d2e00950bd3021d8a4d07d4", url = "https://dl.google.com/go/go1.26.0.windows-amd64.zip"} +"platforms.linux-arm64" = { checksum = "sha256:a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7", url = "https://dl.google.com/go/go1.26.1.linux-arm64.tar.gz"} +"platforms.linux-x64" = { checksum = "sha256:031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a", url = "https://dl.google.com/go/go1.26.1.linux-amd64.tar.gz"} +"platforms.macos-arm64" = { checksum = "sha256:353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f", url = "https://dl.google.com/go/go1.26.1.darwin-arm64.tar.gz"} +"platforms.macos-x64" = { checksum = "sha256:65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08", url = "https://dl.google.com/go/go1.26.1.darwin-amd64.tar.gz"} +"platforms.windows-x64" = { checksum = "sha256:9b68112c913f45b7aebbf13c036721264bbba7e03a642f8f7490c561eebd1ecc", url = "https://dl.google.com/go/go1.26.1.windows-amd64.zip"} [[tools."go:golang.org/x/pkgsite/cmd/pkgsite"]] version = "latest" @@ -24,19 +24,19 @@ version = "0.9.2" backend = "go:mvdan.cc/gofumpt" [[tools.golangci-lint]] -version = "2.10.1" +version = "2.11.3" backend = "aqua:golangci/golangci-lint" -"platforms.linux-arm64" = { checksum = "sha256:6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-linux-arm64.tar.gz"} -"platforms.linux-x64" = { checksum = "sha256:dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-linux-amd64.tar.gz"} -"platforms.macos-arm64" = { checksum = "sha256:03bfadf67e52b441b7ec21305e501c717df93c959836d66c7f97312654acb297", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-darwin-arm64.tar.gz"} -"platforms.macos-x64" = { checksum = "sha256:66fb0da81b8033b477f97eea420d4b46b230ca172b8bb87c6610109f3772b6b6", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-darwin-amd64.tar.gz"} -"platforms.windows-x64" = { checksum = "sha256:c60c87695e79db8e320f0e5be885059859de52bb5ee5f11be5577828570bc2a3", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-windows-amd64.zip"} +"platforms.linux-arm64" = { checksum = "sha256:ee3d95f301359e7d578e6d99c8ad5aeadbabc5a13009a30b2b0df11c8058afe9", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-linux-arm64.tar.gz"} +"platforms.linux-x64" = { checksum = "sha256:87bb8cddbcc825d5778b64e8a91b46c0526b247f4e2f2904dea74ec7450475d1", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-linux-amd64.tar.gz"} +"platforms.macos-arm64" = { checksum = "sha256:30ee39979c516b9d1adca289a3f93429d130c4c0fda5e57d637850894221f6cc", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-darwin-arm64.tar.gz"} +"platforms.macos-x64" = { checksum = "sha256:f93bda1f2cc981fd1326464020494be62f387bbf262706e1b3b644e5afacc440", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-darwin-amd64.tar.gz"} +"platforms.windows-x64" = { checksum = "sha256:cd42e890176bc5cfeb36225a77e66b9410ddd3a59a03551e23f6b210d29e1f67", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-windows-amd64.zip"} [[tools.goreleaser]] -version = "2.14.1" +version = "2.14.3" backend = "aqua:goreleaser/goreleaser" -"platforms.linux-arm64" = { checksum = "sha256:a84d3b27f052c12ad5c8342d7caf1450a7174a305730aed21d72db09301e49a5", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Linux_arm64.tar.gz"} -"platforms.linux-x64" = { checksum = "sha256:2df975a7acbfdeaf888d596cab0024d48ec7fb7d747e1d08b90948b791f40a5f", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Linux_x86_64.tar.gz"} -"platforms.macos-arm64" = { checksum = "sha256:9f2e47f847b4f4177376fc6aa6914fbc7f673f59720076747e738b578c2e896e", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Darwin_all.tar.gz"} -"platforms.macos-x64" = { checksum = "sha256:9f2e47f847b4f4177376fc6aa6914fbc7f673f59720076747e738b578c2e896e", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Darwin_all.tar.gz"} -"platforms.windows-x64" = { checksum = "sha256:d7a3d8ba795e97ab8c4f8003630d300da164adf21fde5a4049440c20f15c3137", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Windows_x86_64.zip"} +"platforms.linux-arm64" = { checksum = "sha256:581a10e53c1176b3e81ee45cf531e02dbf899db0bc7b795669347df4276ce948", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Linux_arm64.tar.gz"} +"platforms.linux-x64" = { checksum = "sha256:dc7faeeeb6da8bdfda788626263a4ae725892a8c7504b975c3234127d4a44579", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Linux_x86_64.tar.gz"} +"platforms.macos-arm64" = { checksum = "sha256:3507798489e107a78aff36b169de48148a335ac26eb3161608d905f3f3a957bd", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Darwin_all.tar.gz"} +"platforms.macos-x64" = { checksum = "sha256:3507798489e107a78aff36b169de48148a335ac26eb3161608d905f3f3a957bd", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Darwin_all.tar.gz"} +"platforms.windows-x64" = { checksum = "sha256:3deea8ff471aa258a2d99f3e5302971d7028647ae8ddaf103257a8113e485a31", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Windows_x86_64.zip"} From d98d5be3a7d8e0c39c3e6d4dea1ba1ad008f286b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 15 Mar 2026 22:01:04 +0100 Subject: [PATCH 42/42] Update go dependencies --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index cb0e935..481f739 100644 --- a/go.mod +++ b/go.mod @@ -11,18 +11,18 @@ require ( github.com/d4l3k/messagediff v1.2.1 // indirect github.com/jarcoal/httpmock v1.0.8 github.com/mccutchen/go-httpbin v1.1.1 - github.com/panjf2000/ants/v2 v2.11.5 + github.com/panjf2000/ants/v2 v2.11.6 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.20.0 // indirect + github.com/prometheus/procfs v0.20.1 // indirect github.com/rs/zerolog v1.34.0 github.com/smira/go-statsd v1.3.4 github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.11.1 github.com/tylertreat/BoomFilters v0.0.0-20251117164519-53813c36cc1b - golang.org/x/crypto v0.48.0 - golang.org/x/net v0.51.0 - golang.org/x/sys v0.41.0 + golang.org/x/crypto v0.49.0 + golang.org/x/net v0.52.0 + golang.org/x/sys v0.42.0 google.golang.org/protobuf v1.36.11 // indirect ) @@ -49,8 +49,8 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/txthinking/runnergroup v0.0.0-20250224021307-5864ffeb65ae // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/sync v0.19.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/sync v0.20.0 // indirect golang.org/x/tools v0.41.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index b61ae52..aa3d085 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-dns v1.3.2 h1:kBLuUZBgkQ4qF4WDXZRQ4rG0Gk6sLVJQ5tESkWrxUa0= github.com/ncruces/go-dns v1.3.2/go.mod h1:tuzixNY8PY/M7yUzcvRbUaeLs3ifIdydpi5H2bfRU+s= -github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg= -github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/panjf2000/ants/v2 v2.11.6 h1:JKsoIUukIoCO0sP0gcOqdyoXmpyKXuU6fC57rODtpug= +github.com/panjf2000/ants/v2 v2.11.6/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -70,8 +70,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= -github.com/prometheus/procfs v0.20.0 h1:AA7aCvjxwAquZAlonN7888f2u4IN8WVeFgBi4k82M4Q= -github.com/prometheus/procfs v0.20.0/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= @@ -105,12 +105,12 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= @@ -119,13 +119,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -135,8 +135,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=