Merge pull request #409 from dolonet/cert-noise-calibration

Add dynamic cert noise calibration for FakeTLS handshake

internal/config/config.go

@@ -50,10 +50,10 @@ type Config struct {
 		Blocklist    ListConfig `json:"blocklist"`
 		Allowlist    ListConfig `json:"allowlist"`
 		Doppelganger struct {
-			URLs       []TypeHttpsURL  `json:"urls"`
+			URLs            []TypeHttpsURL  `json:"urls"`
-			Repeats    TypeConcurrency `json:"repeats_per_raid"`
+			Repeats         TypeConcurrency `json:"repeats_per_raid"`
-			UpdateEach TypeDuration    `json:"raid_each"`
+			UpdateEach      TypeDuration    `json:"raid_each"`
-			DRS        TypeBool        `json:"drs"`
+			DRS             TypeBool        `json:"drs"`
 		} `json:"doppelganger"`
 	} `json:"defense"`
 	Network struct {

internal/config/parse.go

@@ -45,10 +45,10 @@ type tomlConfig struct {
 			UpdateEach          string   `toml:"update-each" json:"updateEach,omitempty"`
 		} `toml:"allowlist" json:"allowlist,omitempty"`
 		Doppelganger struct {
-			URLs       []string `toml:"urls" json:"urls,omitempty"`
+			URLs            []string `toml:"urls" json:"urls,omitempty"`
-			Repeats    uint     `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
+			Repeats         uint     `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
-			UpdateEach string   `toml:"raid-each" json:"raid_each,omitempty"`
+			UpdateEach      string   `toml:"raid-each" json:"raid_each,omitempty"`
-			DRS        bool     `toml:"drs" json:"drs,omitempty"`
+			DRS             bool     `toml:"drs" json:"drs,omitempty"`
 		} `toml:"doppelganger" json:"doppelganger,omitempty"`
 	} `toml:"defense" json:"defense,omitempty"`
 	Network struct {

mtglib/internal/doppel/ganger.go

@@ -2,7 +2,9 @@ package doppel
 
 import (
 	"context"
+	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/9seconds/mtg/v2/essentials"
@@ -12,8 +14,22 @@ const (
 	DoppelGangerMaxDurations  = 4096
 	DoppelGangerScoutRaidEach = 6 * time.Hour
 	DoppelGangerScoutRepeats  = 10
+
+	MinCertSizesToCalculate = 3
 )
 
+// NoiseParams holds the measured cert chain size for FakeTLS noise calibration.
+// If Mean is 0, the caller should use a legacy fallback.
+type NoiseParams struct {
+	Mean   int
+	Jitter int
+}
+
+type scoutRaidResult struct {
+	durations []time.Duration
+	certSizes []int
+}
+
 type gangerConnRequest struct {
 	ret     chan<- Conn
 	payload essentials.Conn
@@ -33,6 +49,9 @@ type Ganger struct {
 
 	stats     *Stats
 	durations []time.Duration
+	certSizes []int
+
+	noiseParams atomic.Pointer[NoiseParams]
 
 	connRequests chan gangerConnRequest
 }
@@ -48,6 +67,16 @@ func (g *Ganger) Run() {
 	})
 }
 
+// NoiseParams returns the current cert-size-based noise parameters.
+// Returns zero-value NoiseParams if not yet measured (caller should use fallback).
+func (g *Ganger) NoiseParams() NoiseParams {
+	if p := g.noiseParams.Load(); p != nil {
+		return *p
+	}
+
+	return NoiseParams{}
+}
+
 func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) {
 	rvChan := make(chan Conn)
 	req := gangerConnRequest{
@@ -81,7 +110,7 @@ func (g *Ganger) run() {
 		}
 	}()
 
-	scoutCollectedChan := make(chan []time.Duration)
+	scoutCollectedChan := make(chan scoutRaidResult)
 	currentScoutCollectedChan := scoutCollectedChan
 
 	updatedStatsChan := make(chan *Stats)
@@ -94,18 +123,29 @@ func (g *Ganger) run() {
 		select {
 		case <-g.ctx.Done():
 			return
-		case durations := <-currentScoutCollectedChan:
+		case result := <-currentScoutCollectedChan:
-			g.durations = append(g.durations, durations...)
+			g.durations = append(g.durations, result.durations...)
 
 			if len(g.durations) > DoppelGangerMaxDurations {
 				copy(g.durations, g.durations[len(g.durations)-DoppelGangerMaxDurations:])
 				g.durations = g.durations[:DoppelGangerMaxDurations]
 			}
 
+			// Update cert sizes and recompute noise params.
+			g.certSizes = append(g.certSizes, result.certSizes...)
+			if len(g.certSizes) > DoppelGangerMaxDurations {
+				g.certSizes = g.certSizes[len(g.certSizes)-DoppelGangerMaxDurations:]
+			}
+
+			if len(g.certSizes) >= MinCertSizesToCalculate {
+				g.updateNoiseParams()
+			}
+
 			if len(g.durations) < MinDurationsToCalculate {
 				continue
 			}
 
+			durations := g.durations
 			currentScoutCollectedChan = nil
 			g.wg.Go(func() {
 				select {
@@ -129,8 +169,45 @@ func (g *Ganger) run() {
 	}
 }
 
-func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
+func (g *Ganger) updateNoiseParams() {
-	durations := []time.Duration{}
+	if len(g.certSizes) == 0 {
+		return
+	}
+
+	sum := 0
+	for _, s := range g.certSizes {
+		sum += s
+	}
+
+	mean := sum / len(g.certSizes)
+
+	maxDev := 0
+	for _, s := range g.certSizes {
+		d := s - mean
+		if d < 0 {
+			d = -d
+		}
+
+		if d > maxDev {
+			maxDev = d
+		}
+	}
+
+	if maxDev < 100 {
+		maxDev = 100
+	}
+
+	np := &NoiseParams{Mean: mean, Jitter: maxDev}
+	g.noiseParams.Store(np)
+
+	g.logger.Info(fmt.Sprintf(
+		"updated noise params: mean=%d jitter=%d samples=%d",
+		mean, maxDev, len(g.certSizes),
+	))
+}
+
+func (g *Ganger) runScoutRaid(rvChan chan<- scoutRaidResult) {
+	var result scoutRaidResult
 
 	for range g.scoutRaidRepeats {
 		learned, err := g.scout.Learn(g.ctx)
@@ -138,13 +215,18 @@ func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
 			g.logger.WarningError("cannot learn", err)
 			continue
 		}
-		durations = append(durations, learned...)
+
+		result.durations = append(result.durations, learned.Durations...)
+
+		if learned.CertSize > 0 {
+			result.certSizes = append(result.certSizes, learned.CertSize)
+		}
 	}
 
 	select {
 	case <-g.ctx.Done():
 		return
-	case rvChan <- durations:
+	case rvChan <- result:
 	}
 }
 

mtglib/internal/doppel/scout.go

@@ -12,36 +12,46 @@ import (
 	"github.com/9seconds/mtg/v2/mtglib/internal/tls"
 )
 
+// ScoutResult holds measurements from a single scout HTTP request.
+type ScoutResult struct {
+	Durations []time.Duration
+	CertSize  int // total ApplicationData bytes during TLS handshake; 0 if unknown
+}
+
 type Scout struct {
 	network Network
 	urls    []string
 }
 
-func (s Scout) Learn(ctx context.Context) ([]time.Duration, error) {
+func (s Scout) Learn(ctx context.Context) (ScoutResult, error) {
-	var durations []time.Duration
+	var combined ScoutResult
 
 	for _, url := range s.urls {
 		learned, err := s.learn(ctx, url)
 		if err != nil {
-			return nil, err
+			return ScoutResult{}, err
 		}
 
-		durations = append(durations, learned...)
+		combined.Durations = append(combined.Durations, learned.Durations...)
+
+		if learned.CertSize > 0 && combined.CertSize == 0 {
+			combined.CertSize = learned.CertSize
+		}
 	}
 
-	return durations, nil
+	return combined, nil
 }
 
-func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
+func (s Scout) learn(ctx context.Context, url string) (ScoutResult, error) {
 	client, results := s.makeClient()
 
 	if !strings.HasPrefix(url, "https://") {
-		return nil, fmt.Errorf("url %s must be https", url)
+		return ScoutResult{}, fmt.Errorf("url %s must be https", url)
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return nil, err
+		return ScoutResult{}, err
 	}
 
 	resp, err := client.Do(req)
@@ -52,10 +62,12 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
 	}
 
 	if err != nil || len(results.data) == 0 {
-		return nil, err
+		return ScoutResult{}, err
 	}
 
-	durations := []time.Duration{}
+	var result ScoutResult
+
+	// Compute inter-record durations (existing logic).
 	lastTimestamp := time.Time{}
 
 	for i, v := range results.data {
@@ -71,11 +83,34 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
 			}
 		}
 
-		durations = append(durations, v.timestamp.Sub(lastTimestamp))
+		result.Durations = append(result.Durations, v.timestamp.Sub(lastTimestamp))
 		lastTimestamp = v.timestamp
 	}
 
-	return durations, nil
+	// Compute cert size: sum of ApplicationData payload between CCS and
+	// the first client Write (which marks the end of server handshake).
+	seenCCS := false
+	boundary := results.writeIndex
+	if boundary < 0 {
+		boundary = len(results.data)
+	}
+
+	for i, v := range results.data {
+		if i >= boundary {
+			break
+		}
+
+		if v.recordType == tls.TypeChangeCipherSpec {
+			seenCCS = true
+			continue
+		}
+
+		if seenCCS && v.recordType == tls.TypeApplicationData {
+			result.CertSize += v.payloadLen
+		}
+	}
+
+	return result, nil
 }
 
 func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) {

mtglib/internal/doppel/scout_conn.go

@@ -14,9 +14,10 @@ type ScoutConn struct {
 
 	results *ScoutConnCollected
 	rawBuf  *bytes.Buffer
+	seenCCS bool
 }
 
-func (s ScoutConn) Read(p []byte) (int, error) {
+func (s *ScoutConn) Read(p []byte) (int, error) {
 	buf := &bytes.Buffer{}
 
 	for {
@@ -31,7 +32,11 @@ func (s ScoutConn) Read(p []byte) (int, error) {
 			return 0, err
 		}
 
-		s.results.Add(recordType)
+		if recordType == tls.TypeChangeCipherSpec {
+			s.seenCCS = true
+		}
+
+		s.results.Add(recordType, int(length))
 		s.rawBuf.Write([]byte{recordType})
 		s.rawBuf.Write(tls.TLSVersion[:])
 
@@ -45,11 +50,19 @@ func (s ScoutConn) Read(p []byte) (int, error) {
 	}
 }
 
-func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) ScoutConn {
+func (s *ScoutConn) Write(p []byte) (int, error) {
+	if s.seenCCS {
+		s.results.MarkWrite()
+	}
+
+	return s.Conn.Write(p)
+}
+
+func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) *ScoutConn {
 	rawBuf := &bytes.Buffer{}
 	rawBuf.Grow(tls.MaxRecordSize)
 
-	return ScoutConn{
+	return &ScoutConn{
 		Conn:    tls.New(conn, false, false),
 		results: results,
 		rawBuf:  rawBuf,

mtglib/internal/doppel/scout_conn_collected.go

@@ -9,21 +9,32 @@ const (
 type ScoutConnResult struct {
 	timestamp  time.Time
 	recordType byte
+	payloadLen int
 }
 
 type ScoutConnCollected struct {
-	data []ScoutConnResult
+	data       []ScoutConnResult
+	writeIndex int // index at which client first wrote post-handshake data; -1 if not set
 }
 
-func (s *ScoutConnCollected) Add(record byte) {
+func (s *ScoutConnCollected) Add(record byte, payloadLen int) {
 	s.data = append(s.data, ScoutConnResult{
 		timestamp:  time.Now(),
 		recordType: record,
+		payloadLen: payloadLen,
 	})
 }
 
+// MarkWrite records the current data length as the handshake boundary.
+func (s *ScoutConnCollected) MarkWrite() {
+	if s.writeIndex < 0 {
+		s.writeIndex = len(s.data)
+	}
+}
+
 func NewScoutConnCollected() *ScoutConnCollected {
 	return &ScoutConnCollected{
-		data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
+		data:       make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
+		writeIndex: -1,
 	}
 }

mtglib/internal/doppel/scout_conn_collected_test.go

@@ -14,7 +14,7 @@ type ScoutConnCollectedTestSuite struct {
 
 func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
 	collected := NewScoutConnCollected()
-	collected.Add(tls.TypeApplicationData)
+	collected.Add(tls.TypeApplicationData, 100)
 
 	suite.Len(collected.data, 1)
 	suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
@@ -23,13 +23,13 @@ func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
 func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
 	collected := NewScoutConnCollected()
 
-	collected.Add(tls.TypeApplicationData)
+	collected.Add(tls.TypeApplicationData, 100)
 
 	time.Sleep(time.Microsecond)
-	collected.Add(tls.TypeApplicationData)
+	collected.Add(tls.TypeApplicationData, 100)
 
 	time.Sleep(time.Microsecond)
-	collected.Add(tls.TypeApplicationData)
+	collected.Add(tls.TypeApplicationData, 100)
 
 	for i := 1; i < len(collected.data); i++ {
 		suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))

mtglib/internal/doppel/scout_test.go

@@ -22,9 +22,9 @@ func (suite *ScoutTestSuite) SetupSuite() {
 }
 
 func (suite *ScoutTestSuite) TestCollectResults() {
-	durations, err := suite.scout.Learn(suite.ctx)
+	result, err := suite.scout.Learn(suite.ctx)
 	suite.NoError(err)
-	suite.Less(3, len(durations))
+	suite.Less(3, len(result.Durations))
 }
 
 func (suite *ScoutTestSuite) TestCollectNothing() {

mtglib/internal/tls/fake/server_side.go

@@ -9,11 +9,18 @@ import (
 	"io"
 	rnd "math/rand/v2"
 
-	"github.com/9seconds/mtg/v2/mtglib/internal/doppel"
 	"github.com/9seconds/mtg/v2/mtglib/internal/tls"
 	"golang.org/x/crypto/curve25519"
 )
 
+// NoiseParams controls the size of the fake ApplicationData record
+// in ServerHello. If Mean is 0, the legacy random range (2500-4700)
+// is used.
+type NoiseParams struct {
+	Mean   int
+	Jitter int
+}
+
 const (
 	TypeHandshakeServer = 0x02
 	ChangeCipherValue   = 0x01
@@ -33,13 +40,13 @@ var serverHelloSuffix = []byte{
 	0x00, 0x20, // 32 bytes of key
 }
 
-func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) error {
+func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello, noise NoiseParams) error {
 	buf := &bytes.Buffer{}
 	buf.Grow(tls.MaxRecordSize)
 
 	generateServerHello(buf, clientHello)
 	generateChangeCipherValue(buf)
-	generateNoise(buf)
+	generateNoise(buf, noise)
 
 	packet := buf.Bytes()
 	digest := hmac.New(sha256.New, secret)
@@ -125,19 +132,31 @@ func generateChangeCipherValue(buf *bytes.Buffer) {
 	buf.WriteByte(ChangeCipherValue)
 }
 
-func generateNoise(buf *bytes.Buffer) {
+// generateNoise writes a single ApplicationData record mimicking the combined
-	data := make(
+// size of a real TLS 1.3 encrypted server handshake (EncryptedExtensions +
-		[]byte,
+// Certificate chain + CertificateVerify + Finished).
-		int64(
+//
-			doppel.TLSRecordSizeStart+rnd.IntN(
+// NOTE: Must be exactly ONE ApplicationData record — the Telegram client reads
-				doppel.TLSRecordSizeAccel-doppel.TLSRecordSizeStart,
+// ServerHello + CCS + 1 ApplicationData and computes HMAC over all three.
-			),
+// Multiple records would cause HMAC mismatch and connection failure.
-		),
+func generateNoise(buf *bytes.Buffer, noise NoiseParams) {
-	)
+	var size int
 
-	if _, err := rand.Read(data[:]); err != nil {
+	if noise.Mean > 0 && noise.Jitter > 0 {
+		// Calibrated: use measured cert chain size ± jitter.
+		size = noise.Mean - noise.Jitter + rnd.IntN(2*noise.Jitter)
+		if size < 1000 {
+			size = 1000
+		}
+	} else {
+		// Legacy fallback: random in 2500-4700 range.
+		size = 2500 + rnd.IntN(2200)
+	}
+
+	data := make([]byte, size)
+	if _, err := rand.Read(data); err != nil {
 		panic(err)
 	}
 
-	tls.WriteRecord(buf, data[:]) //nolint: errcheck
+	tls.WriteRecord(buf, data) //nolint: errcheck
 }

mtglib/internal/tls/fake/server_side_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/9seconds/mtg/v2/mtglib"
-	"github.com/9seconds/mtg/v2/mtglib/internal/doppel"
 	"github.com/9seconds/mtg/v2/mtglib/internal/tls"
 	"github.com/9seconds/mtg/v2/mtglib/internal/tls/fake"
 	"github.com/stretchr/testify/suite"
@@ -39,7 +38,7 @@ func (suite *SendServerHelloTestSuite) SetupTest() {
 }
 
 func (suite *SendServerHelloTestSuite) TestRecordStructure() {
-	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
+	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello, fake.NoiseParams{})
 	suite.NoError(err)
 
 	var rec bytes.Buffer
@@ -59,13 +58,13 @@ func (suite *SendServerHelloTestSuite) TestRecordStructure() {
 	recordType, length, err := tls.ReadRecord(suite.buf, &rec)
 	suite.NoError(err)
 	suite.Equal(byte(tls.TypeApplicationData), recordType)
-	suite.Greater(length, int64(doppel.TLSRecordSizeStart))
+	suite.Greater(length, int64(2500))
 
 	suite.Empty(suite.buf.Bytes())
 }
 
 func (suite *SendServerHelloTestSuite) TestHMAC() {
-	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
+	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello, fake.NoiseParams{})
 	suite.NoError(err)
 
 	packet := make([]byte, suite.buf.Len())
@@ -83,7 +82,7 @@ func (suite *SendServerHelloTestSuite) TestHMAC() {
 }
 
 func (suite *SendServerHelloTestSuite) TestHandshakePayload() {
-	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
+	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello, fake.NoiseParams{})
 	suite.NoError(err)
 
 	packet := suite.buf.Bytes()
@@ -105,7 +104,7 @@ func (suite *SendServerHelloTestSuite) TestHandshakePayload() {
 }
 
 func (suite *SendServerHelloTestSuite) TestChangeCipherSpec() {
-	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
+	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello, fake.NoiseParams{})
 	suite.NoError(err)
 
 	// Skip first record
@@ -124,6 +123,33 @@ func (suite *SendServerHelloTestSuite) TestChangeCipherSpec() {
 	suite.Equal([]byte{fake.ChangeCipherValue}, rec.Bytes())
 }
 
+func (suite *SendServerHelloTestSuite) TestCalibratedNoiseSize() {
+	noise := fake.NoiseParams{Mean: 6480, Jitter: 100}
+	err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello, noise)
+	suite.NoError(err)
+
+	var rec bytes.Buffer
+
+	// Skip ServerHello
+	_, _, err = tls.ReadRecord(suite.buf, &rec)
+	suite.NoError(err)
+
+	// Skip ChangeCipherSpec
+	rec.Reset()
+	_, _, err = tls.ReadRecord(suite.buf, &rec)
+	suite.NoError(err)
+
+	// Read noise ApplicationData
+	rec.Reset()
+	recordType, length, err := tls.ReadRecord(suite.buf, &rec)
+	suite.NoError(err)
+	suite.Equal(byte(tls.TypeApplicationData), recordType)
+
+	// Should be within mean ± jitter range.
+	suite.GreaterOrEqual(length, int64(noise.Mean-noise.Jitter))
+	suite.LessOrEqual(length, int64(noise.Mean+noise.Jitter))
+}
+
 func TestSendServerHello(t *testing.T) {
 	t.Parallel()
 	suite.Run(t, &SendServerHelloTestSuite{})

mtglib/proxy.go

@@ -192,7 +192,10 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool {
 		return false
 	}
 
-	if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello); err != nil {
+	gangerNoise := p.doppelGanger.NoiseParams()
+	noiseParams := fake.NoiseParams{Mean: gangerNoise.Mean, Jitter: gangerNoise.Jitter}
+
+	if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello, noiseParams); err != nil {
 		p.logger.InfoError("cannot send welcome packet", err)
 		return false
 	}

mtglib/proxy_opts.go

@@ -160,6 +160,7 @@ type ProxyOpts struct {
 
 	// DoppelGangerDRS defines if TLS Dynamic Record Sizing is active.
 	DoppelGangerDRS bool
+
 }
 
 func (p ProxyOpts) valid() error {