Move cert noise calibration into doppelganger scout

Instead of a separate cert_probe.go that duplicates the scout's TLS
connection logic, measure the cert chain size directly from the same
HTTPS connections the scout already makes.

Changes:
- Extend ScoutConnResult with payloadLen field
- Add Write interception to ScoutConn for handshake boundary detection
- Scout.learn() now computes cert size (sum of ApplicationData between
  CCS and first client Write) alongside inter-record durations
- Ganger aggregates cert sizes across raids and exposes NoiseParams()
  via atomic pointer for lock-free reads from proxy goroutines
- Proxy reads NoiseParams from Ganger on each handshake instead of
  probing at startup
- Remove cert_probe.go, disk cache, and related config options
  (noise-cache-path, noise-cache-ttl, noise-probe-count)

Falls back to legacy 2500-4700 range until the first scout raid
completes (typically within 1-2 seconds of startup).
This commit is contained in:
Alexey Dolotov
2026-03-27 16:34:42 +03:00
parent 80213ad35d
commit 9dfd992c1d
12 changed files with 177 additions and 363 deletions
+89 -7
View File
@@ -2,7 +2,9 @@ package doppel
import (
"context"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/9seconds/mtg/v2/essentials"
@@ -12,8 +14,22 @@ const (
DoppelGangerMaxDurations = 4096
DoppelGangerScoutRaidEach = 6 * time.Hour
DoppelGangerScoutRepeats = 10
MinCertSizesToCalculate = 3
)
// NoiseParams holds the measured cert chain size for FakeTLS noise calibration.
// If Mean is 0, the caller should use a legacy fallback.
type NoiseParams struct {
Mean int
Jitter int
}
type scoutRaidResult struct {
durations []time.Duration
certSizes []int
}
type gangerConnRequest struct {
ret chan<- Conn
payload essentials.Conn
@@ -33,6 +49,9 @@ type Ganger struct {
stats *Stats
durations []time.Duration
certSizes []int
noiseParams atomic.Pointer[NoiseParams]
connRequests chan gangerConnRequest
}
@@ -48,6 +67,16 @@ func (g *Ganger) Run() {
})
}
// NoiseParams returns the current cert-size-based noise parameters.
// Returns zero-value NoiseParams if not yet measured (caller should use fallback).
func (g *Ganger) NoiseParams() NoiseParams {
if p := g.noiseParams.Load(); p != nil {
return *p
}
return NoiseParams{}
}
func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) {
rvChan := make(chan Conn)
req := gangerConnRequest{
@@ -81,7 +110,7 @@ func (g *Ganger) run() {
}
}()
scoutCollectedChan := make(chan []time.Duration)
scoutCollectedChan := make(chan scoutRaidResult)
currentScoutCollectedChan := scoutCollectedChan
updatedStatsChan := make(chan *Stats)
@@ -94,18 +123,29 @@ func (g *Ganger) run() {
select {
case <-g.ctx.Done():
return
case durations := <-currentScoutCollectedChan:
g.durations = append(g.durations, durations...)
case result := <-currentScoutCollectedChan:
g.durations = append(g.durations, result.durations...)
if len(g.durations) > DoppelGangerMaxDurations {
copy(g.durations, g.durations[len(g.durations)-DoppelGangerMaxDurations:])
g.durations = g.durations[:DoppelGangerMaxDurations]
}
// Update cert sizes and recompute noise params.
g.certSizes = append(g.certSizes, result.certSizes...)
if len(g.certSizes) > DoppelGangerMaxDurations {
g.certSizes = g.certSizes[len(g.certSizes)-DoppelGangerMaxDurations:]
}
if len(g.certSizes) >= MinCertSizesToCalculate {
g.updateNoiseParams()
}
if len(g.durations) < MinDurationsToCalculate {
continue
}
durations := g.durations
currentScoutCollectedChan = nil
g.wg.Go(func() {
select {
@@ -129,8 +169,45 @@ func (g *Ganger) run() {
}
}
func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
durations := []time.Duration{}
func (g *Ganger) updateNoiseParams() {
if len(g.certSizes) == 0 {
return
}
sum := 0
for _, s := range g.certSizes {
sum += s
}
mean := sum / len(g.certSizes)
maxDev := 0
for _, s := range g.certSizes {
d := s - mean
if d < 0 {
d = -d
}
if d > maxDev {
maxDev = d
}
}
if maxDev < 100 {
maxDev = 100
}
np := &NoiseParams{Mean: mean, Jitter: maxDev}
g.noiseParams.Store(np)
g.logger.Info(fmt.Sprintf(
"updated noise params: mean=%d jitter=%d samples=%d",
mean, maxDev, len(g.certSizes),
))
}
func (g *Ganger) runScoutRaid(rvChan chan<- scoutRaidResult) {
var result scoutRaidResult
for range g.scoutRaidRepeats {
learned, err := g.scout.Learn(g.ctx)
@@ -138,13 +215,18 @@ func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
g.logger.WarningError("cannot learn", err)
continue
}
durations = append(durations, learned...)
result.durations = append(result.durations, learned.Durations...)
if learned.CertSize > 0 {
result.certSizes = append(result.certSizes, learned.CertSize)
}
}
select {
case <-g.ctx.Done():
return
case rvChan <- durations:
case rvChan <- result:
}
}
+47 -12
View File
@@ -12,36 +12,46 @@ import (
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
// ScoutResult holds measurements from a single scout HTTP request.
type ScoutResult struct {
Durations []time.Duration
CertSize int // total ApplicationData bytes during TLS handshake; 0 if unknown
}
type Scout struct {
network Network
urls []string
}
func (s Scout) Learn(ctx context.Context) ([]time.Duration, error) {
var durations []time.Duration
func (s Scout) Learn(ctx context.Context) (ScoutResult, error) {
var combined ScoutResult
for _, url := range s.urls {
learned, err := s.learn(ctx, url)
if err != nil {
return nil, err
return ScoutResult{}, err
}
durations = append(durations, learned...)
combined.Durations = append(combined.Durations, learned.Durations...)
if learned.CertSize > 0 && combined.CertSize == 0 {
combined.CertSize = learned.CertSize
}
}
return durations, nil
return combined, nil
}
func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
func (s Scout) learn(ctx context.Context, url string) (ScoutResult, error) {
client, results := s.makeClient()
if !strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("url %s must be https", url)
return ScoutResult{}, fmt.Errorf("url %s must be https", url)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
return ScoutResult{}, err
}
resp, err := client.Do(req)
@@ -52,10 +62,12 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
}
if err != nil || len(results.data) == 0 {
return nil, err
return ScoutResult{}, err
}
durations := []time.Duration{}
var result ScoutResult
// Compute inter-record durations (existing logic).
lastTimestamp := time.Time{}
for i, v := range results.data {
@@ -71,11 +83,34 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
}
}
durations = append(durations, v.timestamp.Sub(lastTimestamp))
result.Durations = append(result.Durations, v.timestamp.Sub(lastTimestamp))
lastTimestamp = v.timestamp
}
return durations, nil
// Compute cert size: sum of ApplicationData payload between CCS and
// the first client Write (which marks the end of server handshake).
seenCCS := false
boundary := results.writeIndex
if boundary < 0 {
boundary = len(results.data)
}
for i, v := range results.data {
if i >= boundary {
break
}
if v.recordType == tls.TypeChangeCipherSpec {
seenCCS = true
continue
}
if seenCCS && v.recordType == tls.TypeApplicationData {
result.CertSize += v.payloadLen
}
}
return result, nil
}
func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) {
+17 -4
View File
@@ -14,9 +14,10 @@ type ScoutConn struct {
results *ScoutConnCollected
rawBuf *bytes.Buffer
seenCCS bool
}
func (s ScoutConn) Read(p []byte) (int, error) {
func (s *ScoutConn) Read(p []byte) (int, error) {
buf := &bytes.Buffer{}
for {
@@ -31,7 +32,11 @@ func (s ScoutConn) Read(p []byte) (int, error) {
return 0, err
}
s.results.Add(recordType)
if recordType == tls.TypeChangeCipherSpec {
s.seenCCS = true
}
s.results.Add(recordType, int(length))
s.rawBuf.Write([]byte{recordType})
s.rawBuf.Write(tls.TLSVersion[:])
@@ -45,11 +50,19 @@ func (s ScoutConn) Read(p []byte) (int, error) {
}
}
func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) ScoutConn {
func (s *ScoutConn) Write(p []byte) (int, error) {
if s.seenCCS {
s.results.MarkWrite()
}
return s.Conn.Write(p)
}
func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) *ScoutConn {
rawBuf := &bytes.Buffer{}
rawBuf.Grow(tls.MaxRecordSize)
return ScoutConn{
return &ScoutConn{
Conn: tls.New(conn, false, false),
results: results,
rawBuf: rawBuf,
+14 -3
View File
@@ -9,21 +9,32 @@ const (
type ScoutConnResult struct {
timestamp time.Time
recordType byte
payloadLen int
}
type ScoutConnCollected struct {
data []ScoutConnResult
data []ScoutConnResult
writeIndex int // index at which client first wrote post-handshake data; -1 if not set
}
func (s *ScoutConnCollected) Add(record byte) {
func (s *ScoutConnCollected) Add(record byte, payloadLen int) {
s.data = append(s.data, ScoutConnResult{
timestamp: time.Now(),
recordType: record,
payloadLen: payloadLen,
})
}
// MarkWrite records the current data length as the handshake boundary.
func (s *ScoutConnCollected) MarkWrite() {
if s.writeIndex < 0 {
s.writeIndex = len(s.data)
}
}
func NewScoutConnCollected() *ScoutConnCollected {
return &ScoutConnCollected{
data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
writeIndex: -1,
}
}
@@ -14,7 +14,7 @@ type ScoutConnCollectedTestSuite struct {
func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
collected := NewScoutConnCollected()
collected.Add(tls.TypeApplicationData)
collected.Add(tls.TypeApplicationData, 100)
suite.Len(collected.data, 1)
suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
@@ -23,13 +23,13 @@ func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
collected := NewScoutConnCollected()
collected.Add(tls.TypeApplicationData)
collected.Add(tls.TypeApplicationData, 100)
time.Sleep(time.Microsecond)
collected.Add(tls.TypeApplicationData)
collected.Add(tls.TypeApplicationData, 100)
time.Sleep(time.Microsecond)
collected.Add(tls.TypeApplicationData)
collected.Add(tls.TypeApplicationData, 100)
for i := 1; i < len(collected.data); i++ {
suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))
+2 -2
View File
@@ -22,9 +22,9 @@ func (suite *ScoutTestSuite) SetupSuite() {
}
func (suite *ScoutTestSuite) TestCollectResults() {
durations, err := suite.scout.Learn(suite.ctx)
result, err := suite.scout.Learn(suite.ctx)
suite.NoError(err)
suite.Less(3, len(durations))
suite.Less(3, len(result.Durations))
}
func (suite *ScoutTestSuite) TestCollectNothing() {
-261
View File
@@ -1,261 +0,0 @@
package fake
import (
"crypto/tls"
"encoding/binary"
"encoding/json"
"fmt"
"net"
"os"
"sync"
"time"
)
const (
probeDialTimeout = 10 * time.Second
probeHandshakeTimeout = 10 * time.Second
defaultProbeCount = 15
defaultCacheTTL = 24 * time.Hour
tlsTypeChangeCipherSpec = 0x14
tlsTypeApplicationData = 0x17
)
// CertProbeResult holds the measured encrypted handshake size.
type CertProbeResult struct {
Mean int `json:"mean"`
Jitter int `json:"jitter"`
}
// CertProbeCache is the on-disk format for cached probe results.
type CertProbeCache struct {
Hostname string `json:"hostname"`
Port int `json:"port"`
Mean int `json:"mean"`
Jitter int `json:"jitter"`
ProbedAt time.Time `json:"probed_at"`
}
// LoadCachedProbe reads a cached probe result from path. Returns the result
// and true if the cache exists, matches hostname:port, and is younger than ttl.
// Otherwise returns zero value and false.
func LoadCachedProbe(path, hostname string, port int, ttl time.Duration) (CertProbeResult, bool) {
if ttl <= 0 {
ttl = defaultCacheTTL
}
data, err := os.ReadFile(path)
if err != nil {
return CertProbeResult{}, false
}
var cache CertProbeCache
if err := json.Unmarshal(data, &cache); err != nil {
return CertProbeResult{}, false
}
if cache.Hostname != hostname || cache.Port != port {
return CertProbeResult{}, false
}
if time.Since(cache.ProbedAt) > ttl {
return CertProbeResult{}, false
}
if cache.Mean <= 0 {
return CertProbeResult{}, false
}
return CertProbeResult{Mean: cache.Mean, Jitter: cache.Jitter}, true
}
// SaveCachedProbe writes a probe result to path as JSON.
func SaveCachedProbe(path, hostname string, port int, result CertProbeResult) error {
cache := CertProbeCache{
Hostname: hostname,
Port: port,
Mean: result.Mean,
Jitter: result.Jitter,
ProbedAt: time.Now(),
}
data, err := json.MarshalIndent(cache, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, data, 0o644) //nolint: gosec
}
// ProbeCertSize connects to hostname:port via TLS multiple times and measures
// the total ApplicationData payload bytes sent by the server during the
// handshake (between ChangeCipherSpec and the first application-level data).
// This corresponds to EncryptedExtensions + Certificate + CertificateVerify +
// Finished in TLS 1.3, which is what the FakeTLS noise must mimic.
func ProbeCertSize(hostname string, port int, count int) (CertProbeResult, error) {
if count <= 0 {
count = defaultProbeCount
}
addr := net.JoinHostPort(hostname, fmt.Sprintf("%d", port))
sizes := make([]int, 0, count)
for i := 0; i < count; i++ {
size, err := probeSingle(addr, hostname)
if err != nil {
if len(sizes) > 0 {
break // use what we have
}
return CertProbeResult{}, fmt.Errorf("probe %d failed: %w", i, err)
}
sizes = append(sizes, size)
}
if len(sizes) == 0 {
return CertProbeResult{}, fmt.Errorf("no successful probes")
}
// Calculate mean and jitter (max deviation from mean).
sum := 0
for _, s := range sizes {
sum += s
}
mean := sum / len(sizes)
maxDev := 0
for _, s := range sizes {
d := s - mean
if d < 0 {
d = -d
}
if d > maxDev {
maxDev = d
}
}
// Ensure minimum jitter of 100 bytes for variability.
if maxDev < 100 {
maxDev = 100
}
return CertProbeResult{Mean: mean, Jitter: maxDev}, nil
}
// probeSingle does one TLS handshake and measures ApplicationData bytes
// received during the handshake.
func probeSingle(addr, hostname string) (int, error) {
rawConn, err := net.DialTimeout("tcp", addr, probeDialTimeout)
if err != nil {
return 0, err
}
defer rawConn.Close() //nolint: errcheck
capture := &recordCapture{conn: rawConn}
tlsConn := tls.Client(capture, &tls.Config{
ServerName: hostname,
MinVersion: tls.VersionTLS12,
})
tlsConn.SetDeadline(time.Now().Add(probeHandshakeTimeout)) //nolint: errcheck
if err := tlsConn.Handshake(); err != nil {
return 0, err
}
tlsConn.Close() //nolint: errcheck
return capture.appDataBytes, nil
}
// recordCapture wraps a net.Conn and parses the raw TLS record stream to
// measure ApplicationData payload sizes sent by the server during handshake.
// It tracks record boundaries by maintaining a state machine over Read calls.
type recordCapture struct {
conn net.Conn
mu sync.Mutex
appDataBytes int
seenCCS bool
done bool
// Record boundary tracking for the read side.
readRemaining int // bytes left in current record payload
readHeaderBuf [5]byte
readHeaderPos int
}
func (rc *recordCapture) Read(p []byte) (int, error) {
n, err := rc.conn.Read(p)
if n > 0 && !rc.done {
rc.mu.Lock()
rc.parseReadBytes(p[:n])
rc.mu.Unlock()
}
return n, err
}
func (rc *recordCapture) parseReadBytes(data []byte) {
for len(data) > 0 {
if rc.readRemaining > 0 {
// Consuming payload of current record.
consume := rc.readRemaining
if consume > len(data) {
consume = len(data)
}
rc.readRemaining -= consume
data = data[consume:]
continue
}
// Accumulate header bytes (5 bytes per record).
need := 5 - rc.readHeaderPos
if need > len(data) {
need = len(data)
}
copy(rc.readHeaderBuf[rc.readHeaderPos:], data[:need])
rc.readHeaderPos += need
data = data[need:]
if rc.readHeaderPos < 5 {
return // incomplete header
}
// Full header available.
recordType := rc.readHeaderBuf[0]
payloadLen := int(binary.BigEndian.Uint16(rc.readHeaderBuf[3:5]))
rc.readHeaderPos = 0
rc.readRemaining = payloadLen
if recordType == tlsTypeChangeCipherSpec {
rc.seenCCS = true
} else if recordType == tlsTypeApplicationData && rc.seenCCS {
rc.appDataBytes += payloadLen
}
}
}
func (rc *recordCapture) Write(p []byte) (int, error) {
// After client writes post-CCS data, server handshake records are done.
if rc.seenCCS && rc.appDataBytes > 0 {
rc.done = true
}
return rc.conn.Write(p)
}
func (rc *recordCapture) Close() error { return rc.conn.Close() }
func (rc *recordCapture) LocalAddr() net.Addr { return rc.conn.LocalAddr() }
func (rc *recordCapture) RemoteAddr() net.Addr { return rc.conn.RemoteAddr() }
func (rc *recordCapture) SetDeadline(t time.Time) error { return rc.conn.SetDeadline(t) }
func (rc *recordCapture) SetReadDeadline(t time.Time) error { return rc.conn.SetReadDeadline(t) }
func (rc *recordCapture) SetWriteDeadline(t time.Time) error { return rc.conn.SetWriteDeadline(t) }
// Ensure recordCapture implements net.Conn.
var _ net.Conn = (*recordCapture)(nil)
+4 -42
View File
@@ -36,7 +36,6 @@ type Proxy struct {
doppelGanger *doppel.Ganger
clientObfuscatror obfuscation.Obfuscator
noiseParams fake.NoiseParams
secret Secret
network Network
antiReplayCache AntiReplayCache
@@ -193,7 +192,10 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool {
return false
}
if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello, p.noiseParams); err != nil {
gangerNoise := p.doppelGanger.NoiseParams()
noiseParams := fake.NoiseParams{Mean: gangerNoise.Mean, Jitter: gangerNoise.Jitter}
if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello, noiseParams); err != nil {
p.logger.InfoError("cannot send welcome packet", err)
return false
}
@@ -324,49 +326,9 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {
logger := opts.getLogger("proxy")
updatersLogger := logger.Named("telegram-updaters")
// Probe the fronting domain's cert chain size for noise calibration.
probeHost := opts.Secret.Host
probePort := opts.getDomainFrontingPort()
noiseParams := fake.NoiseParams{}
probeCount := int(opts.NoiseProbeCount)
if probeCount <= 0 {
probeCount = 15
}
cacheTTL := opts.NoiseCacheTTL
// Try loading from cache first.
if opts.NoiseCachePath != "" {
if cached, ok := fake.LoadCachedProbe(opts.NoiseCachePath, probeHost, probePort, cacheTTL); ok {
noiseParams = fake.NoiseParams(cached)
logger.Info(fmt.Sprintf("cert probe: loaded from cache, host=%s mean=%d jitter=%d",
probeHost, cached.Mean, cached.Jitter))
}
}
// If no cached result, probe live.
if noiseParams.Mean == 0 {
probeResult, probeErr := fake.ProbeCertSize(probeHost, probePort, probeCount)
if probeErr != nil {
logger.WarningError("cert probe failed, using default noise size", probeErr)
} else {
noiseParams = fake.NoiseParams(probeResult)
logger.Info(fmt.Sprintf("cert probe: host=%s mean=%d jitter=%d",
probeHost, probeResult.Mean, probeResult.Jitter))
if opts.NoiseCachePath != "" {
if saveErr := fake.SaveCachedProbe(opts.NoiseCachePath, probeHost, probePort, probeResult); saveErr != nil {
logger.WarningError("failed to save cert probe cache", saveErr)
}
}
}
}
proxy := &Proxy{
ctx: ctx,
ctxCancel: cancel,
noiseParams: noiseParams,
secret: opts.Secret,
network: opts.Network,
antiReplayCache: opts.AntiReplayCache,
-18
View File
@@ -161,24 +161,6 @@ type ProxyOpts struct {
// DoppelGangerDRS defines if TLS Dynamic Record Sizing is active.
DoppelGangerDRS bool
// NoiseProbeCount is the number of TLS connections to make when probing
// the fronting domain's cert chain size for noise calibration.
// Default is 15.
//
// This is an optional setting.
NoiseProbeCount uint
// NoiseCacheTTL is how long a cached cert probe result is considered
// valid. Default is 24 hours.
//
// This is an optional setting.
NoiseCacheTTL time.Duration
// NoiseCachePath is the file path for caching cert probe results
// between restarts. If empty, no caching is performed.
//
// This is an optional setting.
NoiseCachePath string
}
func (p ProxyOpts) valid() error {