mirror of
https://github.com/ScuroNeko/mtg.git
synced 2026-08-31 18:24:02 +03:00
Move cert noise calibration into doppelganger scout
Instead of a separate cert_probe.go that duplicates the scout's TLS connection logic, measure the cert chain size directly from the same HTTPS connections the scout already makes. Changes: - Extend ScoutConnResult with payloadLen field - Add Write interception to ScoutConn for handshake boundary detection - Scout.learn() now computes cert size (sum of ApplicationData between CCS and first client Write) alongside inter-record durations - Ganger aggregates cert sizes across raids and exposes NoiseParams() via atomic pointer for lock-free reads from proxy goroutines - Proxy reads NoiseParams from Ganger on each handshake instead of probing at startup - Remove cert_probe.go, disk cache, and related config options (noise-cache-path, noise-cache-ttl, noise-probe-count) Falls back to legacy 2500-4700 range until the first scout raid completes (typically within 1-2 seconds of startup).
This commit is contained in:
@@ -267,10 +267,6 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen
|
|||||||
DoppelGangerPerRaid: conf.Defense.Doppelganger.Repeats.Get(mtglib.DoppelGangerPerRaid),
|
DoppelGangerPerRaid: conf.Defense.Doppelganger.Repeats.Get(mtglib.DoppelGangerPerRaid),
|
||||||
DoppelGangerEach: conf.Defense.Doppelganger.UpdateEach.Get(mtglib.DoppelGangerEach),
|
DoppelGangerEach: conf.Defense.Doppelganger.UpdateEach.Get(mtglib.DoppelGangerEach),
|
||||||
DoppelGangerDRS: conf.Defense.Doppelganger.DRS.Get(false),
|
DoppelGangerDRS: conf.Defense.Doppelganger.DRS.Get(false),
|
||||||
|
|
||||||
NoiseProbeCount: conf.Defense.Doppelganger.NoiseProbeCount.Get(0),
|
|
||||||
NoiseCacheTTL: conf.Defense.Doppelganger.NoiseCacheTTL.Get(0),
|
|
||||||
NoiseCachePath: conf.Defense.Doppelganger.NoiseCachePath,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
proxy, err := mtglib.NewProxy(opts)
|
proxy, err := mtglib.NewProxy(opts)
|
||||||
|
|||||||
@@ -54,9 +54,6 @@ type Config struct {
|
|||||||
Repeats TypeConcurrency `json:"repeats_per_raid"`
|
Repeats TypeConcurrency `json:"repeats_per_raid"`
|
||||||
UpdateEach TypeDuration `json:"raid_each"`
|
UpdateEach TypeDuration `json:"raid_each"`
|
||||||
DRS TypeBool `json:"drs"`
|
DRS TypeBool `json:"drs"`
|
||||||
NoiseProbeCount TypeConcurrency `json:"noise_probe_count"`
|
|
||||||
NoiseCacheTTL TypeDuration `json:"noise_cache_ttl"`
|
|
||||||
NoiseCachePath string `json:"noise_cache_path"`
|
|
||||||
} `json:"doppelganger"`
|
} `json:"doppelganger"`
|
||||||
} `json:"defense"`
|
} `json:"defense"`
|
||||||
Network struct {
|
Network struct {
|
||||||
|
|||||||
@@ -49,9 +49,6 @@ type tomlConfig struct {
|
|||||||
Repeats uint `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
|
Repeats uint `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
|
||||||
UpdateEach string `toml:"raid-each" json:"raid_each,omitempty"`
|
UpdateEach string `toml:"raid-each" json:"raid_each,omitempty"`
|
||||||
DRS bool `toml:"drs" json:"drs,omitempty"`
|
DRS bool `toml:"drs" json:"drs,omitempty"`
|
||||||
NoiseProbeCount uint `toml:"noise-probe-count" json:"noise_probe_count,omitempty"`
|
|
||||||
NoiseCacheTTL string `toml:"noise-cache-ttl" json:"noise_cache_ttl,omitempty"`
|
|
||||||
NoiseCachePath string `toml:"noise-cache-path" json:"noise_cache_path,omitempty"`
|
|
||||||
} `toml:"doppelganger" json:"doppelganger,omitempty"`
|
} `toml:"doppelganger" json:"doppelganger,omitempty"`
|
||||||
} `toml:"defense" json:"defense,omitempty"`
|
} `toml:"defense" json:"defense,omitempty"`
|
||||||
Network struct {
|
Network struct {
|
||||||
|
|||||||
@@ -2,7 +2,9 @@ package doppel
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/9seconds/mtg/v2/essentials"
|
"github.com/9seconds/mtg/v2/essentials"
|
||||||
@@ -12,8 +14,22 @@ const (
|
|||||||
DoppelGangerMaxDurations = 4096
|
DoppelGangerMaxDurations = 4096
|
||||||
DoppelGangerScoutRaidEach = 6 * time.Hour
|
DoppelGangerScoutRaidEach = 6 * time.Hour
|
||||||
DoppelGangerScoutRepeats = 10
|
DoppelGangerScoutRepeats = 10
|
||||||
|
|
||||||
|
MinCertSizesToCalculate = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// NoiseParams holds the measured cert chain size for FakeTLS noise calibration.
|
||||||
|
// If Mean is 0, the caller should use a legacy fallback.
|
||||||
|
type NoiseParams struct {
|
||||||
|
Mean int
|
||||||
|
Jitter int
|
||||||
|
}
|
||||||
|
|
||||||
|
type scoutRaidResult struct {
|
||||||
|
durations []time.Duration
|
||||||
|
certSizes []int
|
||||||
|
}
|
||||||
|
|
||||||
type gangerConnRequest struct {
|
type gangerConnRequest struct {
|
||||||
ret chan<- Conn
|
ret chan<- Conn
|
||||||
payload essentials.Conn
|
payload essentials.Conn
|
||||||
@@ -33,6 +49,9 @@ type Ganger struct {
|
|||||||
|
|
||||||
stats *Stats
|
stats *Stats
|
||||||
durations []time.Duration
|
durations []time.Duration
|
||||||
|
certSizes []int
|
||||||
|
|
||||||
|
noiseParams atomic.Pointer[NoiseParams]
|
||||||
|
|
||||||
connRequests chan gangerConnRequest
|
connRequests chan gangerConnRequest
|
||||||
}
|
}
|
||||||
@@ -48,6 +67,16 @@ func (g *Ganger) Run() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NoiseParams returns the current cert-size-based noise parameters.
|
||||||
|
// Returns zero-value NoiseParams if not yet measured (caller should use fallback).
|
||||||
|
func (g *Ganger) NoiseParams() NoiseParams {
|
||||||
|
if p := g.noiseParams.Load(); p != nil {
|
||||||
|
return *p
|
||||||
|
}
|
||||||
|
|
||||||
|
return NoiseParams{}
|
||||||
|
}
|
||||||
|
|
||||||
func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) {
|
func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) {
|
||||||
rvChan := make(chan Conn)
|
rvChan := make(chan Conn)
|
||||||
req := gangerConnRequest{
|
req := gangerConnRequest{
|
||||||
@@ -81,7 +110,7 @@ func (g *Ganger) run() {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
scoutCollectedChan := make(chan []time.Duration)
|
scoutCollectedChan := make(chan scoutRaidResult)
|
||||||
currentScoutCollectedChan := scoutCollectedChan
|
currentScoutCollectedChan := scoutCollectedChan
|
||||||
|
|
||||||
updatedStatsChan := make(chan *Stats)
|
updatedStatsChan := make(chan *Stats)
|
||||||
@@ -94,18 +123,29 @@ func (g *Ganger) run() {
|
|||||||
select {
|
select {
|
||||||
case <-g.ctx.Done():
|
case <-g.ctx.Done():
|
||||||
return
|
return
|
||||||
case durations := <-currentScoutCollectedChan:
|
case result := <-currentScoutCollectedChan:
|
||||||
g.durations = append(g.durations, durations...)
|
g.durations = append(g.durations, result.durations...)
|
||||||
|
|
||||||
if len(g.durations) > DoppelGangerMaxDurations {
|
if len(g.durations) > DoppelGangerMaxDurations {
|
||||||
copy(g.durations, g.durations[len(g.durations)-DoppelGangerMaxDurations:])
|
copy(g.durations, g.durations[len(g.durations)-DoppelGangerMaxDurations:])
|
||||||
g.durations = g.durations[:DoppelGangerMaxDurations]
|
g.durations = g.durations[:DoppelGangerMaxDurations]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update cert sizes and recompute noise params.
|
||||||
|
g.certSizes = append(g.certSizes, result.certSizes...)
|
||||||
|
if len(g.certSizes) > DoppelGangerMaxDurations {
|
||||||
|
g.certSizes = g.certSizes[len(g.certSizes)-DoppelGangerMaxDurations:]
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(g.certSizes) >= MinCertSizesToCalculate {
|
||||||
|
g.updateNoiseParams()
|
||||||
|
}
|
||||||
|
|
||||||
if len(g.durations) < MinDurationsToCalculate {
|
if len(g.durations) < MinDurationsToCalculate {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
durations := g.durations
|
||||||
currentScoutCollectedChan = nil
|
currentScoutCollectedChan = nil
|
||||||
g.wg.Go(func() {
|
g.wg.Go(func() {
|
||||||
select {
|
select {
|
||||||
@@ -129,8 +169,45 @@ func (g *Ganger) run() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
|
func (g *Ganger) updateNoiseParams() {
|
||||||
durations := []time.Duration{}
|
if len(g.certSizes) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sum := 0
|
||||||
|
for _, s := range g.certSizes {
|
||||||
|
sum += s
|
||||||
|
}
|
||||||
|
|
||||||
|
mean := sum / len(g.certSizes)
|
||||||
|
|
||||||
|
maxDev := 0
|
||||||
|
for _, s := range g.certSizes {
|
||||||
|
d := s - mean
|
||||||
|
if d < 0 {
|
||||||
|
d = -d
|
||||||
|
}
|
||||||
|
|
||||||
|
if d > maxDev {
|
||||||
|
maxDev = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if maxDev < 100 {
|
||||||
|
maxDev = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
np := &NoiseParams{Mean: mean, Jitter: maxDev}
|
||||||
|
g.noiseParams.Store(np)
|
||||||
|
|
||||||
|
g.logger.Info(fmt.Sprintf(
|
||||||
|
"updated noise params: mean=%d jitter=%d samples=%d",
|
||||||
|
mean, maxDev, len(g.certSizes),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Ganger) runScoutRaid(rvChan chan<- scoutRaidResult) {
|
||||||
|
var result scoutRaidResult
|
||||||
|
|
||||||
for range g.scoutRaidRepeats {
|
for range g.scoutRaidRepeats {
|
||||||
learned, err := g.scout.Learn(g.ctx)
|
learned, err := g.scout.Learn(g.ctx)
|
||||||
@@ -138,13 +215,18 @@ func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
|
|||||||
g.logger.WarningError("cannot learn", err)
|
g.logger.WarningError("cannot learn", err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
durations = append(durations, learned...)
|
|
||||||
|
result.durations = append(result.durations, learned.Durations...)
|
||||||
|
|
||||||
|
if learned.CertSize > 0 {
|
||||||
|
result.certSizes = append(result.certSizes, learned.CertSize)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-g.ctx.Done():
|
case <-g.ctx.Done():
|
||||||
return
|
return
|
||||||
case rvChan <- durations:
|
case rvChan <- result:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,36 +12,46 @@ import (
|
|||||||
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
|
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ScoutResult holds measurements from a single scout HTTP request.
|
||||||
|
type ScoutResult struct {
|
||||||
|
Durations []time.Duration
|
||||||
|
CertSize int // total ApplicationData bytes during TLS handshake; 0 if unknown
|
||||||
|
}
|
||||||
|
|
||||||
type Scout struct {
|
type Scout struct {
|
||||||
network Network
|
network Network
|
||||||
urls []string
|
urls []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s Scout) Learn(ctx context.Context) ([]time.Duration, error) {
|
func (s Scout) Learn(ctx context.Context) (ScoutResult, error) {
|
||||||
var durations []time.Duration
|
var combined ScoutResult
|
||||||
|
|
||||||
for _, url := range s.urls {
|
for _, url := range s.urls {
|
||||||
learned, err := s.learn(ctx, url)
|
learned, err := s.learn(ctx, url)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return ScoutResult{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
durations = append(durations, learned...)
|
combined.Durations = append(combined.Durations, learned.Durations...)
|
||||||
|
|
||||||
|
if learned.CertSize > 0 && combined.CertSize == 0 {
|
||||||
|
combined.CertSize = learned.CertSize
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return durations, nil
|
return combined, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
|
func (s Scout) learn(ctx context.Context, url string) (ScoutResult, error) {
|
||||||
client, results := s.makeClient()
|
client, results := s.makeClient()
|
||||||
|
|
||||||
if !strings.HasPrefix(url, "https://") {
|
if !strings.HasPrefix(url, "https://") {
|
||||||
return nil, fmt.Errorf("url %s must be https", url)
|
return ScoutResult{}, fmt.Errorf("url %s must be https", url)
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return ScoutResult{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
resp, err := client.Do(req)
|
||||||
@@ -52,10 +62,12 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil || len(results.data) == 0 {
|
if err != nil || len(results.data) == 0 {
|
||||||
return nil, err
|
return ScoutResult{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
durations := []time.Duration{}
|
var result ScoutResult
|
||||||
|
|
||||||
|
// Compute inter-record durations (existing logic).
|
||||||
lastTimestamp := time.Time{}
|
lastTimestamp := time.Time{}
|
||||||
|
|
||||||
for i, v := range results.data {
|
for i, v := range results.data {
|
||||||
@@ -71,11 +83,34 @@ func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
durations = append(durations, v.timestamp.Sub(lastTimestamp))
|
result.Durations = append(result.Durations, v.timestamp.Sub(lastTimestamp))
|
||||||
lastTimestamp = v.timestamp
|
lastTimestamp = v.timestamp
|
||||||
}
|
}
|
||||||
|
|
||||||
return durations, nil
|
// Compute cert size: sum of ApplicationData payload between CCS and
|
||||||
|
// the first client Write (which marks the end of server handshake).
|
||||||
|
seenCCS := false
|
||||||
|
boundary := results.writeIndex
|
||||||
|
if boundary < 0 {
|
||||||
|
boundary = len(results.data)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, v := range results.data {
|
||||||
|
if i >= boundary {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if v.recordType == tls.TypeChangeCipherSpec {
|
||||||
|
seenCCS = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if seenCCS && v.recordType == tls.TypeApplicationData {
|
||||||
|
result.CertSize += v.payloadLen
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) {
|
func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) {
|
||||||
|
|||||||
@@ -14,9 +14,10 @@ type ScoutConn struct {
|
|||||||
|
|
||||||
results *ScoutConnCollected
|
results *ScoutConnCollected
|
||||||
rawBuf *bytes.Buffer
|
rawBuf *bytes.Buffer
|
||||||
|
seenCCS bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s ScoutConn) Read(p []byte) (int, error) {
|
func (s *ScoutConn) Read(p []byte) (int, error) {
|
||||||
buf := &bytes.Buffer{}
|
buf := &bytes.Buffer{}
|
||||||
|
|
||||||
for {
|
for {
|
||||||
@@ -31,7 +32,11 @@ func (s ScoutConn) Read(p []byte) (int, error) {
|
|||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
s.results.Add(recordType)
|
if recordType == tls.TypeChangeCipherSpec {
|
||||||
|
s.seenCCS = true
|
||||||
|
}
|
||||||
|
|
||||||
|
s.results.Add(recordType, int(length))
|
||||||
s.rawBuf.Write([]byte{recordType})
|
s.rawBuf.Write([]byte{recordType})
|
||||||
s.rawBuf.Write(tls.TLSVersion[:])
|
s.rawBuf.Write(tls.TLSVersion[:])
|
||||||
|
|
||||||
@@ -45,11 +50,19 @@ func (s ScoutConn) Read(p []byte) (int, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) ScoutConn {
|
func (s *ScoutConn) Write(p []byte) (int, error) {
|
||||||
|
if s.seenCCS {
|
||||||
|
s.results.MarkWrite()
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.Conn.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) *ScoutConn {
|
||||||
rawBuf := &bytes.Buffer{}
|
rawBuf := &bytes.Buffer{}
|
||||||
rawBuf.Grow(tls.MaxRecordSize)
|
rawBuf.Grow(tls.MaxRecordSize)
|
||||||
|
|
||||||
return ScoutConn{
|
return &ScoutConn{
|
||||||
Conn: tls.New(conn, false, false),
|
Conn: tls.New(conn, false, false),
|
||||||
results: results,
|
results: results,
|
||||||
rawBuf: rawBuf,
|
rawBuf: rawBuf,
|
||||||
|
|||||||
@@ -9,21 +9,32 @@ const (
|
|||||||
type ScoutConnResult struct {
|
type ScoutConnResult struct {
|
||||||
timestamp time.Time
|
timestamp time.Time
|
||||||
recordType byte
|
recordType byte
|
||||||
|
payloadLen int
|
||||||
}
|
}
|
||||||
|
|
||||||
type ScoutConnCollected struct {
|
type ScoutConnCollected struct {
|
||||||
data []ScoutConnResult
|
data []ScoutConnResult
|
||||||
|
writeIndex int // index at which client first wrote post-handshake data; -1 if not set
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ScoutConnCollected) Add(record byte) {
|
func (s *ScoutConnCollected) Add(record byte, payloadLen int) {
|
||||||
s.data = append(s.data, ScoutConnResult{
|
s.data = append(s.data, ScoutConnResult{
|
||||||
timestamp: time.Now(),
|
timestamp: time.Now(),
|
||||||
recordType: record,
|
recordType: record,
|
||||||
|
payloadLen: payloadLen,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MarkWrite records the current data length as the handshake boundary.
|
||||||
|
func (s *ScoutConnCollected) MarkWrite() {
|
||||||
|
if s.writeIndex < 0 {
|
||||||
|
s.writeIndex = len(s.data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func NewScoutConnCollected() *ScoutConnCollected {
|
func NewScoutConnCollected() *ScoutConnCollected {
|
||||||
return &ScoutConnCollected{
|
return &ScoutConnCollected{
|
||||||
data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
|
data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
|
||||||
|
writeIndex: -1,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ type ScoutConnCollectedTestSuite struct {
|
|||||||
|
|
||||||
func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
|
func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
|
||||||
collected := NewScoutConnCollected()
|
collected := NewScoutConnCollected()
|
||||||
collected.Add(tls.TypeApplicationData)
|
collected.Add(tls.TypeApplicationData, 100)
|
||||||
|
|
||||||
suite.Len(collected.data, 1)
|
suite.Len(collected.data, 1)
|
||||||
suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
|
suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
|
||||||
@@ -23,13 +23,13 @@ func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
|
|||||||
func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
|
func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
|
||||||
collected := NewScoutConnCollected()
|
collected := NewScoutConnCollected()
|
||||||
|
|
||||||
collected.Add(tls.TypeApplicationData)
|
collected.Add(tls.TypeApplicationData, 100)
|
||||||
|
|
||||||
time.Sleep(time.Microsecond)
|
time.Sleep(time.Microsecond)
|
||||||
collected.Add(tls.TypeApplicationData)
|
collected.Add(tls.TypeApplicationData, 100)
|
||||||
|
|
||||||
time.Sleep(time.Microsecond)
|
time.Sleep(time.Microsecond)
|
||||||
collected.Add(tls.TypeApplicationData)
|
collected.Add(tls.TypeApplicationData, 100)
|
||||||
|
|
||||||
for i := 1; i < len(collected.data); i++ {
|
for i := 1; i < len(collected.data); i++ {
|
||||||
suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))
|
suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))
|
||||||
|
|||||||
@@ -22,9 +22,9 @@ func (suite *ScoutTestSuite) SetupSuite() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (suite *ScoutTestSuite) TestCollectResults() {
|
func (suite *ScoutTestSuite) TestCollectResults() {
|
||||||
durations, err := suite.scout.Learn(suite.ctx)
|
result, err := suite.scout.Learn(suite.ctx)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
suite.Less(3, len(durations))
|
suite.Less(3, len(result.Durations))
|
||||||
}
|
}
|
||||||
|
|
||||||
func (suite *ScoutTestSuite) TestCollectNothing() {
|
func (suite *ScoutTestSuite) TestCollectNothing() {
|
||||||
|
|||||||
@@ -1,261 +0,0 @@
|
|||||||
package fake
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/tls"
|
|
||||||
"encoding/binary"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
probeDialTimeout = 10 * time.Second
|
|
||||||
probeHandshakeTimeout = 10 * time.Second
|
|
||||||
defaultProbeCount = 15
|
|
||||||
defaultCacheTTL = 24 * time.Hour
|
|
||||||
|
|
||||||
tlsTypeChangeCipherSpec = 0x14
|
|
||||||
tlsTypeApplicationData = 0x17
|
|
||||||
)
|
|
||||||
|
|
||||||
// CertProbeResult holds the measured encrypted handshake size.
|
|
||||||
type CertProbeResult struct {
|
|
||||||
Mean int `json:"mean"`
|
|
||||||
Jitter int `json:"jitter"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// CertProbeCache is the on-disk format for cached probe results.
|
|
||||||
type CertProbeCache struct {
|
|
||||||
Hostname string `json:"hostname"`
|
|
||||||
Port int `json:"port"`
|
|
||||||
Mean int `json:"mean"`
|
|
||||||
Jitter int `json:"jitter"`
|
|
||||||
ProbedAt time.Time `json:"probed_at"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadCachedProbe reads a cached probe result from path. Returns the result
|
|
||||||
// and true if the cache exists, matches hostname:port, and is younger than ttl.
|
|
||||||
// Otherwise returns zero value and false.
|
|
||||||
func LoadCachedProbe(path, hostname string, port int, ttl time.Duration) (CertProbeResult, bool) {
|
|
||||||
if ttl <= 0 {
|
|
||||||
ttl = defaultCacheTTL
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return CertProbeResult{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
var cache CertProbeCache
|
|
||||||
if err := json.Unmarshal(data, &cache); err != nil {
|
|
||||||
return CertProbeResult{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if cache.Hostname != hostname || cache.Port != port {
|
|
||||||
return CertProbeResult{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if time.Since(cache.ProbedAt) > ttl {
|
|
||||||
return CertProbeResult{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if cache.Mean <= 0 {
|
|
||||||
return CertProbeResult{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return CertProbeResult{Mean: cache.Mean, Jitter: cache.Jitter}, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// SaveCachedProbe writes a probe result to path as JSON.
|
|
||||||
func SaveCachedProbe(path, hostname string, port int, result CertProbeResult) error {
|
|
||||||
cache := CertProbeCache{
|
|
||||||
Hostname: hostname,
|
|
||||||
Port: port,
|
|
||||||
Mean: result.Mean,
|
|
||||||
Jitter: result.Jitter,
|
|
||||||
ProbedAt: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := json.MarshalIndent(cache, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return os.WriteFile(path, data, 0o644) //nolint: gosec
|
|
||||||
}
|
|
||||||
|
|
||||||
// ProbeCertSize connects to hostname:port via TLS multiple times and measures
|
|
||||||
// the total ApplicationData payload bytes sent by the server during the
|
|
||||||
// handshake (between ChangeCipherSpec and the first application-level data).
|
|
||||||
// This corresponds to EncryptedExtensions + Certificate + CertificateVerify +
|
|
||||||
// Finished in TLS 1.3, which is what the FakeTLS noise must mimic.
|
|
||||||
func ProbeCertSize(hostname string, port int, count int) (CertProbeResult, error) {
|
|
||||||
if count <= 0 {
|
|
||||||
count = defaultProbeCount
|
|
||||||
}
|
|
||||||
|
|
||||||
addr := net.JoinHostPort(hostname, fmt.Sprintf("%d", port))
|
|
||||||
sizes := make([]int, 0, count)
|
|
||||||
|
|
||||||
for i := 0; i < count; i++ {
|
|
||||||
size, err := probeSingle(addr, hostname)
|
|
||||||
if err != nil {
|
|
||||||
if len(sizes) > 0 {
|
|
||||||
break // use what we have
|
|
||||||
}
|
|
||||||
|
|
||||||
return CertProbeResult{}, fmt.Errorf("probe %d failed: %w", i, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
sizes = append(sizes, size)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(sizes) == 0 {
|
|
||||||
return CertProbeResult{}, fmt.Errorf("no successful probes")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate mean and jitter (max deviation from mean).
|
|
||||||
sum := 0
|
|
||||||
for _, s := range sizes {
|
|
||||||
sum += s
|
|
||||||
}
|
|
||||||
|
|
||||||
mean := sum / len(sizes)
|
|
||||||
|
|
||||||
maxDev := 0
|
|
||||||
for _, s := range sizes {
|
|
||||||
d := s - mean
|
|
||||||
if d < 0 {
|
|
||||||
d = -d
|
|
||||||
}
|
|
||||||
|
|
||||||
if d > maxDev {
|
|
||||||
maxDev = d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure minimum jitter of 100 bytes for variability.
|
|
||||||
if maxDev < 100 {
|
|
||||||
maxDev = 100
|
|
||||||
}
|
|
||||||
|
|
||||||
return CertProbeResult{Mean: mean, Jitter: maxDev}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// probeSingle does one TLS handshake and measures ApplicationData bytes
|
|
||||||
// received during the handshake.
|
|
||||||
func probeSingle(addr, hostname string) (int, error) {
|
|
||||||
rawConn, err := net.DialTimeout("tcp", addr, probeDialTimeout)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
defer rawConn.Close() //nolint: errcheck
|
|
||||||
|
|
||||||
capture := &recordCapture{conn: rawConn}
|
|
||||||
|
|
||||||
tlsConn := tls.Client(capture, &tls.Config{
|
|
||||||
ServerName: hostname,
|
|
||||||
MinVersion: tls.VersionTLS12,
|
|
||||||
})
|
|
||||||
tlsConn.SetDeadline(time.Now().Add(probeHandshakeTimeout)) //nolint: errcheck
|
|
||||||
|
|
||||||
if err := tlsConn.Handshake(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
tlsConn.Close() //nolint: errcheck
|
|
||||||
|
|
||||||
return capture.appDataBytes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordCapture wraps a net.Conn and parses the raw TLS record stream to
|
|
||||||
// measure ApplicationData payload sizes sent by the server during handshake.
|
|
||||||
// It tracks record boundaries by maintaining a state machine over Read calls.
|
|
||||||
type recordCapture struct {
|
|
||||||
conn net.Conn
|
|
||||||
mu sync.Mutex
|
|
||||||
appDataBytes int
|
|
||||||
seenCCS bool
|
|
||||||
done bool
|
|
||||||
|
|
||||||
// Record boundary tracking for the read side.
|
|
||||||
readRemaining int // bytes left in current record payload
|
|
||||||
readHeaderBuf [5]byte
|
|
||||||
readHeaderPos int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rc *recordCapture) Read(p []byte) (int, error) {
|
|
||||||
n, err := rc.conn.Read(p)
|
|
||||||
if n > 0 && !rc.done {
|
|
||||||
rc.mu.Lock()
|
|
||||||
rc.parseReadBytes(p[:n])
|
|
||||||
rc.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
return n, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rc *recordCapture) parseReadBytes(data []byte) {
|
|
||||||
for len(data) > 0 {
|
|
||||||
if rc.readRemaining > 0 {
|
|
||||||
// Consuming payload of current record.
|
|
||||||
consume := rc.readRemaining
|
|
||||||
if consume > len(data) {
|
|
||||||
consume = len(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
rc.readRemaining -= consume
|
|
||||||
data = data[consume:]
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Accumulate header bytes (5 bytes per record).
|
|
||||||
need := 5 - rc.readHeaderPos
|
|
||||||
if need > len(data) {
|
|
||||||
need = len(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
copy(rc.readHeaderBuf[rc.readHeaderPos:], data[:need])
|
|
||||||
rc.readHeaderPos += need
|
|
||||||
data = data[need:]
|
|
||||||
|
|
||||||
if rc.readHeaderPos < 5 {
|
|
||||||
return // incomplete header
|
|
||||||
}
|
|
||||||
|
|
||||||
// Full header available.
|
|
||||||
recordType := rc.readHeaderBuf[0]
|
|
||||||
payloadLen := int(binary.BigEndian.Uint16(rc.readHeaderBuf[3:5]))
|
|
||||||
rc.readHeaderPos = 0
|
|
||||||
rc.readRemaining = payloadLen
|
|
||||||
|
|
||||||
if recordType == tlsTypeChangeCipherSpec {
|
|
||||||
rc.seenCCS = true
|
|
||||||
} else if recordType == tlsTypeApplicationData && rc.seenCCS {
|
|
||||||
rc.appDataBytes += payloadLen
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rc *recordCapture) Write(p []byte) (int, error) {
|
|
||||||
// After client writes post-CCS data, server handshake records are done.
|
|
||||||
if rc.seenCCS && rc.appDataBytes > 0 {
|
|
||||||
rc.done = true
|
|
||||||
}
|
|
||||||
|
|
||||||
return rc.conn.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (rc *recordCapture) Close() error { return rc.conn.Close() }
|
|
||||||
func (rc *recordCapture) LocalAddr() net.Addr { return rc.conn.LocalAddr() }
|
|
||||||
func (rc *recordCapture) RemoteAddr() net.Addr { return rc.conn.RemoteAddr() }
|
|
||||||
func (rc *recordCapture) SetDeadline(t time.Time) error { return rc.conn.SetDeadline(t) }
|
|
||||||
func (rc *recordCapture) SetReadDeadline(t time.Time) error { return rc.conn.SetReadDeadline(t) }
|
|
||||||
func (rc *recordCapture) SetWriteDeadline(t time.Time) error { return rc.conn.SetWriteDeadline(t) }
|
|
||||||
|
|
||||||
// Ensure recordCapture implements net.Conn.
|
|
||||||
var _ net.Conn = (*recordCapture)(nil)
|
|
||||||
+4
-42
@@ -36,7 +36,6 @@ type Proxy struct {
|
|||||||
doppelGanger *doppel.Ganger
|
doppelGanger *doppel.Ganger
|
||||||
clientObfuscatror obfuscation.Obfuscator
|
clientObfuscatror obfuscation.Obfuscator
|
||||||
|
|
||||||
noiseParams fake.NoiseParams
|
|
||||||
secret Secret
|
secret Secret
|
||||||
network Network
|
network Network
|
||||||
antiReplayCache AntiReplayCache
|
antiReplayCache AntiReplayCache
|
||||||
@@ -193,7 +192,10 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello, p.noiseParams); err != nil {
|
gangerNoise := p.doppelGanger.NoiseParams()
|
||||||
|
noiseParams := fake.NoiseParams{Mean: gangerNoise.Mean, Jitter: gangerNoise.Jitter}
|
||||||
|
|
||||||
|
if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello, noiseParams); err != nil {
|
||||||
p.logger.InfoError("cannot send welcome packet", err)
|
p.logger.InfoError("cannot send welcome packet", err)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -324,49 +326,9 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {
|
|||||||
logger := opts.getLogger("proxy")
|
logger := opts.getLogger("proxy")
|
||||||
updatersLogger := logger.Named("telegram-updaters")
|
updatersLogger := logger.Named("telegram-updaters")
|
||||||
|
|
||||||
// Probe the fronting domain's cert chain size for noise calibration.
|
|
||||||
probeHost := opts.Secret.Host
|
|
||||||
probePort := opts.getDomainFrontingPort()
|
|
||||||
noiseParams := fake.NoiseParams{}
|
|
||||||
|
|
||||||
probeCount := int(opts.NoiseProbeCount)
|
|
||||||
if probeCount <= 0 {
|
|
||||||
probeCount = 15
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheTTL := opts.NoiseCacheTTL
|
|
||||||
|
|
||||||
// Try loading from cache first.
|
|
||||||
if opts.NoiseCachePath != "" {
|
|
||||||
if cached, ok := fake.LoadCachedProbe(opts.NoiseCachePath, probeHost, probePort, cacheTTL); ok {
|
|
||||||
noiseParams = fake.NoiseParams(cached)
|
|
||||||
logger.Info(fmt.Sprintf("cert probe: loaded from cache, host=%s mean=%d jitter=%d",
|
|
||||||
probeHost, cached.Mean, cached.Jitter))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no cached result, probe live.
|
|
||||||
if noiseParams.Mean == 0 {
|
|
||||||
probeResult, probeErr := fake.ProbeCertSize(probeHost, probePort, probeCount)
|
|
||||||
if probeErr != nil {
|
|
||||||
logger.WarningError("cert probe failed, using default noise size", probeErr)
|
|
||||||
} else {
|
|
||||||
noiseParams = fake.NoiseParams(probeResult)
|
|
||||||
logger.Info(fmt.Sprintf("cert probe: host=%s mean=%d jitter=%d",
|
|
||||||
probeHost, probeResult.Mean, probeResult.Jitter))
|
|
||||||
|
|
||||||
if opts.NoiseCachePath != "" {
|
|
||||||
if saveErr := fake.SaveCachedProbe(opts.NoiseCachePath, probeHost, probePort, probeResult); saveErr != nil {
|
|
||||||
logger.WarningError("failed to save cert probe cache", saveErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
proxy := &Proxy{
|
proxy := &Proxy{
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
ctxCancel: cancel,
|
ctxCancel: cancel,
|
||||||
noiseParams: noiseParams,
|
|
||||||
secret: opts.Secret,
|
secret: opts.Secret,
|
||||||
network: opts.Network,
|
network: opts.Network,
|
||||||
antiReplayCache: opts.AntiReplayCache,
|
antiReplayCache: opts.AntiReplayCache,
|
||||||
|
|||||||
@@ -161,24 +161,6 @@ type ProxyOpts struct {
|
|||||||
// DoppelGangerDRS defines if TLS Dynamic Record Sizing is active.
|
// DoppelGangerDRS defines if TLS Dynamic Record Sizing is active.
|
||||||
DoppelGangerDRS bool
|
DoppelGangerDRS bool
|
||||||
|
|
||||||
// NoiseProbeCount is the number of TLS connections to make when probing
|
|
||||||
// the fronting domain's cert chain size for noise calibration.
|
|
||||||
// Default is 15.
|
|
||||||
//
|
|
||||||
// This is an optional setting.
|
|
||||||
NoiseProbeCount uint
|
|
||||||
|
|
||||||
// NoiseCacheTTL is how long a cached cert probe result is considered
|
|
||||||
// valid. Default is 24 hours.
|
|
||||||
//
|
|
||||||
// This is an optional setting.
|
|
||||||
NoiseCacheTTL time.Duration
|
|
||||||
|
|
||||||
// NoiseCachePath is the file path for caching cert probe results
|
|
||||||
// between restarts. If empty, no caching is performed.
|
|
||||||
//
|
|
||||||
// This is an optional setting.
|
|
||||||
NoiseCachePath string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p ProxyOpts) valid() error {
|
func (p ProxyOpts) valid() error {
|
||||||
|
|||||||
Reference in New Issue
Block a user