REPOSITORY / ScuroNeko/mtg

Compare commits

DIFF REPOSITORY

Compare commits

...
122 Commits
Author SHA1 Message Date
9seconds fc72de9e39 Merge remote-tracking branch 'origin/stable' into v2 2026-03-19 18:52:35 +01:00
9seconds cb627f2a66 Merge remote-tracking branch 'origin/master' into stable 2026-03-19 18:52:11 +01:00
Sergei ArkhipovandGitHub 9ba6df0d1c Merge pull request #383 from 9seconds/avoid-double-buffering
Avoid double buffering in TLS hot path
2026-03-19 17:46:36 +01:00
9seconds 4a8d099aca Remove unused buffer 2026-03-19 17:39:57 +01:00
9seconds feb57004e1 Fix reslicing 2026-03-19 17:39:48 +01:00
9seconds cb436efd87 Avoid double buffering in TLS hot path 2026-03-19 17:37:51 +01:00
Sergei ArkhipovandGitHub 24148ea95c Merge pull request #382 from 9seconds/write-cond
Optimize waiting time for TLS chunker
2026-03-19 15:51:11 +01:00
9seconds 724904f50d Wait in doppel.Conn if there is anything to write 2026-03-19 15:42:00 +01:00
9seconds a23ae05f3b Remove SyncWrite 2026-03-19 13:47:08 +01:00
Sergei ArkhipovandGitHub b153a55149 Merge pull request #379 from 9seconds/fix-telegram-ips
Show ip of telegram endpoints in event stream
2026-03-18 22:46:47 +01:00
9seconds 913a38d13a Show real IP of the telegram endpoint in event stream 2026-03-18 22:05:34 +01:00
9seconds dc81f7981c Merge remote-tracking branch 'origin/stable' into v2 2026-03-16 23:56:10 +01:00
9seconds 9d5fd989e5 Merge remote-tracking branch 'origin/master' into stable 2026-03-16 23:55:56 +01:00
Sergei ArkhipovandGitHub 81703233b0 Merge pull request #368 from 9seconds/flake-tests
Fix flaky test
2026-03-16 23:55:01 +01:00
9seconds eb7720b11e Fix flaky test 2026-03-16 23:44:06 +01:00
Sergei ArkhipovandGitHub df7ddc3d6a Merge pull request #367 from saleacy/patch-1
fix: ensure network.Dial and MakeHTTPClient use socks5 proxy
2026-03-16 23:38:43 +01:00
saleacyandGitHub 3bc1e415f9 fix: ensure network.Dial and MakeHTTPClient use socks5 proxy
The package `network/v2/proxy_network.go` does not wrap `network.Dial`
and `network.MakeHTTPClient`, which causes them to bypass the SOCKS5
proxy and initiate TCP connections directly from the local machine.
2026-03-17 01:35:18 +08:00
Sergei ArkhipovandGitHub 306fa19ad6 Merge pull request #366 from Maks-2012/patch-1
Fix preferIPOnlyIPv6
2026-03-16 15:31:09 +01:00
Maks-2012andGitHub 079252d810 Fix preferIPOnlyIPv6 2026-03-16 16:10:38 +03:00
9seconds d0502e7083 Merge remote-tracking branch 'origin/stable' into v2 2026-03-15 22:04:35 +01:00
9seconds 30aa9d3a44 Merge remote-tracking branch 'origin/master' into stable 2026-03-15 22:04:09 +01:00
9seconds d98d5be3a7 Update go dependencies 2026-03-15 22:01:04 +01:00
9seconds 94ac9d6ffa Update tooling 2026-03-15 22:00:08 +01:00
9seconds c036558df4 Update docs 2026-03-15 21:58:04 +01:00
Sergei ArkhipovandGitHub 4167dec5e1 Merge pull request #365 from 9seconds/best-practices
Add document about best practices
2026-03-15 21:52:16 +01:00
9seconds 57529904e3 Add document about best practices 2026-03-15 21:50:43 +01:00
Sergei ArkhipovandGitHub ed4e511560 Merge pull request #363 from 9seconds/relay-buffer
Decrease relay buffer size
2026-03-14 14:15:52 +01:00
9seconds bb90bcc127 Decrease relay buffer size
Even if it makes sense to have a huge buffers, we do artificial delays
now. In that case we could achieve the same results with a lower buffer.
If not, then we won't send a packet bigger that this value
2026-03-14 14:06:48 +01:00
9seconds 879ebd132d Formatting of the README 2026-03-14 09:44:07 +01:00
Sergei ArkhipovandGitHub 58e6dfd603 Merge pull request #361 from 9seconds/docs
Add docs about doppelganger
2026-03-14 09:43:19 +01:00
9seconds e675baa860 Add docs about doppelganger 2026-03-14 09:42:28 +01:00
Sergei ArkhipovandGitHub 4a17849763 Merge pull request #360 from 9seconds/fix-android
Have a noise of variable size
2026-03-13 17:11:46 +01:00
9seconds 7adfc0352e Have a noise of variable size 2026-03-13 17:08:44 +01:00
Sergei ArkhipovandGitHub a5d602b538 Merge pull request #359 from 9seconds/fix-android
Fix android ping
2026-03-13 16:46:18 +01:00
9seconds b35c212d65 Have a noise of variable size 2026-03-13 16:39:16 +01:00
9seconds 45b0964afd Fix android ping 2026-03-13 16:20:18 +01:00
Sergei ArkhipovandGitHub 6d8d2961e8 Merge pull request #358 from 9seconds/doppel-drs
Make DRS optional
2026-03-13 14:51:15 +01:00
9seconds 4138cc6494 Use SyncWrite call to send a packet to proxy 2026-03-13 14:44:56 +01:00
9seconds 33c0fa9bf7 Add SyncWrite method to doppel.Conn 2026-03-13 11:30:52 +01:00
9seconds 21d7522356 Make DRS optional 2026-03-13 11:04:01 +01:00
9seconds ea71fe81b2 Update stats defaults 2026-03-13 08:59:59 +01:00
Sergei ArkhipovandGitHub 0871c2d790 Merge pull request #357 from 9seconds/access-native-dialer
Access command should use native dialer instead of network one
2026-03-13 08:47:24 +01:00
9seconds 8df1629882 Access command should use native dialer instead of network one 2026-03-13 08:42:15 +01:00
9seconds ad945c77a9 Enable caching for govulncheck 2026-03-12 23:01:26 +01:00
Sergei ArkhipovandGitHub a051518def Merge pull request #356 from 9seconds/gh-cache
Speed up Github actions
2026-03-12 22:51:59 +01:00
9seconds 4dca1d2b07 Add cache mount for apk downloads 2026-03-12 22:46:19 +01:00
9seconds 287a794772 Always run tests with race detector 2026-03-12 22:22:09 +01:00
9seconds d4822989ec Add count number to covtest 2026-03-12 22:22:00 +01:00
Sergei ArkhipovandGitHub d6a17c502f Merge pull request #355 from 9seconds/scout-native
Use native dialer for scout
2026-03-12 22:14:52 +01:00
9seconds 3db1be0687 Use cache to speed up jobs 2026-03-12 22:13:17 +01:00
9seconds 991346621c Speedup docker builds by using cache 2026-03-12 22:13:00 +01:00
9seconds cc63e762e3 Use native dialer for scout 2026-03-12 21:47:03 +01:00
9seconds a85348d6be Use curl as user agent 2026-03-12 21:43:04 +01:00
Sergei ArkhipovandGitHub 7762cf3a55 Merge pull request #353 from 9seconds/domain-fronting-native
Use native dialer to communicate with fronting domain
2026-03-12 21:40:38 +01:00
9seconds 62cba24071 Use native dialer to communicate with fronting domain 2026-03-12 20:51:02 +01:00
9seconds 94e4179fb7 Add NativeDialer method to mtglib.Network 2026-03-12 19:25:53 +01:00
Sergei ArkhipovandGitHub 6493688282 Merge pull request #352 from 9seconds/doppleganger
Doppleganger
2026-03-12 19:18:44 +01:00
9seconds ffd3ab03cc Formatting with gofumpt 2026-03-12 19:13:33 +01:00
9seconds 00403e3a94 Remove old faketls package 2026-03-12 19:11:39 +01:00
9seconds 0bfc1ef2d4 Fix double TLS wrapping for noise 2026-03-12 19:07:11 +01:00
9seconds 7aa01dcebe Limit min number of collected stats 2026-03-12 19:07:11 +01:00
9seconds 1c0e847247 Set statistics measured from real life 2026-03-12 19:07:11 +01:00
9seconds 83ff4ee266 Align scout defaults with config defaults 2026-03-12 19:07:11 +01:00
9seconds 7a58c74cfe Correctly process doppel conns 2026-03-12 19:07:11 +01:00
9seconds d43d6692d7 Align naming about scout raids 2026-03-12 19:07:11 +01:00
9seconds 75392941da Propagate settings to Proxy 2026-03-12 19:07:11 +01:00
9seconds 25ad776b6f Propagate doppelganger to config 2026-03-12 19:07:11 +01:00
9seconds 5557393b38 Add config section 2026-03-12 19:07:11 +01:00
9seconds bb49c6a55d Integrate new fake package and doppel into proxy 2026-03-12 19:07:11 +01:00
9seconds 37f8d18be5 Add server side of things 2026-03-12 19:07:11 +01:00
9seconds 59557059df ReadClientHello function 2026-03-12 19:07:10 +01:00
9seconds 1182b9ef6f Add doppel and tls packages 2026-03-12 19:07:10 +01:00
9seconds c886ffdd81 Add TrustTLS method to networkHTTPTransport 2026-03-11 23:21:41 +01:00
Sergei ArkhipovandGitHub e9b38a5b74 Merge pull request #348 from 9seconds/bundle-config 2026-03-05 19:09:48 +01:00
9seconds 036b10be67 Include example config into distribution bundle 2026-03-05 17:14:52 +01:00
9seconds fefc479f94 Merge remote-tracking branch 'origin/stable' into v2 2026-02-28 14:52:05 +01:00
9seconds 9f12620dba Merge remote-tracking branch 'origin/master' into stable 2026-02-28 14:51:26 +01:00
9seconds 23aa2eefad More details on DOH-IP option 2026-02-28 14:50:48 +01:00
9seconds ddc34bf918 Comment out doh-ip 2026-02-28 14:48:37 +01:00
9seconds f5244c2bfd Merge remote-tracking branch 'origin/stable' into v2 2026-02-28 14:47:43 +01:00
9seconds 9946cdfeb5 Merge remote-tracking branch 'origin/master' into stable 2026-02-28 14:47:25 +01:00
Sergei ArkhipovandGitHub ffe052617b Merge pull request #346 from 9seconds/simple-network
Simplify network implementation
2026-02-27 16:25:23 +01:00
9seconds 700417b1d0 Integrate DNS resolver in our codebase 2026-02-27 16:23:24 +01:00
9seconds 897e6bf505 Propagate DNS setting to configuration 2026-02-27 16:23:24 +01:00
9seconds 317d7380cb Add support for custom DNS resolvers 2026-02-27 16:08:28 +01:00
9seconds 1151291535 Use network v2 instead of network v1 2026-02-27 15:52:29 +01:00
9seconds 42927c8bdc Add v2 network package 2026-02-27 15:50:21 +01:00
9seconds 7d748077e9 Merge remote-tracking branch 'origin/stable' into v2 2026-02-26 13:53:32 +01:00
9seconds f61a63a59b Merge remote-tracking branch 'origin/master' into stable 2026-02-26 13:53:17 +01:00
9seconds 282896be09 Update own dependencies 2026-02-26 13:52:26 +01:00
9seconds 3809f034ad Update go packages 2026-02-26 13:51:29 +01:00
Sergei ArkhipovandGitHub 6feef02d08 Merge pull request #344 from 9seconds/blocked-telegram
Make auto updating optional
2026-02-26 13:46:01 +01:00
Sergei ArkhipovandGitHub 5549a82d19 Merge pull request #345 from 9seconds/fix-no-addresses
Do not panic if no addresses were found
2026-02-26 13:35:47 +01:00
9seconds 189dce07c6 Make auto updating optional 2026-02-26 13:35:05 +01:00
9seconds 449f84133a Do not panic if no addresses were found 2026-02-26 12:31:30 +01:00
9seconds 9c122f942f Always use latest govulncheck to check vulnerabilities 2026-02-24 19:06:16 +01:00
9seconds 136eea551f Merge remote-tracking branch 'origin/stable' into v2 2026-02-24 18:46:18 +01:00
9seconds e6fa5906c9 Merge remote-tracking branch 'origin/master' into stable 2026-02-24 18:45:46 +01:00
9seconds 42f612f49e Use go tag for 1.26 2026-02-24 18:08:45 +01:00
Sergei ArkhipovandGitHub d7db8ca98b Merge pull request #339 from 9seconds/domain-fronting-config-grouping
Domain fronting config grouping
2026-02-24 18:07:02 +01:00
9seconds 1cb225f52c Introduce [domain-fronting] config 2026-02-24 18:05:12 +01:00
9seconds af72b2a574 Delete obsoleted setting 2026-02-24 16:56:42 +01:00
Sergei ArkhipovandGitHub 2cbee5d453 Merge pull request #338 from 9seconds/proxy-proto-front
Add support for domain fronting proxy protocol
2026-02-24 16:53:17 +01:00
9seconds cde313b359 Add support for domain fronting proxy protocol 2026-02-24 16:44:35 +01:00
Sergei ArkhipovandGitHub 58cb0b2caf Merge pull request #336 from 9seconds/obfuscated2
Fetch DC203 from Telegram
2026-02-24 16:41:06 +01:00
9seconds bb320e9d89 Update fuzz tests 2026-02-24 15:27:19 +01:00
Sergei ArkhipovandGitHub f6d2f2ffd8 Merge pull request #337 from 9seconds/govulncheck
Check for vulnerabilities
2026-02-24 15:25:06 +01:00
9seconds 5fe3fdd73c Check for vulnerabilities 2026-02-24 14:21:16 +01:00
9seconds 5b91edf5c4 Fix tests 2026-02-24 13:58:16 +01:00
9seconds 8b34c1b104 Merge remote-tracking branch 'origin/master' into obfuscated2 2026-02-24 13:37:10 +01:00
9seconds 36c766b331 Fix lint issues 2026-02-24 13:35:06 +01:00
9seconds e4a9a96309 Remove mentioning of DC overrides 2026-02-24 13:32:06 +01:00
9seconds 94d46d2c65 Add fetching of addresses from proxyGetConfig endpoint 2026-02-24 12:55:16 +01:00
9seconds 908842063a Do not use additional bytes buffer for faketls 2026-02-23 10:27:01 +01:00
9seconds e50cee5748 Do not use unnecessary lock in connRewind 2026-02-23 10:12:25 +01:00
9seconds ee524abdb5 Remove redundant copyBufferPool from relay 2026-02-23 10:12:25 +01:00
9seconds 3e75e4fa63 Delete old obfuscated2 package 2026-02-23 10:12:25 +01:00
9seconds 140e9dfc2e Integrate obfuscation package 2026-02-23 10:12:25 +01:00
9seconds d0065d35c2 Add new obfuscation package 2026-02-23 10:12:25 +01:00
Sergei ArkhipovandGitHub 45ce5c2f61 Merge pull request #334 from ivulit/master 2026-02-20 20:33:10 +01:00
ivulit 21129b6e00 Add domain-fronting-ip to example config 2026-02-20 12:34:22 +03:00
ivulit bf38f9f8af Add domain-fronting-ip option
Allow specifying an explicit IP address for the domain fronting host
instead of relying on DNS resolution. Useful when DNS resolution of
the fronting hostname is blocked.

The hostname from the secret is still used for SNI in TLS handshake.
2026-02-20 12:34:17 +03:00
147 changed files with 6503 additions and 2472 deletions
+33 -11
View File
@@ -48,6 +48,16 @@ jobs:
- uses: jdx/mise-action@v3
name: Install mise
- name: Cache Go modules and build
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Run tests
run: mise tasks run covtest
@@ -69,6 +79,16 @@ jobs:
- uses: jdx/mise-action@v3
name: Install mise
- name: Cache Go modules and build
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Run fuzzing
run: mise tasks run 'test:fuzz:*'
@@ -86,6 +106,16 @@ jobs:
- uses: jdx/mise-action@v3
name: Install mise
- name: Cache Go modules and build
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Run linter
run: mise tasks run lint
@@ -123,14 +153,6 @@ jobs:
- name: Setup BuildX
uses: docker/setup-buildx-action@v3
- name: Setup cache
uses: actions/cache@v5
with:
path: /tmp/buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
@@ -147,7 +169,7 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v2
uses: docker/build-push-action@v6
with:
pull: true
context: .
@@ -155,5 +177,5 @@ jobs:
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=local,src=/tmp/buildx-cache
cache-to: type=local,dest=/tmp/buildx-cache
cache-from: type=gha
cache-to: type=gha,mode=max
+42
View File
@@ -0,0 +1,42 @@
---
name: Vulnerability checks
permissions:
actions: read
checks: read
contents: read
deployments: read
issues: read
discussions: read
pull-requests: read
repository-projects: read
security-events: read
statuses: read
on:
push:
pull_request:
schedule: # daily at 10:22 UTC
- cron: '22 10 * * *'
workflow_dispatch:
jobs:
vuln:
name: Test vulnerabilities
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
with:
submodules: recursive
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version-file: go.mod
cache: true
- name: Check for vulnerabilities
run: |
go run golang.org/x/vuln/cmd/govulncheck@latest ./...
+2
View File
@@ -54,6 +54,8 @@ archives:
- LICENSE
- README.md
- SECURITY.md
- BEST_PRACTICES.md
- example.config.toml
gomod:
proxy: true
+13 -16
View File
@@ -1,6 +1,7 @@
[tools]
"go:golang.org/x/pkgsite/cmd/pkgsite" = "latest"
"go:golang.org/x/tools/gopls" = "latest"
"go:golang.org/x/vuln/cmd/govulncheck" = "latest"
"go:mvdan.cc/gofumpt" = "latest"
go = "latest"
golangci-lint = "latest"
@@ -19,20 +20,24 @@ run = "go build"
description = "Update dependencies"
run = [
"go get -u",
"go mod tidy -go=1.25"
"go mod tidy -go=1.26"
]
[tasks.lint]
description = "Run linter"
run = "golangci-lint run"
[tasks.vuln]
description = "Test for vulnerabilities"
run = "govulncheck ./..."
[tasks.test]
description = "Run tests"
run = "go test -v ./..."
run = "go test -v -race ./..."
[tasks.covtest]
description = "Run tests with code coverage"
run = "go test -coverprofile=coverage.txt -covermode=atomic -parallel 2 -race -v ./..."
run = "go test -coverprofile=coverage.txt -covermode=atomic -count=2 -race -v ./..."
[tasks.test-all]
description = "Run all tests"
@@ -43,23 +48,15 @@ depends = [
[tasks."test:fuzz:client-hello"]
description = "Run fuzzy test for ClientHello"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzClientHello ./mtglib/internal/faketls"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzReadClientHello ./mtglib/internal/tls/fake"
[tasks."test:fuzz:client-handshake"]
description = "Run fuzzy test for ClientHandshake"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzClientHandshake ./mtglib/internal/obfuscated2"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzClientServerHandshake ./mtglib/internal/obfuscation"
[tasks."test:fuzz:server-generate-handshake-frame"]
description = "Run fuzzy test for ServerGenerateHandshakeFrame"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzServerGenerateHandshakeFrame ./mtglib/internal/obfuscated2"
[tasks."test:fuzz:server-receive"]
description = "Run fuzzy test for ServerReceive"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzServerReceive ./mtglib/internal/obfuscated2"
[tasks."test:fuzz:server-send"]
description = "Run fuzzy test for ServerSend"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzServerSend ./mtglib/internal/obfuscated2"
[tasks."test:fuzz:server-handshake-frame"]
description = "Run fuzzy test for GenerateHandshakeFrame"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzGenerateHandshakeFrame ./mtglib/internal/obfuscation"
[tasks.static]
description = "Build static binary"
+55
View File
@@ -0,0 +1,55 @@
# Best practices
This is unfortunate, but since 2018 many things were changed. Most of them
became way worse. Previous iterations of censorship systems were very dumb,
DPI were primitive and filtered very obvious things. Nowadays they are
way more intelligent and it is very naive to treat them frivolously.
In 2026 is not enough to pretend that your mtg installation is a Microsoft
website that sits in Amsterdam Digital Ocean location. Now your installation
has to be a website that is mtg in disguise. Yes, it requires a bit more effort
but this effort is probably less than rotating proxies each other day.
mtproto traffic, even with FakeTLS, has its specifics that are probably
very well known by DPI systems. These specifics are not something unique but
could mark an IP address as suspicious. Now let's think:
1. You have a proxy in Amsterdam Digital Ocean that tells it is microsoft.com
how hard could it be to find out that this is probably fake? 1 or probably 2
DNS queries for `microsoft.com`? In case of some CDN, there are ECS-powered
resolvers that are very capable to return results from POV of some subnets.
If censor sees no relevant results, will they be afraid to block IP?
2. You have a proxy in Amsterdam Digital Ocean that tells it is a website from
the same public subnet. But not the same. Would it be hard to make these DNS
queries and ban IP?
The correct way of having this proxy is following:
1. Register a domain name
2. Get some VPS, probably in your domestic location
3. Set that domain name from a step 1 to IP address of that VPS
4. Generate a couple of HTML pages by LLMs or even copy them from elsewhere
5. Set some webserver and issue TLS certificates with Let's Encrypt or any other
name
6. Set mtg before this webserver.
7. Use sing-box or anything like that to provide local socks5 interface and
have VPNized uplinks
8. Set up mtg to use socks5 from a 7 step.
In that case you will get a match of DNS and SNI in requests. As a side effect,
your proxy will work with XTLS and its friends: XTLS in sniff mode ignores
IP address a client wants to connect to. Instead, it reads SNI and connect
to resolved address: a clever idea if user does not have a trustworthy DNS
set up.
Yes, this is much longer that usual technique, and requires more effort. But
this is could probably be very well automated to some reasonable extent.
Unfortunately, this is a best practice right now.
Do not also forget about other implementation, like
[telemt](https://github.com/telemt/telemt). Try everything. Use VPNs. It does
not really matter which project you are going to use as long it helps you to
stay connected.
_March 2026._
+11 -6
View File
@@ -5,14 +5,19 @@ FROM golang:1.26-alpine AS build
ENV CGO_ENABLED=0
RUN set -x \
&& apk --no-cache --update add \
bash \
ca-certificates \
git
RUN --mount=type=cache,target=/var/cache/apk \
set -x \
&& apk --update add \
bash \
ca-certificates \
git
COPY go.mod go.sum /app/
WORKDIR /app
RUN go mod download
COPY . /app
WORKDIR /app
RUN set -x \
&& version="$(git describe --exact-match HEAD 2>/dev/null || git describe --tags --always)" \
+78
View File
@@ -38,6 +38,33 @@ goal: to give a possibility to connect to Telegram in a restricted,
censored environment. But it does it slightly differently in details
that probably matter.
* **Domain fronting**
For years mtg supports domain fronting. This technique means that it fallbacks
to accessing a real website in case if request fails. It could fail by many
reasons: anti-replay protection, accidental access to the webserver or
stale request. Anyway, if mtg rejects this request, it does not break a
connection. It connects to the websites and replicates everything that client
has sent, and simply proxies it back as is. Users will see a response from
the real website, _byte-to-byte identical_ to the response of the real netloc.
* **Doppelganger**
mtg also is a doppelganger of the website it fronts. Sure, with domain fronting
users will see replies of the real website in case if something will go wrong.
But what about such cases when _everything is fine_?
In that case mtg mimics TLS connection statistical characteristics as close as
possible. Different application have different statistics of their patterns.
Big CDN steadily pumping the data, small websites burst with short easily
compressiable chunks of traffic.
mtg artificially emulates those delays to be statistically indistinguishable
from the real website even if it covers connection of the very specific app.
It also follows 2 most common patterns of traffic chunking, so censors
will have to put more resources to find out that we have Telegram here
but not a hookah webshop served by nginx.
* **Resource-efficient**
It has to be resource-efficient. It does not mean that you will see
@@ -93,6 +120,8 @@ that probably matter.
software (written in Golang) with a minimum effort + you can replace
some parts with those you want.
Please also to read about [best practices](https://github.com/9seconds/mtg/blob/master/BEST_PRACTICES.md).
### Version 2
If you use version 1.x before, you are probably noticed some major
@@ -398,6 +427,55 @@ or if you are using docker:
$ docker exec mtg-proxy /mtg access /config.toml
```
## Doppelganger
mtg can mimic real websites, please take a look at relevant section in example
config file.
mtg comes with some very good precollected statistics coming from
[ok.ru](https://ok.ru/). It does not mean that you have to cover yourself
by pretending that mtg is _ok.ru_. **Do not do that: ok.ru comes from very specific
ASNs, but not from VPS providers you are going to use.** What I want to say
is that defaults are very good enough to use as is because ok.ru for public
pages has a very generic profile of TLS packets delay.
But for better results it is recommended to teach mtg about the website you
will use as a domain front. In order to do that, you need to specify URLs
from this website. Just go to it, open WebDeveloper console and pick up
random URLs. For better results they have to be **from the same domain name
you are going to use as a disguise** but serve light and heavy content: pages,
images etc. Do not use many, 2-3 will probably work.
mtg will crawl these pages periodically, accumulating statistics and
using it as you go.
```toml
[defense.doppelganger]
urls = [
"https://lalala.com/index.html",
"https://lalala.com/contacts.html",
]
```
This is not very necessary. Keep in mind these rules:
1. If you are not sure what is this all about, do nothing. Defaults are good.
2. All URLs must be HTTPS
3. All URLs should be from the same domain name (but this is not a rule)
4. Do not use a lot of pages. Use _different_ pages. mtg will start using this
statistics when it will accumulate enough anyway.
5. These URLs should be directly accessible from mtg without proxies whatsoever
6. Do not create huge raids. mtg will repeatedly crawl in raids, making N repeats.
Do not use high N, you do not want to be noticeable.
7. It makes no sense to have small delay between raids. Usually webservers
do not update their TLS settings each hour.
8. If you have some specific knowledge if webserver is using
[TLS Dynamic Record Sizing](https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/), you
can use a very specific setting. This are Cloudflare, Go standard webservers,
[caddy](https://caddyserver.com/) and [H2O](https://h2o.examp1e.net/). If so,
you can enable `drs` setting.
9. **If you are not sure, touch nothing!**
## Metrics
Out of the box, mtg works with
+25
View File
@@ -24,3 +24,28 @@ type Conn interface {
CloseableReader
CloseableWriter
}
type netConnWrapper struct {
net.Conn
}
func (n netConnWrapper) CloseRead() error {
if conn, ok := n.Conn.(CloseableReader); ok {
return conn.CloseRead()
}
return n.Close()
}
func (n netConnWrapper) CloseWrite() error {
if conn, ok := n.Conn.(CloseableWriter); ok {
return conn.CloseWrite()
}
return n.Close()
}
// WrapConn wraps a generic [net.Conn] into Conn.
func WrapNetConn(conn net.Conn) Conn {
return netConnWrapper{conn}
}
+145 -31
View File
@@ -36,13 +36,6 @@ bind-to = "0.0.0.0:3128"
# All other incoming connections are going to be dropped.
concurrency = 8192
# A size of user-space buffer for TCP to use. Since we do 2 connections,
# then we have tcp-buffer * (4 + 2) per each connection: read/write for
# each connection + 2 copy buffers to pump the data between sockets.
#
# Deprecated: this setting is no longer makes any effect.
# tcp-buffer = "4kb"
# Sometimes you want to enforce mtg to use some types of
# IP connectivity to Telegram. We have 4 modes:
# - prefer-ipv6:
@@ -55,9 +48,35 @@ concurrency = 8192
# Only ipv4 connectivity is used
prefer-ip = "prefer-ipv6"
# If this setting is set, then mtg will try to get proxy updates from Telegram
# Usually this is completely fine to have it disabled, because mtg has a list
# of some core proxies hardcoded.
auto-update = false
# FakeTLS uses domain fronting protection. So it needs to know a port to
# access.
domain-fronting-port = 443
#
# Deprecated: use [domain-fronting] configuration block. If relevant option
# is defined there, this one would be ignored.
# domain-fronting-port = 443
# By default, mtg resolves the fronting hostname (from the secret) via DNS
# to establish a TCP connection. If DNS resolution of that hostname is blocked,
# you can specify an IP address to connect to directly. The hostname is still
# used for SNI in the TLS handshake.
#
# default value is not set (DNS resolution is used).
#
# Deprecated: use [domain-fronting] configuration block. If relevant option
# is defined there, this one would be ignored.
# domain-fronting-ip = "10.0.0.10"
# This makes a communication between both fronting website and mtg to use
# proxy protocol.
#
# Deprecated: use [domain-fronting] configuration block. If relevant option
# is defined there, this one would be ignored.
# domain-fronting-proxy-protocol = false
# FakeTLS can compare timestamps to prevent probes. Each message has
# encrypted timestamp. So, mtg can compare this timestamp and decide if
@@ -80,14 +99,26 @@ tolerate-time-skewness = "5s"
# Otherwise, chose a new DC.
allow-fallback-on-unknown-dc = false
# Telegram uses different DCs for different purposes. Unfortunately, most of
# DCs are not public, and dependent on a location of the current user, so
# mtg cannot know upfront about all of them, and how to access them. It has
# a default list of DCs, including some CDN IPs, but it is possible that some
# of them are not working for you. In this case, you can override them here.
[[dc-overrides]]
dc = 101
ips = ["127.0.0.1:443"]
# This section is relevant to communication with fronting domain. Usually
# you do not need to setup anything here but there are plenty of cases, especially
# if you put mtg behind load balancer, when some specific configuration is
# required.
[domain-fronting]
# By default, mtg resolves the fronting hostname (from the secret) via DNS
# to establish a TCP connection. If DNS resolution of that hostname is blocked,
# you can specify an IP address to connect to directly. The hostname is still
# used for SNI in the TLS handshake.
#
# default value is not set (DNS resolution is used).
# ip = "10.10.10.11"
# FakeTLS uses domain fronting protection. So it needs to know a port to
# access. Default value is 443
# port = 443
# This makes a communication between both fronting website and mtg to use
# proxy protocol.
# proxy-protocol = false
# network defines different network-related settings
[network]
@@ -103,7 +134,42 @@ ips = ["127.0.0.1:443"]
# it has to access.
#
# By default we use Cloudflare.
doh-ip = "1.1.1.1"
#
# DEPRECATED option:
# If dns option is specified, it will be used instead. No default is defined
# anymore. But if you set this option, this is literally the same providing
# https:// URL to `dns`.
#
# In other words:
#
# doh-ip = "1.1.1.1"
#
# is literally the same as
#
# dns = "https://1.1.1.1"
# doh-ip = "1.1.1.1"
# Starting from mtg v2.1.12 we have changed a configuration for DNS. Now it
# supports DNS-over-HTTPS, DNS-over-TLS, custom UDP resolver and system
# resolver.
#
# Here is how to define DNS-over-HTTPS:
# - https://1.1.1.1
# - https://1.1.1.1/dns-query
# - https://cloudflare-dns.com/dns-query
# - https://cloudflare-dns.com
#
# Here is how to define DNS-over-TLS:
# - tls://1.1.1.1
# - tls://cloudflare-dns.com
#
# Here is how to define a custom UDP resolver (we support only IPs here)
# - 1.1.1.1
# - udp://1.1.1.1
#
# If you set it to empty string, default resolver will be used.
# But please comment out doh-ip
dns = "https://1.1.1.1"
# mtg can work via proxies (for now, we support only socks5). Proxy
# configuration is done via list. So, you can specify many proxies
@@ -118,25 +184,13 @@ doh-ip = "1.1.1.1"
#
# Proxy configuration is done via ordinary URI schema:
#
# socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s
# socks5://user:password@host:port
#
# Only socks5 proxy is used. user/password is optional. As you can
# see, you can specify some parameters in GET query. These parameters
# configure circuit breaker.
#
# open_threshold means a number of errors which should happen so we stop
# use a proxy.
#
# half_open_timeout means a time period (in Golang duration notation)
# after which we can retry with this proxy
#
# reset_failures_timeout means a time period when we flush out errors
# when circuit breaker in closed state.
#
# Please see https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker
# on details about circuit breakers.
proxies = [
# "socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s"
# "socks5://user:password@host:port"
]
# network timeouts define different settings for timeouts. tcp timeout
@@ -152,6 +206,66 @@ tcp = "5s"
http = "10s"
idle = "1m"
# mtg has to mimic real websites. It does not mean domain fronting, it also
# means that traffic characteristics should be similar to real world traffic.
# websites and applications behave differently, their traffic patterns are also
# different. Applications do bursts of RPC-style messages (or JSON communication,
# does not really matter), while websites pump heavy content in HTTP2 streams
#
# It means that statistically there is a different between traffic shape:
# delays between packets are also different.
# In order to avoid censorship detection based on these patterns, there is a
# mtg subsystem called "Doppelganger" that aims to mimic website statistics
# as close as it could.
#
# Delays between TLS packets are not constant. There are many factors
# that come in play. Application should generate some response, it could
# send some headers first and stream content with chunked encoding. So
# some first packets could come as soon as possible, with some delays
# after first ones. Such phenomenon is described by different statistic
# distribution. There are 2 distribution that describe it: lognormal
# distribution and Weibul distribution. Lognormal is all about steady streams
# of heavy content like a video. Weibul is great about short bursts like
# user who requested a static page an a couple of images.
[defense.doppelganger]
# This is a list of URLs that would be crawled by mtg to approximate delay
# statistics. They MUST be HTTPS urls.
#
# You can come to the website and collect different URLs, with light and
# heavy content. We recommend to search for CDNs.
urls = [
# "https://st-ok.cdn-vk.ru/res/react/vendor/clsx-2.1.1-amd.js"
]
# A collection is done in raids. Each raid makes this number of requests to
# each URL in this list. Do not use a huge number, 10 is probably ok.
repeats-per-raid = 10
# This is a duration between each raid. It makes no sense to have a small number
# here as you would start to make a noticeable activity. Usually traffic patterns
# do not change a lot, so do not expect different results if you request
# each 10 minutes.
raid-each = "6h"
# This enables dynamic tls record sizing.
#
# Some modern stacks and platforms start to use the technique that is called
# DRS. They start with small TLS packets and ramp up eventually. First packets
# are usually about MTU size, after that we get 4k and eventually max size.
# This is done with a good intention: to minimize a time to the first byte,
# so application could start doing something with the data right after first
# RTT.
#
# Apparently, about 90% of application do not employ this technique, they use
# max size always: nginx, apache, java stuff. But Golang tools, angie and
# some specific patches activate this technique.
#
# In order to mimic a real website we need to know something about software
# it uses. Usually nobody cares: openssl does 16384, Python does it, nginx
# does it. So this setting is disabled by default.
#
# https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/
# https://aws.github.io/s2n-tls/usage-guide/ch08-record-sizes.html
# https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch
drs = false
# Some countries do active probing on Telegram connections. This technique
# allows to protect from such effort.
#
+10 -8
View File
@@ -1,6 +1,6 @@
module github.com/9seconds/mtg/v2
go 1.25
go 1.26
require (
github.com/OneOfOne/xxhash v1.2.8
@@ -11,24 +11,26 @@ require (
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/jarcoal/httpmock v1.0.8
github.com/mccutchen/go-httpbin v1.1.1
github.com/panjf2000/ants/v2 v2.11.5
github.com/panjf2000/ants/v2 v2.11.6
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/rs/zerolog v1.34.0
github.com/smira/go-statsd v1.3.4
github.com/stretchr/objx v0.5.2 // indirect
github.com/stretchr/testify v1.11.1
github.com/tylertreat/BoomFilters v0.0.0-20251117164519-53813c36cc1b
golang.org/x/crypto v0.48.0
golang.org/x/net v0.49.0 // indirect
golang.org/x/sys v0.41.0
golang.org/x/crypto v0.49.0
golang.org/x/net v0.52.0
golang.org/x/sys v0.42.0
google.golang.org/protobuf v1.36.11 // indirect
)
require (
github.com/ncruces/go-dns v1.3.2
github.com/pelletier/go-toml/v2 v2.2.4
github.com/pires/go-proxyproto v0.11.0
github.com/things-go/go-socks5 v0.1.0
github.com/txthinking/socks5 v0.0.0-20251011041537-5c31f201a10e
github.com/yl2chen/cidranger v1.0.2
)
@@ -47,8 +49,8 @@ require (
github.com/prometheus/client_model v0.6.2 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/txthinking/runnergroup v0.0.0-20250224021307-5864ffeb65ae // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/sync v0.19.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/tools v0.41.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+18 -14
View File
@@ -51,8 +51,10 @@ github.com/miekg/dns v1.1.51 h1:0+Xg7vObnhrz/4ZCZcZh7zPXlmU0aveS2HDBd0m0qSo=
github.com/miekg/dns v1.1.51/go.mod h1:2Z9d3CP1LQWihRZUf29mQ19yDThaI4DAYzte2CaQW5c=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/ncruces/go-dns v1.3.2 h1:kBLuUZBgkQ4qF4WDXZRQ4rG0Gk6sLVJQ5tESkWrxUa0=
github.com/ncruces/go-dns v1.3.2/go.mod h1:tuzixNY8PY/M7yUzcvRbUaeLs3ifIdydpi5H2bfRU+s=
github.com/panjf2000/ants/v2 v2.11.6 h1:JKsoIUukIoCO0sP0gcOqdyoXmpyKXuU6fC57rODtpug=
github.com/panjf2000/ants/v2 v2.11.6/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
@@ -68,8 +70,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
@@ -89,6 +91,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/things-go/go-socks5 v0.1.0 h1:4f5dz0iMQ6cA4wseFmyLmCHmg3SWJTW92ndrKS6oERg=
github.com/things-go/go-socks5 v0.1.0/go.mod h1:Riabiyu52kLsla0YmJqunt1c1JEl6iXSr4bRd7swFEA=
github.com/txthinking/runnergroup v0.0.0-20210608031112-152c7c4432bf/go.mod h1:CLUSJbazqETbaR+i0YAhXBICV9TrKH93pziccMhmhpM=
github.com/txthinking/runnergroup v0.0.0-20250224021307-5864ffeb65ae h1:ArVM1jICfm7g4E4dBet+KHUFMLuxmj1Nxdp/tr3ByCU=
github.com/txthinking/runnergroup v0.0.0-20250224021307-5864ffeb65ae/go.mod h1:cldYm15/XHcGt7ndItnEWHwFZo7dinU+2QoyjfErhsI=
@@ -101,12 +105,12 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
@@ -115,13 +119,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -131,8 +135,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+6 -1
View File
@@ -101,8 +101,13 @@ func (a *Access) Run(cli *CLI, version string) error {
}
func (a *Access) getIP(ntw mtglib.Network, protocol string) net.IP {
dialer := ntw.NativeDialer()
client := ntw.MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error) {
return ntw.DialContext(ctx, protocol, address) //nolint: wrapcheck
conn, err := dialer.DialContext(ctx, protocol, address)
if err != nil {
return nil, err
}
return essentials.WrapNetConn(conn), err
})
req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) //nolint: noctx
+39 -38
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"net"
"net/url"
"os"
"github.com/9seconds/mtg/v2/antireplay"
@@ -16,7 +15,7 @@ import (
"github.com/9seconds/mtg/v2/ipblocklist/files"
"github.com/9seconds/mtg/v2/logger"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/network"
"github.com/9seconds/mtg/v2/network/v2"
"github.com/9seconds/mtg/v2/stats"
"github.com/pires/go-proxyproto"
"github.com/rs/zerolog"
@@ -40,43 +39,41 @@ func makeLogger(conf *config.Config) mtglib.Logger {
}
func makeNetwork(conf *config.Config, version string) (mtglib.Network, error) {
tcpTimeout := conf.Network.Timeout.TCP.Get(network.DefaultTimeout)
httpTimeout := conf.Network.Timeout.HTTP.Get(network.DefaultHTTPTimeout)
dohIP := conf.Network.DOHIP.Get(net.ParseIP(network.DefaultDOHHostname)).String()
userAgent := "mtg/" + version
baseDialer, err := network.NewDefaultDialer(tcpTimeout, 0)
resolver, err := network.GetDNS(conf.GetDNS())
if err != nil {
return nil, fmt.Errorf("cannot build a default dialer: %w", err)
return nil, fmt.Errorf("cannot create DNS resolver: %w", err)
}
if len(conf.Network.Proxies) == 0 {
return network.NewNetwork(baseDialer, userAgent, dohIP, httpTimeout) //nolint: wrapcheck
}
base := network.New(
resolver,
"",
conf.Network.Timeout.TCP.Get(0),
conf.Network.Timeout.HTTP.Get(0),
conf.Network.Timeout.Idle.Get(0),
)
proxyURLs := make([]*url.URL, 0, len(conf.Network.Proxies))
for _, v := range conf.Network.Proxies {
if value := v.Get(nil); value != nil {
proxyURLs = append(proxyURLs, value)
}
}
if len(proxyURLs) == 1 {
socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0])
proxyDialers := make([]mtglib.Network, len(conf.Network.Proxies))
for idx, v := range conf.Network.Proxies {
value, err := network.NewProxyNetwork(base, v.Get(nil))
if err != nil {
return nil, fmt.Errorf("cannot build socks5 dialer: %w", err)
return nil, fmt.Errorf("cannot use %v for proxy url: %w", v.Get(nil), err)
}
return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout) //nolint: wrapcheck
proxyDialers[idx] = value
}
socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs)
switch len(proxyDialers) {
case 0:
return base, nil
case 1:
return proxyDialers[0], nil
}
value, err := network.Join(proxyDialers...)
if err != nil {
return nil, fmt.Errorf("cannot build socks5 dialer: %w", err)
panic(err)
}
return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout) //nolint: wrapcheck
return value, nil
}
func makeAntiReplayCache(conf *config.Config) mtglib.AntiReplayCache {
@@ -242,12 +239,9 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen
return fmt.Errorf("cannot build ip allowlist: %w", err)
}
dcOverrides := map[int][]string{}
for _, override := range conf.DCOverrides {
dcid := override.DC.Get()
for _, addr := range override.IPs {
dcOverrides[dcid] = append(dcOverrides[dcid], addr.Get(""))
}
doppelGangerURLs := make([]string, len(conf.Defense.Doppelganger.URLs))
for i, v := range conf.Defense.Doppelganger.URLs {
doppelGangerURLs[i] = v.String()
}
opts := mtglib.ProxyOpts{
@@ -258,13 +252,20 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen
IPAllowlist: allowlist,
EventStream: eventStream,
Secret: conf.Secret,
DomainFrontingPort: conf.DomainFrontingPort.Get(mtglib.DefaultDomainFrontingPort),
PreferIP: conf.PreferIP.Get(mtglib.DefaultPreferIP),
Secret: conf.Secret,
DomainFrontingPort: conf.GetDomainFrontingPort(mtglib.DefaultDomainFrontingPort),
DomainFrontingIP: conf.GetDomainFrontingIP(nil),
DomainFrontingProxyProtocol: conf.GetDomainFrontingProxyProtocol(false),
PreferIP: conf.PreferIP.Get(mtglib.DefaultPreferIP),
AutoUpdate: conf.AutoUpdate.Get(false),
AllowFallbackOnUnknownDC: conf.AllowFallbackOnUnknownDC.Get(false),
TolerateTimeSkewness: conf.TolerateTimeSkewness.Value,
DCOverrides: dcOverrides,
DoppelGangerURLs: doppelGangerURLs,
DoppelGangerPerRaid: conf.Defense.Doppelganger.Repeats.Get(mtglib.DoppelGangerPerRaid),
DoppelGangerEach: conf.Defense.Doppelganger.UpdateEach.Get(mtglib.DoppelGangerEach),
DoppelGangerDRS: conf.Defense.Doppelganger.DRS.Get(false),
}
proxy, err := mtglib.NewProxy(opts)
+7
View File
@@ -18,6 +18,7 @@ type SimpleRun struct {
TCPBuffer string `kong:"name='tcp-buffer',short='b',default='4KB',help='Deprecated and ignored'"` //nolint: lll
PreferIP string `kong:"name='prefer-ip',short='i',default='prefer-ipv6',help='IP preference. By default we prefer IPv6 with fallback to IPv4.'"` //nolint: lll
DomainFrontingPort uint64 `kong:"name='domain-fronting-port',short='p',default='443',help='A port to access for domain fronting.'"` //nolint: lll
DomainFrontingIP string `kong:"name='domain-fronting-ip',help='An IP address to use for domain fronting instead of resolving the hostname via DNS.'"` //nolint: lll
DOHIP net.IP `kong:"name='doh-ip',short='n',default='1.1.1.1',help='IP address of DNS-over-HTTP to use.'"` //nolint: lll
Timeout time.Duration `kong:"name='timeout',short='t',default='10s',help='Network timeout to use'"` //nolint: lll
Socks5Proxies []string `kong:"name='socks5-proxy',short='s',help='Socks5 proxies to use for network access.'"` //nolint: lll
@@ -47,6 +48,12 @@ func (s *SimpleRun) Run(cli *CLI, version string) error { //nolint: cyclop,funle
return fmt.Errorf("incorrect domain-fronting-port: %w", err)
}
if s.DomainFrontingIP != "" {
if err := conf.DomainFrontingIP.Set(s.DomainFrontingIP); err != nil {
return fmt.Errorf("incorrect domain-fronting-ip: %w", err)
}
}
if err := conf.Network.DOHIP.Set(s.DOHIP.String()); err != nil {
return fmt.Errorf("incorrect doh-ip: %w", err)
}
+60 -16
View File
@@ -4,6 +4,8 @@ import (
"bytes"
"encoding/json"
"fmt"
"net"
"net/url"
"github.com/9seconds/mtg/v2/mtglib"
)
@@ -21,24 +23,38 @@ type ListConfig struct {
}
type Config struct {
Debug TypeBool `json:"debug"`
AllowFallbackOnUnknownDC TypeBool `json:"allowFallbackOnUnknownDc"`
Secret mtglib.Secret `json:"secret"`
BindTo TypeHostPort `json:"bindTo"`
ProxyProtocolListener TypeBool `json:"proxyProtocolListener"`
PreferIP TypePreferIP `json:"preferIp"`
DomainFrontingPort TypePort `json:"domainFrontingPort"`
TolerateTimeSkewness TypeDuration `json:"tolerateTimeSkewness"`
Concurrency TypeConcurrency `json:"concurrency"`
Defense struct {
Debug TypeBool `json:"debug"`
AllowFallbackOnUnknownDC TypeBool `json:"allowFallbackOnUnknownDc"`
Secret mtglib.Secret `json:"secret"`
BindTo TypeHostPort `json:"bindTo"`
ProxyProtocolListener TypeBool `json:"proxyProtocolListener"`
PreferIP TypePreferIP `json:"preferIp"`
AutoUpdate TypeBool `json:"autoUpdate"`
DomainFrontingPort TypePort `json:"domainFrontingPort"`
DomainFrontingIP TypeIP `json:"domainFrontingIp"`
DomainFrontingProxyProtocol TypeBool `json:"domainFrontingProxyProtocol"`
TolerateTimeSkewness TypeDuration `json:"tolerateTimeSkewness"`
Concurrency TypeConcurrency `json:"concurrency"`
DomainFronting struct {
IP TypeIP `json:"ip"`
Port TypePort `json:"port"`
ProxyProtocol TypeBool `json:"proxyProtocol"`
} `json:"domainFronting"`
Defense struct {
AntiReplay struct {
Optional
MaxSize TypeBytes `json:"maxSize"`
ErrorRate TypeErrorRate `json:"errorRate"`
} `json:"antiReplay"`
Blocklist ListConfig `json:"blocklist"`
Allowlist ListConfig `json:"allowlist"`
Blocklist ListConfig `json:"blocklist"`
Allowlist ListConfig `json:"allowlist"`
Doppelganger struct {
URLs []TypeHttpsURL `json:"urls"`
Repeats TypeConcurrency `json:"repeats_per_raid"`
UpdateEach TypeDuration `json:"raid_each"`
DRS TypeBool `json:"drs"`
} `json:"doppelganger"`
} `json:"defense"`
Network struct {
Timeout struct {
@@ -47,6 +63,7 @@ type Config struct {
Idle TypeDuration `json:"idle"`
} `json:"timeout"`
DOHIP TypeIP `json:"dohIp"`
DNS TypeDNSURI `json:"dns"`
Proxies []TypeProxyURL `json:"proxies"`
} `json:"network"`
Stats struct {
@@ -65,10 +82,37 @@ type Config struct {
MetricPrefix TypeMetricPrefix `json:"metricPrefix"`
} `json:"prometheus"`
} `json:"stats"`
DCOverrides []struct {
DC TypeDC `json:"dc"`
IPs []TypeHostPort `json:"ips"`
} `json:"dcOverrides"`
}
func (c *Config) GetDNS() *url.URL {
var dohURL *url.URL
if dohIP := c.Network.DOHIP.Get(nil); dohIP != nil {
dohURL, _ = url.Parse("https://" + dohIP.String())
}
return c.Network.DNS.Get(dohURL)
}
func (c *Config) GetDomainFrontingPort(defaultValue uint) uint {
if port := c.DomainFronting.Port.Get(0); port != 0 {
return port
}
return c.DomainFrontingPort.Get(defaultValue)
}
func (c *Config) GetDomainFrontingIP(defaultValue net.IP) string {
if ip := c.DomainFronting.IP.Get(nil); ip != nil {
return ip.String()
}
if ip := c.DomainFrontingIP.Get(defaultValue); ip != nil {
return ip.String()
}
return ""
}
func (c *Config) GetDomainFrontingProxyProtocol(defaultValue bool) bool {
return c.DomainFronting.ProxyProtocol.Get(false) || c.DomainFrontingProxyProtocol.Get(defaultValue)
}
func (c *Config) Validate() error {
+25 -14
View File
@@ -9,16 +9,24 @@ import (
)
type tomlConfig struct {
Debug bool `toml:"debug" json:"debug,omitempty"`
AllowFallbackOnUnknownDC bool `toml:"allow-fallback-on-unknown-dc" json:"allowFallbackOnUnknownDc,omitempty"`
Secret string `toml:"secret" json:"secret"`
BindTo string `toml:"bind-to" json:"bindTo"`
ProxyProtocolListener bool `toml:"proxy-protocol-listener" json:"proxyProtocolListener"`
PreferIP string `toml:"prefer-ip" json:"preferIp,omitempty"`
DomainFrontingPort uint `toml:"domain-fronting-port" json:"domainFrontingPort,omitempty"`
TolerateTimeSkewness string `toml:"tolerate-time-skewness" json:"tolerateTimeSkewness,omitempty"`
Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"`
Defense struct {
Debug bool `toml:"debug" json:"debug,omitempty"`
AllowFallbackOnUnknownDC bool `toml:"allow-fallback-on-unknown-dc" json:"allowFallbackOnUnknownDc,omitempty"`
Secret string `toml:"secret" json:"secret"`
BindTo string `toml:"bind-to" json:"bindTo"`
ProxyProtocolListener bool `toml:"proxy-protocol-listener" json:"proxyProtocolListener"`
PreferIP string `toml:"prefer-ip" json:"preferIp,omitempty"`
AutoUpdate bool `toml:"auto-update" json:"autoUpdate,omitempty"`
DomainFrontingPort uint `toml:"domain-fronting-port" json:"domainFrontingPort,omitempty"`
DomainFrontingIP string `toml:"domain-fronting-ip" json:"domainFrontingIp,omitempty"`
DomainFrontingProxyProtocol bool `toml:"domain-fronting-proxy-protocol" json:"domainFrontingProxyProtocol,omitempty"`
TolerateTimeSkewness string `toml:"tolerate-time-skewness" json:"tolerateTimeSkewness,omitempty"`
Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"`
DomainFronting struct {
IP string `toml:"ip" json:"ip,omitempty"`
Port uint `toml:"port" json:"port,omitempty"`
ProxyProtocol bool `toml:"proxy-protocol" json:"proxyProtocol,omitempty"`
} `toml:"domain-fronting" json:"domainFronting,omitempty"`
Defense struct {
AntiReplay struct {
Enabled bool `toml:"enabled" json:"enabled,omitempty"`
MaxSize string `toml:"max-size" json:"maxSize,omitempty"`
@@ -36,6 +44,12 @@ type tomlConfig struct {
URLs []string `toml:"urls" json:"urls,omitempty"`
UpdateEach string `toml:"update-each" json:"updateEach,omitempty"`
} `toml:"allowlist" json:"allowlist,omitempty"`
Doppelganger struct {
URLs []string `toml:"urls" json:"urls,omitempty"`
Repeats uint `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
UpdateEach string `toml:"raid-each" json:"raid_each,omitempty"`
DRS bool `toml:"drs" json:"drs,omitempty"`
} `toml:"doppelganger" json:"doppelganger,omitempty"`
} `toml:"defense" json:"defense,omitempty"`
Network struct {
Timeout struct {
@@ -44,6 +58,7 @@ type tomlConfig struct {
Idle string `toml:"idle" json:"idle,omitempty"`
} `toml:"timeout" json:"timeout,omitempty"`
DOHIP string `toml:"doh-ip" json:"dohIp,omitempty"`
DNS string `toml:"dns" json:"dns,omitempty"`
Proxies []string `toml:"proxies" json:"proxies,omitempty"`
} `toml:"network" json:"network,omitempty"`
Stats struct {
@@ -60,10 +75,6 @@ type tomlConfig struct {
MetricPrefix string `toml:"metric-prefix" json:"metricPrefix,omitempty"`
} `toml:"prometheus" json:"prometheus,omitempty"`
} `toml:"stats" json:"stats,omitempty"`
DCOverrides []struct {
DC uint `toml:"dc" json:"dc"`
IPs []string `toml:"ips" json:"ips"`
} `toml:"dc-overrides" json:"dcOverrides,omitempty"`
}
func Parse(rawData []byte) (*Config, error) {
+69
View File
@@ -0,0 +1,69 @@
package config
import (
"fmt"
"net"
"net/url"
)
type TypeDNSURI struct {
Value *url.URL
}
func (t *TypeDNSURI) Set(value string) error {
parsed, err := url.Parse(value)
if err != nil {
return fmt.Errorf("value is not URI: %w", err)
}
if parsed.Host == "" {
parsed.Host = parsed.Path
parsed.Path = ""
parsed.Scheme = "udp"
}
switch parsed.Scheme {
case "https", "tls":
case "udp":
if ip := net.ParseIP(parsed.Hostname()); ip == nil {
return fmt.Errorf("simple DNS must IP address: %s", parsed.Hostname())
}
default:
return fmt.Errorf("unsupported DNS type %s", parsed.Scheme)
}
if parsed.Scheme != "https" && parsed.Path != "" {
return fmt.Errorf("path is supported only for DoH: %s", parsed)
}
if parsed.User != nil {
return fmt.Errorf("used info is not supported: %s", parsed.User.String())
}
t.Value = parsed
return nil
}
func (t *TypeDNSURI) Get(defaultValue *url.URL) *url.URL {
if t.Value != nil {
return t.Value
}
return defaultValue
}
func (t *TypeDNSURI) UnmarshalText(data []byte) error {
return t.Set(string(data))
}
func (t TypeDNSURI) MarshalText() ([]byte, error) {
return []byte(t.String()), nil
}
func (t TypeDNSURI) String() string {
if t.Value == nil {
return ""
}
return t.Value.String()
}
+117
View File
@@ -0,0 +1,117 @@
package config_test
import (
"encoding/json"
"testing"
"github.com/9seconds/mtg/v2/internal/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type typeDNSURITestStruct struct {
Value config.TypeDNSURI `json:"value"`
}
type TypeDNSURITestSuite struct {
suite.Suite
}
func (suite *TypeDNSURITestSuite) TestUnmarshalFail() {
testData := []string{
"xx",
"ppar",
"",
"dns://hahaha",
"udp://xcxxcv",
"udp://1.1.1.1/xcv",
"1.1.1.1/xxx",
"tls://dns/xx",
"tls://1.1.1.1/xx",
"https://user:password@1.1.1.1",
"tls://user:password@1.1.1.1",
"udp://user:password@1.1.1.1",
}
for _, v := range testData {
data, err := json.Marshal(map[string]string{
"value": v,
})
suite.NoError(err)
suite.T().Run(v, func(t *testing.T) {
assert.Error(t, json.Unmarshal(data, &typeDNSURITestStruct{}))
})
}
}
func (suite *TypeDNSURITestSuite) TestUnmarshalOk() {
testData := []string{
"1.1.1.1",
"tls://1.1.1.1",
"tls://dns.google",
"https://1.1.1.1",
"https://1.1.1.1/dns-query",
"https://dns.google",
"https://dns.google/dns-query",
"udp://1.1.1.1",
}
for _, v := range testData {
data, err := json.Marshal(map[string]string{
"value": v,
})
suite.NoError(err)
suite.T().Run(v, func(t *testing.T) {
testStruct := &typeDNSURITestStruct{}
assert.NoError(t, json.Unmarshal(data, testStruct))
if v == "1.1.1.1" {
v = "udp://" + v
}
assert.Equal(t, v, testStruct.Value.String())
})
}
}
func (suite *TypeDNSURITestSuite) TestMarshalOk() {
testData := []string{
"tls://1.1.1.1",
"tls://dns.google",
"https://1.1.1.1",
"https://1.1.1.1/dns-query",
}
for _, v := range testData {
suite.T().Run(v, func(t *testing.T) {
testStruct := &typePreferIPTestStruct{
Value: config.TypePreferIP{
Value: v,
},
}
encodedJSON, err := json.Marshal(testStruct)
assert.NoError(t, err)
expectedJSON, err := json.Marshal(map[string]string{
"value": v,
})
assert.NoError(t, err)
assert.JSONEq(t, string(expectedJSON), string(encodedJSON))
})
}
}
func (suite *TypeDNSURITestSuite) TestGet() {
value := config.TypeDNSURI{}
suite.Nil(value.Get(nil))
suite.NoError(value.Set("tls://1.1.1.1"))
suite.NotNil(value.Get(nil))
}
func TestDNSURI(t *testing.T) {
t.Parallel()
suite.Run(t, &TypeDNSURITestSuite{})
}
+53
View File
@@ -0,0 +1,53 @@
package config
import (
"fmt"
"net/url"
)
type TypeHttpsURL struct {
Value *url.URL
}
func (t *TypeHttpsURL) Set(value string) error {
parsedURL, err := url.Parse(value)
if err != nil {
return fmt.Errorf("value is not correct URL (%s): %w", value, err)
}
if parsedURL.Host == "" {
return fmt.Errorf("url has to have a schema: %s", value)
}
if parsedURL.Scheme != "https" {
return fmt.Errorf("unsupported schema: %s", parsedURL.Scheme)
}
t.Value = parsedURL
return nil
}
func (t *TypeHttpsURL) Get(defaultValue *url.URL) *url.URL {
if t.Value == nil {
return defaultValue
}
return t.Value
}
func (t *TypeHttpsURL) UnmarshalText(data []byte) error {
return t.Set(string(data))
}
func (t TypeHttpsURL) MarshalText() ([]byte, error) {
return []byte(t.String()), nil
}
func (t TypeHttpsURL) String() string {
if t.Value == nil {
return ""
}
return t.Value.String()
}
+100
View File
@@ -0,0 +1,100 @@
package config_test
import (
"encoding/json"
"net/url"
"testing"
"github.com/9seconds/mtg/v2/internal/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type typeHttpsURLTestStruct struct {
Value config.TypeHttpsURL `json:"value"`
}
type HttpsURLTestSuite struct {
suite.Suite
}
func (suite *HttpsURLTestSuite) TestUnmarshalFail() {
testData := []string{
"",
"https://",
"://lala",
"/path",
"http://example.com",
"socks5://example.com",
}
for _, v := range testData {
data, err := json.Marshal(map[string]string{
"value": v,
})
suite.NoError(err)
suite.T().Run(v, func(t *testing.T) {
assert.Error(t, json.Unmarshal(data, &typeHttpsURLTestStruct{}))
})
}
}
func (suite *HttpsURLTestSuite) TestUnmarshalOk() {
testData := map[string]string{
"https://example.com": "https://example.com",
"https://example.com:8443": "https://example.com:8443",
"https://example.com/path?q=1": "https://example.com/path?q=1",
"https://user:pass@example.com": "https://user:pass@example.com",
}
for k, v := range testData {
value := v
data, err := json.Marshal(map[string]string{
"value": k,
})
suite.NoError(err)
suite.T().Run(k, func(t *testing.T) {
testStruct := &typeHttpsURLTestStruct{}
assert.NoError(t, json.Unmarshal(data, testStruct))
parsed, _ := url.Parse(value)
assert.Equal(t, parsed.Scheme, testStruct.Value.Get(nil).Scheme)
assert.Equal(t, parsed.Host, testStruct.Value.Get(nil).Host)
assert.Equal(t, parsed.RawQuery, testStruct.Value.Get(nil).RawQuery)
assert.Equal(t, parsed.Path, testStruct.Value.Get(nil).Path)
})
}
}
func (suite *HttpsURLTestSuite) TestMarshalOk() {
parsed, _ := url.Parse("https://example.com/path?q=1")
testStruct := &typeHttpsURLTestStruct{
Value: config.TypeHttpsURL{
Value: parsed,
},
}
encodedJSON, err := json.Marshal(testStruct)
suite.NoError(err)
suite.JSONEq(`{"value": "https://example.com/path?q=1"}`,
string(encodedJSON))
}
func (suite *HttpsURLTestSuite) TestGet() {
emptyURL := &url.URL{}
value := config.TypeHttpsURL{}
suite.Equal(emptyURL, value.Get(emptyURL))
value.Value = &url.URL{}
suite.Equal(value.Value, value.Get(emptyURL))
}
func TestTypeHttpsURL(t *testing.T) {
t.Parallel()
suite.Run(t, &HttpsURLTestSuite{})
}
+8 -4
View File
@@ -15,20 +15,24 @@ type TypeProxyURL struct {
func (t *TypeProxyURL) Set(value string) error {
parsedURL, err := url.Parse(value)
if err != nil {
return fmt.Errorf("value is not corect URL (%s): %w", value, err)
return fmt.Errorf("value is not correct URL (%s): %w", value, err)
}
if parsedURL.Host == "" {
return fmt.Errorf("url has to have a schema: %s", value)
}
if parsedURL.Scheme != "socks5" {
switch parsedURL.Scheme {
case "socks5", "socks5h":
default:
return fmt.Errorf("unsupported schema: %s", parsedURL.Scheme)
}
if _, _, err := net.SplitHostPort(parsedURL.Host); err != nil {
parsedURL.Host = net.JoinHostPort(parsedURL.Host,
typeProxyURLDefaultSOCKS5Port)
parsedURL.Host = net.JoinHostPort(
parsedURL.Host,
typeProxyURLDefaultSOCKS5Port,
)
}
t.Value = parsedURL
+5
View File
@@ -2,6 +2,7 @@ package testlib
import (
"context"
"net"
"net/http"
"github.com/9seconds/mtg/v2/essentials"
@@ -24,6 +25,10 @@ func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address st
return args.Get(0).(essentials.Conn), args.Error(1) //nolint: wrapcheck, forcetypeassert
}
func (m *MtglibNetworkMock) NativeDialer() *net.Dialer {
return m.Called().Get(0).(*net.Dialer)
}
func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context,
network, address string) (essentials.Conn, error),
) *http.Client {
+22 -18
View File
@@ -1,11 +1,11 @@
[[tools.go]]
version = "1.26.0"
version = "1.26.1"
backend = "core:go"
"platforms.linux-arm64" = { checksum = "sha256:bd03b743eb6eb4193ea3c3fd3956546bf0e3ca5b7076c8226334afe6b75704cd", url = "https://dl.google.com/go/go1.26.0.linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:aac1b08a0fb0c4e0a7c1555beb7b59180b05dfc5a3d62e40e9de90cd42f88235", url = "https://dl.google.com/go/go1.26.0.linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:b1640525dfe68f066d56f200bef7bf4dce955a1a893bd061de6754c211431023", url = "https://dl.google.com/go/go1.26.0.darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:1ca28b7703cbea05a65b2a1d92d6b308610ef92f8824578a0874f2e60c9d5a22", url = "https://dl.google.com/go/go1.26.0.darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:9bbe0fc64236b2b51f6255c05c4232532b8ecc0e6d2e00950bd3021d8a4d07d4", url = "https://dl.google.com/go/go1.26.0.windows-amd64.zip"}
"platforms.linux-arm64" = { checksum = "sha256:a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7", url = "https://dl.google.com/go/go1.26.1.linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a", url = "https://dl.google.com/go/go1.26.1.linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f", url = "https://dl.google.com/go/go1.26.1.darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08", url = "https://dl.google.com/go/go1.26.1.darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:9b68112c913f45b7aebbf13c036721264bbba7e03a642f8f7490c561eebd1ecc", url = "https://dl.google.com/go/go1.26.1.windows-amd64.zip"}
[[tools."go:golang.org/x/pkgsite/cmd/pkgsite"]]
version = "latest"
@@ -15,24 +15,28 @@ backend = "go:golang.org/x/pkgsite/cmd/pkgsite"
version = "0.21.1"
backend = "go:golang.org/x/tools/gopls"
[[tools."go:golang.org/x/vuln/cmd/govulncheck"]]
version = "1.1.4"
backend = "go:golang.org/x/vuln/cmd/govulncheck"
[[tools."go:mvdan.cc/gofumpt"]]
version = "0.9.2"
backend = "go:mvdan.cc/gofumpt"
[[tools.golangci-lint]]
version = "2.10.1"
version = "2.11.3"
backend = "aqua:golangci/golangci-lint"
"platforms.linux-arm64" = { checksum = "sha256:6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:03bfadf67e52b441b7ec21305e501c717df93c959836d66c7f97312654acb297", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:66fb0da81b8033b477f97eea420d4b46b230ca172b8bb87c6610109f3772b6b6", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:c60c87695e79db8e320f0e5be885059859de52bb5ee5f11be5577828570bc2a3", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-windows-amd64.zip"}
"platforms.linux-arm64" = { checksum = "sha256:ee3d95f301359e7d578e6d99c8ad5aeadbabc5a13009a30b2b0df11c8058afe9", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:87bb8cddbcc825d5778b64e8a91b46c0526b247f4e2f2904dea74ec7450475d1", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:30ee39979c516b9d1adca289a3f93429d130c4c0fda5e57d637850894221f6cc", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:f93bda1f2cc981fd1326464020494be62f387bbf262706e1b3b644e5afacc440", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:cd42e890176bc5cfeb36225a77e66b9410ddd3a59a03551e23f6b210d29e1f67", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-windows-amd64.zip"}
[[tools.goreleaser]]
version = "2.13.3"
version = "2.14.3"
backend = "aqua:goreleaser/goreleaser"
"platforms.linux-arm64" = { checksum = "sha256:156656d0f874542d618568bd50afd3d33ced2e8aab2c60cc7c21e1b9fa52031e", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.13.3/goreleaser_Linux_arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:4b66f2f78f78561330350651ade557b70328664718490f37834749073af21d20", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.13.3/goreleaser_Linux_x86_64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:5516c37779efb3935d5b213cda3b0b9025ae94ddbcb51df6919acbcdef4194b0", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.13.3/goreleaser_Darwin_all.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:5516c37779efb3935d5b213cda3b0b9025ae94ddbcb51df6919acbcdef4194b0", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.13.3/goreleaser_Darwin_all.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:c5586c4ed749ca358ad61ed73ee4b8039cfa68daae8c23e69fb086d549dfb31d", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.13.3/goreleaser_Windows_x86_64.zip"}
"platforms.linux-arm64" = { checksum = "sha256:581a10e53c1176b3e81ee45cf531e02dbf899db0bc7b795669347df4276ce948", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Linux_arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:dc7faeeeb6da8bdfda788626263a4ae725892a8c7504b975c3234127d4a44579", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Linux_x86_64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:3507798489e107a78aff36b169de48148a335ac26eb3161608d905f3f3a957bd", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Darwin_all.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:3507798489e107a78aff36b169de48148a335ac26eb3161608d905f3f3a957bd", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Darwin_all.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:3deea8ff471aa258a2d99f3e5302971d7028647ae8ddaf103257a8113e485a31", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Windows_x86_64.zip"}
+38 -10
View File
@@ -3,10 +3,12 @@ package mtglib
import (
"bytes"
"context"
"fmt"
"io"
"sync"
"net"
"github.com/9seconds/mtg/v2/essentials"
"github.com/pires/go-proxyproto"
)
type connTraffic struct {
@@ -40,22 +42,15 @@ func (c connTraffic) Write(b []byte) (int, error) {
type connRewind struct {
essentials.Conn
active io.Reader
buf bytes.Buffer
mutex sync.RWMutex
active io.Reader
}
func (c *connRewind) Read(p []byte) (int, error) {
c.mutex.RLock()
defer c.mutex.RUnlock()
return c.active.Read(p) //nolint: wrapcheck
return c.active.Read(p)
}
func (c *connRewind) Rewind() {
c.mutex.Lock()
defer c.mutex.Unlock()
c.active = io.MultiReader(&c.buf, c.Conn)
}
@@ -67,3 +62,36 @@ func newConnRewind(conn essentials.Conn) *connRewind {
return rv
}
type connProxyProtocol struct {
essentials.Conn
sourceAddr net.Addr
headersWritten bool
}
func (c *connProxyProtocol) Write(p []byte) (int, error) {
if !c.headersWritten {
headers := proxyproto.HeaderProxyFromAddrs(2, c.sourceAddr, c.RemoteAddr())
toSend, err := headers.Format()
if err != nil {
panic(err)
}
if _, err := c.Conn.Write(toSend); err != nil {
return 0, fmt.Errorf("cannot send proxy protocol header: %w", err)
}
c.headersWritten = true
}
return c.Conn.Write(p)
}
func newConnProxyProtocol(source, target essentials.Conn) *connProxyProtocol {
return &connProxyProtocol{
Conn: target,
sourceAddr: source.RemoteAddr(),
}
}
+96
View File
@@ -1,14 +1,17 @@
package mtglib
import (
"bufio"
"bytes"
"context"
"errors"
"io"
"net"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/pires/go-proxyproto"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
@@ -200,6 +203,94 @@ func (suite *ConnRewindTestSuite) TestRead() {
suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, data)
}
type ConnProxyProtocolTestSuite struct {
suite.Suite
sourceConnMock *testlib.EssentialsConnMock
targetConnMock *testlib.EssentialsConnMock
conn *connProxyProtocol
}
func (suite *ConnProxyProtocolTestSuite) SetupTest() {
suite.sourceConnMock = &testlib.EssentialsConnMock{}
suite.targetConnMock = &testlib.EssentialsConnMock{}
localAddr := &net.TCPAddr{
IP: net.ParseIP("127.0.0.1").To4(),
}
remoteAddr := &net.TCPAddr{
IP: net.ParseIP("127.0.0.2").To4(),
}
suite.sourceConnMock.
On("RemoteAddr").
Return(localAddr)
suite.targetConnMock.
On("RemoteAddr").
Maybe().
Return(remoteAddr)
suite.conn = newConnProxyProtocol(suite.sourceConnMock, suite.targetConnMock)
}
func (suite *ConnProxyProtocolTestSuite) TestRead() {
value := []byte{1, 2, 3, 4, 5}
toRead := make([]byte, len(value))
suite.targetConnMock.
On("Read", mock.AnythingOfType("[]uint8")).
Once().
Return(len(toRead), nil).
Run(func(args mock.Arguments) {
arr := args.Get(0).([]byte)
copy(arr, value)
})
n, err := suite.conn.Read(toRead)
suite.Equal(len(value), n)
suite.NoError(err)
suite.Equal(value, toRead)
}
func (suite *ConnProxyProtocolTestSuite) TestWrite() {
value := []byte{1, 2, 3, 4, 5}
buf := &bytes.Buffer{}
bufReader := bufio.NewReader(buf)
suite.targetConnMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(28, nil).
Run(func(args mock.Arguments) {
arr := args.Get(0).([]byte)
buf.Write(arr)
})
_, err := suite.conn.Write(value)
suite.NoError(err)
header, err := proxyproto.Read(bufReader)
suite.NoError(err)
sourceAddr, destAddr, ok := header.TCPAddrs()
suite.True(ok)
suite.Equal(suite.sourceConnMock.RemoteAddr(), sourceAddr)
suite.Equal(suite.targetConnMock.RemoteAddr(), destAddr)
read, _ := io.ReadAll(bufReader)
suite.Equal(value, read)
_, err = suite.conn.Write(value)
suite.NoError(err)
read, _ = io.ReadAll(bufReader)
suite.Equal(value, read)
}
func (suite *ConnProxyProtocolTestSuite) TearDownTest() {
suite.sourceConnMock.AssertExpectations(suite.T())
suite.targetConnMock.AssertExpectations(suite.T())
}
func TestConnTraffic(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTrafficTestSuite{})
@@ -209,3 +300,8 @@ func TestConnRewind(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnRewindTestSuite{})
}
func TestConnProxyProtocol(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnProxyProtocolTestSuite{})
}
+12 -1
View File
@@ -99,6 +99,13 @@ const (
// reads from Telegram after which connection will be terminated. This is
// required to abort stale connections.
TCPRelayReadTimeout = 20 * time.Second
// DoppelGangerPerRaid defines a number of requests to each URL
// per raid.
DoppelGangerPerRaid = 10
// DoppelGangerEach defines a time period between each crawl attempt.
DoppelGangerEach = 6 * time.Hour
)
// Network defines a knowledge how to work with a network. It may sound fun but
@@ -117,13 +124,17 @@ type Network interface {
// Dial establishes context-free TCP connections.
Dial(network, address string) (essentials.Conn, error)
// DialContext dials using a context. This is a preferrable way of
// DialContext dials using a context. This is a preferable way of
// establishing TCP connections.
DialContext(ctx context.Context, network, address string) (essentials.Conn, error)
// MakeHTTPClient build an HTTP client with given dial function. If nothing is
// provided, then DialContext of this interface is going to be used.
MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client
// NativeDialer returns a configured instance of native dialer that
// skips proxy connections or any other irrelevant settings.
NativeDialer() *net.Dialer
}
// AntiReplayCache is an interface that is used to detect replay attacks based
+10 -3
View File
@@ -1,10 +1,17 @@
package dc
import (
"fmt"
"github.com/9seconds/mtg/v2/mtglib/internal/obfuscation"
)
type Addr struct {
Network string
Address string
Network string
Address string
Obfuscator obfuscation.Obfuscator
}
func (d Addr) String() string {
return d.Address
return fmt.Sprintf("addr=%s, secret=%v", d.Address, d.Obfuscator.Secret)
}
-16
View File
@@ -1,16 +0,0 @@
package dc_test
import (
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/dc"
"github.com/stretchr/testify/assert"
)
func TestAddr(t *testing.T) {
t.Parallel()
addr := dc.Addr{Network: "tcp4", Address: "127.0.0.1:443"}
assert.Equal(t, "127.0.0.1:443", addr.String())
}
+61 -50
View File
@@ -1,5 +1,10 @@
package dc
import (
"context"
"time"
)
type preferIP uint8
const (
@@ -10,7 +15,18 @@ const (
)
const (
// Default DC to connect to if not sure.
DefaultDC = 2
// How often should we request updates from
// https://core.telegram.org/getProxyConfig
PublicConfigUpdateEach = time.Hour
PublicConfigUpdateURLv4 = "https://core.telegram.org/getProxyConfig"
PublicConfigUpdateURLv6 = "https://core.telegram.org/getProxyConfigV6"
// How often should we extract hosts from Telegram using help.getConfig
// method.
OwnConfigUpdateEach = time.Hour
)
type Logger interface {
@@ -18,56 +34,51 @@ type Logger interface {
WarningError(msg string, err error)
}
var (
// https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/mtproto_dc_options.cpp#L30
defaultDCAddrSet = dcAddrSet{
v4: map[int][]Addr{
1: {
{Network: "tcp4", Address: "149.154.175.50:443"},
},
2: {
{Network: "tcp4", Address: "149.154.167.51:443"},
{Network: "tcp4", Address: "95.161.76.100:443"},
},
3: {
{Network: "tcp4", Address: "149.154.175.100:443"},
},
4: {
{Network: "tcp4", Address: "149.154.167.91:443"},
},
5: {
{Network: "tcp4", Address: "149.154.171.5:443"},
},
},
v6: map[int][]Addr{
1: {
{Network: "tcp6", Address: "[2001:b28:f23d:f001::a]:443"},
},
2: {
{Network: "tcp6", Address: "[2001:67c:04e8:f002::a]:443"},
},
3: {
{Network: "tcp6", Address: "[2001:b28:f23d:f003::a]:443"},
},
4: {
{Network: "tcp6", Address: "[2001:67c:04e8:f004::a]:443"},
},
5: {
{Network: "tcp6", Address: "[2001:b28:f23f:f005::a]:443"},
},
},
}
type Updater interface {
Run(ctx context.Context)
}
defaultDCOverridesAddrSet = dcAddrSet{
v4: map[int][]Addr{
203: {
{Network: "tcp4", Address: "91.105.192.100:443"},
},
// https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/mtproto_dc_options.cpp#L30
var defaultDCAddrSet = dcAddrSet{
v4: map[int][]Addr{
1: {
{Network: "tcp4", Address: "149.154.175.50:443"},
},
v6: map[int][]Addr{
203: {
{Network: "tcp6", Address: "[2a0a:f280:0203:000a:5000:0000:0000:0100]:443"},
},
2: {
{Network: "tcp4", Address: "149.154.167.51:443"},
{Network: "tcp4", Address: "95.161.76.100:443"},
},
}
)
3: {
{Network: "tcp4", Address: "149.154.175.100:443"},
},
4: {
{Network: "tcp4", Address: "149.154.167.91:443"},
},
5: {
{Network: "tcp4", Address: "149.154.171.5:443"},
},
203: {
{Network: "tcp4", Address: "91.105.192.100:443"},
},
},
v6: map[int][]Addr{
1: {
{Network: "tcp6", Address: "[2001:b28:f23d:f001::a]:443"},
},
2: {
{Network: "tcp6", Address: "[2001:67c:04e8:f002::a]:443"},
},
3: {
{Network: "tcp6", Address: "[2001:b28:f23d:f003::a]:443"},
},
4: {
{Network: "tcp6", Address: "[2001:67c:04e8:f004::a]:443"},
},
5: {
{Network: "tcp6", Address: "[2001:b28:f23f:f005::a]:443"},
},
203: {
{Network: "tcp6", Address: "[2a0a:f280:0203:000a:5000:0000:0000:0100]:443"},
},
},
}
+43
View File
@@ -0,0 +1,43 @@
package dc
import (
"context"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type LoggerMock struct {
mock.Mock
}
func (m *LoggerMock) Info(msg string) {
m.Called(msg)
}
func (m *LoggerMock) WarningError(msg string, err error) {
m.Called(msg, err)
}
type UpdaterTestSuiteBase struct {
suite.Suite
ctx context.Context
ctxCancel context.CancelFunc
loggerMock *LoggerMock
}
func (s *UpdaterTestSuiteBase) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
s.loggerMock = &LoggerMock{}
s.loggerMock.On("Info", mock.AnythingOfType("string"))
s.loggerMock.On("WarningError", mock.AnythingOfType("string"), mock.Anything)
s.ctx = ctx
s.ctxCancel = cancel
}
func (s *UpdaterTestSuiteBase) TearDownTest() {
s.ctxCancel()
}
@@ -0,0 +1,93 @@
package dc
import (
"bufio"
"context"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
)
var publicConfigRe = regexp.MustCompile(`^\s*proxy_for\s+(\d+)\s+(\S+?)?;\s*$`)
type PublicConfigUpdater struct {
updater
http *http.Client
tg *Telegram
}
func (p *PublicConfigUpdater) Run(ctx context.Context, url, network string) {
p.run(ctx, func() error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
panic(err)
}
resp, err := p.http.Do(req)
if err != nil {
if resp != nil {
io.Copy(io.Discard, resp.Body) //nolint: errcheck
resp.Body.Close() //nolint: errcheck
}
return fmt.Errorf("cannot fetch url %s: %w", url, err)
}
if resp.StatusCode >= http.StatusBadRequest {
return fmt.Errorf("unexpected status code from %s: %d", url, resp.StatusCode)
}
scanner := bufio.NewScanner(resp.Body)
addrs := map[int][]Addr{}
for scanner.Scan() {
matches := publicConfigRe.FindStringSubmatch(scanner.Text())
if len(matches) != 3 {
continue
}
dc, err := strconv.Atoi(matches[1])
if err != nil {
continue
}
switch dc {
// this is a list of DC we currently support. Other are ignored.
case 203: // CDN DC
p.logger.Info(fmt.Sprintf("found %s address for DC %d", matches[2], dc))
addrs[dc] = append(addrs[dc], Addr{
Network: network,
Address: matches[2],
})
}
}
if err := scanner.Err(); err != nil {
return fmt.Errorf("cannot read response body from %s: %w", url, err)
}
p.tg.lock.Lock()
defer p.tg.lock.Unlock()
if network == "tcp4" {
p.tg.view.publicConfigs.v4 = addrs
} else {
p.tg.view.publicConfigs.v6 = addrs
}
return nil
})
}
func NewPublicConfigUpdater(tg *Telegram, logger Logger, client *http.Client) *PublicConfigUpdater {
return &PublicConfigUpdater{
updater: updater{
logger: logger,
period: PublicConfigUpdateEach,
},
http: client,
tg: tg,
}
}
@@ -0,0 +1,113 @@
package dc
import (
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type PublicConfigUpdaterTestSuite struct {
UpdaterTestSuiteBase
u *PublicConfigUpdater
lock sync.Mutex
srv *httptest.Server
responseHandler func(w http.ResponseWriter)
}
func (s *PublicConfigUpdaterTestSuite) SetupSuite() {
s.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.lock.Lock()
s.responseHandler(w)
s.lock.Unlock()
}))
}
func (s *PublicConfigUpdaterTestSuite) TearDownSuite() {
s.srv.Close()
}
func (s *PublicConfigUpdaterTestSuite) SetupTest() {
s.UpdaterTestSuiteBase.SetupTest()
tg, err := New("prefer-ipv4")
require.NoError(s.T(), err)
s.u = NewPublicConfigUpdater(tg, s.loggerMock, s.srv.Client())
}
func (s *PublicConfigUpdaterTestSuite) Test502StatusCode() {
s.responseHandler = func(w http.ResponseWriter) {
w.WriteHeader(http.StatusBadGateway)
}
s.u.Run(s.ctx, s.srv.URL, "tcp4")
time.Sleep(100 * time.Millisecond)
s.ctxCancel()
s.u.Wait()
s.Len(s.u.tg.view.publicConfigs.v4, 0)
}
func (s *PublicConfigUpdaterTestSuite) TestEmptyFile() {
s.responseHandler = func(w http.ResponseWriter) {
w.WriteHeader(http.StatusOK)
}
s.u.Run(s.ctx, s.srv.URL, "tcp4")
time.Sleep(100 * time.Millisecond)
s.ctxCancel()
s.u.Wait()
s.Len(s.u.tg.view.publicConfigs.v4, 0)
}
func (s *PublicConfigUpdaterTestSuite) TestGarbage() {
result := `
proxy_for -1 -1;
proxy_for 100 100.10.0.0:3333;
lala 0 0
`
s.responseHandler = func(w http.ResponseWriter) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(result)) //nolint: errcheck
}
s.u.Run(s.ctx, s.srv.URL, "tcp4")
time.Sleep(100 * time.Millisecond)
s.ctxCancel()
s.u.Wait()
s.Len(s.u.tg.view.publicConfigs.v4, 0)
}
func (s *PublicConfigUpdaterTestSuite) TestOk() {
result := `
proxy_for 203 100.10.0.0:3333;
proxy_for -100 101.10.0.0:3333;
`
s.responseHandler = func(w http.ResponseWriter) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(result)) //nolint: errcheck
}
s.u.Run(s.ctx, s.srv.URL, "tcp4")
time.Sleep(100 * time.Millisecond)
s.ctxCancel()
s.u.Wait()
s.Len(s.u.tg.view.publicConfigs.v4, 1)
s.Len(s.u.tg.view.publicConfigs.v4[203], 1)
s.Equal("100.10.0.0:3333", s.u.tg.view.publicConfigs.v4[203][0].Address)
}
func TestPublicConfigUpdater(t *testing.T) {
suite.Run(t, &PublicConfigUpdaterTestSuite{})
}
+7 -36
View File
@@ -2,21 +2,25 @@ package dc
import (
"fmt"
"net"
"strings"
"sync"
)
type Telegram struct {
lock sync.RWMutex
view dcView
preferIP preferIP
}
func (t *Telegram) GetAddresses(dc int) []Addr {
t.lock.RLock()
defer t.lock.RUnlock()
switch t.preferIP {
case preferIPOnlyIPv4:
return t.view.getV4(dc)
case preferIPOnlyIPv6:
return t.view.getV4(dc)
return t.view.getV6(dc)
case preferIPPreferIPv4:
return append(t.view.getV4(dc), t.view.getV6(dc)...)
}
@@ -24,7 +28,7 @@ func (t *Telegram) GetAddresses(dc int) []Addr {
return append(t.view.getV6(dc), t.view.getV4(dc)...)
}
func New(ipPreference string, userOverrides map[int][]string) (*Telegram, error) {
func New(ipPreference string) (*Telegram, error) {
var pref preferIP
switch strings.ToLower(ipPreference) {
@@ -40,40 +44,7 @@ func New(ipPreference string, userOverrides map[int][]string) (*Telegram, error)
return nil, fmt.Errorf("unknown ip preference %s", ipPreference)
}
overrides := dcAddrSet{
v4: map[int][]Addr{},
v6: map[int][]Addr{},
}
for dc, addrs := range userOverrides {
for _, addr := range addrs {
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, fmt.Errorf("incorrect host %s: %w", addr, err)
}
parsed := net.ParseIP(host)
if parsed == nil {
return nil, fmt.Errorf("incorrect host %s", addr)
}
if parsed.To4() != nil {
overrides.v4[dc] = append(overrides.v4[dc], Addr{
Network: "tcp4",
Address: addr,
})
} else {
overrides.v6[dc] = append(overrides.v6[dc], Addr{
Network: "tcp6",
Address: addr,
})
}
}
}
return &Telegram{
view: dcView{
overrides: overrides,
},
preferIP: pref,
}, nil
}
+47
View File
@@ -0,0 +1,47 @@
package dc
import (
"context"
"sync"
"time"
)
type updater struct {
wg sync.WaitGroup
logger Logger
period time.Duration
}
func (u *updater) Wait() {
u.wg.Wait()
}
func (u *updater) run(ctx context.Context, callback func() error) {
u.wg.Go(func() {
ticker := time.NewTicker(u.period)
defer func() {
ticker.Stop()
select {
case <-ticker.C:
default:
}
}()
for {
u.logger.Info("start update")
if err := callback(); err != nil {
u.logger.WarningError("cannot update", err)
}
u.logger.Info("updated")
select {
case <-ctx.Done():
u.logger.Info("stop updating")
return
case <-ticker.C:
}
}
})
}
+55
View File
@@ -0,0 +1,55 @@
package dc
import (
"sync"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type UpdaterTestSuite struct {
UpdaterTestSuiteBase
u updater
}
func (s *UpdaterTestSuite) SetupTest() {
s.UpdaterTestSuiteBase.SetupTest()
s.u = updater{
logger: s.loggerMock,
period: 100 * time.Millisecond,
}
}
func (s *UpdaterTestSuite) TestPeriodicUpdates() {
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
lock := &sync.Mutex{}
collected := []time.Time{}
go s.u.run(s.ctx, func() error {
select {
case <-s.ctx.Done():
case value := <-ticker.C:
lock.Lock()
collected = append(collected, value)
lock.Unlock()
}
return nil
})
s.Eventually(func() bool {
lock.Lock()
defer lock.Unlock()
return len(collected) == 3
}, time.Second, 10*time.Millisecond)
}
func TestUpdater(t *testing.T) {
t.Parallel()
suite.Run(t, &UpdaterTestSuite{})
}
+3 -5
View File
@@ -1,20 +1,18 @@
package dc
type dcView struct {
overrides dcAddrSet
publicConfigs dcAddrSet
}
func (d dcView) getV4(dc int) []Addr {
addrs := d.overrides.getV4(dc)
addrs = append(addrs, defaultDCOverridesAddrSet.getV4(dc)...)
addrs := d.publicConfigs.getV4(dc)
addrs = append(addrs, defaultDCAddrSet.getV4(dc)...)
return addrs
}
func (d dcView) getV6(dc int) []Addr {
addrs := d.overrides.getV6(dc)
addrs = append(addrs, defaultDCOverridesAddrSet.getV6(dc)...)
addrs := d.publicConfigs.getV6(dc)
addrs = append(addrs, defaultDCAddrSet.getV6(dc)...)
return addrs
+9 -9
View File
@@ -16,7 +16,7 @@ type ViewTestSuite struct {
func (suite *ViewTestSuite) SetupSuite() {
suite.view = dcView{
overrides: dcAddrSet{
publicConfigs: dcAddrSet{
v4: map[int][]Addr{
111: {
{Network: "tcp4", Address: "127.0.0.1:443"},
@@ -37,15 +37,15 @@ func (suite *ViewTestSuite) SetupSuite() {
func (suite *ViewTestSuite) TestGetV4() {
testData := map[int][]Addr{
111: {
{"tcp4", "127.0.0.1:443"},
{Network: "tcp4", Address: "127.0.0.1:443"},
},
203: {
{"tcp4", "127.0.0.2:443"},
{"tcp4", "91.105.192.100:443"},
{Network: "tcp4", Address: "127.0.0.2:443"},
{Network: "tcp4", Address: "91.105.192.100:443"},
},
2: {
{"tcp4", "149.154.167.51:443"},
{"tcp4", "95.161.76.100:443"},
{Network: "tcp4", Address: "149.154.167.51:443"},
{Network: "tcp4", Address: "95.161.76.100:443"},
},
}
@@ -60,11 +60,11 @@ func (suite *ViewTestSuite) TestGetV6() {
testData := map[int][]Addr{
111: {},
203: {
{"tcp6", "xxx"},
{"tcp6", "[2a0a:f280:0203:000a:5000:0000:0000:0100]:443"},
{Network: "tcp6", Address: "xxx"},
{Network: "tcp6", Address: "[2a0a:f280:0203:000a:5000:0000:0000:0100]:443"},
},
1: {
{"tcp6", "[2001:b28:f23d:f001::a]:443"},
{Network: "tcp6", Address: "[2001:b28:f23d:f001::a]:443"},
},
}
+35
View File
@@ -0,0 +1,35 @@
package doppel
import (
"context"
"time"
)
type Clock struct {
stats *Stats
tick chan struct{}
}
func (c Clock) Start(ctx context.Context) {
tickTock := time.NewTimer(c.stats.Delay())
defer func() {
tickTock.Stop()
select {
case <-tickTock.C:
default:
}
}()
for {
select {
case <-ctx.Done():
return
case <-tickTock.C:
select {
case <-ctx.Done():
case c.tick <- struct{}{}:
}
tickTock.Reset(c.stats.Delay())
}
}
}
+80
View File
@@ -0,0 +1,80 @@
package doppel
import (
"context"
"sync"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type ClockTestSuite struct {
suite.Suite
clock Clock
wg sync.WaitGroup
ctx context.Context
ctxCancel context.CancelFunc
}
func (suite *ClockTestSuite) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
suite.ctx = ctx
suite.ctxCancel = cancel
suite.clock = Clock{
stats: &Stats{
k: StatsDefaultK,
lambda: StatsDefaultLambda,
},
tick: make(chan struct{}),
}
suite.wg.Go(func() {
suite.clock.Start(suite.ctx)
})
}
func (suite *ClockTestSuite) TearDownTest() {
suite.ctxCancel()
suite.wg.Wait()
}
func (suite *ClockTestSuite) TestTicks() {
received := 0
for range 3 {
select {
case <-suite.clock.tick:
received++
case <-time.After(2 * time.Second):
suite.Fail("timed out waiting for tick")
}
}
suite.Equal(3, received)
}
func (suite *ClockTestSuite) TestStopsOnCancel() {
select {
case <-suite.clock.tick:
case <-time.After(2 * time.Second):
suite.Fail("timed out waiting for first tick")
}
suite.ctxCancel()
time.Sleep(50 * time.Millisecond)
select {
case <-suite.clock.tick:
suite.Fail("received tick after cancel")
default:
}
}
func TestClock(t *testing.T) {
t.Parallel()
suite.Run(t, &ClockTestSuite{})
}
+116
View File
@@ -0,0 +1,116 @@
package doppel
import (
"bytes"
"context"
"sync"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
type Conn struct {
essentials.Conn
p *connPayload
}
type connPayload struct {
ctx context.Context
ctxCancel context.CancelCauseFunc
clock Clock
wg sync.WaitGroup
writeStream bytes.Buffer
writtenCond sync.Cond
done bool
}
func (c Conn) Write(p []byte) (int, error) {
if len(p) == 0 {
return 0, context.Cause(c.p.ctx)
}
c.p.writtenCond.L.Lock()
c.p.writeStream.Write(p)
c.p.writtenCond.L.Unlock()
c.p.writtenCond.Signal()
return len(p), context.Cause(c.p.ctx)
}
func (c Conn) Start() {
c.p.wg.Go(func() {
c.start()
})
}
func (c Conn) start() {
buf := [tls.MaxRecordSize]byte{}
for {
select {
case <-c.p.ctx.Done():
return
case <-c.p.clock.tick:
}
size := c.p.clock.stats.Size()
c.p.writtenCond.L.Lock()
for c.p.writeStream.Len() == 0 && !c.p.done {
c.p.writtenCond.Wait()
}
n, _ := c.p.writeStream.Read(buf[tls.SizeHeader : tls.SizeHeader+size])
c.p.writtenCond.L.Unlock()
if n == 0 {
continue
}
if err := tls.WriteRecordInPlace(c.Conn, buf[:], n); err != nil {
c.p.ctxCancel(err)
return
}
}
}
func (c Conn) Stop() {
c.p.ctxCancel(nil)
c.p.writtenCond.L.Lock()
c.p.done = true
c.p.writtenCond.L.Unlock()
c.p.writtenCond.Broadcast()
c.p.wg.Wait()
}
func NewConn(ctx context.Context, conn essentials.Conn, stats *Stats) Conn {
ctx, cancel := context.WithCancelCause(ctx)
rv := Conn{
Conn: conn,
p: &connPayload{
ctx: ctx,
ctxCancel: cancel,
writtenCond: sync.Cond{
L: &sync.Mutex{},
},
clock: Clock{
stats: stats,
tick: make(chan struct{}),
},
},
}
rv.p.writeStream.Grow(tls.DefaultBufferSize)
rv.p.wg.Go(func() {
rv.p.clock.Start(ctx)
})
rv.p.wg.Go(func() {
rv.start()
})
return rv
}
+194
View File
@@ -0,0 +1,194 @@
package doppel
import (
"bytes"
"context"
"encoding/binary"
"errors"
"io"
"sync"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ConnMock struct {
testlib.EssentialsConnMock
mu sync.Mutex
writeBuffer bytes.Buffer
}
func (m *ConnMock) Write(p []byte) (int, error) {
args := m.Called(p)
if err := args.Error(1); err != nil {
return args.Int(0), err
}
m.mu.Lock()
defer m.mu.Unlock()
return m.writeBuffer.Write(p)
}
func (m *ConnMock) Written() []byte {
m.mu.Lock()
defer m.mu.Unlock()
return bytes.Clone(m.writeBuffer.Bytes())
}
type ConnTestSuite struct {
suite.Suite
connMock *ConnMock
ctx context.Context
ctxCancel context.CancelFunc
}
func (suite *ConnTestSuite) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
suite.ctx = ctx
suite.ctxCancel = cancel
suite.connMock = &ConnMock{}
}
func (suite *ConnTestSuite) TearDownTest() {
suite.ctxCancel()
suite.connMock.AssertExpectations(suite.T())
}
func (suite *ConnTestSuite) makeConn() Conn {
return NewConn(suite.ctx, suite.connMock, &Stats{
k: 2.0,
lambda: 0.01,
})
}
func (suite *ConnTestSuite) TestWriteBuffersData() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
defer c.Stop()
n, err := c.Write([]byte{1, 2, 3})
suite.NoError(err)
suite.Equal(3, n)
}
func (suite *ConnTestSuite) TestWriteOutputsTLSRecords() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
payload := []byte("hello doppelganger")
_, err := c.Write(payload)
suite.NoError(err)
suite.Eventually(func() bool {
return len(suite.connMock.Written()) > 0
}, 2*time.Second, time.Millisecond)
c.Stop()
assembled := &bytes.Buffer{}
reader := bytes.NewReader(suite.connMock.Written())
for {
header := make([]byte, tls.SizeHeader)
if _, err := io.ReadFull(reader, header); err != nil {
break
}
suite.Equal(byte(tls.TypeApplicationData), header[0])
suite.Equal(tls.TLSVersion[:], header[tls.SizeRecordType:tls.SizeRecordType+tls.SizeVersion])
length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:])
suite.Greater(length, uint16(0))
rec := make([]byte, length)
_, err := io.ReadFull(reader, rec)
suite.NoError(err)
assembled.Write(rec)
}
suite.Equal(payload, assembled.Bytes())
}
func (suite *ConnTestSuite) TestWriteReturnsErrorAfterStop() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
c.Stop()
time.Sleep(10 * time.Millisecond)
_, err := c.Write([]byte{1})
suite.Error(err)
}
func (suite *ConnTestSuite) TestStopDoesNotDeadlockWhenStartIsWaiting() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
for range 100 {
func() {
ctx, cancel := context.WithCancel(suite.ctx)
defer cancel()
c := NewConn(ctx, suite.connMock, &Stats{
k: 2.0,
lambda: 0.01,
})
done := make(chan struct{})
go func() {
defer close(done)
c.Stop()
}()
select {
case <-done:
case <-time.After(2 * time.Second):
suite.Fail("Stop() deadlocked: start() likely stuck in writtenCond.Wait()")
}
}()
}
}
func (suite *ConnTestSuite) TestStopOnUnderlyingWriteError() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, errors.New("connection reset")).
Maybe()
c := suite.makeConn()
_, _ = c.Write([]byte("data"))
suite.Eventually(func() bool {
_, err := c.Write([]byte{1})
return err != nil
}, 2*time.Second, time.Millisecond)
}
func TestConn(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTestSuite{})
}
+185
View File
@@ -0,0 +1,185 @@
package doppel
import (
"context"
"sync"
"time"
"github.com/9seconds/mtg/v2/essentials"
)
const (
DoppelGangerMaxDurations = 4096
DoppelGangerScoutRaidEach = 6 * time.Hour
DoppelGangerScoutRepeats = 10
)
type gangerConnRequest struct {
ret chan<- Conn
payload essentials.Conn
}
type Ganger struct {
ctx context.Context
ctxCancel context.CancelFunc
logger Logger
wg sync.WaitGroup
scout Scout
scoutRaidEach time.Duration
scoutRaidRepeats int
drs bool
stats *Stats
durations []time.Duration
connRequests chan gangerConnRequest
}
func (g *Ganger) Shutdown() {
g.ctxCancel()
g.wg.Wait()
}
func (g *Ganger) Run() {
g.wg.Go(func() {
g.run()
})
}
func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) {
rvChan := make(chan Conn)
req := gangerConnRequest{
ret: rvChan,
payload: conn,
}
defer close(req.ret)
select {
case <-g.ctx.Done():
return Conn{}, context.Cause(g.ctx)
case g.connRequests <- req:
}
select {
case <-g.ctx.Done():
return Conn{}, context.Cause(g.ctx)
case conn := <-rvChan:
return conn, nil
}
}
func (g *Ganger) run() {
scoutTicker := time.NewTicker(g.scoutRaidEach)
defer func() {
scoutTicker.Stop()
select {
case <-scoutTicker.C:
default:
}
}()
scoutCollectedChan := make(chan []time.Duration)
currentScoutCollectedChan := scoutCollectedChan
updatedStatsChan := make(chan *Stats)
g.wg.Go(func() {
g.runScoutRaid(scoutCollectedChan)
})
for {
select {
case <-g.ctx.Done():
return
case durations := <-currentScoutCollectedChan:
g.durations = append(g.durations, durations...)
if len(g.durations) > DoppelGangerMaxDurations {
copy(g.durations, g.durations[len(g.durations)-DoppelGangerMaxDurations:])
g.durations = g.durations[:DoppelGangerMaxDurations]
}
if len(g.durations) < MinDurationsToCalculate {
continue
}
currentScoutCollectedChan = nil
g.wg.Go(func() {
select {
case <-g.ctx.Done():
case updatedStatsChan <- NewStats(durations, g.drs):
}
})
case stats := <-updatedStatsChan:
g.stats = stats
currentScoutCollectedChan = scoutCollectedChan
case <-scoutTicker.C:
g.wg.Go(func() {
g.runScoutRaid(scoutCollectedChan)
})
case req := <-g.connRequests:
select {
case <-g.ctx.Done():
case req.ret <- NewConn(g.ctx, req.payload, g.stats):
}
}
}
}
func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
durations := []time.Duration{}
for range g.scoutRaidRepeats {
learned, err := g.scout.Learn(g.ctx)
if err != nil {
g.logger.WarningError("cannot learn", err)
continue
}
durations = append(durations, learned...)
}
select {
case <-g.ctx.Done():
return
case rvChan <- durations:
}
}
func NewGanger(
ctx context.Context,
network Network,
logger Logger,
scoutEach time.Duration,
scoutRepeats int,
urls []string,
drs bool,
) *Ganger {
ctx, cancel := context.WithCancel(ctx)
if scoutEach == 0 {
scoutEach = DoppelGangerScoutRaidEach
}
if scoutRepeats == 0 {
scoutRepeats = DoppelGangerScoutRepeats
}
return &Ganger{
ctx: ctx,
ctxCancel: cancel,
logger: logger,
scoutRaidEach: scoutEach,
scoutRaidRepeats: scoutRepeats,
drs: drs,
stats: &Stats{
k: StatsDefaultK,
lambda: StatsDefaultLambda,
drs: drs,
},
scout: NewScout(network, urls),
connRequests: make(chan gangerConnRequest),
}
}
+107
View File
@@ -0,0 +1,107 @@
package doppel
import (
"bytes"
"sync"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type GangerTestSuite struct {
TLSServerTestSuite
log *LoggerMock
g *Ganger
}
func (suite *GangerTestSuite) SetupTest() {
suite.TLSServerTestSuite.SetupTest()
suite.log = &LoggerMock{}
suite.log.
On("Info", mock.AnythingOfType("string")).
Maybe()
suite.log.
On("WarningError", mock.AnythingOfType("string"), mock.Anything).
Maybe()
suite.g = NewGanger(suite.ctx, suite.network, suite.log, time.Hour, 1, suite.urls, true)
suite.g.Run()
}
func (suite *GangerTestSuite) TearDownTest() {
suite.g.Shutdown()
suite.log.AssertExpectations(suite.T())
suite.TLSServerTestSuite.TearDownTest()
}
func (suite *GangerTestSuite) TestNewConnAfterShutdown() {
suite.g.Shutdown()
connMock := &testlib.EssentialsConnMock{}
_, err := suite.g.NewConn(connMock)
suite.Error(err)
}
func (suite *GangerTestSuite) TestNewConnWhileRunning() {
connMock := &testlib.EssentialsConnMock{}
connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
connMock.On("Close").
Return(nil).
Maybe()
conn, err := suite.g.NewConn(connMock)
suite.NoError(err)
conn.Stop()
}
func (suite *GangerTestSuite) TestNewConnWriteProducesTLSRecords() {
var (
mu sync.Mutex
buf bytes.Buffer
)
connMock := &testlib.EssentialsConnMock{}
connMock.On("Write", mock.AnythingOfType("[]uint8")).
Run(func(args mock.Arguments) {
mu.Lock()
buf.Write(args.Get(0).([]byte))
mu.Unlock()
}).
Return(0, nil).
Maybe()
connMock.On("Close").
Return(nil).
Maybe()
conn, err := suite.g.NewConn(connMock)
suite.NoError(err)
payload := bytes.Repeat([]byte("x"), 512)
_, err = conn.Write(payload)
suite.NoError(err)
time.Sleep(500 * time.Millisecond)
conn.Stop()
mu.Lock()
written := buf.Bytes()
mu.Unlock()
suite.NotEmpty(written)
}
func TestGanger(t *testing.T) {
t.Parallel()
suite.Run(t, &GangerTestSuite{})
}
+43
View File
@@ -0,0 +1,43 @@
package doppel
import (
"context"
"net"
"net/http"
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
const (
// Please see Stats description
// https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/
// https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch
TLSRecordSizeStart = 1450
TLSRecordSizeAccel = 4096
TLSRecordSizeMax = 16384 - tls.SizeHeader
TLSCounterAccelAfter = 40
TLSCounterMaxAfter = TLSCounterAccelAfter + 20
TLSRecordSizeResetAfter = time.Second
)
// copypasted from mtglib
type Network interface {
// Dial establishes context-free TCP connections.
Dial(network, address string) (essentials.Conn, error)
// DialContext dials using a context. This is a preferable way of
// establishing TCP connections.
DialContext(ctx context.Context, network, address string) (essentials.Conn, error)
// MakeHTTPClient build an HTTP client with given dial function. If nothing is
// provided, then DialContext of this interface is going to be used.
MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client
// NativeDialer returns a configured instance of native dialer that
// skips proxy connections or any other irrelevant settings.
NativeDialer() *net.Dialer
}
+107
View File
@@ -0,0 +1,107 @@
package doppel
import (
"context"
"crypto/tls"
"net"
"net/http"
"net/http/httptest"
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type SimpleNetwork struct{}
func (s SimpleNetwork) Dial(network, address string) (essentials.Conn, error) {
return s.DialContext(context.Background(), network, address)
}
func (s SimpleNetwork) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) {
d := &net.Dialer{}
conn, err := d.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return conn.(*net.TCPConn), nil
}
func (s SimpleNetwork) NativeDialer() *net.Dialer {
return &net.Dialer{}
}
func (s SimpleNetwork) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client {
if dialFunc == nil {
dialFunc = s.DialContext
}
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, //nolint: gosec
},
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
return dialFunc(ctx, network, address)
},
},
}
}
type TLSServerTestSuite struct {
suite.Suite
tlsServer *httptest.Server
ctx context.Context
ctxCancel context.CancelFunc
network SimpleNetwork
urls []string
}
func (suite *TLSServerTestSuite) SetupSuite() {
suite.tlsServer = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusOK)
w.Header().Add("Hello", "how long")
if _, err := w.Write([]byte{1, 2, 3}); err != nil {
panic(err)
}
time.Sleep(5 * time.Millisecond)
if _, err := w.Write([]byte{1, 2, 3}); err != nil {
panic(err)
}
}))
suite.urls = []string{suite.tlsServer.URL}
}
func (suite *TLSServerTestSuite) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
suite.ctx = ctx
suite.ctxCancel = cancel
}
func (suite *TLSServerTestSuite) TearDownTest() {
suite.ctxCancel()
suite.tlsServer.CloseClientConnections()
}
func (suite *TLSServerTestSuite) TearDownSuite() {
suite.tlsServer.Close()
}
type LoggerMock struct {
mock.Mock
}
func (l *LoggerMock) Info(msg string) {
l.Called(msg)
}
func (l *LoggerMock) WarningError(msg string, err error) {
l.Called(msg, err)
}
+6
View File
@@ -0,0 +1,6 @@
package doppel
type Logger interface {
Info(msg string)
WarningError(msg string, err error)
}
+105
View File
@@ -0,0 +1,105 @@
package doppel
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
type Scout struct {
network Network
urls []string
}
func (s Scout) Learn(ctx context.Context) ([]time.Duration, error) {
var durations []time.Duration
for _, url := range s.urls {
learned, err := s.learn(ctx, url)
if err != nil {
return nil, err
}
durations = append(durations, learned...)
}
return durations, nil
}
func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
client, results := s.makeClient()
if !strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("url %s must be https", url)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if resp != nil {
io.Copy(io.Discard, resp.Body) //nolint: errcheck
resp.Body.Close() //nolint: errcheck
client.CloseIdleConnections()
}
if err != nil || len(results.data) == 0 {
return nil, err
}
durations := []time.Duration{}
lastTimestamp := time.Time{}
for i, v := range results.data {
if v.recordType != tls.TypeApplicationData {
continue
}
if lastTimestamp.IsZero() {
if i > 0 {
lastTimestamp = results.data[i-1].timestamp
} else {
lastTimestamp = v.timestamp
}
}
durations = append(durations, v.timestamp.Sub(lastTimestamp))
lastTimestamp = v.timestamp
}
return durations, nil
}
func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) {
dialer := s.network.NativeDialer()
collected := NewScoutConnCollected()
client := s.network.MakeHTTPClient(func(
ctx context.Context,
network string,
address string,
) (essentials.Conn, error) {
conn, err := dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return NewScoutConn(essentials.WrapNetConn(conn), collected), nil
})
return client, collected
}
func NewScout(network Network, urls []string) Scout {
return Scout{
network: network,
urls: urls,
}
}
+57
View File
@@ -0,0 +1,57 @@
package doppel
import (
"bytes"
"encoding/binary"
"io"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
type ScoutConn struct {
tls.Conn
results *ScoutConnCollected
rawBuf *bytes.Buffer
}
func (s ScoutConn) Read(p []byte) (int, error) {
buf := &bytes.Buffer{}
for {
if n, err := s.rawBuf.Read(p); err == nil {
return n, nil
}
s.rawBuf.Reset()
recordType, length, err := tls.ReadRecord(s.Conn, buf)
if err != nil {
return 0, err
}
s.results.Add(recordType)
s.rawBuf.Write([]byte{recordType})
s.rawBuf.Write(tls.TLSVersion[:])
if err := binary.Write(s.rawBuf, binary.BigEndian, uint16(length)); err != nil {
return 0, err
}
if _, err := io.Copy(s.rawBuf, buf); err != nil {
return 0, err
}
}
}
func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) ScoutConn {
rawBuf := &bytes.Buffer{}
rawBuf.Grow(tls.MaxRecordSize)
return ScoutConn{
Conn: tls.New(conn, false, false),
results: results,
rawBuf: rawBuf,
}
}
@@ -0,0 +1,29 @@
package doppel
import "time"
const (
ScoutConnCollectedPreallocSize = 100
)
type ScoutConnResult struct {
timestamp time.Time
recordType byte
}
type ScoutConnCollected struct {
data []ScoutConnResult
}
func (s *ScoutConnCollected) Add(record byte) {
s.data = append(s.data, ScoutConnResult{
timestamp: time.Now(),
recordType: record,
})
}
func NewScoutConnCollected() *ScoutConnCollected {
return &ScoutConnCollected{
data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
}
}
@@ -0,0 +1,42 @@
package doppel
import (
"testing"
"time"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/stretchr/testify/suite"
)
type ScoutConnCollectedTestSuite struct {
suite.Suite
}
func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
collected := NewScoutConnCollected()
collected.Add(tls.TypeApplicationData)
suite.Len(collected.data, 1)
suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
}
func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
collected := NewScoutConnCollected()
collected.Add(tls.TypeApplicationData)
time.Sleep(time.Microsecond)
collected.Add(tls.TypeApplicationData)
time.Sleep(time.Microsecond)
collected.Add(tls.TypeApplicationData)
for i := 1; i < len(collected.data); i++ {
suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))
}
}
func TestScoutConnCollected(t *testing.T) {
t.Parallel()
suite.Run(t, &ScoutConnCollectedTestSuite{})
}
+39
View File
@@ -0,0 +1,39 @@
package doppel
import (
"testing"
"github.com/stretchr/testify/suite"
)
type ScoutTestSuite struct {
TLSServerTestSuite
scout Scout
}
func (suite *ScoutTestSuite) SetupSuite() {
suite.TLSServerTestSuite.SetupSuite()
suite.scout = Scout{
network: suite.network,
urls: suite.urls,
}
}
func (suite *ScoutTestSuite) TestCollectResults() {
durations, err := suite.scout.Learn(suite.ctx)
suite.NoError(err)
suite.Less(3, len(durations))
}
func (suite *ScoutTestSuite) TestCollectNothing() {
suite.ctxCancel()
_, err := suite.scout.Learn(suite.ctx)
suite.Error(err)
}
func TestScout(t *testing.T) {
suite.Run(t, &ScoutTestSuite{})
}
+170
View File
@@ -0,0 +1,170 @@
package doppel
import (
"math"
"math/rand/v2"
"time"
)
const (
StatsBisectTimes = 70
StatsLowK = 0.01
StatsHighK = 10.0
// do not calculate statistics if we have < than this number of durations
MinDurationsToCalculate = 100
// these values are taken from ok.ru. measured from moscow site.
StatsDefaultK = 0.37846373895785335
StatsDefaultLambda = 1.73177086015485
// how many bytes should we drift
DRSNoise = 100
)
// Stats is responsible for generating values that are distributed according
// to some statistical distribution.
//
// It follows several ideas:
// 1. Based on nginx and Cloudflare behaviour, even if server is eager
// to send a lot, they all start with small TLS packets that are
// approximately MTU-sized. After
// 2. After ~40 TLS records, server considers TCP session as somewhat solid
// and reliable and ramps up to 4096.
// 3. After ~20 TLS records more it jumps to the max 16384 bytes and keep
// this size as long as it can
// 4. If there is no any byte within a connection for a longer time period,
// this counter resets.
//
// This is called Dynamic TLS Record Sizing
// - https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/
// - https://community.f5.com/kb/technicalarticles/boosting-tls-performance-with-dynamic-record-sizing-on-big-ip/280798
// - https://www.igvita.com/2013/10/24/optimizing-tls-record-size-and-buffering-latency/
//
// And this optimized for the very first byte, so web browsers could start to
// render as early as possible, showing user some preliminary results, optimizing
// for perceived latency.
//
// Since this is very typical for the website, we also aim for that.
//
// Another important idea is how delays between TLS packets are distributed.
// In case of sending huge heavy content with max sized record, delays have
// lognormal distribution. But a nature of a typical website shows that
// it eagers to deliver as fast as it can in a few very first records and
// could possibly slow down later.
//
// This is perfectly described by Weibull distribution:
// - https://en.wikipedia.org/wiki/Weibull_distribution
// - https://ieeexplore.ieee.org/document/6662948
// - https://www.researchgate.net/publication/224621285_Traffic_modelling_and_cost_optimization_for_transmitting_traffic_messages_over_a_hybrid_broadcast_and_cellular_network
// - https://ir.uitm.edu.my/id/eprint/105386/1/105386.pdf
//
// In other word, a combination of Dynamic TLS Record Sizing hints us for
// Weibull distribution.
//
// But we also have to keep in mind that DRS is not well spread yet. In most cases
// users still rely on OpenSSL or webserver defaults. OpenSSL chunks with
// biggest packet sizes, nginx relies on static setting that is 16k by default.
// Thus, dynamic sizing has to be present but we cannot oblige users to use that.
type Stats struct {
sizeLastRequested time.Time
sizeCounter int
// https://en.wikipedia.org/wiki/Shape_parameter
k float64
// https://en.wikipedia.org/wiki/Scale_parameter
lambda float64
// Dynamic Record Sizing
drs bool
}
func (d *Stats) Delay() time.Duration {
// u ∈ (0, 1], avoids ln(0)
u := 1.0 - rand.Float64()
// X = λ·(-ln U)^(1/k)
generated := d.lambda * math.Pow(-math.Log(u), 1.0/d.k)
// generated is in milliseconds
return time.Duration(generated * float64(time.Millisecond))
}
func (d *Stats) Size() int {
if time.Since(d.sizeLastRequested) > TLSRecordSizeResetAfter {
d.sizeCounter = 0
}
if !d.drs {
return TLSRecordSizeMax
}
d.sizeLastRequested = time.Now()
d.sizeCounter++
switch {
case d.sizeCounter <= TLSCounterAccelAfter:
return TLSRecordSizeStart - rand.IntN(DRSNoise)
case d.sizeCounter <= TLSCounterMaxAfter:
return TLSRecordSizeAccel - rand.IntN(DRSNoise)
}
return TLSRecordSizeMax
}
func NewStats(durations []time.Duration, drs bool) *Stats {
n := float64(len(durations))
// in milliseconds
durFloats := make([]float64, len(durations))
for i, v := range durations {
durFloats[i] = float64(v.Microseconds()) / 1000.0
}
// The bisection solves the standard Weibull MLE equation for shape
// parameter k. There is no any good formula for doing that so we
// approximate it by several bisections. The number of operations
// is statically defined by a constant.
sumLog := 0.0
for _, v := range durFloats {
sumLog += math.Log(v)
}
lowK := StatsLowK
highK := StatsHighK
for range StatsBisectTimes {
midK := (lowK + highK) / 2.0
sumXK := 0.0
sumXKLog := 0.0
for _, v := range durFloats {
xk := math.Pow(v, midK)
sumXK += xk
sumXKLog += xk * math.Log(v)
}
if (1.0/midK)+(sumLog/n)-(sumXKLog/sumXK) > 0 {
lowK = midK
} else {
highK = midK
}
}
k := (lowK + highK) / 2
sumXK := 0.0
for _, v := range durFloats {
sumXK += math.Pow(v, k)
}
// λ = (Σxᵢᵏ / n)^(1/k)
lambda := math.Pow(sumXK/n, 1.0/k)
return &Stats{
k: k,
lambda: lambda,
drs: drs,
}
}
+219
View File
@@ -0,0 +1,219 @@
package doppel
import (
"math"
"math/rand/v2"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type StatsTestSuite struct {
suite.Suite
}
func (suite *StatsTestSuite) GenWeibull(k, lambda float64, n int, seed uint64) []time.Duration {
rng := rand.New(rand.NewPCG(seed, 0))
samples := make([]time.Duration, n)
for i := range samples {
u := 1.0 - rng.Float64()
ms := lambda * math.Pow(-math.Log(u), 1.0/k)
d := time.Duration(ms * float64(time.Millisecond))
if d < time.Microsecond {
time.Sleep(time.Microsecond)
d = time.Microsecond
}
samples[i] = d
}
return samples
}
func (suite *StatsTestSuite) TestNewStatsRecoverParameters() {
knownK := 1.5
knownLambda := 100.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 42)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.1)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestNewStatsExponentialCase() {
// When k=1, Weibull reduces to exponential distribution.
knownK := 1.0
knownLambda := 50.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 123)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.1)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestNewStatsSmallK() {
// k < 1 produces a heavy-tailed distribution typical for network delays.
// Lambda must be large enough so samples stay above microsecond precision
// after time.Duration round-trip.
knownK := 0.6
knownLambda := 100.0
samples := suite.GenWeibull(knownK, knownLambda, 10000, 99)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.05)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestNewStatsLargeK() {
// k > 1: light tail, concentrated around the mode.
knownK := 5.0
knownLambda := 200.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 77)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.3)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestDelayNonNegative() {
stats := &Stats{
k: 1.5,
lambda: 100.0,
}
for range 200 {
dur := stats.Delay()
suite.GreaterOrEqual(dur, time.Duration(0))
}
}
func (suite *StatsTestSuite) TestDelayDistributionMean() {
// Weibull mean = λ · Γ(1 + 1/k)
k := 2.0
lambda := 50.0
stats := &Stats{k: k, lambda: lambda}
n := 50000
sum := 0.0
for range n {
dur := stats.Delay()
sum += float64(dur) / float64(time.Millisecond)
}
sampleMean := sum / float64(n)
expectedMean := lambda * math.Gamma(1.0+1.0/k)
suite.InDelta(expectedMean, sampleMean, expectedMean*0.05)
}
func (suite *StatsTestSuite) TestNewStatsRoundTrip() {
// Estimate parameters from data, then verify that Delay samples
// from the fitted distribution have approximately the same mean.
knownK := 1.2
knownLambda := 80.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 555)
stats := NewStats(samples, true)
n := 50000
sum := 0.0
for range n {
dur := stats.Delay()
sum += float64(dur) / float64(time.Millisecond)
}
sampleMean := sum / float64(n)
expectedMean := knownLambda * math.Gamma(1.0+1.0/knownK)
suite.InDelta(expectedMean, sampleMean, expectedMean*0.05)
}
func (suite *StatsTestSuite) TestSizeStartPhase() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
for range TLSCounterAccelAfter {
size := stats.Size()
suite.GreaterOrEqual(size, TLSRecordSizeStart-DRSNoise)
suite.LessOrEqual(size, TLSRecordSizeStart)
}
}
func (suite *StatsTestSuite) TestSizeAccelPhase() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
for range TLSCounterAccelAfter {
stats.Size()
}
for range TLSCounterMaxAfter - TLSCounterAccelAfter {
size := stats.Size()
suite.GreaterOrEqual(size, TLSRecordSizeAccel-DRSNoise)
suite.LessOrEqual(size, TLSRecordSizeAccel)
}
}
func (suite *StatsTestSuite) TestSizeMaxPhase() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
for range TLSCounterMaxAfter {
stats.Size()
}
for range 20 {
size := stats.Size()
suite.Equal(TLSRecordSizeMax, size)
}
}
func (suite *StatsTestSuite) TestSizeResetsAfterInactivity() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
// Advance past start phase.
for range TLSCounterMaxAfter {
stats.Size()
}
suite.Equal(TLSRecordSizeMax, stats.Size())
// Simulate inactivity by backdating sizeLastRequested.
stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond)
size := stats.Size()
suite.GreaterOrEqual(size, TLSRecordSizeStart-DRSNoise)
suite.LessOrEqual(size, TLSRecordSizeStart)
}
func (suite *StatsTestSuite) TestSizeNoDRSAlwaysMax() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: false}
for range TLSCounterMaxAfter + 20 {
suite.Equal(TLSRecordSizeMax, stats.Size())
}
}
func (suite *StatsTestSuite) TestSizeNoDRSIgnoresCounter() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: false}
// Even after many calls, always returns max.
for range 200 {
suite.Equal(TLSRecordSizeMax, stats.Size())
}
// Inactivity has no effect either.
stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond)
suite.Equal(TLSRecordSizeMax, stats.Size())
}
func TestStats(t *testing.T) {
t.Parallel()
suite.Run(t, &StatsTestSuite{})
}
-134
View File
@@ -1,134 +0,0 @@
package faketls
import (
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/binary"
"fmt"
"time"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
)
type ClientHello struct {
Time time.Time
Random [RandomLen]byte
SessionID []byte
Host string
CipherSuite uint16
}
func (c ClientHello) Valid(hostname string, tolerateTimeSkewness time.Duration) error {
if c.Host != "" && c.Host != hostname {
return fmt.Errorf("incorrect hostname %s", hostname)
}
now := time.Now()
timeDiff := now.Sub(c.Time)
if timeDiff < 0 {
timeDiff = -timeDiff
}
if timeDiff > tolerateTimeSkewness {
return fmt.Errorf("incorrect timestamp. got=%d, now=%d, diff=%s",
c.Time.Unix(), now.Unix(), timeDiff.String())
}
return nil
}
func ParseClientHello(secret, handshake []byte) (ClientHello, error) {
hello := ClientHello{}
if len(handshake) < ClientHelloMinLen {
return hello, fmt.Errorf("lengh of handshake is too small: %d", len(handshake))
}
if handshake[0] != HandshakeTypeClient {
return hello, fmt.Errorf("unknown handshake type %#x", handshake[0])
}
handshakeSizeBytes := [4]byte{0, handshake[1], handshake[2], handshake[3]}
handshakeLength := binary.BigEndian.Uint32(handshakeSizeBytes[:])
if len(handshake)-4 != int(handshakeLength) {
return hello,
fmt.Errorf("incorrect handshake size. manifested=%d, real=%d",
handshakeLength, len(handshake)-4)
}
copy(hello.Random[:], handshake[ClientHelloRandomOffset:])
copy(handshake[ClientHelloRandomOffset:], clientHelloEmptyRandom)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeHandshake
rec.Version = record.Version10
rec.Payload.Write(handshake)
// mac is calculated for the whole record, not only
// for the payload part
mac := hmac.New(sha256.New, secret)
rec.Dump(mac) //nolint: errcheck
computedRandom := mac.Sum(nil)
for i := range RandomLen {
computedRandom[i] ^= hello.Random[i]
}
if subtle.ConstantTimeCompare(clientHelloEmptyRandom[:RandomLen-4], computedRandom[:RandomLen-4]) != 1 {
return hello, ErrBadDigest
}
timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:]))
hello.Time = time.Unix(timestamp, 0)
parseSessionID(&hello, handshake)
parseCipherSuite(&hello, handshake)
parseSNI(&hello, handshake)
return hello, nil
}
func parseSessionID(hello *ClientHello, handshake []byte) {
hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset])
copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:])
}
func parseCipherSuite(hello *ClientHello, handshake []byte) {
cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3
hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2])
}
func parseSNI(hello *ClientHello, handshake []byte) {
cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 1
handshake = handshake[cipherSuiteOffset:]
cipherSuiteLength := binary.BigEndian.Uint16(handshake[:2])
handshake = handshake[2+cipherSuiteLength:]
compressionMethodsLength := int(handshake[0])
handshake = handshake[1+compressionMethodsLength:]
extensionsLength := binary.BigEndian.Uint16(handshake[:2])
handshake = handshake[2 : 2+extensionsLength]
for len(handshake) > 0 {
if binary.BigEndian.Uint16(handshake[:2]) != ExtensionSNI {
extensionsLength := binary.BigEndian.Uint16(handshake[2:4])
handshake = handshake[4+extensionsLength:]
continue
}
hostnameLength := binary.BigEndian.Uint16(handshake[7:9])
handshake = handshake[9:]
hello.Host = string(handshake[:int(hostnameLength)])
return
}
}
@@ -1,21 +0,0 @@
package faketls_test
import (
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/stretchr/testify/require"
)
var FuzzClientHelloSecret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
func FuzzClientHello(f *testing.F) {
f.Add([]byte{1, 2, 3})
f.Fuzz(func(t *testing.T, frame []byte) {
_, err := faketls.ParseClientHello(FuzzClientHelloSecret, frame)
// a probability of having != err is almost negligible
require.Error(t, err)
})
}
@@ -1,191 +0,0 @@
package faketls_test
import (
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type ClientHelloSnapshot struct {
Time int `json:"time"`
Random string `json:"random"`
SessionID string `json:"sessionId"`
Host string `json:"host"`
CipherSuite int `json:"cipherSuite"`
Full string `json:"full"`
}
func (c ClientHelloSnapshot) GetTime() time.Time {
return time.Unix(int64(c.Time), 0)
}
func (c ClientHelloSnapshot) GetRandom() []byte {
data, _ := base64.StdEncoding.DecodeString(c.Random)
return data
}
func (c ClientHelloSnapshot) GetSessionID() []byte {
data, _ := base64.StdEncoding.DecodeString(c.SessionID)
return data
}
func (c ClientHelloSnapshot) GetHost() string {
return c.Host
}
func (c ClientHelloSnapshot) GetCipherSuite() uint16 {
return uint16(c.CipherSuite)
}
func (c ClientHelloSnapshot) GetFull() []byte {
data, _ := base64.StdEncoding.DecodeString(c.Full)
return data
}
type ClientHelloTestSuite struct {
suite.Suite
secret mtglib.Secret
}
func (suite *ClientHelloTestSuite) SetupSuite() {
parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d")
if err != nil {
panic(err)
}
suite.secret = parsed
}
func (suite *ClientHelloTestSuite) TestEmptyHandshake() {
_, err := faketls.ParseClientHello(suite.secret.Key[:], nil)
suite.Error(err)
}
func (suite *ClientHelloTestSuite) TestIncorrectHandshakeType() {
data := make([]byte, 1024)
data[0] = 0x02
_, err := faketls.ParseClientHello(suite.secret.Key[:], data)
suite.Error(err)
}
func (suite *ClientHelloTestSuite) TestIncorrectLength() {
data := make([]byte, 1024)
data[0] = 0x01
data[1] = 0xff
data[2] = 0xff
_, err := faketls.ParseClientHello(suite.secret.Key[:], data)
suite.Error(err)
}
func (suite *ClientHelloTestSuite) TestSnapshotOk() {
files, err := os.ReadDir("testdata")
suite.NoError(err)
testData := []string{}
for _, v := range files {
if strings.HasPrefix(v.Name(), "client-hello-ok") {
testData = append(testData, v.Name())
}
}
for _, name := range testData {
path := filepath.Join("testdata", name)
suite.T().Run(name, func(t *testing.T) {
fileData, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &ClientHelloSnapshot{}
assert.NoError(t, json.Unmarshal(fileData, snapshot))
hello, err := faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull())
assert.NoError(t, err)
assert.WithinDuration(t, snapshot.GetTime(), hello.Time, time.Second)
assert.Equal(t, snapshot.GetRandom(), hello.Random[:])
assert.Equal(t, snapshot.GetSessionID(), hello.SessionID)
assert.Equal(t, snapshot.GetHost(), hello.Host)
assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite)
})
}
}
func (suite *ClientHelloTestSuite) TestSnapshotBad() {
files, err := os.ReadDir("testdata")
suite.NoError(err)
testData := []string{}
for _, v := range files {
if strings.HasPrefix(v.Name(), "client-hello-bad") {
testData = append(testData, v.Name())
}
}
for _, name := range testData {
path := filepath.Join("testdata", name)
suite.T().Run(name, func(t *testing.T) {
fileData, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &ClientHelloSnapshot{}
assert.NoError(t, json.Unmarshal(fileData, snapshot))
_, err = faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull())
assert.Error(t, err)
})
}
}
func (suite *ClientHelloTestSuite) TestValidateHostname() {
hello := faketls.ClientHello{
Time: time.Now(),
}
suite.NoError(hello.Valid("hostname", time.Second))
hello.Host = "hostname"
suite.Error(hello.Valid("hostname2", time.Second))
suite.NoError(hello.Valid("hostname", time.Second))
}
func (suite *ClientHelloTestSuite) TestValidateTime() {
testData := []time.Duration{
-2 * time.Second,
2 * time.Second,
}
for _, v := range testData {
value := v
suite.T().Run(value.String(), func(t *testing.T) {
hello := faketls.ClientHello{
Host: "hostname",
Time: time.Now().Add(value),
}
suite.Error(hello.Valid("hostname", 500*time.Millisecond))
suite.Error(hello.Valid("hostname", time.Second))
suite.NoError(hello.Valid("hostname", 3*time.Second))
})
}
}
func TestClientHello(t *testing.T) {
t.Parallel()
suite.Run(t, &ClientHelloTestSuite{})
}
-73
View File
@@ -1,73 +0,0 @@
package faketls
import (
"bytes"
"fmt"
"math/rand/v2"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
)
type Conn struct {
essentials.Conn
readBuffer bytes.Buffer
}
func (c *Conn) Read(p []byte) (int, error) {
if n, _ := c.readBuffer.Read(p); n > 0 {
return n, nil
}
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
for {
if err := rec.Read(c.Conn); err != nil {
return 0, err //nolint: wrapcheck
}
switch rec.Type { //nolint: exhaustive
case record.TypeApplicationData:
rec.Payload.WriteTo(&c.readBuffer) //nolint: errcheck
return c.readBuffer.Read(p) //nolint: wrapcheck
case record.TypeChangeCipherSpec:
default:
return 0, fmt.Errorf("unsupported record type %v", rec.Type)
}
}
}
func (c *Conn) Write(p []byte) (int, error) {
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeApplicationData
rec.Version = record.Version12
sendBuffer := acquireBytesBuffer()
defer releaseBytesBuffer(sendBuffer)
lenP := len(p)
for len(p) > 0 {
chunkSize := rand.IntN(record.TLSMaxRecordSize)
if chunkSize > len(p) || chunkSize == 0 {
chunkSize = len(p)
}
rec.Payload.Reset()
rec.Payload.Write(p[:chunkSize])
rec.Dump(sendBuffer) //nolint: errcheck
p = p[chunkSize:]
}
if _, err := c.Conn.Write(sendBuffer.Bytes()); err != nil {
return 0, err //nolint: wrapcheck
}
return lenP, nil
}
-153
View File
@@ -1,153 +0,0 @@
package faketls_test
import (
"bytes"
"crypto/rand"
"errors"
"io"
"testing"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ConnMock struct {
testlib.EssentialsConnMock
readBuffer bytes.Buffer
writeBuffer bytes.Buffer
}
func (m *ConnMock) Read(p []byte) (int, error) {
m.Called(p)
return m.readBuffer.Read(p) //nolint: wrapcheck
}
func (m *ConnMock) Write(p []byte) (int, error) {
m.Called(p)
return m.writeBuffer.Write(p) //nolint: wrapcheck
}
type ConnTestSuite struct {
suite.Suite
connMock *ConnMock
c *faketls.Conn
}
func (suite *ConnTestSuite) SetupTest() {
suite.connMock = &ConnMock{}
suite.c = &faketls.Conn{
Conn: suite.connMock,
}
}
func (suite *ConnTestSuite) TearDownTest() {
suite.connMock.AssertExpectations(suite.T())
}
func (suite *ConnTestSuite) TestRead() {
suite.connMock.On("Read", mock.Anything).Return(0, nil)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeChangeCipherSpec
rec.Version = record.Version12
rec.Payload.WriteByte(0x01)
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeApplicationData
rec.Version = record.Version12
rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
resultBuffer := &bytes.Buffer{}
buf := make([]byte, 2)
for {
n, err := suite.c.Read(buf)
if errors.Is(err, io.EOF) {
break
}
resultBuffer.Write(buf[:n])
}
suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, resultBuffer.Bytes())
}
func (suite *ConnTestSuite) TestReadUnexpected() {
suite.connMock.On("Read", mock.Anything).Return(0, nil)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeChangeCipherSpec
rec.Version = record.Version12
rec.Payload.WriteByte(0x01)
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeHandshake
rec.Version = record.Version12
rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
buf := make([]byte, 2)
for {
_, err := suite.c.Read(buf)
switch {
case err == nil:
case errors.Is(err, io.EOF):
suite.FailNow("unexpected to finish")
default:
return
}
}
}
func (suite *ConnTestSuite) TestWrite() {
suite.connMock.On("Write", mock.Anything).Return(0, nil)
dataToRec := make([]byte, record.TLSMaxRecordSize*2)
rand.Read(dataToRec) //nolint: staticcheck, errcheck
n, err := suite.c.Write(dataToRec)
suite.NoError(err)
suite.Equal(len(dataToRec), n)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
buf := &bytes.Buffer{}
for {
if err := rec.Read(&suite.connMock.writeBuffer); err != nil {
break
}
suite.Equal(record.TypeApplicationData, rec.Type)
suite.Equal(record.Version12, rec.Version)
rec.Payload.WriteTo(buf) //nolint: errcheck
}
suite.Equal(dataToRec, buf.Bytes())
}
func TestConn(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTestSuite{})
}
-59
View File
@@ -1,59 +0,0 @@
package faketls
import (
"bytes"
"errors"
)
const (
// RandomLen defines a size of the random digest in TLS Hellos.
RandomLen = 32
// ClientHelloRandomOffset is an offset in ClientHello record where
// random digest is started.
ClientHelloRandomOffset = 6
// ClientHelloSessionIDOffset is an offset in ClientHello record where
// SessionID is started.
ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen
// ClientHelloMinLen is a minimal possible length of
// ClientHello record.
ClientHelloMinLen = 6
// WelcomePacketRandomOffset is an offset of random in ServerHello
// packet (including record envelope).
WelcomePacketRandomOffset = 11
// HandshakeTypeClient is a value representing a client handshake.
HandshakeTypeClient = 0x01
// HandshakeTypeServer is a value representing a server handshake.
HandshakeTypeServer = 0x02
// ChangeCipherValue is a value representing a change cipher
// specification record.
ChangeCipherValue = 0x01
// ExtensionSNI is a value for TLS extension 'SNI'.
ExtensionSNI = 0x00
)
var (
// ErrBadDigest is returned if given TLS Client Hello mismatches with a
// derived one.
ErrBadDigest = errors.New("bad digest")
serverHelloSuffix = []byte{
0x00, // no compression
0x00, 0x2e, // 46 bytes of data
0x00, 0x2b, // Extension - Supported Versions
0x00, 0x02, // 2 bytes are following
0x03, 0x04, // TLS 1.3
0x00, 0x33, // Extension - Key Share
0x00, 0x24, // 36 bytes
0x00, 0x1d, // x25519 curve
0x00, 0x20, // 32 bytes of key
}
clientHelloEmptyRandom = bytes.Repeat([]byte{0}, RandomLen)
)
-21
View File
@@ -1,21 +0,0 @@
package faketls
import (
"bytes"
"sync"
)
var bytesBufferPool = sync.Pool{
New: func() any {
return &bytes.Buffer{}
},
}
func acquireBytesBuffer() *bytes.Buffer {
return bytesBufferPool.Get().(*bytes.Buffer) //nolint: forcetypeassert
}
func releaseBytesBuffer(b *bytes.Buffer) {
b.Reset()
bytesBufferPool.Put(b)
}
-84
View File
@@ -1,84 +0,0 @@
package record
import "fmt"
const TLSMaxRecordSize = 65535 // max uint16
type Type uint8
const (
// TypeChangeCipherSpec defines a byte value of the TLS record when a
// peer wants to change a specifications of the chosen cipher.
TypeChangeCipherSpec Type = 0x14
// TypeHandshake defines a byte value of the TLS record when a peer
// initiates a new TLS connection and wants to make a handshake
// ceremony.
TypeHandshake Type = 0x16
// TypeApplicationData defines a byte value of the TLS record when a
// peer sends an user data, not a control frames.
TypeApplicationData Type = 0x17
)
func (t Type) String() string {
switch t {
case TypeChangeCipherSpec:
return "changeCipher(0x14)"
case TypeHandshake:
return "handshake(0x16)"
case TypeApplicationData:
return "applicationData(0x17)"
}
return fmt.Sprintf("unknown(%#x)", byte(t))
}
func (t Type) Valid() error {
switch t {
case TypeChangeCipherSpec, TypeHandshake, TypeApplicationData:
return nil
}
return fmt.Errorf("unknown type %#x", byte(t))
}
type Version uint16
const (
// Version10 defines a TLS1.0.
Version10 Version = 769 // 0x03 0x01
// Version11 defines a TLS1.1.
Version11 Version = 770 // 0x03 0x02
// Version12 defines a TLS1.2.
Version12 Version = 771 // 0x03 0x03
// Version13 defines a TLS1.3.
Version13 Version = 772 // 0x03 0x04
)
func (v Version) String() string {
switch v {
case Version10:
return "tls1.0"
case Version11:
return "tls1.1"
case Version12:
return "tls1.2"
case Version13:
return "tls1.3"
}
return fmt.Sprintf("tls?(%d)", uint16(v))
}
func (v Version) Valid() error {
switch v {
case Version10, Version11, Version12, Version13:
return nil
}
return fmt.Errorf("unknown version %d", uint16(v))
}
@@ -1,79 +0,0 @@
package record_test
import (
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/suite"
)
type TypeTestSuite struct {
suite.Suite
}
func (suite *TypeTestSuite) TestChangeCipherSpec() {
suite.Contains(record.TypeChangeCipherSpec.String(), "changeCipher")
suite.Contains(record.TypeChangeCipherSpec.String(), "0x14")
suite.NoError(record.TypeChangeCipherSpec.Valid())
}
func (suite *TypeTestSuite) TestHandshake() {
suite.Contains(record.TypeHandshake.String(), "handshake")
suite.Contains(record.TypeHandshake.String(), "0x16")
suite.NoError(record.TypeHandshake.Valid())
}
func (suite *TypeTestSuite) TestApplicationData() {
suite.Contains(record.TypeApplicationData.String(), "applicationData")
suite.Contains(record.TypeApplicationData.String(), "0x17")
suite.NoError(record.TypeApplicationData.Valid())
}
func (suite *TypeTestSuite) TestUnknown() {
value := record.Type(0x20)
suite.Contains(value.String(), "unknown")
suite.Contains(value.String(), "0x20")
suite.Error(value.Valid())
}
type VersionTestSuite struct {
suite.Suite
}
func (suite *VersionTestSuite) Test10() {
suite.Equal("tls1.0", record.Version10.String())
suite.NoError(record.Version10.Valid())
}
func (suite *VersionTestSuite) Test11() {
suite.Equal("tls1.1", record.Version11.String())
suite.NoError(record.Version11.Valid())
}
func (suite *VersionTestSuite) Test12() {
suite.Equal("tls1.2", record.Version12.String())
suite.NoError(record.Version12.Valid())
}
func (suite *VersionTestSuite) Test13() {
suite.Equal("tls1.3", record.Version13.String())
suite.NoError(record.Version13.Valid())
}
func (suite *VersionTestSuite) TestUnknown() {
value := record.Version(900)
suite.Equal("tls?(900)", value.String())
suite.Error(value.Valid())
}
func TestType(t *testing.T) {
t.Parallel()
suite.Run(t, &TypeTestSuite{})
}
func TestVersion(t *testing.T) {
t.Parallel()
suite.Run(t, &VersionTestSuite{})
}
-20
View File
@@ -1,20 +0,0 @@
package record
import (
"sync"
)
var recordPool = sync.Pool{
New: func() any {
return &Record{}
},
}
func AcquireRecord() *Record {
return recordPool.Get().(*Record) //nolint: forcetypeassert
}
func ReleaseRecord(r *Record) {
r.Reset()
recordPool.Put(r)
}
-86
View File
@@ -1,86 +0,0 @@
package record
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"io"
)
type Record struct {
Type Type
Version Version
Payload bytes.Buffer
}
func (r *Record) String() string {
return fmt.Sprintf("<tlsRecord(type=%v, version=%v, payload=%s)>",
r.Type,
r.Version,
base64.StdEncoding.EncodeToString(r.Payload.Bytes()))
}
func (r *Record) Reset() {
r.Payload.Reset()
}
func (r *Record) Read(reader io.Reader) error {
r.Reset()
buf := [2]byte{}
if _, err := io.ReadFull(reader, buf[:1]); err != nil {
return fmt.Errorf("cannot read type: %w", err)
}
r.Type = Type(buf[0])
if err := r.Type.Valid(); err != nil {
return fmt.Errorf("invalid type: %w", err)
}
if _, err := io.ReadFull(reader, buf[:]); err != nil {
return fmt.Errorf("cannot read version: %w", err)
}
r.Version = Version(binary.BigEndian.Uint16(buf[:]))
if err := r.Version.Valid(); err != nil {
return fmt.Errorf("invalid version: %w", err)
}
if _, err := io.ReadFull(reader, buf[:]); err != nil {
return fmt.Errorf("cannot read payload length: %w", err)
}
length := int64(binary.BigEndian.Uint16(buf[:]))
if _, err := io.CopyN(&r.Payload, reader, length); err != nil {
return fmt.Errorf("cannot read payload: %w", err)
}
return nil
}
func (r *Record) Dump(writer io.Writer) error {
buf := [2]byte{byte(r.Type), 0}
if _, err := writer.Write(buf[:1]); err != nil {
return fmt.Errorf("cannot dump record type: %w", err)
}
binary.BigEndian.PutUint16(buf[:], uint16(r.Version))
if _, err := writer.Write(buf[:]); err != nil {
return fmt.Errorf("cannot dump version: %w", err)
}
binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len()))
if _, err := writer.Write(buf[:]); err != nil {
return fmt.Errorf("cannot dump payload length: %w", err)
}
if _, err := writer.Write(r.Payload.Bytes()); err != nil {
return fmt.Errorf("cannot dump record: %w", err)
}
return nil
}
@@ -1,110 +0,0 @@
package record_test
import (
"bytes"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type RecordTestSnapshot struct {
Type int `json:"type"`
Version int `json:"version"`
Payload string `json:"payload"`
Record string `json:"record"`
}
func (r RecordTestSnapshot) RecordBytes() []byte {
data, _ := base64.StdEncoding.DecodeString(r.Record)
return data
}
func (r RecordTestSnapshot) PayloadBytes() []byte {
data, _ := base64.StdEncoding.DecodeString(r.Payload)
return data
}
type RecordTestSuite struct {
suite.Suite
r *record.Record
buf *bytes.Buffer
}
func (suite *RecordTestSuite) SetupTest() {
suite.r = record.AcquireRecord()
suite.buf = &bytes.Buffer{}
}
func (suite *RecordTestSuite) TearDownTest() {
record.ReleaseRecord(suite.r)
suite.buf.Reset()
}
func (suite *RecordTestSuite) TestIdempotent() {
suite.r.Type = record.TypeApplicationData
suite.r.Version = record.Version13
suite.r.Payload.Write([]byte{1, 2, 3})
suite.NoError(suite.r.Dump(suite.buf))
suite.r.Reset()
suite.NoError(suite.r.Read(suite.buf))
suite.Equal(0, suite.buf.Len())
suite.Equal(record.TypeApplicationData, suite.r.Type)
suite.Equal(record.Version13, suite.r.Version)
suite.Equal([]byte{1, 2, 3}, suite.r.Payload.Bytes())
}
func (suite *RecordTestSuite) TestString() {
_ = suite.r.String()
}
func (suite *RecordTestSuite) TestSnapshot() {
files, err := os.ReadDir("testdata")
suite.NoError(err)
testData := map[string]string{}
for _, f := range files {
testData[f.Name()] = filepath.Join("testdata", f.Name())
}
for name, pathV := range testData {
path := pathV
suite.T().Run(name, func(t *testing.T) {
data, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &RecordTestSnapshot{}
assert.NoError(t, json.Unmarshal(data, snapshot))
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
assert.NoError(t, rec.Read(bytes.NewReader(snapshot.RecordBytes())))
assert.Equal(t, snapshot.Type, int(rec.Type))
assert.Equal(t, snapshot.Version, int(rec.Version))
assert.Equal(t, snapshot.PayloadBytes(), rec.Payload.Bytes())
buf := &bytes.Buffer{}
assert.NoError(t, rec.Dump(buf))
assert.Equal(t, snapshot.RecordBytes(), buf.Bytes())
})
}
}
func TestRecord(t *testing.T) {
t.Parallel()
suite.Run(t, &RecordTestSuite{})
}
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 772,
"payload": "sxS+0oAyk+NBv0LLVtQOp9WSx4CweyUZPz01tQ0o4oyp8aaBl6/kMFvLq3q52KE8lCiKejLw2NxVBUkE+4izCf2gLx9qfr81opWnqJTChWzcDijvttbq9cmtDFNL+odKsS3v1/TfYEFtPsoRPrJRmOHRAnqnf49Y5Q==",
"record": "FAMEAHmzFL7SgDKT40G/QstW1A6n1ZLHgLB7JRk/PTW1DSjijKnxpoGXr+QwW8urernYoTyUKIp6MvDY3FUFSQT7iLMJ/aAvH2p+vzWilaeolMKFbNwOKO+21ur1ya0MU0v6h0qxLe/X9N9gQW0+yhE+slGY4dECeqd/j1jl"
}
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 772,
"payload": "waNH223htyxCBKAb6hm0u/SK/9mhI8Ck91nfWob7QMOaIREogrDYREJH4Djcp47XrpAlEaUIDiCvoFLVJ/LK1nYs4swzfHSSl/+Aj1eqPA63XqPa8EG4FAbf0DwjwXxV9qVIhvP9b2TafKbzr4Yb5GCygzFRb/zawA==",
"record": "FgMEAHnBo0fbbeG3LEIEoBvqGbS79Ir/2aEjwKT3Wd9ahvtAw5ohESiCsNhEQkfgONynjteukCURpQgOIK+gUtUn8srWdizizDN8dJKX/4CPV6o8Drdeo9rwQbgUBt/QPCPBfFX2pUiG8/1vZNp8pvOvhhvkYLKDMVFv/NrA"
}
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 769,
"payload": "jmJ0o1E5+ehAHHYAbCo4AMV03X7RSivYl250s06nD9CO44fyjaoGELz0N7IeCg1jFKcRVSCRmYYmiIY9wydn2fXOJhKif8B0BlM3qhbethYgyP+l1S8hyyETpIiOtiiiOnAJwl1D1j9OryFiJFSdRRXReIMZ4CPqPg==",
"record": "FwMBAHmOYnSjUTn56EAcdgBsKjgAxXTdftFKK9iXbnSzTqcP0I7jh/KNqgYQvPQ3sh4KDWMUpxFVIJGZhiaIhj3DJ2fZ9c4mEqJ/wHQGUzeqFt62FiDI/6XVLyHLIROkiI62KKI6cAnCXUPWP06vIWIkVJ1FFdF4gxngI+o+"
}
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 769,
"payload": "hBnpBnNUdlqe/rKXa7Judcz79u7AkUgSGOycn8EqvbkZpVxnI31rNOvAsPZqG+GF7DWJ3R7H2ETmFmrpnyyng32MjSs1jptmV1oAs63zTADD7sVipgid9AJHwfl4CrC3FIQr43IPMYd29JPOl5bqu/SfrgI16PBiJw==",
"record": "FgMBAHmEGekGc1R2Wp7+spdrsm51zPv27sCRSBIY7JyfwSq9uRmlXGcjfWs068Cw9mob4YXsNYndHsfYROYWaumfLKeDfYyNKzWOm2ZXWgCzrfNMAMPuxWKmCJ30AkfB+XgKsLcUhCvjcg8xh3b0k86Xluq79J+uAjXo8GIn"
}
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 770,
"payload": "Vm/C+DO56czlbtR915aHzsugSyDtp8CtojF9w1jKY0efyyfcLrNuhNg/pZm3gQ7v2BBbL1UJ97v/RIjST+5gRIfg3bBN1BE9hkf+N2AYY2lHLi0yeInHB0zFWPeHscsDopDFadIi5KtC8HvbEMuK+kK8POVk5tN9UQ==",
"record": "FwMCAHlWb8L4M7npzOVu1H3XlofOy6BLIO2nwK2iMX3DWMpjR5/LJ9wus26E2D+lmbeBDu/YEFsvVQn3u/9EiNJP7mBEh+DdsE3UET2GR/43YBhjaUcuLTJ4iccHTMVY94exywOikMVp0iLkq0Lwe9sQy4r6Qrw85WTm031R"
}
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 770,
"payload": "ajPzpsgk4gwm2stRQKbllvKRLdI7vmyaj1uxEJ/kKoQnQSPumdDNKD618U2Cq6PVd0/b+9YtH67Uzx1QxtpKuby5fUXqw06WUuDAQsmjq7F26EkE5FND6rQUjUPC+e1U0dF4TQzOUSS4IAkFQPAaVehUVTRxVWa/0g==",
"record": "FgMCAHlqM/OmyCTiDCbay1FApuWW8pEt0ju+bJqPW7EQn+QqhCdBI+6Z0M0oPrXxTYKro9V3T9v71i0frtTPHVDG2kq5vLl9RerDTpZS4MBCyaOrsXboSQTkU0PqtBSNQ8L57VTR0XhNDM5RJLggCQVA8BpV6FRVNHFVZr/S"
}
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 771,
"payload": "d1Hiv1NYVgEDR9mtJyv9j8mg3dWqfUpeKfOsL+jzSDfVIxeDiJZFLDT50TjNW44/yEOVEX/Y/pk+wnc7E8aCEiwGwAvB+Insw1UCJ2ejt689VWLo2u4klGVKTHuOpUvdGVTc7Lo4FAt91KQSPLYB5iqxomjEv5e3Vg==",
"record": "FAMDAHl3UeK/U1hWAQNH2a0nK/2PyaDd1ap9Sl4p86wv6PNIN9UjF4OIlkUsNPnROM1bjj/IQ5URf9j+mT7CdzsTxoISLAbAC8H4iezDVQInZ6O3rz1VYuja7iSUZUpMe46lS90ZVNzsujgUC33UpBI8tgHmKrGiaMS/l7dW"
}
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 771,
"payload": "wbdU1CbrzuAJDsh6CFjGyE+AFArJj/Wmsa2wtDyW0kRuE2vUO8gg+nXkg0kkoz0WnvQEOdaswfJIaVrloD78yoyeQVfBB+VUP/63vqn60v5ccaQEn0jLdxgLjiTAxKDQDxCTMRoLnFE2ZZf28zw+HfqpIxiOZs8LhQ==",
"record": "FwMDAHnBt1TUJuvO4AkOyHoIWMbIT4AUCsmP9aaxrbC0PJbSRG4Ta9Q7yCD6deSDSSSjPRae9AQ51qzB8khpWuWgPvzKjJ5BV8EH5VQ//re+qfrS/lxxpASfSMt3GAuOJMDEoNAPEJMxGgucUTZll/bzPD4d+qkjGI5mzwuF"
}
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 772,
"payload": "qqnBMb1Af3zZt4DPHpVRuIiON9ODGJUNFicFjranORh67L/HI4D6HnHyycZFUSBOw2FjMBF6UialY8snOYaRKrQmQzuUNg1Ztq7yAZ+Lgj3TBarR6OMlYhEAY0Px9Xv1UuJ0YcvQx33gdM1skJ5HBR3yZvEKNJV1LA==",
"record": "FwMEAHmqqcExvUB/fNm3gM8elVG4iI4304MYlQ0WJwWOtqc5GHrsv8cjgPoecfLJxkVRIE7DYWMwEXpSJqVjyyc5hpEqtCZDO5Q2DVm2rvIBn4uCPdMFqtHo4yViEQBjQ/H1e/VS4nRhy9DHfeB0zWyQnkcFHfJm8Qo0lXUs"
}
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 769,
"payload": "NEe735TuQFp7bWpFQhASas/e1XaySvus0ovXmkfCbFq334MyFHq2eDMadziXsfu/GfBjoYggvk0LgYUeoAkBNKR0dfSovjSndaqmIUonoWl+6sZObiGZkRIMwuY2q4Eaw4/iuDu/pZhjRW/iAIH+YH7cyk/1tgdJDg==",
"record": "FAMBAHk0R7vflO5AWnttakVCEBJqz97VdrJK+6zSi9eaR8JsWrffgzIUerZ4Mxp3OJex+78Z8GOhiCC+TQuBhR6gCQE0pHR19Ki+NKd1qqYhSiehaX7qxk5uIZmREgzC5jargRrDj+K4O7+lmGNFb+IAgf5gftzKT/W2B0kO"
}
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 771,
"payload": "wrXjZrPm3OSyzO0klv6/G+z2PDloR/colS/RlWwQE31Vb2xm8YkEchDDKwlc/KPLD73qMoz3MQOQLtSLc8LhVYp+l7L9jz49yTaVKtBI5UuGbo09snsKxFCgCyYUBETKabATBQtiaEu/D8dmF4Yk/2ww4sEb8DwKLQ==",
"record": "FgMDAHnCteNms+bc5LLM7SSW/r8b7PY8OWhH9yiVL9GVbBATfVVvbGbxiQRyEMMrCVz8o8sPveoyjPcxA5Au1ItzwuFVin6Xsv2PPj3JNpUq0EjlS4ZujT2yewrEUKALJhQERMppsBMFC2JoS78Px2YXhiT/bDDiwRvwPAot"
}
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 770,
"payload": "OU5s8Sa11hpXWEarWzFlX55IZt3Eo+F4AMbQ/2RwB4rfHS/JNl8n63OR4oYs9QXw3RfCrYJuU9n6Xn+I/+7ZzAgZ0PbLSXW1PrLtttdfmhTErK90b49YEWdY9na4g++NMkKykwgXvY1hNxZIHX/qawEWJgxXUR3DdQ==",
"record": "FAMCAHk5TmzxJrXWGldYRqtbMWVfnkhm3cSj4XgAxtD/ZHAHit8dL8k2Xyfrc5Hihiz1BfDdF8Ktgm5T2fpef4j/7tnMCBnQ9stJdbU+su2211+aFMSsr3Rvj1gRZ1j2driD740yQrKTCBe9jWE3Fkgdf+prARYmDFdRHcN1"
}
@@ -1,8 +0,0 @@
{
"time": 1617181365,
"random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=",
"sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCACq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFANAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
@@ -1,8 +0,0 @@
{
"time": 1617181365,
"random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=",
"sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
@@ -1,8 +0,0 @@
{
"time": 1617181352,
"random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=",
"sessionId": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDoYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12osgFGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8gANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAga6CocpFP8Qd4YCFR9pkaCr97po2ALj0P5nI9Nnb3UWMALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
@@ -1,8 +0,0 @@
{
"time": 1617181352,
"random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=",
"sessionId": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMD5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1rogjxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgrulAaqUdKeVYM0F+pu6on/h6LBpOyzOKG4xFIKcoFk4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
@@ -1,8 +0,0 @@
{
"time": 1617181365,
"random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=",
"sessionId": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMD8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQg00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAUANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAg/9P7140NtKzjyDwBf99mOy1+FjRPAPHTNQ9WxHOKpV4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
@@ -1,8 +0,0 @@
{
"time": 1617181352,
"random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=",
"sessionId": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDzja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4gqPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnAANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgXviLRAqAYJ8xOLdlcsUhldI4Xl0g/s9+y2Qrd8raPEgALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
-92
View File
@@ -1,92 +0,0 @@
package faketls
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"io"
mrand "math/rand/v2"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"golang.org/x/crypto/curve25519"
)
func SendWelcomePacket(writer io.Writer, secret []byte, clientHello ClientHello) error {
buf := acquireBytesBuffer()
defer releaseBytesBuffer(buf)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeHandshake
rec.Version = record.Version12
generateServerHello(&rec.Payload, clientHello)
rec.Dump(buf) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeChangeCipherSpec
rec.Version = record.Version12
rec.Payload.WriteByte(ChangeCipherValue)
rec.Dump(buf) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeApplicationData
rec.Version = record.Version12
if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.IntN(3092))); err != nil {
panic(err)
}
rec.Dump(buf) //nolint: errcheck
packet := buf.Bytes()
mac := hmac.New(sha256.New, secret)
mac.Write(clientHello.Random[:])
mac.Write(packet)
copy(packet[WelcomePacketRandomOffset:], mac.Sum(nil))
if _, err := writer.Write(packet); err != nil {
return err //nolint: wrapcheck
}
return nil
}
func generateServerHello(writer io.Writer, clientHello ClientHello) {
bodyBuf := acquireBytesBuffer()
defer releaseBytesBuffer(bodyBuf)
sliceBuf := [2]byte{}
digest := [RandomLen]byte{}
binary.BigEndian.PutUint16(sliceBuf[:], uint16(record.Version12))
bodyBuf.Write(sliceBuf[:])
bodyBuf.Write(digest[:])
bodyBuf.WriteByte(byte(len(clientHello.SessionID)))
bodyBuf.Write(clientHello.SessionID)
binary.BigEndian.PutUint16(sliceBuf[:], clientHello.CipherSuite)
bodyBuf.Write(sliceBuf[:])
bodyBuf.Write(serverHelloSuffix)
scalar := [32]byte{}
if _, err := rand.Read(scalar[:]); err != nil {
panic(err)
}
curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint)
bodyBuf.Write(curve)
header := [4]byte{0, 0, 0, 0}
binary.BigEndian.PutUint32(header[:], uint32(bodyBuf.Len()))
header[0] = HandshakeTypeServer
writer.Write(header[:]) //nolint: errcheck
bodyBuf.WriteTo(writer) //nolint: errcheck
}
-82
View File
@@ -1,82 +0,0 @@
package faketls_test
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"testing"
"time"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/suite"
)
type WelcomeTestSuite struct {
suite.Suite
h *faketls.ClientHello
buf *bytes.Buffer
secret mtglib.Secret
}
func (suite *WelcomeTestSuite) SetupTest() {
suite.h = &faketls.ClientHello{
Time: time.Now(),
Host: "google.com",
CipherSuite: 4867,
SessionID: make([]byte, 32),
}
_, err := rand.Read(suite.h.SessionID) //nolint: staticcheck
suite.NoError(err)
_, err = rand.Read(suite.h.Random[:]) //nolint: staticcheck
suite.NoError(err)
suite.buf = &bytes.Buffer{}
suite.secret = mtglib.GenerateSecret("google.com")
}
func (suite *WelcomeTestSuite) TestOk() {
suite.NoError(faketls.SendWelcomePacket(suite.buf, suite.secret.Key[:], *suite.h))
welcomePacket := []byte{}
welcomePacket = append(welcomePacket, suite.buf.Bytes()...)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
suite.NoError(rec.Read(suite.buf))
suite.Equal(record.TypeHandshake, rec.Type)
suite.Equal(record.Version12, rec.Version)
suite.NoError(rec.Read(suite.buf))
suite.Equal(record.TypeChangeCipherSpec, rec.Type)
suite.Equal(record.Version12, rec.Version)
suite.NoError(rec.Read(suite.buf))
suite.Equal(record.TypeApplicationData, rec.Type)
suite.Equal(record.Version12, rec.Version)
suite.Empty(suite.buf.Bytes())
random := make([]byte, 32)
copy(random, welcomePacket[11:])
empty := make([]byte, 32)
copy(welcomePacket[11:], empty)
mac := hmac.New(sha256.New, suite.secret.Key[:])
mac.Write(suite.h.Random[:])
mac.Write(welcomePacket)
suite.Equal(random, mac.Sum(nil))
}
func TestWelcome(t *testing.T) {
t.Parallel()
suite.Run(t, &WelcomeTestSuite{})
}
@@ -1,54 +0,0 @@
package obfuscated2
import (
"crypto/cipher"
"crypto/subtle"
"encoding/hex"
"fmt"
"io"
)
type clientHandhakeFrame struct {
handshakeFrame
}
func (c *clientHandhakeFrame) decryptor(secret []byte) cipher.Stream {
hasher := acquireSha256Hasher()
defer releaseSha256Hasher(hasher)
hasher.Write(c.key())
hasher.Write(secret)
return makeAesCtr(hasher.Sum(nil), c.iv())
}
func (c *clientHandhakeFrame) encryptor(secret []byte) cipher.Stream {
invertedHandshake := c.invert()
hasher := acquireSha256Hasher()
defer releaseSha256Hasher(hasher)
hasher.Write(invertedHandshake.key())
hasher.Write(secret)
return makeAesCtr(hasher.Sum(nil), invertedHandshake.iv())
}
func ClientHandshake(secret []byte, reader io.Reader) (int, cipher.Stream, cipher.Stream, error) {
handshake := clientHandhakeFrame{}
if _, err := io.ReadFull(reader, handshake.data[:]); err != nil {
return 0, nil, nil, fmt.Errorf("cannot read frame: %w", err)
}
decryptor := handshake.decryptor(secret)
encryptor := handshake.encryptor(secret)
decryptor.XORKeyStream(handshake.data[:], handshake.data[:])
if val := handshake.connectionType(); subtle.ConstantTimeCompare(handshakeConnectionType, val) != 1 {
return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(val))
}
return handshake.dc(), encryptor, decryptor, nil
}
@@ -1,32 +0,0 @@
package obfuscated2
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
)
var FuzzClientHandshakeSecret = []byte{1, 2, 3}
func FuzzClientHandshake(f *testing.F) {
f.Add([]byte{1, 2, 3})
f.Fuzz(func(t *testing.T, frame []byte) {
data := bytes.NewReader(frame)
if _, _, _, err := ClientHandshake(FuzzClientHandshakeSecret, data); err != nil {
return
}
handshake := clientHandhakeFrame{}
require.Len(t, frame, handshakeFrameLen)
copy(handshake.data[:], frame)
decryptor := handshake.decryptor(FuzzClientHandshakeSecret)
decryptor.XORKeyStream(handshake.data[:], handshake.data[:])
require.Equal(t, handshakeConnectionType, handshake.connectionType())
})
}
@@ -1,89 +0,0 @@
package obfuscated2_test
import (
"bytes"
"testing"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ClientHandshakeTestSuite struct {
suite.Suite
SnapshotTestSuite
}
func (suite *ClientHandshakeTestSuite) SetupSuite() {
suite.NoError(suite.IngestSnapshots(".", "client-handshake-snapshot-"))
}
func (suite *ClientHandshakeTestSuite) TestCannotRead() {
buf := bytes.NewBuffer([]byte{1, 2, 3})
_, _, _, err := obfuscated2.ClientHandshake([]byte{1, 2, 3}, buf) //nolint: dogsled
suite.Error(err)
}
func (suite *ClientHandshakeTestSuite) TestOk() {
for nameV, snapshotV := range suite.snapshots {
snapshot := snapshotV
suite.T().Run(nameV, func(t *testing.T) {
buf := bytes.NewBuffer(snapshot.Frame.data)
dc, encryptor, decryptor, err := obfuscated2.ClientHandshake(
snapshot.Secret.data, buf)
assert.NoError(t, err)
assert.EqualValues(t, snapshot.DC, dc)
writeData := make([]byte, len(snapshot.Encrypted.Text.data))
readData := make([]byte, len(snapshot.Decrypted.Text.data))
connMock := &testlib.EssentialsConnMock{}
connMock.On("Read", mock.Anything).
Once().
Return(len(snapshot.Decrypted.Text.data), nil).
Run(func(args mock.Arguments) {
arr, ok := args.Get(0).([]byte)
suite.True(ok)
copy(arr, snapshot.Decrypted.Cipher.data)
})
connMock.On("Write", mock.Anything).
Once().
Return(len(snapshot.Encrypted.Text.data), nil).
Run(func(args mock.Arguments) {
arr, ok := args.Get(0).([]byte)
suite.True(ok)
copy(writeData, arr)
})
conn := obfuscated2.Conn{
Conn: connMock,
Encryptor: encryptor,
Decryptor: decryptor,
}
n, err := conn.Read(readData)
assert.Equal(t, len(readData), n)
assert.NoError(t, err)
assert.Equal(t, snapshot.Decrypted.Text.data, readData)
n, err = conn.Write(snapshot.Encrypted.Text.data)
assert.Equal(t, len(writeData), n)
assert.NoError(t, err)
assert.Equal(t, snapshot.Encrypted.Cipher.data, writeData)
connMock.AssertExpectations(t)
})
}
}
func TestClientHandshake(t *testing.T) {
t.Parallel()
suite.Run(t, &ClientHandshakeTestSuite{})
}
-37
View File
@@ -1,37 +0,0 @@
package obfuscated2
import (
"crypto/cipher"
"github.com/9seconds/mtg/v2/essentials"
)
type Conn struct {
essentials.Conn
Encryptor cipher.Stream
Decryptor cipher.Stream
}
func (c Conn) Read(p []byte) (int, error) {
n, err := c.Conn.Read(p)
if err != nil {
return n, err //nolint: wrapcheck
}
c.Decryptor.XORKeyStream(p, p[:n])
return n, nil
}
func (c Conn) Write(p []byte) (int, error) {
buf := acquireBytesBuffer()
defer releaseBytesBuffer(buf)
buf.Write(p)
payload := buf.Bytes()
c.Encryptor.XORKeyStream(payload, payload)
return c.Conn.Write(payload) //nolint: wrapcheck
}
@@ -1,71 +0,0 @@
package obfuscated2
const (
// DefaultDC defines a number of the default DC to use. This value used
// only if a value from obfuscated2 handshake frame is 0 (default).
DefaultDC = 2
handshakeFrameLen = 64
handshakeFrameLenKey = 32
handshakeFrameLenIV = 16
handshakeFrameLenConnectionType = 4
handshakeFrameOffsetStart = 8
handshakeFrameOffsetKey = handshakeFrameOffsetStart
handshakeFrameOffsetIV = handshakeFrameOffsetKey + handshakeFrameLenKey
handshakeFrameOffsetConnectionType = handshakeFrameOffsetIV + handshakeFrameLenIV
handshakeFrameOffsetDC = handshakeFrameOffsetConnectionType + handshakeFrameLenConnectionType
)
// Connection-Type: Secure. We support only fake tls.
var handshakeConnectionType = []byte{0xdd, 0xdd, 0xdd, 0xdd}
// A structure of obfuscated2 handshake frame is following:
//
// [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd].
//
// - 8 bytes of noise
// - 32 bytes of AES Key
// - 16 bytes of AES IV
// - 4 bytes of 'connection type' - this has some setting like a connection type
// - 2 bytes of 'DC'. DC is little endian int16
// - 2 bytes of noise
type handshakeFrame struct {
data [handshakeFrameLen]byte
}
func (h *handshakeFrame) dc() int {
idx := int16(h.data[handshakeFrameOffsetDC]) | int16(h.data[handshakeFrameOffsetDC+1])<<8 //nolint: lll // little endian for int16 is here
switch {
case idx > 0:
return int(idx)
case idx < 0:
return -int(idx)
default:
return DefaultDC
}
}
func (h *handshakeFrame) key() []byte {
return h.data[handshakeFrameOffsetKey:handshakeFrameOffsetIV]
}
func (h *handshakeFrame) iv() []byte {
return h.data[handshakeFrameOffsetIV:handshakeFrameOffsetConnectionType]
}
func (h *handshakeFrame) connectionType() []byte {
return h.data[handshakeFrameOffsetConnectionType:handshakeFrameOffsetDC]
}
func (h *handshakeFrame) invert() handshakeFrame {
copyFrame := *h
for i := range handshakeFrameLenKey + handshakeFrameLenIV {
copyFrame.data[handshakeFrameOffsetKey+i] = h.data[handshakeFrameOffsetConnectionType-1-i]
}
return copyFrame
}
@@ -1,73 +0,0 @@
package obfuscated2
import (
"crypto/rand"
"encoding/base64"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type HandshakeFrameTestSuite struct {
suite.Suite
}
func (suite *HandshakeFrameTestSuite) Decode(value string) []byte {
v, err := base64.RawStdEncoding.DecodeString(value)
suite.NoError(err)
return v
}
func (suite *HandshakeFrameTestSuite) Encode(value []byte) string {
return base64.RawStdEncoding.EncodeToString(value)
}
func (suite *HandshakeFrameTestSuite) TestOk() {
hf := handshakeFrame{}
testFrame := suite.Decode(
"L9TmCzzxl9bPKODBpZeVM/qqNUxQ/axxBup1S2ymbIfUd6f7YSyzzM9EmTFv2/XzGqJGEHuj2zofmUGBLghu5g")
copy(hf.data[:], testFrame)
suite.Equal("zyjgwaWXlTP6qjVMUP2scQbqdUtspmyH1Hen+2Ess8w", suite.Encode(hf.key()))
suite.Equal("z0SZMW/b9fMaokYQe6PbOg", suite.Encode(hf.iv()))
suite.Equal("H5lBgQ", suite.Encode(hf.connectionType()))
suite.EqualValues(2094, hf.dc())
inverted := hf.invert()
suite.Equal("OtujexBGohrz9dtvMZlEz8yzLGH7p3fUh2ymbEt16gY", suite.Encode(inverted.key()))
suite.Equal("caz9UEw1qvozlZelweAozw", suite.Encode(inverted.iv()))
suite.Equal("H5lBgQ", suite.Encode(inverted.connectionType()))
suite.EqualValues(2094, inverted.dc())
}
func (suite *HandshakeFrameTestSuite) TestDC() {
testData := map[int16]int{
1: 1,
-1: 1,
0: DefaultDC,
}
for k, v := range testData {
incoming := k
expected := v
suite.T().Run(strconv.Itoa(int(incoming)), func(t *testing.T) {
frame := handshakeFrame{}
rand.Read(frame.data[:]) //nolint: errcheck
frame.data[handshakeFrameOffsetDC] = byte(incoming)
frame.data[handshakeFrameOffsetDC+1] = byte(incoming >> 8)
assert.Equal(t, expected, frame.dc())
})
}
}
func TestHandshakeFrame(t *testing.T) {
t.Parallel()
suite.Run(t, &HandshakeFrameTestSuite{})
}
-137
View File
@@ -1,137 +0,0 @@
package obfuscated2_test
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"encoding/base64"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2"
"github.com/stretchr/testify/require"
)
type snapshotBytes struct {
data []byte
}
func (s snapshotBytes) MarshalText() ([]byte, error) {
if len(s.data) == 0 {
return nil, nil
}
return []byte(base64.RawStdEncoding.EncodeToString(s.data)), nil
}
func (s *snapshotBytes) UnmarshalText(data []byte) error {
val, err := base64.RawStdEncoding.DecodeString(string(data))
if err != nil {
return fmt.Errorf("cannot unmarshal %v: %w", len(val), err)
}
s.data = val
return nil
}
type Obfuscated2Snapshot struct {
Secret snapshotBytes `json:"secret"`
Frame snapshotBytes `json:"frame"`
DC int16 `json:"dc"`
Encrypted struct {
Text snapshotBytes `json:"text"`
Cipher snapshotBytes `json:"cipher"`
} `json:"encrypted"`
Decrypted struct {
Text snapshotBytes `json:"text"`
Cipher snapshotBytes `json:"cipher"`
} `json:"decrypted"`
}
type SnapshotTestSuite struct {
snapshots map[string]*Obfuscated2Snapshot
}
type ServerHandshakeTestData struct {
connMock *testlib.EssentialsConnMock
proxyConn obfuscated2.Conn
encryptor cipher.Stream
decryptor cipher.Stream
}
func (suite *SnapshotTestSuite) IngestSnapshots(dirname, namePrefix string) error {
suite.snapshots = map[string]*Obfuscated2Snapshot{}
files, err := os.ReadDir(filepath.Join("testdata", dirname))
if err != nil {
return fmt.Errorf("cannot ingest snapshots: %w", err)
}
for _, v := range files {
if !strings.HasPrefix(v.Name(), namePrefix) {
continue
}
filename := filepath.Join("testdata", dirname, v.Name())
contents, err := os.ReadFile(filename)
if err != nil {
return fmt.Errorf("cannot read %s: %w", filename, err)
}
value := &Obfuscated2Snapshot{}
if err := json.Unmarshal(contents, value); err != nil {
return fmt.Errorf("cannot unmarshal %s: %w", filename, err)
}
suite.snapshots[v.Name()] = value
}
return nil
}
func NewServerHandshakeTestData(t *testing.T) ServerHandshakeTestData {
buf := &bytes.Buffer{}
connMock := &testlib.EssentialsConnMock{}
handshakeEnc, handshakeDec, err := obfuscated2.ServerHandshake(buf)
require.NoError(t, err)
serverEncrypted := buf.Bytes()
decBlock, _ := aes.NewCipher(serverEncrypted[8 : 8+32])
decryptor := cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16])
serverDecrypted := make([]byte, len(serverEncrypted))
decryptor.XORKeyStream(serverDecrypted, serverEncrypted)
require.Equal(t, "3d3d3Q",
base64.RawStdEncoding.EncodeToString(serverDecrypted[8+32+16:8+32+16+4]))
serverEncryptedReverted := make([]byte, len(serverEncrypted))
for i := range 32 + 16 {
serverEncryptedReverted[8+i] = serverEncrypted[8+32+16-1-i]
}
encBlock, _ := aes.NewCipher(serverEncryptedReverted[8 : 8+32])
encryptor := cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16])
return ServerHandshakeTestData{
connMock: connMock,
proxyConn: obfuscated2.Conn{
Conn: connMock,
Encryptor: handshakeEnc,
Decryptor: handshakeDec,
},
encryptor: encryptor,
decryptor: decryptor,
}
}
-39
View File
@@ -1,39 +0,0 @@
package obfuscated2
import (
"bytes"
"crypto/sha256"
"hash"
"sync"
)
var (
sha256HasherPool = sync.Pool{
New: func() any {
return sha256.New()
},
}
bytesBufferPool = sync.Pool{
New: func() any {
return &bytes.Buffer{}
},
}
)
func acquireSha256Hasher() hash.Hash {
return sha256HasherPool.Get().(hash.Hash) //nolint: forcetypeassert
}
func releaseSha256Hasher(h hash.Hash) {
h.Reset()
sha256HasherPool.Put(h)
}
func acquireBytesBuffer() *bytes.Buffer {
return bytesBufferPool.Get().(*bytes.Buffer) //nolint: forcetypeassert
}
func releaseBytesBuffer(buf *bytes.Buffer) {
buf.Reset()
bytesBufferPool.Put(buf)
}
@@ -1,67 +0,0 @@
package obfuscated2
import (
"crypto/cipher"
"crypto/rand"
"encoding/binary"
"fmt"
"io"
)
type serverHandshakeFrame struct {
handshakeFrame
}
func (s *serverHandshakeFrame) decryptor() cipher.Stream {
invertedHandshake := s.invert()
return makeAesCtr(invertedHandshake.key(), invertedHandshake.iv())
}
func (s *serverHandshakeFrame) encryptor() cipher.Stream {
return makeAesCtr(s.key(), s.iv())
}
func ServerHandshake(writer io.Writer) (cipher.Stream, cipher.Stream, error) {
handshake := generateServerHanshakeFrame()
copyHandshake := handshake
encryptor := handshake.encryptor()
decryptor := handshake.decryptor()
encryptor.XORKeyStream(handshake.data[:], handshake.data[:])
copy(handshake.key(), copyHandshake.key())
copy(handshake.iv(), copyHandshake.iv())
if _, err := writer.Write(handshake.data[:]); err != nil {
return nil, nil, fmt.Errorf("cannot send a handshake frame to telegram: %w", err)
}
return encryptor, decryptor, nil
}
func generateServerHanshakeFrame() serverHandshakeFrame {
frame := serverHandshakeFrame{}
for {
if _, err := rand.Read(frame.data[:]); err != nil {
panic(err)
}
if frame.data[0] == 0xef { // taken from tg sources
continue
}
switch binary.LittleEndian.Uint32(frame.data[:4]) {
case 0x44414548, 0x54534f50, 0x20544547, 0x4954504f, 0xeeeeeeee: // taken from tg sources
continue
}
if frame.data[4]|frame.data[5]|frame.data[6]|frame.data[7] == 0 {
continue
}
copy(frame.connectionType(), handshakeConnectionType)
return frame
}
}
@@ -1,58 +0,0 @@
package obfuscated2_test
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func FuzzServerSend(f *testing.F) {
f.Add([]byte{1, 2, 3, 4, 5})
f.Fuzz(func(t *testing.T, data []byte) {
handshakeData := NewServerHandshakeTestData(t)
handshakeData.connMock.
On("Write", mock.Anything).
Return(len(data), nil).
Once().
Run(func(args mock.Arguments) {
message := make([]byte, len(data))
handshakeData.decryptor.XORKeyStream(message, args.Get(0).([]byte)) //nolint: forcetypeassert
assert.Equal(t, message, data)
})
n, err := handshakeData.proxyConn.Write(data)
assert.EqualValues(t, len(data), n)
assert.NoError(t, err)
handshakeData.connMock.AssertExpectations(t)
})
}
func FuzzServerReceive(f *testing.F) {
f.Add([]byte{1, 2, 3, 4, 5})
f.Fuzz(func(t *testing.T, data []byte) {
handshakeData := NewServerHandshakeTestData(t)
buffer := make([]byte, len(data))
handshakeData.connMock.
On("Read", mock.Anything).
Return(len(data), nil).
Once().
Run(func(args mock.Arguments) {
message := make([]byte, len(data))
handshakeData.encryptor.XORKeyStream(message, data)
copy(args.Get(0).([]byte), message) //nolint: forcetypeassert
})
n, err := handshakeData.proxyConn.Read(buffer)
assert.EqualValues(t, len(data), n)
assert.NoError(t, err)
assert.Equal(t, data, buffer)
handshakeData.connMock.AssertExpectations(t)
})
}
@@ -1,65 +0,0 @@
package obfuscated2_test
import (
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ServerHandshakeTestSuite struct {
suite.Suite
data ServerHandshakeTestData
}
func (suite *ServerHandshakeTestSuite) SetupTest() {
suite.data = NewServerHandshakeTestData(suite.T())
}
func (suite *ServerHandshakeTestSuite) TearDownTest() {
suite.data.connMock.AssertExpectations(suite.T())
}
func (suite *ServerHandshakeTestSuite) TestSendToTelegram() {
messageToTelegram := []byte{10, 11, 12, 13, 14, 'a'}
suite.data.connMock.
On("Write", mock.Anything).
Return(len(messageToTelegram), nil).
Once().
Run(func(args mock.Arguments) {
message := make([]byte, len(messageToTelegram))
suite.data.decryptor.XORKeyStream(message, args.Get(0).([]byte)) //nolint: forcetypeassert
suite.Equal(messageToTelegram, message)
})
n, err := suite.data.proxyConn.Write(messageToTelegram)
suite.EqualValues(len(messageToTelegram), n)
suite.NoError(err)
}
func (suite *ServerHandshakeTestSuite) TestRecieveFromTelegram() {
messageFromTelegram := []byte{10, 11, 12, 13, 14, 'a'}
buffer := make([]byte, len(messageFromTelegram))
suite.data.connMock.
On("Read", mock.Anything).
Return(len(messageFromTelegram), nil).
Once().
Run(func(args mock.Arguments) {
message := make([]byte, len(messageFromTelegram))
suite.data.encryptor.XORKeyStream(message, messageFromTelegram)
copy(args.Get(0).([]byte), message) //nolint: forcetypeassert
})
n, err := suite.data.proxyConn.Read(buffer)
suite.EqualValues(len(messageFromTelegram), n)
suite.NoError(err)
suite.Equal(messageFromTelegram, buffer)
}
func TestServerHandshake(t *testing.T) {
t.Parallel()
suite.Run(t, &ServerHandshakeTestSuite{})
}
-15
View File
@@ -1,15 +0,0 @@
package obfuscated2
import (
"crypto/aes"
"crypto/cipher"
)
func makeAesCtr(key, iv []byte) cipher.Stream {
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
return cipher.NewCTR(block, iv)
}
+34
View File
@@ -0,0 +1,34 @@
package obfuscation
import (
"crypto/cipher"
"github.com/9seconds/mtg/v2/essentials"
)
type conn struct {
essentials.Conn
sendCipher cipher.Stream
recvCipher cipher.Stream
}
func (c conn) Read(p []byte) (int, error) {
n, err := c.Conn.Read(p)
if err != nil {
return n, err
}
c.recvCipher.XORKeyStream(p, p[:n])
return n, nil
}
func (c conn) Write(p []byte) (int, error) {
// yes, this is a bit violent and goes against a contract in io.Writer
// but we do it to avoid creating a new buffer just to perform this
// encryption.
c.sendCipher.XORKeyStream(p, p)
return c.Conn.Write(p)
}
+102
View File
@@ -0,0 +1,102 @@
package obfuscation
import (
"crypto/aes"
"crypto/cipher"
"encoding/hex"
"testing"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ConnTestSuite struct {
suite.Suite
secret []byte
}
func (s *ConnTestSuite) SetupSuite() {
secret := [32]byte{}
s.secret = secret[:]
}
func (s *ConnTestSuite) TestRead() {
testData := map[string]string{
"data1": "b8f4b41993",
"": "",
"___": "83ca9f",
}
for incoming, outgoing := range testData {
s.T().Run(incoming, func(t *testing.T) {
connMock := &testlib.EssentialsConnMock{}
testConn := s.makeConn(connMock)
data := make([]byte, len(incoming))
connMock.On("Read", make([]byte, len(incoming))).Return(len(incoming), nil).Run(func(args mock.Arguments) {
arg := args.Get(0).([]byte)
copy(arg, []byte(incoming))
})
n, err := testConn.Read(data)
assert.Equal(t, len(data), n)
assert.NoError(t, err)
assert.Equal(t, outgoing, hex.EncodeToString(data))
connMock.AssertExpectations(t)
})
}
}
func (s *ConnTestSuite) TestWrite() {
testData := map[string]string{
"b8f4b41993": "data1",
"": "",
"83ca9f": "___",
}
for incoming, outgoing := range testData {
s.T().Run(incoming, func(t *testing.T) {
connMock := &testlib.EssentialsConnMock{}
testConn := s.makeConn(connMock)
toWrite, _ := hex.DecodeString(incoming)
data := make([]byte, len(toWrite))
connMock.On("Write", []byte(outgoing)).Return(len(toWrite), nil)
n, err := testConn.Write(toWrite)
assert.Equal(t, len(data), n)
assert.NoError(t, err)
connMock.AssertExpectations(t)
})
}
}
func (s *ConnTestSuite) makeConn(rawConn *testlib.EssentialsConnMock) essentials.Conn {
rblock, err := aes.NewCipher(s.secret)
if err != nil {
panic(err)
}
wblock, err := aes.NewCipher(s.secret)
if err != nil {
panic(err)
}
return conn{
Conn: rawConn,
sendCipher: cipher.NewCTR(wblock, s.secret[:aes.BlockSize]),
recvCipher: cipher.NewCTR(rblock, s.secret[:aes.BlockSize]),
}
}
func TestConn(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTestSuite{})
}
@@ -0,0 +1,111 @@
package obfuscation
import (
"crypto/rand"
"encoding/binary"
"slices"
)
// https://core.telegram.org/mtproto/mtproto-transports#transport-obfuscation
const (
// default DC is nothing is selected
defaultDC = 2
// the length of the handshake frame. Always 64 bytes
hfLen = 64
hfLenKey = 32
hfLenIV = 16
hfLenConnectionType = 4
// A structure of obfuscated handshake frame is following:
//
// [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd].
//
// - 8 bytes of noise
// - 32 bytes of AES Key
// - 16 bytes of AES IV
// - 4 bytes of 'connection type' - this has some setting like a connection type
// - 2 bytes of 'DC'. DC is little endian int16
// - 2 bytes of noise
hfOffsetKey = 8
hfOffsetIV = hfOffsetKey + hfLenKey
hfOffsetConnectionType = hfOffsetIV + hfLenIV
hfOffsetDC = hfOffsetConnectionType + hfLenConnectionType
)
// Connection-Type: Secure. We support only fake tls.
var hfConnectionType = [hfLenConnectionType]byte{0xdd, 0xdd, 0xdd, 0xdd}
type handshakeFrame struct {
data [hfLen]byte
}
func (h *handshakeFrame) key() []byte {
return h.data[hfOffsetKey : hfOffsetKey+hfLenKey]
}
func (h *handshakeFrame) iv() []byte {
return h.data[hfOffsetIV : hfOffsetIV+hfLenIV]
}
func (h *handshakeFrame) connectionType() []byte {
return h.data[hfOffsetConnectionType : hfOffsetConnectionType+hfLenConnectionType]
}
func (h *handshakeFrame) dcSlice() []byte {
return h.data[hfOffsetDC : hfOffsetDC+2]
}
func (h *handshakeFrame) dc() int {
idx := int16(binary.LittleEndian.Uint16(h.dcSlice()))
switch {
case idx > 0:
return int(idx)
case idx < 0:
return -int(idx)
}
return defaultDC
}
func (h *handshakeFrame) revert() {
slices.Reverse(h.data[hfOffsetKey:hfOffsetConnectionType])
}
func generateHandshake(dc int) handshakeFrame {
frame := handshakeFrame{}
for {
if _, err := rand.Read(frame.data[:]); err != nil {
panic(err)
}
// https://github.com/tdlib/td/blob/master/td/mtproto/TcpTransport.cpp#L157-L158.
if frame.data[0] == 0xef { // abridged header
// https://core.telegram.org/mtproto/mtproto-transports#abridged
continue
}
switch binary.LittleEndian.Uint32(frame.data[:4]) {
case 0x44414548, // HEAD
0x54534f50, // POST
0x20544547, // GET
0x4954504f, // OPTI
0x02010316, // ????
0xdddddddd, // PaddedIntermediate header
0xeeeeeeee: // Intermediate header
continue
}
if frame.data[4]|frame.data[5]|frame.data[6]|frame.data[7] == 0 {
continue
}
copy(frame.connectionType(), hfConnectionType[:])
binary.LittleEndian.PutUint16(frame.dcSlice(), uint16(dc))
return frame
}
}

Some files were not shown because too many files have changed in this diff Show More