ScuroNeko/mtg
1
0
Fork 0
mirror of https://github.com/ScuroNeko/mtg.git synced 2026-08-31 11:54:01 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into stable

Browse Source
This commit is contained in:
9seconds
2026-03-15 22:04:09 +01:00
parent 9f12620dba d98d5be3a7
commit 30aa9d3a44
96 changed files with 3920 additions and 1427 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yaml
+33 -11
View File
@@ -48,6 +48,16 @@ jobs:
- uses: jdx/mise-action@v3
name: Install mise
- name: Cache Go modules and build
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Run tests
run: mise tasks run covtest
@@ -69,6 +79,16 @@ jobs:
- uses: jdx/mise-action@v3
name: Install mise
- name: Cache Go modules and build
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Run fuzzing
run: mise tasks run 'test:fuzz:*'
@@ -86,6 +106,16 @@ jobs:
- uses: jdx/mise-action@v3
name: Install mise
- name: Cache Go modules and build
uses: actions/cache@v5
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Run linter
run: mise tasks run lint
@@ -123,14 +153,6 @@ jobs:
- name: Setup BuildX
uses: docker/setup-buildx-action@v3
- name: Setup cache
uses: actions/cache@v5
with:
path: /tmp/buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
@@ -147,7 +169,7 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v2
uses: docker/build-push-action@v6
with:
pull: true
context: .
@@ -155,5 +177,5 @@ jobs:
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=local,src=/tmp/buildx-cache
cache-to: type=local,dest=/tmp/buildx-cache
cache-from: type=gha
cache-to: type=gha,mode=max
.github/workflows/govulncheck.yml
+1
View File
@@ -35,6 +35,7 @@ jobs:
uses: actions/setup-go@v6
with:
go-version-file: go.mod
cache: true
- name: Check for vulnerabilities
run: |
.goreleaser.yml
+2
View File
@@ -54,6 +54,8 @@ archives:
- LICENSE
- README.md
- SECURITY.md
- BEST_PRACTICES.md
- example.config.toml
gomod:
proxy: true
.mise.toml
+3 -3
View File
@@ -33,11 +33,11 @@ run = "govulncheck ./..."
[tasks.test]
description = "Run tests"
run = "go test -v ./..."
run = "go test -v -race ./..."
[tasks.covtest]
description = "Run tests with code coverage"
run = "go test -coverprofile=coverage.txt -covermode=atomic -parallel 2 -race -v ./..."
run = "go test -coverprofile=coverage.txt -covermode=atomic -count=2 -race -v ./..."
[tasks.test-all]
description = "Run all tests"
@@ -48,7 +48,7 @@ depends = [
[tasks."test:fuzz:client-hello"]
description = "Run fuzzy test for ClientHello"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzClientHello ./mtglib/internal/faketls"
run = "go test -v {{ vars.fuzzflags }} -fuzz=FuzzReadClientHello ./mtglib/internal/tls/fake"
[tasks."test:fuzz:client-handshake"]
description = "Run fuzzy test for ClientHandshake"
BEST_PRACTICES.md
+55
View File
@@ -0,0 +1,55 @@
# Best practices
This is unfortunate, but since 2018 many things were changed. Most of them
became way worse. Previous iterations of censorship systems were very dumb,
DPI were primitive and filtered very obvious things. Nowadays they are
way more intelligent and it is very naive to treat them frivolously.
In 2026 is not enough to pretend that your mtg installation is a Microsoft
website that sits in Amsterdam Digital Ocean location. Now your installation
has to be a website that is mtg in disguise. Yes, it requires a bit more effort
but this effort is probably less than rotating proxies each other day.
mtproto traffic, even with FakeTLS, has its specifics that are probably
very well known by DPI systems. These specifics are not something unique but
could mark an IP address as suspicious. Now let's think:
1. You have a proxy in Amsterdam Digital Ocean that tells it is microsoft.com
how hard could it be to find out that this is probably fake? 1 or probably 2
DNS queries for `microsoft.com`? In case of some CDN, there are ECS-powered
resolvers that are very capable to return results from POV of some subnets.
If censor sees no relevant results, will they be afraid to block IP?
2. You have a proxy in Amsterdam Digital Ocean that tells it is a website from
the same public subnet. But not the same. Would it be hard to make these DNS
queries and ban IP?
The correct way of having this proxy is following:
1. Register a domain name
2. Get some VPS, probably in your domestic location
3. Set that domain name from a step 1 to IP address of that VPS
4. Generate a couple of HTML pages by LLMs or even copy them from elsewhere
5. Set some webserver and issue TLS certificates with Let's Encrypt or any other
name
6. Set mtg before this webserver.
7. Use sing-box or anything like that to provide local socks5 interface and
have VPNized uplinks
8. Set up mtg to use socks5 from a 7 step.
In that case you will get a match of DNS and SNI in requests. As a side effect,
your proxy will work with XTLS and its friends: XTLS in sniff mode ignores
IP address a client wants to connect to. Instead, it reads SNI and connect
to resolved address: a clever idea if user does not have a trustworthy DNS
set up.
Yes, this is much longer that usual technique, and requires more effort. But
this is could probably be very well automated to some reasonable extent.
Unfortunately, this is a best practice right now.
Do not also forget about other implementation, like
[telemt](https://github.com/telemt/telemt). Try everything. Use VPNs. It does
not really matter which project you are going to use as long it helps you to
stay connected.
_March 2026._
Dockerfile
+11 -6
View File
@@ -5,14 +5,19 @@ FROM golang:1.26-alpine AS build
ENV CGO_ENABLED=0
RUN set -x \
&& apk --no-cache --update add \
bash \
ca-certificates \
git
RUN --mount=type=cache,target=/var/cache/apk \
set -x \
&& apk --update add \
bash \
ca-certificates \
git
COPY go.mod go.sum /app/
WORKDIR /app
RUN go mod download
COPY . /app
WORKDIR /app
RUN set -x \
&& version="$(git describe --exact-match HEAD 2>/dev/null || git describe --tags --always)" \
README.md
+78
View File
@@ -38,6 +38,33 @@ goal: to give a possibility to connect to Telegram in a restricted,
censored environment. But it does it slightly differently in details
that probably matter.
* **Domain fronting**
For years mtg supports domain fronting. This technique means that it fallbacks
to accessing a real website in case if request fails. It could fail by many
reasons: anti-replay protection, accidental access to the webserver or
stale request. Anyway, if mtg rejects this request, it does not break a
connection. It connects to the websites and replicates everything that client
has sent, and simply proxies it back as is. Users will see a response from
the real website, _byte-to-byte identical_ to the response of the real netloc.
* **Doppelganger**
mtg also is a doppelganger of the website it fronts. Sure, with domain fronting
users will see replies of the real website in case if something will go wrong.
But what about such cases when _everything is fine_?
In that case mtg mimics TLS connection statistical characteristics as close as
possible. Different application have different statistics of their patterns.
Big CDN steadily pumping the data, small websites burst with short easily
compressiable chunks of traffic.
mtg artificially emulates those delays to be statistically indistinguishable
from the real website even if it covers connection of the very specific app.
It also follows 2 most common patterns of traffic chunking, so censors
will have to put more resources to find out that we have Telegram here
but not a hookah webshop served by nginx.
* **Resource-efficient**
It has to be resource-efficient. It does not mean that you will see
@@ -93,6 +120,8 @@ that probably matter.
software (written in Golang) with a minimum effort + you can replace
some parts with those you want.
Please also to read about [best practices](https://github.com/9seconds/mtg/blob/master/BEST_PRACTICES.md).
### Version 2
If you use version 1.x before, you are probably noticed some major
@@ -398,6 +427,55 @@ or if you are using docker:
$ docker exec mtg-proxy /mtg access /config.toml
```
## Doppelganger
mtg can mimic real websites, please take a look at relevant section in example
config file.
mtg comes with some very good precollected statistics coming from
[ok.ru](https://ok.ru/). It does not mean that you have to cover yourself
by pretending that mtg is _ok.ru_. **Do not do that: ok.ru comes from very specific
ASNs, but not from VPS providers you are going to use.** What I want to say
is that defaults are very good enough to use as is because ok.ru for public
pages has a very generic profile of TLS packets delay.
But for better results it is recommended to teach mtg about the website you
will use as a domain front. In order to do that, you need to specify URLs
from this website. Just go to it, open WebDeveloper console and pick up
random URLs. For better results they have to be **from the same domain name
you are going to use as a disguise** but serve light and heavy content: pages,
images etc. Do not use many, 2-3 will probably work.
mtg will crawl these pages periodically, accumulating statistics and
using it as you go.
```toml
[defense.doppelganger]
urls = [
"https://lalala.com/index.html",
"https://lalala.com/contacts.html",
]
```
This is not very necessary. Keep in mind these rules:
1. If you are not sure what is this all about, do nothing. Defaults are good.
2. All URLs must be HTTPS
3. All URLs should be from the same domain name (but this is not a rule)
4. Do not use a lot of pages. Use _different_ pages. mtg will start using this
statistics when it will accumulate enough anyway.
5. These URLs should be directly accessible from mtg without proxies whatsoever
6. Do not create huge raids. mtg will repeatedly crawl in raids, making N repeats.
Do not use high N, you do not want to be noticeable.
7. It makes no sense to have small delay between raids. Usually webservers
do not update their TLS settings each hour.
8. If you have some specific knowledge if webserver is using
[TLS Dynamic Record Sizing](https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/), you
can use a very specific setting. This are Cloudflare, Go standard webservers,
[caddy](https://caddyserver.com/) and [H2O](https://h2o.examp1e.net/). If so,
you can enable `drs` setting.
9. **If you are not sure, touch nothing!**
## Metrics
Out of the box, mtg works with
example.config.toml
+60
View File
@@ -206,6 +206,66 @@ tcp = "5s"
http = "10s"
idle = "1m"
# mtg has to mimic real websites. It does not mean domain fronting, it also
# means that traffic characteristics should be similar to real world traffic.
# websites and applications behave differently, their traffic patterns are also
# different. Applications do bursts of RPC-style messages (or JSON communication,
# does not really matter), while websites pump heavy content in HTTP2 streams
#
# It means that statistically there is a different between traffic shape:
# delays between packets are also different.
# In order to avoid censorship detection based on these patterns, there is a
# mtg subsystem called "Doppelganger" that aims to mimic website statistics
# as close as it could.
#
# Delays between TLS packets are not constant. There are many factors
# that come in play. Application should generate some response, it could
# send some headers first and stream content with chunked encoding. So
# some first packets could come as soon as possible, with some delays
# after first ones. Such phenomenon is described by different statistic
# distribution. There are 2 distribution that describe it: lognormal
# distribution and Weibul distribution. Lognormal is all about steady streams
# of heavy content like a video. Weibul is great about short bursts like
# user who requested a static page an a couple of images.
[defense.doppelganger]
# This is a list of URLs that would be crawled by mtg to approximate delay
# statistics. They MUST be HTTPS urls.
#
# You can come to the website and collect different URLs, with light and
# heavy content. We recommend to search for CDNs.
urls = [
# "https://st-ok.cdn-vk.ru/res/react/vendor/clsx-2.1.1-amd.js"
]
# A collection is done in raids. Each raid makes this number of requests to
# each URL in this list. Do not use a huge number, 10 is probably ok.
repeats-per-raid = 10
# This is a duration between each raid. It makes no sense to have a small number
# here as you would start to make a noticeable activity. Usually traffic patterns
# do not change a lot, so do not expect different results if you request
# each 10 minutes.
raid-each = "6h"
# This enables dynamic tls record sizing.
#
# Some modern stacks and platforms start to use the technique that is called
# DRS. They start with small TLS packets and ramp up eventually. First packets
# are usually about MTU size, after that we get 4k and eventually max size.
# This is done with a good intention: to minimize a time to the first byte,
# so application could start doing something with the data right after first
# RTT.
#
# Apparently, about 90% of application do not employ this technique, they use
# max size always: nginx, apache, java stuff. But Golang tools, angie and
# some specific patches activate this technique.
#
# In order to mimic a real website we need to know something about software
# it uses. Usually nobody cares: openssl does 16384, Python does it, nginx
# does it. So this setting is disabled by default.
#
# https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/
# https://aws.github.io/s2n-tls/usage-guide/ch08-record-sizes.html
# https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch
drs = false
# Some countries do active probing on Telegram connections. This technique
# allows to protect from such effort.
#
go.mod
+7 -7
View File
@@ -11,18 +11,18 @@ require (
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/jarcoal/httpmock v1.0.8
github.com/mccutchen/go-httpbin v1.1.1
github.com/panjf2000/ants/v2 v2.11.5
github.com/panjf2000/ants/v2 v2.11.6
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.20.0 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/rs/zerolog v1.34.0
github.com/smira/go-statsd v1.3.4
github.com/stretchr/objx v0.5.2 // indirect
github.com/stretchr/testify v1.11.1
github.com/tylertreat/BoomFilters v0.0.0-20251117164519-53813c36cc1b
golang.org/x/crypto v0.48.0
golang.org/x/net v0.51.0
golang.org/x/sys v0.41.0
golang.org/x/crypto v0.49.0
golang.org/x/net v0.52.0
golang.org/x/sys v0.42.0
google.golang.org/protobuf v1.36.11 // indirect
)
@@ -49,8 +49,8 @@ require (
github.com/prometheus/client_model v0.6.2 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/txthinking/runnergroup v0.0.0-20250224021307-5864ffeb65ae // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
golang.org/x/sync v0.19.0 // indirect
go.yaml.in/yaml/v2 v2.4.4 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/tools v0.41.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum
+14 -14
View File
@@ -53,8 +53,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/ncruces/go-dns v1.3.2 h1:kBLuUZBgkQ4qF4WDXZRQ4rG0Gk6sLVJQ5tESkWrxUa0=
github.com/ncruces/go-dns v1.3.2/go.mod h1:tuzixNY8PY/M7yUzcvRbUaeLs3ifIdydpi5H2bfRU+s=
github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg=
github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/panjf2000/ants/v2 v2.11.6 h1:JKsoIUukIoCO0sP0gcOqdyoXmpyKXuU6fC57rODtpug=
github.com/panjf2000/ants/v2 v2.11.6/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
@@ -70,8 +70,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
github.com/prometheus/procfs v0.20.0 h1:AA7aCvjxwAquZAlonN7888f2u4IN8WVeFgBi4k82M4Q=
github.com/prometheus/procfs v0.20.0/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
@@ -105,12 +105,12 @@ github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
@@ -119,13 +119,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -135,8 +135,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
internal/cli/access.go
+6 -1
View File
@@ -101,8 +101,13 @@ func (a *Access) Run(cli *CLI, version string) error {
}
func (a *Access) getIP(ntw mtglib.Network, protocol string) net.IP {
dialer := ntw.NativeDialer()
client := ntw.MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error) {
return ntw.DialContext(ctx, protocol, address) //nolint: wrapcheck
conn, err := dialer.DialContext(ctx, protocol, address)
if err != nil {
return nil, err
}
return essentials.WrapNetConn(conn), err
})
req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) //nolint: noctx
internal/cli/run_proxy.go
+12 -2
View File
@@ -46,13 +46,13 @@ func makeNetwork(conf *config.Config, version string) (mtglib.Network, error) {
base := network.New(
resolver,
"mtg/"+version,
"",
conf.Network.Timeout.TCP.Get(0),
conf.Network.Timeout.HTTP.Get(0),
conf.Network.Timeout.Idle.Get(0),
)
proxyDialers := make([]network.Network, len(conf.Network.Proxies))
proxyDialers := make([]mtglib.Network, len(conf.Network.Proxies))
for idx, v := range conf.Network.Proxies {
value, err := network.NewProxyNetwork(base, v.Get(nil))
if err != nil {
@@ -239,6 +239,11 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen
return fmt.Errorf("cannot build ip allowlist: %w", err)
}
doppelGangerURLs := make([]string, len(conf.Defense.Doppelganger.URLs))
for i, v := range conf.Defense.Doppelganger.URLs {
doppelGangerURLs[i] = v.String()
}
opts := mtglib.ProxyOpts{
Logger: logger,
Network: ntw,
@@ -256,6 +261,11 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen
AllowFallbackOnUnknownDC: conf.AllowFallbackOnUnknownDC.Get(false),
TolerateTimeSkewness: conf.TolerateTimeSkewness.Value,
DoppelGangerURLs: doppelGangerURLs,
DoppelGangerPerRaid: conf.Defense.Doppelganger.Repeats.Get(mtglib.DoppelGangerPerRaid),
DoppelGangerEach: conf.Defense.Doppelganger.UpdateEach.Get(mtglib.DoppelGangerEach),
DoppelGangerDRS: conf.Defense.Doppelganger.DRS.Get(false),
}
proxy, err := mtglib.NewProxy(opts)
internal/config/config.go
+8 -2
View File
@@ -47,8 +47,14 @@ type Config struct {
MaxSize TypeBytes `json:"maxSize"`
ErrorRate TypeErrorRate `json:"errorRate"`
} `json:"antiReplay"`
Blocklist ListConfig `json:"blocklist"`
Allowlist ListConfig `json:"allowlist"`
Blocklist ListConfig `json:"blocklist"`
Allowlist ListConfig `json:"allowlist"`
Doppelganger struct {
URLs []TypeHttpsURL `json:"urls"`
Repeats TypeConcurrency `json:"repeats_per_raid"`
UpdateEach TypeDuration `json:"raid_each"`
DRS TypeBool `json:"drs"`
} `json:"doppelganger"`
} `json:"defense"`
Network struct {
Timeout struct {
internal/config/parse.go
+6
View File
@@ -44,6 +44,12 @@ type tomlConfig struct {
URLs []string `toml:"urls" json:"urls,omitempty"`
UpdateEach string `toml:"update-each" json:"updateEach,omitempty"`
} `toml:"allowlist" json:"allowlist,omitempty"`
Doppelganger struct {
URLs []string `toml:"urls" json:"urls,omitempty"`
Repeats uint `toml:"repeats-per-raid" json:"repeats_per_raid,omitempty"`
UpdateEach string `toml:"raid-each" json:"raid_each,omitempty"`
DRS bool `toml:"drs" json:"drs,omitempty"`
} `toml:"doppelganger" json:"doppelganger,omitempty"`
} `toml:"defense" json:"defense,omitempty"`
Network struct {
Timeout struct {
internal/config/type_https_url.go
+53
View File
@@ -0,0 +1,53 @@
package config
import (
"fmt"
"net/url"
)
type TypeHttpsURL struct {
Value *url.URL
}
func (t *TypeHttpsURL) Set(value string) error {
parsedURL, err := url.Parse(value)
if err != nil {
return fmt.Errorf("value is not correct URL (%s): %w", value, err)
}
if parsedURL.Host == "" {
return fmt.Errorf("url has to have a schema: %s", value)
}
if parsedURL.Scheme != "https" {
return fmt.Errorf("unsupported schema: %s", parsedURL.Scheme)
}
t.Value = parsedURL
return nil
}
func (t *TypeHttpsURL) Get(defaultValue *url.URL) *url.URL {
if t.Value == nil {
return defaultValue
}
return t.Value
}
func (t *TypeHttpsURL) UnmarshalText(data []byte) error {
return t.Set(string(data))
}
func (t TypeHttpsURL) MarshalText() ([]byte, error) {
return []byte(t.String()), nil
}
func (t TypeHttpsURL) String() string {
if t.Value == nil {
return ""
}
return t.Value.String()
}
internal/config/type_https_url_test.go
+100
View File
@@ -0,0 +1,100 @@
package config_test
import (
"encoding/json"
"net/url"
"testing"
"github.com/9seconds/mtg/v2/internal/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type typeHttpsURLTestStruct struct {
Value config.TypeHttpsURL `json:"value"`
}
type HttpsURLTestSuite struct {
suite.Suite
}
func (suite *HttpsURLTestSuite) TestUnmarshalFail() {
testData := []string{
"",
"https://",
"://lala",
"/path",
"http://example.com",
"socks5://example.com",
}
for _, v := range testData {
data, err := json.Marshal(map[string]string{
"value": v,
})
suite.NoError(err)
suite.T().Run(v, func(t *testing.T) {
assert.Error(t, json.Unmarshal(data, &typeHttpsURLTestStruct{}))
})
}
}
func (suite *HttpsURLTestSuite) TestUnmarshalOk() {
testData := map[string]string{
"https://example.com": "https://example.com",
"https://example.com:8443": "https://example.com:8443",
"https://example.com/path?q=1": "https://example.com/path?q=1",
"https://user:pass@example.com": "https://user:pass@example.com",
}
for k, v := range testData {
value := v
data, err := json.Marshal(map[string]string{
"value": k,
})
suite.NoError(err)
suite.T().Run(k, func(t *testing.T) {
testStruct := &typeHttpsURLTestStruct{}
assert.NoError(t, json.Unmarshal(data, testStruct))
parsed, _ := url.Parse(value)
assert.Equal(t, parsed.Scheme, testStruct.Value.Get(nil).Scheme)
assert.Equal(t, parsed.Host, testStruct.Value.Get(nil).Host)
assert.Equal(t, parsed.RawQuery, testStruct.Value.Get(nil).RawQuery)
assert.Equal(t, parsed.Path, testStruct.Value.Get(nil).Path)
})
}
}
func (suite *HttpsURLTestSuite) TestMarshalOk() {
parsed, _ := url.Parse("https://example.com/path?q=1")
testStruct := &typeHttpsURLTestStruct{
Value: config.TypeHttpsURL{
Value: parsed,
},
}
encodedJSON, err := json.Marshal(testStruct)
suite.NoError(err)
suite.JSONEq(`{"value": "https://example.com/path?q=1"}`,
string(encodedJSON))
}
func (suite *HttpsURLTestSuite) TestGet() {
emptyURL := &url.URL{}
value := config.TypeHttpsURL{}
suite.Equal(emptyURL, value.Get(emptyURL))
value.Value = &url.URL{}
suite.Equal(value.Value, value.Get(emptyURL))
}
func TestTypeHttpsURL(t *testing.T) {
t.Parallel()
suite.Run(t, &HttpsURLTestSuite{})
}
internal/testlib/mtglib_network_mock.go
+5
View File
@@ -2,6 +2,7 @@ package testlib
import (
"context"
"net"
"net/http"
"github.com/9seconds/mtg/v2/essentials"
@@ -24,6 +25,10 @@ func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address st
return args.Get(0).(essentials.Conn), args.Error(1) //nolint: wrapcheck, forcetypeassert
}
func (m *MtglibNetworkMock) NativeDialer() *net.Dialer {
return m.Called().Get(0).(*net.Dialer)
}
func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context,
network, address string) (essentials.Conn, error),
) *http.Client {
mise.lock
+18 -18
View File
@@ -1,11 +1,11 @@
[[tools.go]]
version = "1.26.0"
version = "1.26.1"
backend = "core:go"
"platforms.linux-arm64" = { checksum = "sha256:bd03b743eb6eb4193ea3c3fd3956546bf0e3ca5b7076c8226334afe6b75704cd", url = "https://dl.google.com/go/go1.26.0.linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:aac1b08a0fb0c4e0a7c1555beb7b59180b05dfc5a3d62e40e9de90cd42f88235", url = "https://dl.google.com/go/go1.26.0.linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:b1640525dfe68f066d56f200bef7bf4dce955a1a893bd061de6754c211431023", url = "https://dl.google.com/go/go1.26.0.darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:1ca28b7703cbea05a65b2a1d92d6b308610ef92f8824578a0874f2e60c9d5a22", url = "https://dl.google.com/go/go1.26.0.darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:9bbe0fc64236b2b51f6255c05c4232532b8ecc0e6d2e00950bd3021d8a4d07d4", url = "https://dl.google.com/go/go1.26.0.windows-amd64.zip"}
"platforms.linux-arm64" = { checksum = "sha256:a290581cfe4fe28ddd737dde3095f3dbeb7f2e4065cab4eae44dfc53b760c2f7", url = "https://dl.google.com/go/go1.26.1.linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:031f088e5d955bab8657ede27ad4e3bc5b7c1ba281f05f245bcc304f327c987a", url = "https://dl.google.com/go/go1.26.1.linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:353df43a7811ce284c8938b5f3c7df40b7bfb6f56cb165b150bc40b5e2dd541f", url = "https://dl.google.com/go/go1.26.1.darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:65773dab2f8cc4cd23d93ba6d0a805de150ca0b78378879292be0b903b8cdd08", url = "https://dl.google.com/go/go1.26.1.darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:9b68112c913f45b7aebbf13c036721264bbba7e03a642f8f7490c561eebd1ecc", url = "https://dl.google.com/go/go1.26.1.windows-amd64.zip"}
[[tools."go:golang.org/x/pkgsite/cmd/pkgsite"]]
version = "latest"
@@ -24,19 +24,19 @@ version = "0.9.2"
backend = "go:mvdan.cc/gofumpt"
[[tools.golangci-lint]]
version = "2.10.1"
version = "2.11.3"
backend = "aqua:golangci/golangci-lint"
"platforms.linux-arm64" = { checksum = "sha256:6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:03bfadf67e52b441b7ec21305e501c717df93c959836d66c7f97312654acb297", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:66fb0da81b8033b477f97eea420d4b46b230ca172b8bb87c6610109f3772b6b6", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:c60c87695e79db8e320f0e5be885059859de52bb5ee5f11be5577828570bc2a3", url = "https://github.com/golangci/golangci-lint/releases/download/v2.10.1/golangci-lint-2.10.1-windows-amd64.zip"}
"platforms.linux-arm64" = { checksum = "sha256:ee3d95f301359e7d578e6d99c8ad5aeadbabc5a13009a30b2b0df11c8058afe9", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-linux-arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:87bb8cddbcc825d5778b64e8a91b46c0526b247f4e2f2904dea74ec7450475d1", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-linux-amd64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:30ee39979c516b9d1adca289a3f93429d130c4c0fda5e57d637850894221f6cc", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-darwin-arm64.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:f93bda1f2cc981fd1326464020494be62f387bbf262706e1b3b644e5afacc440", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-darwin-amd64.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:cd42e890176bc5cfeb36225a77e66b9410ddd3a59a03551e23f6b210d29e1f67", url = "https://github.com/golangci/golangci-lint/releases/download/v2.11.3/golangci-lint-2.11.3-windows-amd64.zip"}
[[tools.goreleaser]]
version = "2.14.1"
version = "2.14.3"
backend = "aqua:goreleaser/goreleaser"
"platforms.linux-arm64" = { checksum = "sha256:a84d3b27f052c12ad5c8342d7caf1450a7174a305730aed21d72db09301e49a5", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Linux_arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:2df975a7acbfdeaf888d596cab0024d48ec7fb7d747e1d08b90948b791f40a5f", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Linux_x86_64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:9f2e47f847b4f4177376fc6aa6914fbc7f673f59720076747e738b578c2e896e", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Darwin_all.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:9f2e47f847b4f4177376fc6aa6914fbc7f673f59720076747e738b578c2e896e", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Darwin_all.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:d7a3d8ba795e97ab8c4f8003630d300da164adf21fde5a4049440c20f15c3137", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.1/goreleaser_Windows_x86_64.zip"}
"platforms.linux-arm64" = { checksum = "sha256:581a10e53c1176b3e81ee45cf531e02dbf899db0bc7b795669347df4276ce948", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Linux_arm64.tar.gz"}
"platforms.linux-x64" = { checksum = "sha256:dc7faeeeb6da8bdfda788626263a4ae725892a8c7504b975c3234127d4a44579", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Linux_x86_64.tar.gz"}
"platforms.macos-arm64" = { checksum = "sha256:3507798489e107a78aff36b169de48148a335ac26eb3161608d905f3f3a957bd", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Darwin_all.tar.gz"}
"platforms.macos-x64" = { checksum = "sha256:3507798489e107a78aff36b169de48148a335ac26eb3161608d905f3f3a957bd", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Darwin_all.tar.gz"}
"platforms.windows-x64" = { checksum = "sha256:3deea8ff471aa258a2d99f3e5302971d7028647ae8ddaf103257a8113e485a31", url = "https://github.com/goreleaser/goreleaser/releases/download/v2.14.3/goreleaser_Windows_x86_64.zip"}
mtglib/init.go
+12 -1
View File
@@ -99,6 +99,13 @@ const (
// reads from Telegram after which connection will be terminated. This is
// required to abort stale connections.
TCPRelayReadTimeout = 20 * time.Second
// DoppelGangerPerRaid defines a number of requests to each URL
// per raid.
DoppelGangerPerRaid = 10
// DoppelGangerEach defines a time period between each crawl attempt.
DoppelGangerEach = 6 * time.Hour
)
// Network defines a knowledge how to work with a network. It may sound fun but
@@ -117,13 +124,17 @@ type Network interface {
// Dial establishes context-free TCP connections.
Dial(network, address string) (essentials.Conn, error)
// DialContext dials using a context. This is a preferrable way of
// DialContext dials using a context. This is a preferable way of
// establishing TCP connections.
DialContext(ctx context.Context, network, address string) (essentials.Conn, error)
// MakeHTTPClient build an HTTP client with given dial function. If nothing is
// provided, then DialContext of this interface is going to be used.
MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client
// NativeDialer returns a configured instance of native dialer that
// skips proxy connections or any other irrelevant settings.
NativeDialer() *net.Dialer
}
// AntiReplayCache is an interface that is used to detect replay attacks based
mtglib/internal/doppel/clock.go
+35
View File
@@ -0,0 +1,35 @@
package doppel
import (
"context"
"time"
)
type Clock struct {
stats *Stats
tick chan struct{}
}
func (c Clock) Start(ctx context.Context) {
tickTock := time.NewTimer(c.stats.Delay())
defer func() {
tickTock.Stop()
select {
case <-tickTock.C:
default:
}
}()
for {
select {
case <-ctx.Done():
return
case <-tickTock.C:
select {
case <-ctx.Done():
case c.tick <- struct{}{}:
}
tickTock.Reset(c.stats.Delay())
}
}
}
mtglib/internal/doppel/clock_test.go
+80
View File
@@ -0,0 +1,80 @@
package doppel
import (
"context"
"sync"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type ClockTestSuite struct {
suite.Suite
clock Clock
wg sync.WaitGroup
ctx context.Context
ctxCancel context.CancelFunc
}
func (suite *ClockTestSuite) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
suite.ctx = ctx
suite.ctxCancel = cancel
suite.clock = Clock{
stats: &Stats{
k: StatsDefaultK,
lambda: StatsDefaultLambda,
},
tick: make(chan struct{}),
}
suite.wg.Go(func() {
suite.clock.Start(suite.ctx)
})
}
func (suite *ClockTestSuite) TearDownTest() {
suite.ctxCancel()
suite.wg.Wait()
}
func (suite *ClockTestSuite) TestTicks() {
received := 0
for range 3 {
select {
case <-suite.clock.tick:
received++
case <-time.After(2 * time.Second):
suite.Fail("timed out waiting for tick")
}
}
suite.Equal(3, received)
}
func (suite *ClockTestSuite) TestStopsOnCancel() {
select {
case <-suite.clock.tick:
case <-time.After(2 * time.Second):
suite.Fail("timed out waiting for first tick")
}
suite.ctxCancel()
time.Sleep(50 * time.Millisecond)
select {
case <-suite.clock.tick:
suite.Fail("received tick after cancel")
default:
}
}
func TestClock(t *testing.T) {
t.Parallel()
suite.Run(t, &ClockTestSuite{})
}
mtglib/internal/doppel/conn.go
+130
View File
@@ -0,0 +1,130 @@
package doppel
import (
"bytes"
"context"
"sync"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
type Conn struct {
essentials.Conn
p *connPayload
}
type connPayload struct {
ctx context.Context
ctxCancel context.CancelCauseFunc
clock Clock
wg sync.WaitGroup
syncWriteLock sync.RWMutex
writeStream bytes.Buffer
writeCond *sync.Cond
}
func (c Conn) Write(p []byte) (int, error) {
c.p.syncWriteLock.RLock()
defer c.p.syncWriteLock.RUnlock()
c.p.writeCond.L.Lock()
c.p.writeStream.Write(p)
c.p.writeCond.L.Unlock()
return len(p), context.Cause(c.p.ctx)
}
func (c Conn) SyncWrite(p []byte) (int, error) {
c.p.syncWriteLock.Lock()
defer c.p.syncWriteLock.Unlock()
c.p.writeCond.L.Lock()
// wait until buffer is exhausted
for c.p.writeStream.Len() != 0 && context.Cause(c.p.ctx) == nil {
c.p.writeCond.Wait()
}
c.p.writeStream.Write(p)
c.p.writeCond.L.Unlock()
if err := context.Cause(c.p.ctx); err != nil {
return len(p), err
}
c.p.writeCond.L.Lock()
// wait until data will be sent
for c.p.writeStream.Len() != 0 && context.Cause(c.p.ctx) == nil {
c.p.writeCond.Wait()
}
c.p.writeCond.L.Unlock()
return len(p), context.Cause(c.p.ctx)
}
func (c Conn) Start() {
c.p.wg.Go(func() {
c.start()
})
}
func (c Conn) start() {
defer c.p.writeCond.Broadcast()
buf := [tls.MaxRecordSize]byte{}
for {
select {
case <-c.p.ctx.Done():
return
case <-c.p.clock.tick:
}
c.p.writeCond.L.Lock()
n, err := c.p.writeStream.Read(buf[:c.p.clock.stats.Size()])
c.p.writeCond.L.Unlock()
if n == 0 || err != nil {
continue
}
if err := tls.WriteRecord(c.Conn, buf[:n]); err != nil {
c.p.ctxCancel(err)
return
}
c.p.writeCond.Signal()
}
}
func (c Conn) Stop() {
c.p.ctxCancel(nil)
c.p.wg.Wait()
}
func NewConn(ctx context.Context, conn essentials.Conn, stats *Stats) Conn {
ctx, cancel := context.WithCancelCause(ctx)
rv := Conn{
Conn: conn,
p: &connPayload{
ctx: ctx,
ctxCancel: cancel,
writeCond: sync.NewCond(&sync.Mutex{}),
clock: Clock{
stats: stats,
tick: make(chan struct{}),
},
},
}
rv.p.writeStream.Grow(tls.DefaultBufferSize)
rv.p.wg.Go(func() {
rv.p.clock.Start(ctx)
})
rv.p.wg.Go(func() {
rv.start()
})
return rv
}
mtglib/internal/doppel/conn_test.go
+293
View File
@@ -0,0 +1,293 @@
package doppel
import (
"bytes"
"context"
"encoding/binary"
"errors"
"io"
"sync"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ConnMock struct {
testlib.EssentialsConnMock
mu sync.Mutex
writeBuffer bytes.Buffer
}
func (m *ConnMock) Write(p []byte) (int, error) {
args := m.Called(p)
if err := args.Error(1); err != nil {
return args.Int(0), err
}
m.mu.Lock()
defer m.mu.Unlock()
return m.writeBuffer.Write(p)
}
func (m *ConnMock) Written() []byte {
m.mu.Lock()
defer m.mu.Unlock()
return bytes.Clone(m.writeBuffer.Bytes())
}
type ConnTestSuite struct {
suite.Suite
connMock *ConnMock
ctx context.Context
ctxCancel context.CancelFunc
}
func (suite *ConnTestSuite) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
suite.ctx = ctx
suite.ctxCancel = cancel
suite.connMock = &ConnMock{}
}
func (suite *ConnTestSuite) TearDownTest() {
suite.ctxCancel()
suite.connMock.AssertExpectations(suite.T())
}
func (suite *ConnTestSuite) makeConn() Conn {
return NewConn(suite.ctx, suite.connMock, &Stats{
k: 2.0,
lambda: 0.01,
})
}
func (suite *ConnTestSuite) TestWriteBuffersData() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
defer c.Stop()
n, err := c.Write([]byte{1, 2, 3})
suite.NoError(err)
suite.Equal(3, n)
}
func (suite *ConnTestSuite) TestWriteOutputsTLSRecords() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
payload := []byte("hello doppelganger")
_, err := c.Write(payload)
suite.NoError(err)
suite.Eventually(func() bool {
return len(suite.connMock.Written()) > 0
}, 2*time.Second, time.Millisecond)
c.Stop()
assembled := &bytes.Buffer{}
reader := bytes.NewReader(suite.connMock.Written())
for {
header := make([]byte, tls.SizeHeader)
if _, err := io.ReadFull(reader, header); err != nil {
break
}
suite.Equal(byte(tls.TypeApplicationData), header[0])
suite.Equal(tls.TLSVersion[:], header[tls.SizeRecordType:tls.SizeRecordType+tls.SizeVersion])
length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:])
suite.Greater(length, uint16(0))
rec := make([]byte, length)
_, err := io.ReadFull(reader, rec)
suite.NoError(err)
assembled.Write(rec)
}
suite.Equal(payload, assembled.Bytes())
}
func (suite *ConnTestSuite) TestWriteReturnsErrorAfterStop() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
c.Stop()
time.Sleep(10 * time.Millisecond)
_, err := c.Write([]byte{1})
suite.Error(err)
}
func (suite *ConnTestSuite) TestStopOnUnderlyingWriteError() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, errors.New("connection reset")).
Maybe()
c := suite.makeConn()
_, _ = c.Write([]byte("data"))
suite.Eventually(func() bool {
_, err := c.Write([]byte{1})
return err != nil
}, 2*time.Second, time.Millisecond)
}
func (suite *ConnTestSuite) TestSyncWriteDataSent() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
defer c.Stop()
payload := []byte("sync hello")
n, err := c.SyncWrite(payload)
suite.NoError(err)
suite.Equal(len(payload), n)
// SyncWrite returns only after data is flushed to the wire.
assembled := &bytes.Buffer{}
reader := bytes.NewReader(suite.connMock.Written())
for {
header := make([]byte, tls.SizeHeader)
if _, err := io.ReadFull(reader, header); err != nil {
break
}
suite.Equal(byte(tls.TypeApplicationData), header[0])
length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:])
rec := make([]byte, length)
_, err := io.ReadFull(reader, rec)
suite.NoError(err)
assembled.Write(rec)
}
suite.Equal(payload, assembled.Bytes())
}
func (suite *ConnTestSuite) TestSyncWriteDrainsBufferFirst() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
defer c.Stop()
// Buffer some data via async Write.
_, err := c.Write([]byte("first"))
suite.NoError(err)
// SyncWrite must drain "first" before sending "second".
n, err := c.SyncWrite([]byte("second"))
suite.NoError(err)
suite.Equal(6, n)
// All data should be on the wire now.
assembled := &bytes.Buffer{}
reader := bytes.NewReader(suite.connMock.Written())
for {
header := make([]byte, tls.SizeHeader)
if _, err := io.ReadFull(reader, header); err != nil {
break
}
length := binary.BigEndian.Uint16(header[tls.SizeRecordType+tls.SizeVersion:])
rec := make([]byte, length)
_, err := io.ReadFull(reader, rec)
suite.NoError(err)
assembled.Write(rec)
}
suite.Equal([]byte("firstsecond"), assembled.Bytes())
}
func (suite *ConnTestSuite) TestSyncWriteBlocksAsyncWrite() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
defer c.Stop()
// Start SyncWrite — it holds exclusive lock.
syncDone := make(chan struct{})
go func() {
defer close(syncDone)
c.SyncWrite([]byte("exclusive")) //nolint: errcheck
}()
// Give SyncWrite time to acquire the lock.
time.Sleep(10 * time.Millisecond)
// Async Write should block until SyncWrite completes.
writeDone := make(chan struct{})
go func() {
defer close(writeDone)
c.Write([]byte("blocked")) //nolint: errcheck
}()
// SyncWrite should finish first.
<-syncDone
select {
case <-writeDone:
// Write completed after SyncWrite — correct.
case <-time.After(2 * time.Second):
suite.Fail("async Write did not unblock after SyncWrite completed")
}
}
func (suite *ConnTestSuite) TestSyncWriteReturnsErrorAfterStop() {
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
c := suite.makeConn()
c.Stop()
time.Sleep(10 * time.Millisecond)
_, err := c.SyncWrite([]byte("too late"))
suite.Error(err)
}
func TestConn(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTestSuite{})
}
mtglib/internal/doppel/ganger.go
+184
View File
@@ -0,0 +1,184 @@
package doppel
import (
"context"
"sync"
"time"
"github.com/9seconds/mtg/v2/essentials"
)
const (
DoppelGangerMaxDurations = 4096
DoppelGangerScoutRaidEach = 6 * time.Hour
DoppelGangerScoutRepeats = 10
)
type gangerConnRequest struct {
ret chan<- Conn
payload essentials.Conn
}
type Ganger struct {
ctx context.Context
ctxCancel context.CancelFunc
logger Logger
wg sync.WaitGroup
scout Scout
scoutRaidEach time.Duration
scoutRaidRepeats int
drs bool
stats *Stats
durations []time.Duration
connRequests chan gangerConnRequest
}
func (g *Ganger) Shutdown() {
g.ctxCancel()
g.wg.Wait()
}
func (g *Ganger) Run() {
g.wg.Go(func() {
g.run()
})
}
func (g *Ganger) NewConn(conn essentials.Conn) (Conn, error) {
rvChan := make(chan Conn)
req := gangerConnRequest{
ret: rvChan,
payload: conn,
}
defer close(req.ret)
select {
case <-g.ctx.Done():
return Conn{}, context.Cause(g.ctx)
case g.connRequests <- req:
}
select {
case <-g.ctx.Done():
return Conn{}, context.Cause(g.ctx)
case conn := <-rvChan:
return conn, nil
}
}
func (g *Ganger) run() {
scoutTicker := time.NewTicker(g.scoutRaidEach)
defer func() {
scoutTicker.Stop()
select {
case <-scoutTicker.C:
default:
}
}()
scoutCollectedChan := make(chan []time.Duration)
currentScoutCollectedChan := scoutCollectedChan
updatedStatsChan := make(chan *Stats)
g.wg.Go(func() {
g.runScoutRaid(scoutCollectedChan)
})
for {
select {
case <-g.ctx.Done():
return
case durations := <-currentScoutCollectedChan:
g.durations = append(g.durations, durations...)
if len(g.durations) > DoppelGangerMaxDurations {
g.durations = g.durations[len(g.durations)-DoppelGangerMaxDurations:]
}
if len(g.durations) < MinDurationsToCalculate {
continue
}
currentScoutCollectedChan = nil
g.wg.Go(func() {
select {
case <-g.ctx.Done():
case updatedStatsChan <- NewStats(durations, g.drs):
}
})
case stats := <-updatedStatsChan:
g.stats = stats
currentScoutCollectedChan = scoutCollectedChan
case <-scoutTicker.C:
g.wg.Go(func() {
g.runScoutRaid(scoutCollectedChan)
})
case req := <-g.connRequests:
select {
case <-g.ctx.Done():
case req.ret <- NewConn(g.ctx, req.payload, g.stats):
}
}
}
}
func (g *Ganger) runScoutRaid(rvChan chan<- []time.Duration) {
durations := []time.Duration{}
for range g.scoutRaidRepeats {
learned, err := g.scout.Learn(g.ctx)
if err != nil {
g.logger.WarningError("cannot learn", err)
continue
}
durations = append(durations, learned...)
}
select {
case <-g.ctx.Done():
return
case rvChan <- durations:
}
}
func NewGanger(
ctx context.Context,
network Network,
logger Logger,
scoutEach time.Duration,
scoutRepeats int,
urls []string,
drs bool,
) *Ganger {
ctx, cancel := context.WithCancel(ctx)
if scoutEach == 0 {
scoutEach = DoppelGangerScoutRaidEach
}
if scoutRepeats == 0 {
scoutRepeats = DoppelGangerScoutRepeats
}
return &Ganger{
ctx: ctx,
ctxCancel: cancel,
logger: logger,
scoutRaidEach: scoutEach,
scoutRaidRepeats: scoutRepeats,
drs: drs,
stats: &Stats{
k: StatsDefaultK,
lambda: StatsDefaultLambda,
drs: drs,
},
scout: NewScout(network, urls),
connRequests: make(chan gangerConnRequest),
}
}
mtglib/internal/doppel/ganger_test.go
+107
View File
@@ -0,0 +1,107 @@
package doppel
import (
"bytes"
"sync"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type GangerTestSuite struct {
TLSServerTestSuite
log *LoggerMock
g *Ganger
}
func (suite *GangerTestSuite) SetupTest() {
suite.TLSServerTestSuite.SetupTest()
suite.log = &LoggerMock{}
suite.log.
On("Info", mock.AnythingOfType("string")).
Maybe()
suite.log.
On("WarningError", mock.AnythingOfType("string"), mock.Anything).
Maybe()
suite.g = NewGanger(suite.ctx, suite.network, suite.log, time.Hour, 1, suite.urls, true)
suite.g.Run()
}
func (suite *GangerTestSuite) TearDownTest() {
suite.g.Shutdown()
suite.log.AssertExpectations(suite.T())
suite.TLSServerTestSuite.TearDownTest()
}
func (suite *GangerTestSuite) TestNewConnAfterShutdown() {
suite.g.Shutdown()
connMock := &testlib.EssentialsConnMock{}
_, err := suite.g.NewConn(connMock)
suite.Error(err)
}
func (suite *GangerTestSuite) TestNewConnWhileRunning() {
connMock := &testlib.EssentialsConnMock{}
connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(0, nil).
Maybe()
connMock.On("Close").
Return(nil).
Maybe()
conn, err := suite.g.NewConn(connMock)
suite.NoError(err)
conn.Stop()
}
func (suite *GangerTestSuite) TestNewConnWriteProducesTLSRecords() {
var (
mu sync.Mutex
buf bytes.Buffer
)
connMock := &testlib.EssentialsConnMock{}
connMock.On("Write", mock.AnythingOfType("[]uint8")).
Run(func(args mock.Arguments) {
mu.Lock()
buf.Write(args.Get(0).([]byte))
mu.Unlock()
}).
Return(0, nil).
Maybe()
connMock.On("Close").
Return(nil).
Maybe()
conn, err := suite.g.NewConn(connMock)
suite.NoError(err)
payload := bytes.Repeat([]byte("x"), 512)
_, err = conn.Write(payload)
suite.NoError(err)
time.Sleep(500 * time.Millisecond)
conn.Stop()
mu.Lock()
written := buf.Bytes()
mu.Unlock()
suite.NotEmpty(written)
}
func TestGanger(t *testing.T) {
t.Parallel()
suite.Run(t, &GangerTestSuite{})
}
mtglib/internal/doppel/init.go
+43
View File
@@ -0,0 +1,43 @@
package doppel
import (
"context"
"net"
"net/http"
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
const (
// Please see Stats description
// https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/
// https://github.com/cloudflare/sslconfig/blob/master/patches/nginx__dynamic_tls_records.patch
TLSRecordSizeStart = 1450
TLSRecordSizeAccel = 4096
TLSRecordSizeMax = 16384 - tls.SizeHeader
TLSCounterAccelAfter = 40
TLSCounterMaxAfter = TLSCounterAccelAfter + 20
TLSRecordSizeResetAfter = time.Second
)
// copypasted from mtglib
type Network interface {
// Dial establishes context-free TCP connections.
Dial(network, address string) (essentials.Conn, error)
// DialContext dials using a context. This is a preferable way of
// establishing TCP connections.
DialContext(ctx context.Context, network, address string) (essentials.Conn, error)
// MakeHTTPClient build an HTTP client with given dial function. If nothing is
// provided, then DialContext of this interface is going to be used.
MakeHTTPClient(func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client
// NativeDialer returns a configured instance of native dialer that
// skips proxy connections or any other irrelevant settings.
NativeDialer() *net.Dialer
}
mtglib/internal/doppel/init_test.go
+107
View File
@@ -0,0 +1,107 @@
package doppel
import (
"context"
"crypto/tls"
"net"
"net/http"
"net/http/httptest"
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type SimpleNetwork struct{}
func (s SimpleNetwork) Dial(network, address string) (essentials.Conn, error) {
return s.DialContext(context.Background(), network, address)
}
func (s SimpleNetwork) DialContext(ctx context.Context, network, address string) (essentials.Conn, error) {
d := &net.Dialer{}
conn, err := d.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return conn.(*net.TCPConn), nil
}
func (s SimpleNetwork) NativeDialer() *net.Dialer {
return &net.Dialer{}
}
func (s SimpleNetwork) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (essentials.Conn, error)) *http.Client {
if dialFunc == nil {
dialFunc = s.DialContext
}
return &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true, //nolint: gosec
},
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
return dialFunc(ctx, network, address)
},
},
}
}
type TLSServerTestSuite struct {
suite.Suite
tlsServer *httptest.Server
ctx context.Context
ctxCancel context.CancelFunc
network SimpleNetwork
urls []string
}
func (suite *TLSServerTestSuite) SetupSuite() {
suite.tlsServer = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusOK)
w.Header().Add("Hello", "how long")
if _, err := w.Write([]byte{1, 2, 3}); err != nil {
panic(err)
}
time.Sleep(5 * time.Millisecond)
if _, err := w.Write([]byte{1, 2, 3}); err != nil {
panic(err)
}
}))
suite.urls = []string{suite.tlsServer.URL}
}
func (suite *TLSServerTestSuite) SetupTest() {
ctx, cancel := context.WithCancel(context.Background())
suite.ctx = ctx
suite.ctxCancel = cancel
}
func (suite *TLSServerTestSuite) TearDownTest() {
suite.ctxCancel()
suite.tlsServer.CloseClientConnections()
}
func (suite *TLSServerTestSuite) TearDownSuite() {
suite.tlsServer.Close()
}
type LoggerMock struct {
mock.Mock
}
func (l *LoggerMock) Info(msg string) {
l.Called(msg)
}
func (l *LoggerMock) WarningError(msg string, err error) {
l.Called(msg, err)
}
mtglib/internal/doppel/logger.go
+6
View File
@@ -0,0 +1,6 @@
package doppel
type Logger interface {
Info(msg string)
WarningError(msg string, err error)
}
mtglib/internal/doppel/scout.go
+105
View File
@@ -0,0 +1,105 @@
package doppel
import (
"context"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
type Scout struct {
network Network
urls []string
}
func (s Scout) Learn(ctx context.Context) ([]time.Duration, error) {
var durations []time.Duration
for _, url := range s.urls {
learned, err := s.learn(ctx, url)
if err != nil {
return nil, err
}
durations = append(durations, learned...)
}
return durations, nil
}
func (s Scout) learn(ctx context.Context, url string) ([]time.Duration, error) {
client, results := s.makeClient()
if !strings.HasPrefix(url, "https://") {
return nil, fmt.Errorf("url %s must be https", url)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if resp != nil {
io.Copy(io.Discard, resp.Body) //nolint: errcheck
resp.Body.Close() //nolint: errcheck
client.CloseIdleConnections()
}
if err != nil || len(results.data) == 0 {
return nil, err
}
durations := []time.Duration{}
lastTimestamp := time.Time{}
for i, v := range results.data {
if v.recordType != tls.TypeApplicationData {
continue
}
if lastTimestamp.IsZero() {
if i > 0 {
lastTimestamp = results.data[i-1].timestamp
} else {
lastTimestamp = v.timestamp
}
}
durations = append(durations, v.timestamp.Sub(lastTimestamp))
lastTimestamp = v.timestamp
}
return durations, nil
}
func (s Scout) makeClient() (*http.Client, *ScoutConnCollected) {
dialer := s.network.NativeDialer()
collected := NewScoutConnCollected()
client := s.network.MakeHTTPClient(func(
ctx context.Context,
network string,
address string,
) (essentials.Conn, error) {
conn, err := dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return NewScoutConn(essentials.WrapNetConn(conn), collected), nil
})
return client, collected
}
func NewScout(network Network, urls []string) Scout {
return Scout{
network: network,
urls: urls,
}
}
mtglib/internal/doppel/scout_conn.go
+57
View File
@@ -0,0 +1,57 @@
package doppel
import (
"bytes"
"encoding/binary"
"io"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
type ScoutConn struct {
tls.Conn
results *ScoutConnCollected
rawBuf *bytes.Buffer
}
func (s ScoutConn) Read(p []byte) (int, error) {
buf := &bytes.Buffer{}
for {
if n, err := s.rawBuf.Read(p); err == nil {
return n, nil
}
s.rawBuf.Reset()
recordType, length, err := tls.ReadRecord(s.Conn, buf)
if err != nil {
return 0, err
}
s.results.Add(recordType)
s.rawBuf.Write([]byte{recordType})
s.rawBuf.Write(tls.TLSVersion[:])
if err := binary.Write(s.rawBuf, binary.BigEndian, uint16(length)); err != nil {
return 0, err
}
if _, err := io.Copy(s.rawBuf, buf); err != nil {
return 0, err
}
}
}
func NewScoutConn(conn essentials.Conn, results *ScoutConnCollected) ScoutConn {
rawBuf := &bytes.Buffer{}
rawBuf.Grow(tls.MaxRecordSize)
return ScoutConn{
Conn: tls.New(conn, false, false),
results: results,
rawBuf: rawBuf,
}
}
mtglib/internal/doppel/scout_conn_collected.go
+29
View File
@@ -0,0 +1,29 @@
package doppel
import "time"
const (
ScoutConnCollectedPreallocSize = 100
)
type ScoutConnResult struct {
timestamp time.Time
recordType byte
}
type ScoutConnCollected struct {
data []ScoutConnResult
}
func (s *ScoutConnCollected) Add(record byte) {
s.data = append(s.data, ScoutConnResult{
timestamp: time.Now(),
recordType: record,
})
}
func NewScoutConnCollected() *ScoutConnCollected {
return &ScoutConnCollected{
data: make([]ScoutConnResult, 0, ScoutConnCollectedPreallocSize),
}
}
mtglib/internal/doppel/scout_conn_collected_test.go
+42
View File
@@ -0,0 +1,42 @@
package doppel
import (
"testing"
"time"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/stretchr/testify/suite"
)
type ScoutConnCollectedTestSuite struct {
suite.Suite
}
func (suite *ScoutConnCollectedTestSuite) TestAddSingle() {
collected := NewScoutConnCollected()
collected.Add(tls.TypeApplicationData)
suite.Len(collected.data, 1)
suite.Equal(byte(tls.TypeApplicationData), collected.data[0].recordType)
}
func (suite *ScoutConnCollectedTestSuite) TestAddTimestampsAreMonotonic() {
collected := NewScoutConnCollected()
collected.Add(tls.TypeApplicationData)
time.Sleep(time.Microsecond)
collected.Add(tls.TypeApplicationData)
time.Sleep(time.Microsecond)
collected.Add(tls.TypeApplicationData)
for i := 1; i < len(collected.data); i++ {
suite.True(collected.data[i].timestamp.After(collected.data[i-1].timestamp))
}
}
func TestScoutConnCollected(t *testing.T) {
t.Parallel()
suite.Run(t, &ScoutConnCollectedTestSuite{})
}
mtglib/internal/doppel/scout_test.go
+39
View File
@@ -0,0 +1,39 @@
package doppel
import (
"testing"
"github.com/stretchr/testify/suite"
)
type ScoutTestSuite struct {
TLSServerTestSuite
scout Scout
}
func (suite *ScoutTestSuite) SetupSuite() {
suite.TLSServerTestSuite.SetupSuite()
suite.scout = Scout{
network: suite.network,
urls: suite.urls,
}
}
func (suite *ScoutTestSuite) TestCollectResults() {
durations, err := suite.scout.Learn(suite.ctx)
suite.NoError(err)
suite.Less(3, len(durations))
}
func (suite *ScoutTestSuite) TestCollectNothing() {
suite.ctxCancel()
_, err := suite.scout.Learn(suite.ctx)
suite.Error(err)
}
func TestScout(t *testing.T) {
suite.Run(t, &ScoutTestSuite{})
}
mtglib/internal/doppel/stats.go
+170
View File
@@ -0,0 +1,170 @@
package doppel
import (
"math"
"math/rand/v2"
"time"
)
const (
StatsBisectTimes = 70
StatsLowK = 0.01
StatsHighK = 10.0
// do not calculate statistics if we have < than this number of durations
MinDurationsToCalculate = 100
// these values are taken from ok.ru. measured from moscow site.
StatsDefaultK = 0.37846373895785335
StatsDefaultLambda = 1.73177086015485
// how many bytes should we drift
DRSNoise = 100
)
// Stats is responsible for generating values that are distributed according
// to some statistical distribution.
//
// It follows several ideas:
// 1. Based on nginx and Cloudflare behaviour, even if server is eager
// to send a lot, they all start with small TLS packets that are
// approximately MTU-sized. After
// 2. After ~40 TLS records, server considers TCP session as somewhat solid
// and reliable and ramps up to 4096.
// 3. After ~20 TLS records more it jumps to the max 16384 bytes and keep
// this size as long as it can
// 4. If there is no any byte within a connection for a longer time period,
// this counter resets.
//
// This is called Dynamic TLS Record Sizing
// - https://blog.cloudflare.com/optimizing-tls-over-tcp-to-reduce-latency/
// - https://community.f5.com/kb/technicalarticles/boosting-tls-performance-with-dynamic-record-sizing-on-big-ip/280798
// - https://www.igvita.com/2013/10/24/optimizing-tls-record-size-and-buffering-latency/
//
// And this optimized for the very first byte, so web browsers could start to
// render as early as possible, showing user some preliminary results, optimizing
// for perceived latency.
//
// Since this is very typical for the website, we also aim for that.
//
// Another important idea is how delays between TLS packets are distributed.
// In case of sending huge heavy content with max sized record, delays have
// lognormal distribution. But a nature of a typical website shows that
// it eagers to deliver as fast as it can in a few very first records and
// could possibly slow down later.
//
// This is perfectly described by Weibull distribution:
// - https://en.wikipedia.org/wiki/Weibull_distribution
// - https://ieeexplore.ieee.org/document/6662948
// - https://www.researchgate.net/publication/224621285_Traffic_modelling_and_cost_optimization_for_transmitting_traffic_messages_over_a_hybrid_broadcast_and_cellular_network
// - https://ir.uitm.edu.my/id/eprint/105386/1/105386.pdf
//
// In other word, a combination of Dynamic TLS Record Sizing hints us for
// Weibull distribution.
//
// But we also have to keep in mind that DRS is not well spread yet. In most cases
// users still rely on OpenSSL or webserver defaults. OpenSSL chunks with
// biggest packet sizes, nginx relies on static setting that is 16k by default.
// Thus, dynamic sizing has to be present but we cannot oblige users to use that.
type Stats struct {
sizeLastRequested time.Time
sizeCounter int
// https://en.wikipedia.org/wiki/Shape_parameter
k float64
// https://en.wikipedia.org/wiki/Scale_parameter
lambda float64
// Dynamic Record Sizing
drs bool
}
func (d *Stats) Delay() time.Duration {
// u ∈ (0, 1], avoids ln(0)
u := 1.0 - rand.Float64()
// X = λ·(-ln U)^(1/k)
generated := d.lambda * math.Pow(-math.Log(u), 1.0/d.k)
// generated is in milliseconds
return time.Duration(generated * float64(time.Millisecond))
}
func (d *Stats) Size() int {
if time.Since(d.sizeLastRequested) > TLSRecordSizeResetAfter {
d.sizeCounter = 0
}
if !d.drs {
return TLSRecordSizeMax
}
d.sizeLastRequested = time.Now()
d.sizeCounter++
switch {
case d.sizeCounter <= TLSCounterAccelAfter:
return TLSRecordSizeStart - rand.IntN(DRSNoise)
case d.sizeCounter <= TLSCounterMaxAfter:
return TLSRecordSizeAccel - rand.IntN(DRSNoise)
}
return TLSRecordSizeMax
}
func NewStats(durations []time.Duration, drs bool) *Stats {
n := float64(len(durations))
// in milliseconds
durFloats := make([]float64, len(durations))
for i, v := range durations {
durFloats[i] = float64(v.Microseconds()) / 1000.0
}
// The bisection solves the standard Weibull MLE equation for shape
// parameter k. There is no any good formula for doing that so we
// approximate it by several bisections. The number of operations
// is statically defined by a constant.
sumLog := 0.0
for _, v := range durFloats {
sumLog += math.Log(v)
}
lowK := StatsLowK
highK := StatsHighK
for range StatsBisectTimes {
midK := (lowK + highK) / 2.0
sumXK := 0.0
sumXKLog := 0.0
for _, v := range durFloats {
xk := math.Pow(v, midK)
sumXK += xk
sumXKLog += xk * math.Log(v)
}
if (1.0/midK)+(sumLog/n)-(sumXKLog/sumXK) > 0 {
lowK = midK
} else {
highK = midK
}
}
k := (lowK + highK) / 2
sumXK := 0.0
for _, v := range durFloats {
sumXK += math.Pow(v, k)
}
// λ = (Σxᵢᵏ / n)^(1/k)
lambda := math.Pow(sumXK/n, 1.0/k)
return &Stats{
k: k,
lambda: lambda,
drs: drs,
}
}
mtglib/internal/doppel/stats_test.go
+219
View File
@@ -0,0 +1,219 @@
package doppel
import (
"math"
"math/rand/v2"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type StatsTestSuite struct {
suite.Suite
}
func (suite *StatsTestSuite) GenWeibull(k, lambda float64, n int, seed uint64) []time.Duration {
rng := rand.New(rand.NewPCG(seed, 0))
samples := make([]time.Duration, n)
for i := range samples {
u := 1.0 - rng.Float64()
ms := lambda * math.Pow(-math.Log(u), 1.0/k)
d := time.Duration(ms * float64(time.Millisecond))
if d < time.Microsecond {
time.Sleep(time.Microsecond)
d = time.Microsecond
}
samples[i] = d
}
return samples
}
func (suite *StatsTestSuite) TestNewStatsRecoverParameters() {
knownK := 1.5
knownLambda := 100.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 42)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.1)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestNewStatsExponentialCase() {
// When k=1, Weibull reduces to exponential distribution.
knownK := 1.0
knownLambda := 50.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 123)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.1)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestNewStatsSmallK() {
// k < 1 produces a heavy-tailed distribution typical for network delays.
// Lambda must be large enough so samples stay above microsecond precision
// after time.Duration round-trip.
knownK := 0.6
knownLambda := 100.0
samples := suite.GenWeibull(knownK, knownLambda, 10000, 99)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.05)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestNewStatsLargeK() {
// k > 1: light tail, concentrated around the mode.
knownK := 5.0
knownLambda := 200.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 77)
stats := NewStats(samples, true)
suite.InDelta(knownK, stats.k, 0.3)
suite.InDelta(knownLambda, stats.lambda, 5.0)
}
func (suite *StatsTestSuite) TestDelayNonNegative() {
stats := &Stats{
k: 1.5,
lambda: 100.0,
}
for range 200 {
dur := stats.Delay()
suite.GreaterOrEqual(dur, time.Duration(0))
}
}
func (suite *StatsTestSuite) TestDelayDistributionMean() {
// Weibull mean = λ · Γ(1 + 1/k)
k := 2.0
lambda := 50.0
stats := &Stats{k: k, lambda: lambda}
n := 50000
sum := 0.0
for range n {
dur := stats.Delay()
sum += float64(dur) / float64(time.Millisecond)
}
sampleMean := sum / float64(n)
expectedMean := lambda * math.Gamma(1.0+1.0/k)
suite.InDelta(expectedMean, sampleMean, expectedMean*0.05)
}
func (suite *StatsTestSuite) TestNewStatsRoundTrip() {
// Estimate parameters from data, then verify that Delay samples
// from the fitted distribution have approximately the same mean.
knownK := 1.2
knownLambda := 80.0
samples := suite.GenWeibull(knownK, knownLambda, 5000, 555)
stats := NewStats(samples, true)
n := 50000
sum := 0.0
for range n {
dur := stats.Delay()
sum += float64(dur) / float64(time.Millisecond)
}
sampleMean := sum / float64(n)
expectedMean := knownLambda * math.Gamma(1.0+1.0/knownK)
suite.InDelta(expectedMean, sampleMean, expectedMean*0.05)
}
func (suite *StatsTestSuite) TestSizeStartPhase() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
for range TLSCounterAccelAfter {
size := stats.Size()
suite.GreaterOrEqual(size, TLSRecordSizeStart-DRSNoise)
suite.LessOrEqual(size, TLSRecordSizeStart)
}
}
func (suite *StatsTestSuite) TestSizeAccelPhase() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
for range TLSCounterAccelAfter {
stats.Size()
}
for range TLSCounterMaxAfter - TLSCounterAccelAfter {
size := stats.Size()
suite.GreaterOrEqual(size, TLSRecordSizeAccel-DRSNoise)
suite.LessOrEqual(size, TLSRecordSizeAccel)
}
}
func (suite *StatsTestSuite) TestSizeMaxPhase() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
for range TLSCounterMaxAfter {
stats.Size()
}
for range 20 {
size := stats.Size()
suite.Equal(TLSRecordSizeMax, size)
}
}
func (suite *StatsTestSuite) TestSizeResetsAfterInactivity() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: true}
// Advance past start phase.
for range TLSCounterMaxAfter {
stats.Size()
}
suite.Equal(TLSRecordSizeMax, stats.Size())
// Simulate inactivity by backdating sizeLastRequested.
stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond)
size := stats.Size()
suite.GreaterOrEqual(size, TLSRecordSizeStart-DRSNoise)
suite.LessOrEqual(size, TLSRecordSizeStart)
}
func (suite *StatsTestSuite) TestSizeNoDRSAlwaysMax() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: false}
for range TLSCounterMaxAfter + 20 {
suite.Equal(TLSRecordSizeMax, stats.Size())
}
}
func (suite *StatsTestSuite) TestSizeNoDRSIgnoresCounter() {
stats := &Stats{k: 1.0, lambda: 1.0, drs: false}
// Even after many calls, always returns max.
for range 200 {
suite.Equal(TLSRecordSizeMax, stats.Size())
}
// Inactivity has no effect either.
stats.sizeLastRequested = time.Now().Add(-TLSRecordSizeResetAfter - time.Millisecond)
suite.Equal(TLSRecordSizeMax, stats.Size())
}
func TestStats(t *testing.T) {
t.Parallel()
suite.Run(t, &StatsTestSuite{})
}
mtglib/internal/faketls/client_hello.go
-134
View File
@@ -1,134 +0,0 @@
package faketls
import (
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/binary"
"fmt"
"time"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
)
type ClientHello struct {
Time time.Time
Random [RandomLen]byte
SessionID []byte
Host string
CipherSuite uint16
}
func (c ClientHello) Valid(hostname string, tolerateTimeSkewness time.Duration) error {
if c.Host != "" && c.Host != hostname {
return fmt.Errorf("incorrect hostname %s", hostname)
}
now := time.Now()
timeDiff := now.Sub(c.Time)
if timeDiff < 0 {
timeDiff = -timeDiff
}
if timeDiff > tolerateTimeSkewness {
return fmt.Errorf("incorrect timestamp. got=%d, now=%d, diff=%s",
c.Time.Unix(), now.Unix(), timeDiff.String())
}
return nil
}
func ParseClientHello(secret, handshake []byte) (ClientHello, error) {
hello := ClientHello{}
if len(handshake) < ClientHelloMinLen {
return hello, fmt.Errorf("lengh of handshake is too small: %d", len(handshake))
}
if handshake[0] != HandshakeTypeClient {
return hello, fmt.Errorf("unknown handshake type %#x", handshake[0])
}
handshakeSizeBytes := [4]byte{0, handshake[1], handshake[2], handshake[3]}
handshakeLength := binary.BigEndian.Uint32(handshakeSizeBytes[:])
if len(handshake)-4 != int(handshakeLength) {
return hello,
fmt.Errorf("incorrect handshake size. manifested=%d, real=%d",
handshakeLength, len(handshake)-4)
}
copy(hello.Random[:], handshake[ClientHelloRandomOffset:])
copy(handshake[ClientHelloRandomOffset:], clientHelloEmptyRandom)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeHandshake
rec.Version = record.Version10
rec.Payload.Write(handshake)
// mac is calculated for the whole record, not only
// for the payload part
mac := hmac.New(sha256.New, secret)
rec.Dump(mac) //nolint: errcheck
computedRandom := mac.Sum(nil)
for i := range RandomLen {
computedRandom[i] ^= hello.Random[i]
}
if subtle.ConstantTimeCompare(clientHelloEmptyRandom[:RandomLen-4], computedRandom[:RandomLen-4]) != 1 {
return hello, ErrBadDigest
}
timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:]))
hello.Time = time.Unix(timestamp, 0)
parseSessionID(&hello, handshake)
parseCipherSuite(&hello, handshake)
parseSNI(&hello, handshake)
return hello, nil
}
func parseSessionID(hello *ClientHello, handshake []byte) {
hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset])
copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:])
}
func parseCipherSuite(hello *ClientHello, handshake []byte) {
cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3
hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2])
}
func parseSNI(hello *ClientHello, handshake []byte) {
cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 1
handshake = handshake[cipherSuiteOffset:]
cipherSuiteLength := binary.BigEndian.Uint16(handshake[:2])
handshake = handshake[2+cipherSuiteLength:]
compressionMethodsLength := int(handshake[0])
handshake = handshake[1+compressionMethodsLength:]
extensionsLength := binary.BigEndian.Uint16(handshake[:2])
handshake = handshake[2 : 2+extensionsLength]
for len(handshake) > 0 {
if binary.BigEndian.Uint16(handshake[:2]) != ExtensionSNI {
extensionsLength := binary.BigEndian.Uint16(handshake[2:4])
handshake = handshake[4+extensionsLength:]
continue
}
hostnameLength := binary.BigEndian.Uint16(handshake[7:9])
handshake = handshake[9:]
hello.Host = string(handshake[:int(hostnameLength)])
return
}
}
mtglib/internal/faketls/client_hello_fuzz_test.go
-21
View File
@@ -1,21 +0,0 @@
package faketls_test
import (
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/stretchr/testify/require"
)
var FuzzClientHelloSecret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
func FuzzClientHello(f *testing.F) {
f.Add([]byte{1, 2, 3})
f.Fuzz(func(t *testing.T, frame []byte) {
_, err := faketls.ParseClientHello(FuzzClientHelloSecret, frame)
// a probability of having != err is almost negligible
require.Error(t, err)
})
}
mtglib/internal/faketls/client_hello_test.go
-191
View File
@@ -1,191 +0,0 @@
package faketls_test
import (
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type ClientHelloSnapshot struct {
Time int `json:"time"`
Random string `json:"random"`
SessionID string `json:"sessionId"`
Host string `json:"host"`
CipherSuite int `json:"cipherSuite"`
Full string `json:"full"`
}
func (c ClientHelloSnapshot) GetTime() time.Time {
return time.Unix(int64(c.Time), 0)
}
func (c ClientHelloSnapshot) GetRandom() []byte {
data, _ := base64.StdEncoding.DecodeString(c.Random)
return data
}
func (c ClientHelloSnapshot) GetSessionID() []byte {
data, _ := base64.StdEncoding.DecodeString(c.SessionID)
return data
}
func (c ClientHelloSnapshot) GetHost() string {
return c.Host
}
func (c ClientHelloSnapshot) GetCipherSuite() uint16 {
return uint16(c.CipherSuite)
}
func (c ClientHelloSnapshot) GetFull() []byte {
data, _ := base64.StdEncoding.DecodeString(c.Full)
return data
}
type ClientHelloTestSuite struct {
suite.Suite
secret mtglib.Secret
}
func (suite *ClientHelloTestSuite) SetupSuite() {
parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d")
if err != nil {
panic(err)
}
suite.secret = parsed
}
func (suite *ClientHelloTestSuite) TestEmptyHandshake() {
_, err := faketls.ParseClientHello(suite.secret.Key[:], nil)
suite.Error(err)
}
func (suite *ClientHelloTestSuite) TestIncorrectHandshakeType() {
data := make([]byte, 1024)
data[0] = 0x02
_, err := faketls.ParseClientHello(suite.secret.Key[:], data)
suite.Error(err)
}
func (suite *ClientHelloTestSuite) TestIncorrectLength() {
data := make([]byte, 1024)
data[0] = 0x01
data[1] = 0xff
data[2] = 0xff
_, err := faketls.ParseClientHello(suite.secret.Key[:], data)
suite.Error(err)
}
func (suite *ClientHelloTestSuite) TestSnapshotOk() {
files, err := os.ReadDir("testdata")
suite.NoError(err)
testData := []string{}
for _, v := range files {
if strings.HasPrefix(v.Name(), "client-hello-ok") {
testData = append(testData, v.Name())
}
}
for _, name := range testData {
path := filepath.Join("testdata", name)
suite.T().Run(name, func(t *testing.T) {
fileData, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &ClientHelloSnapshot{}
assert.NoError(t, json.Unmarshal(fileData, snapshot))
hello, err := faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull())
assert.NoError(t, err)
assert.WithinDuration(t, snapshot.GetTime(), hello.Time, time.Second)
assert.Equal(t, snapshot.GetRandom(), hello.Random[:])
assert.Equal(t, snapshot.GetSessionID(), hello.SessionID)
assert.Equal(t, snapshot.GetHost(), hello.Host)
assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite)
})
}
}
func (suite *ClientHelloTestSuite) TestSnapshotBad() {
files, err := os.ReadDir("testdata")
suite.NoError(err)
testData := []string{}
for _, v := range files {
if strings.HasPrefix(v.Name(), "client-hello-bad") {
testData = append(testData, v.Name())
}
}
for _, name := range testData {
path := filepath.Join("testdata", name)
suite.T().Run(name, func(t *testing.T) {
fileData, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &ClientHelloSnapshot{}
assert.NoError(t, json.Unmarshal(fileData, snapshot))
_, err = faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull())
assert.Error(t, err)
})
}
}
func (suite *ClientHelloTestSuite) TestValidateHostname() {
hello := faketls.ClientHello{
Time: time.Now(),
}
suite.NoError(hello.Valid("hostname", time.Second))
hello.Host = "hostname"
suite.Error(hello.Valid("hostname2", time.Second))
suite.NoError(hello.Valid("hostname", time.Second))
}
func (suite *ClientHelloTestSuite) TestValidateTime() {
testData := []time.Duration{
-2 * time.Second,
2 * time.Second,
}
for _, v := range testData {
value := v
suite.T().Run(value.String(), func(t *testing.T) {
hello := faketls.ClientHello{
Host: "hostname",
Time: time.Now().Add(value),
}
suite.Error(hello.Valid("hostname", 500*time.Millisecond))
suite.Error(hello.Valid("hostname", time.Second))
suite.NoError(hello.Valid("hostname", 3*time.Second))
})
}
}
func TestClientHello(t *testing.T) {
t.Parallel()
suite.Run(t, &ClientHelloTestSuite{})
}
mtglib/internal/faketls/conn.go
-72
View File
@@ -1,72 +0,0 @@
package faketls
import (
"bytes"
"fmt"
"math/rand/v2"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
)
type Conn struct {
essentials.Conn
readBuffer bytes.Buffer
}
func (c *Conn) Read(p []byte) (int, error) {
if n, _ := c.readBuffer.Read(p); n > 0 {
return n, nil
}
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
for {
if err := rec.Read(c.Conn); err != nil {
return 0, err //nolint: wrapcheck
}
switch rec.Type { //nolint: exhaustive
case record.TypeApplicationData:
rec.Payload.WriteTo(&c.readBuffer) //nolint: errcheck
return c.readBuffer.Read(p) //nolint: wrapcheck
case record.TypeChangeCipherSpec:
default:
return 0, fmt.Errorf("unsupported record type %v", rec.Type)
}
}
}
func (c *Conn) Write(p []byte) (int, error) {
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeApplicationData
rec.Version = record.Version12
written := 0
for len(p) > 0 {
chunkSize := rand.IntN(record.TLSMaxRecordSize)
if chunkSize > len(p) || chunkSize == 0 {
chunkSize = len(p)
}
rec.Payload.Reset()
rec.Payload.Write(p[:chunkSize])
err := rec.Dump(c.Conn)
written += chunkSize
if err != nil {
return written, err
}
p = p[chunkSize:]
}
return written, nil
}
mtglib/internal/faketls/conn_test.go
-153
View File
@@ -1,153 +0,0 @@
package faketls_test
import (
"bytes"
"crypto/rand"
"errors"
"io"
"testing"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ConnMock struct {
testlib.EssentialsConnMock
readBuffer bytes.Buffer
writeBuffer bytes.Buffer
}
func (m *ConnMock) Read(p []byte) (int, error) {
m.Called(p)
return m.readBuffer.Read(p) //nolint: wrapcheck
}
func (m *ConnMock) Write(p []byte) (int, error) {
m.Called(p)
return m.writeBuffer.Write(p) //nolint: wrapcheck
}
type ConnTestSuite struct {
suite.Suite
connMock *ConnMock
c *faketls.Conn
}
func (suite *ConnTestSuite) SetupTest() {
suite.connMock = &ConnMock{}
suite.c = &faketls.Conn{
Conn: suite.connMock,
}
}
func (suite *ConnTestSuite) TearDownTest() {
suite.connMock.AssertExpectations(suite.T())
}
func (suite *ConnTestSuite) TestRead() {
suite.connMock.On("Read", mock.Anything).Return(0, nil)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeChangeCipherSpec
rec.Version = record.Version12
rec.Payload.WriteByte(0x01)
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeApplicationData
rec.Version = record.Version12
rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
resultBuffer := &bytes.Buffer{}
buf := make([]byte, 2)
for {
n, err := suite.c.Read(buf)
if errors.Is(err, io.EOF) {
break
}
resultBuffer.Write(buf[:n])
}
suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, resultBuffer.Bytes())
}
func (suite *ConnTestSuite) TestReadUnexpected() {
suite.connMock.On("Read", mock.Anything).Return(0, nil)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeChangeCipherSpec
rec.Version = record.Version12
rec.Payload.WriteByte(0x01)
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeHandshake
rec.Version = record.Version12
rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
rec.Dump(&suite.connMock.readBuffer) //nolint: errcheck
buf := make([]byte, 2)
for {
_, err := suite.c.Read(buf)
switch {
case err == nil:
case errors.Is(err, io.EOF):
suite.FailNow("unexpected to finish")
default:
return
}
}
}
func (suite *ConnTestSuite) TestWrite() {
suite.connMock.On("Write", mock.Anything).Return(0, nil)
dataToRec := make([]byte, record.TLSMaxRecordSize*2)
rand.Read(dataToRec) //nolint: staticcheck, errcheck
n, err := suite.c.Write(dataToRec)
suite.NoError(err)
suite.Equal(len(dataToRec), n)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
buf := &bytes.Buffer{}
for {
if err := rec.Read(&suite.connMock.writeBuffer); err != nil {
break
}
suite.Equal(record.TypeApplicationData, rec.Type)
suite.Equal(record.Version12, rec.Version)
rec.Payload.WriteTo(buf) //nolint: errcheck
}
suite.Equal(dataToRec, buf.Bytes())
}
func TestConn(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTestSuite{})
}
mtglib/internal/faketls/init.go
-59
View File
@@ -1,59 +0,0 @@
package faketls
import (
"bytes"
"errors"
)
const (
// RandomLen defines a size of the random digest in TLS Hellos.
RandomLen = 32
// ClientHelloRandomOffset is an offset in ClientHello record where
// random digest is started.
ClientHelloRandomOffset = 6
// ClientHelloSessionIDOffset is an offset in ClientHello record where
// SessionID is started.
ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen
// ClientHelloMinLen is a minimal possible length of
// ClientHello record.
ClientHelloMinLen = 6
// WelcomePacketRandomOffset is an offset of random in ServerHello
// packet (including record envelope).
WelcomePacketRandomOffset = 11
// HandshakeTypeClient is a value representing a client handshake.
HandshakeTypeClient = 0x01
// HandshakeTypeServer is a value representing a server handshake.
HandshakeTypeServer = 0x02
// ChangeCipherValue is a value representing a change cipher
// specification record.
ChangeCipherValue = 0x01
// ExtensionSNI is a value for TLS extension 'SNI'.
ExtensionSNI = 0x00
)
var (
// ErrBadDigest is returned if given TLS Client Hello mismatches with a
// derived one.
ErrBadDigest = errors.New("bad digest")
serverHelloSuffix = []byte{
0x00, // no compression
0x00, 0x2e, // 46 bytes of data
0x00, 0x2b, // Extension - Supported Versions
0x00, 0x02, // 2 bytes are following
0x03, 0x04, // TLS 1.3
0x00, 0x33, // Extension - Key Share
0x00, 0x24, // 36 bytes
0x00, 0x1d, // x25519 curve
0x00, 0x20, // 32 bytes of key
}
clientHelloEmptyRandom = bytes.Repeat([]byte{0}, RandomLen)
)
mtglib/internal/faketls/record/init.go
-84
View File
@@ -1,84 +0,0 @@
package record
import "fmt"
const TLSMaxRecordSize = 65535 // max uint16
type Type uint8
const (
// TypeChangeCipherSpec defines a byte value of the TLS record when a
// peer wants to change a specifications of the chosen cipher.
TypeChangeCipherSpec Type = 0x14
// TypeHandshake defines a byte value of the TLS record when a peer
// initiates a new TLS connection and wants to make a handshake
// ceremony.
TypeHandshake Type = 0x16
// TypeApplicationData defines a byte value of the TLS record when a
// peer sends an user data, not a control frames.
TypeApplicationData Type = 0x17
)
func (t Type) String() string {
switch t {
case TypeChangeCipherSpec:
return "changeCipher(0x14)"
case TypeHandshake:
return "handshake(0x16)"
case TypeApplicationData:
return "applicationData(0x17)"
}
return fmt.Sprintf("unknown(%#x)", byte(t))
}
func (t Type) Valid() error {
switch t {
case TypeChangeCipherSpec, TypeHandshake, TypeApplicationData:
return nil
}
return fmt.Errorf("unknown type %#x", byte(t))
}
type Version uint16
const (
// Version10 defines a TLS1.0.
Version10 Version = 769 // 0x03 0x01
// Version11 defines a TLS1.1.
Version11 Version = 770 // 0x03 0x02
// Version12 defines a TLS1.2.
Version12 Version = 771 // 0x03 0x03
// Version13 defines a TLS1.3.
Version13 Version = 772 // 0x03 0x04
)
func (v Version) String() string {
switch v {
case Version10:
return "tls1.0"
case Version11:
return "tls1.1"
case Version12:
return "tls1.2"
case Version13:
return "tls1.3"
}
return fmt.Sprintf("tls?(%d)", uint16(v))
}
func (v Version) Valid() error {
switch v {
case Version10, Version11, Version12, Version13:
return nil
}
return fmt.Errorf("unknown version %d", uint16(v))
}
mtglib/internal/faketls/record/init_test.go
-79
View File
@@ -1,79 +0,0 @@
package record_test
import (
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/suite"
)
type TypeTestSuite struct {
suite.Suite
}
func (suite *TypeTestSuite) TestChangeCipherSpec() {
suite.Contains(record.TypeChangeCipherSpec.String(), "changeCipher")
suite.Contains(record.TypeChangeCipherSpec.String(), "0x14")
suite.NoError(record.TypeChangeCipherSpec.Valid())
}
func (suite *TypeTestSuite) TestHandshake() {
suite.Contains(record.TypeHandshake.String(), "handshake")
suite.Contains(record.TypeHandshake.String(), "0x16")
suite.NoError(record.TypeHandshake.Valid())
}
func (suite *TypeTestSuite) TestApplicationData() {
suite.Contains(record.TypeApplicationData.String(), "applicationData")
suite.Contains(record.TypeApplicationData.String(), "0x17")
suite.NoError(record.TypeApplicationData.Valid())
}
func (suite *TypeTestSuite) TestUnknown() {
value := record.Type(0x20)
suite.Contains(value.String(), "unknown")
suite.Contains(value.String(), "0x20")
suite.Error(value.Valid())
}
type VersionTestSuite struct {
suite.Suite
}
func (suite *VersionTestSuite) Test10() {
suite.Equal("tls1.0", record.Version10.String())
suite.NoError(record.Version10.Valid())
}
func (suite *VersionTestSuite) Test11() {
suite.Equal("tls1.1", record.Version11.String())
suite.NoError(record.Version11.Valid())
}
func (suite *VersionTestSuite) Test12() {
suite.Equal("tls1.2", record.Version12.String())
suite.NoError(record.Version12.Valid())
}
func (suite *VersionTestSuite) Test13() {
suite.Equal("tls1.3", record.Version13.String())
suite.NoError(record.Version13.Valid())
}
func (suite *VersionTestSuite) TestUnknown() {
value := record.Version(900)
suite.Equal("tls?(900)", value.String())
suite.Error(value.Valid())
}
func TestType(t *testing.T) {
t.Parallel()
suite.Run(t, &TypeTestSuite{})
}
func TestVersion(t *testing.T) {
t.Parallel()
suite.Run(t, &VersionTestSuite{})
}
mtglib/internal/faketls/record/pools.go
-20
View File
@@ -1,20 +0,0 @@
package record
import (
"sync"
)
var recordPool = sync.Pool{
New: func() any {
return &Record{}
},
}
func AcquireRecord() *Record {
return recordPool.Get().(*Record) //nolint: forcetypeassert
}
func ReleaseRecord(r *Record) {
r.Reset()
recordPool.Put(r)
}
mtglib/internal/faketls/record/record.go
-86
View File
@@ -1,86 +0,0 @@
package record
import (
"bytes"
"encoding/base64"
"encoding/binary"
"fmt"
"io"
)
type Record struct {
Type Type
Version Version
Payload bytes.Buffer
}
func (r *Record) String() string {
return fmt.Sprintf("<tlsRecord(type=%v, version=%v, payload=%s)>",
r.Type,
r.Version,
base64.StdEncoding.EncodeToString(r.Payload.Bytes()))
}
func (r *Record) Reset() {
r.Payload.Reset()
}
func (r *Record) Read(reader io.Reader) error {
r.Reset()
buf := [2]byte{}
if _, err := io.ReadFull(reader, buf[:1]); err != nil {
return fmt.Errorf("cannot read type: %w", err)
}
r.Type = Type(buf[0])
if err := r.Type.Valid(); err != nil {
return fmt.Errorf("invalid type: %w", err)
}
if _, err := io.ReadFull(reader, buf[:]); err != nil {
return fmt.Errorf("cannot read version: %w", err)
}
r.Version = Version(binary.BigEndian.Uint16(buf[:]))
if err := r.Version.Valid(); err != nil {
return fmt.Errorf("invalid version: %w", err)
}
if _, err := io.ReadFull(reader, buf[:]); err != nil {
return fmt.Errorf("cannot read payload length: %w", err)
}
length := int64(binary.BigEndian.Uint16(buf[:]))
if _, err := io.CopyN(&r.Payload, reader, length); err != nil {
return fmt.Errorf("cannot read payload: %w", err)
}
return nil
}
func (r *Record) Dump(writer io.Writer) error {
buf := [2]byte{byte(r.Type), 0}
if _, err := writer.Write(buf[:1]); err != nil {
return fmt.Errorf("cannot dump record type: %w", err)
}
binary.BigEndian.PutUint16(buf[:], uint16(r.Version))
if _, err := writer.Write(buf[:]); err != nil {
return fmt.Errorf("cannot dump version: %w", err)
}
binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len()))
if _, err := writer.Write(buf[:]); err != nil {
return fmt.Errorf("cannot dump payload length: %w", err)
}
if _, err := writer.Write(r.Payload.Bytes()); err != nil {
return fmt.Errorf("cannot dump record: %w", err)
}
return nil
}
mtglib/internal/faketls/record/record_test.go
-110
View File
@@ -1,110 +0,0 @@
package record_test
import (
"bytes"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
type RecordTestSnapshot struct {
Type int `json:"type"`
Version int `json:"version"`
Payload string `json:"payload"`
Record string `json:"record"`
}
func (r RecordTestSnapshot) RecordBytes() []byte {
data, _ := base64.StdEncoding.DecodeString(r.Record)
return data
}
func (r RecordTestSnapshot) PayloadBytes() []byte {
data, _ := base64.StdEncoding.DecodeString(r.Payload)
return data
}
type RecordTestSuite struct {
suite.Suite
r *record.Record
buf *bytes.Buffer
}
func (suite *RecordTestSuite) SetupTest() {
suite.r = record.AcquireRecord()
suite.buf = &bytes.Buffer{}
}
func (suite *RecordTestSuite) TearDownTest() {
record.ReleaseRecord(suite.r)
suite.buf.Reset()
}
func (suite *RecordTestSuite) TestIdempotent() {
suite.r.Type = record.TypeApplicationData
suite.r.Version = record.Version13
suite.r.Payload.Write([]byte{1, 2, 3})
suite.NoError(suite.r.Dump(suite.buf))
suite.r.Reset()
suite.NoError(suite.r.Read(suite.buf))
suite.Equal(0, suite.buf.Len())
suite.Equal(record.TypeApplicationData, suite.r.Type)
suite.Equal(record.Version13, suite.r.Version)
suite.Equal([]byte{1, 2, 3}, suite.r.Payload.Bytes())
}
func (suite *RecordTestSuite) TestString() {
_ = suite.r.String()
}
func (suite *RecordTestSuite) TestSnapshot() {
files, err := os.ReadDir("testdata")
suite.NoError(err)
testData := map[string]string{}
for _, f := range files {
testData[f.Name()] = filepath.Join("testdata", f.Name())
}
for name, pathV := range testData {
path := pathV
suite.T().Run(name, func(t *testing.T) {
data, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &RecordTestSnapshot{}
assert.NoError(t, json.Unmarshal(data, snapshot))
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
assert.NoError(t, rec.Read(bytes.NewReader(snapshot.RecordBytes())))
assert.Equal(t, snapshot.Type, int(rec.Type))
assert.Equal(t, snapshot.Version, int(rec.Version))
assert.Equal(t, snapshot.PayloadBytes(), rec.Payload.Bytes())
buf := &bytes.Buffer{}
assert.NoError(t, rec.Dump(buf))
assert.Equal(t, snapshot.RecordBytes(), buf.Bytes())
})
}
}
func TestRecord(t *testing.T) {
t.Parallel()
suite.Run(t, &RecordTestSuite{})
}
mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 772,
"payload": "sxS+0oAyk+NBv0LLVtQOp9WSx4CweyUZPz01tQ0o4oyp8aaBl6/kMFvLq3q52KE8lCiKejLw2NxVBUkE+4izCf2gLx9qfr81opWnqJTChWzcDijvttbq9cmtDFNL+odKsS3v1/TfYEFtPsoRPrJRmOHRAnqnf49Y5Q==",
"record": "FAMEAHmzFL7SgDKT40G/QstW1A6n1ZLHgLB7JRk/PTW1DSjijKnxpoGXr+QwW8urernYoTyUKIp6MvDY3FUFSQT7iLMJ/aAvH2p+vzWilaeolMKFbNwOKO+21ur1ya0MU0v6h0qxLe/X9N9gQW0+yhE+slGY4dECeqd/j1jl"
}
mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 772,
"payload": "waNH223htyxCBKAb6hm0u/SK/9mhI8Ck91nfWob7QMOaIREogrDYREJH4Djcp47XrpAlEaUIDiCvoFLVJ/LK1nYs4swzfHSSl/+Aj1eqPA63XqPa8EG4FAbf0DwjwXxV9qVIhvP9b2TafKbzr4Yb5GCygzFRb/zawA==",
"record": "FgMEAHnBo0fbbeG3LEIEoBvqGbS79Ir/2aEjwKT3Wd9ahvtAw5ohESiCsNhEQkfgONynjteukCURpQgOIK+gUtUn8srWdizizDN8dJKX/4CPV6o8Drdeo9rwQbgUBt/QPCPBfFX2pUiG8/1vZNp8pvOvhhvkYLKDMVFv/NrA"
}
mtglib/internal/faketls/record/testdata/736f358216afe91f.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 769,
"payload": "jmJ0o1E5+ehAHHYAbCo4AMV03X7RSivYl250s06nD9CO44fyjaoGELz0N7IeCg1jFKcRVSCRmYYmiIY9wydn2fXOJhKif8B0BlM3qhbethYgyP+l1S8hyyETpIiOtiiiOnAJwl1D1j9OryFiJFSdRRXReIMZ4CPqPg==",
"record": "FwMBAHmOYnSjUTn56EAcdgBsKjgAxXTdftFKK9iXbnSzTqcP0I7jh/KNqgYQvPQ3sh4KDWMUpxFVIJGZhiaIhj3DJ2fZ9c4mEqJ/wHQGUzeqFt62FiDI/6XVLyHLIROkiI62KKI6cAnCXUPWP06vIWIkVJ1FFdF4gxngI+o+"
}
mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 769,
"payload": "hBnpBnNUdlqe/rKXa7Judcz79u7AkUgSGOycn8EqvbkZpVxnI31rNOvAsPZqG+GF7DWJ3R7H2ETmFmrpnyyng32MjSs1jptmV1oAs63zTADD7sVipgid9AJHwfl4CrC3FIQr43IPMYd29JPOl5bqu/SfrgI16PBiJw==",
"record": "FgMBAHmEGekGc1R2Wp7+spdrsm51zPv27sCRSBIY7JyfwSq9uRmlXGcjfWs068Cw9mob4YXsNYndHsfYROYWaumfLKeDfYyNKzWOm2ZXWgCzrfNMAMPuxWKmCJ30AkfB+XgKsLcUhCvjcg8xh3b0k86Xluq79J+uAjXo8GIn"
}
mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 770,
"payload": "Vm/C+DO56czlbtR915aHzsugSyDtp8CtojF9w1jKY0efyyfcLrNuhNg/pZm3gQ7v2BBbL1UJ97v/RIjST+5gRIfg3bBN1BE9hkf+N2AYY2lHLi0yeInHB0zFWPeHscsDopDFadIi5KtC8HvbEMuK+kK8POVk5tN9UQ==",
"record": "FwMCAHlWb8L4M7npzOVu1H3XlofOy6BLIO2nwK2iMX3DWMpjR5/LJ9wus26E2D+lmbeBDu/YEFsvVQn3u/9EiNJP7mBEh+DdsE3UET2GR/43YBhjaUcuLTJ4iccHTMVY94exywOikMVp0iLkq0Lwe9sQy4r6Qrw85WTm031R"
}
mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 770,
"payload": "ajPzpsgk4gwm2stRQKbllvKRLdI7vmyaj1uxEJ/kKoQnQSPumdDNKD618U2Cq6PVd0/b+9YtH67Uzx1QxtpKuby5fUXqw06WUuDAQsmjq7F26EkE5FND6rQUjUPC+e1U0dF4TQzOUSS4IAkFQPAaVehUVTRxVWa/0g==",
"record": "FgMCAHlqM/OmyCTiDCbay1FApuWW8pEt0ju+bJqPW7EQn+QqhCdBI+6Z0M0oPrXxTYKro9V3T9v71i0frtTPHVDG2kq5vLl9RerDTpZS4MBCyaOrsXboSQTkU0PqtBSNQ8L57VTR0XhNDM5RJLggCQVA8BpV6FRVNHFVZr/S"
}
mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 771,
"payload": "d1Hiv1NYVgEDR9mtJyv9j8mg3dWqfUpeKfOsL+jzSDfVIxeDiJZFLDT50TjNW44/yEOVEX/Y/pk+wnc7E8aCEiwGwAvB+Insw1UCJ2ejt689VWLo2u4klGVKTHuOpUvdGVTc7Lo4FAt91KQSPLYB5iqxomjEv5e3Vg==",
"record": "FAMDAHl3UeK/U1hWAQNH2a0nK/2PyaDd1ap9Sl4p86wv6PNIN9UjF4OIlkUsNPnROM1bjj/IQ5URf9j+mT7CdzsTxoISLAbAC8H4iezDVQInZ6O3rz1VYuja7iSUZUpMe46lS90ZVNzsujgUC33UpBI8tgHmKrGiaMS/l7dW"
}
mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 771,
"payload": "wbdU1CbrzuAJDsh6CFjGyE+AFArJj/Wmsa2wtDyW0kRuE2vUO8gg+nXkg0kkoz0WnvQEOdaswfJIaVrloD78yoyeQVfBB+VUP/63vqn60v5ccaQEn0jLdxgLjiTAxKDQDxCTMRoLnFE2ZZf28zw+HfqpIxiOZs8LhQ==",
"record": "FwMDAHnBt1TUJuvO4AkOyHoIWMbIT4AUCsmP9aaxrbC0PJbSRG4Ta9Q7yCD6deSDSSSjPRae9AQ51qzB8khpWuWgPvzKjJ5BV8EH5VQ//re+qfrS/lxxpASfSMt3GAuOJMDEoNAPEJMxGgucUTZll/bzPD4d+qkjGI5mzwuF"
}
mtglib/internal/faketls/record/testdata/b0acd44296056b54.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 23,
"version": 772,
"payload": "qqnBMb1Af3zZt4DPHpVRuIiON9ODGJUNFicFjranORh67L/HI4D6HnHyycZFUSBOw2FjMBF6UialY8snOYaRKrQmQzuUNg1Ztq7yAZ+Lgj3TBarR6OMlYhEAY0Px9Xv1UuJ0YcvQx33gdM1skJ5HBR3yZvEKNJV1LA==",
"record": "FwMEAHmqqcExvUB/fNm3gM8elVG4iI4304MYlQ0WJwWOtqc5GHrsv8cjgPoecfLJxkVRIE7DYWMwEXpSJqVjyyc5hpEqtCZDO5Q2DVm2rvIBn4uCPdMFqtHo4yViEQBjQ/H1e/VS4nRhy9DHfeB0zWyQnkcFHfJm8Qo0lXUs"
}
mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 769,
"payload": "NEe735TuQFp7bWpFQhASas/e1XaySvus0ovXmkfCbFq334MyFHq2eDMadziXsfu/GfBjoYggvk0LgYUeoAkBNKR0dfSovjSndaqmIUonoWl+6sZObiGZkRIMwuY2q4Eaw4/iuDu/pZhjRW/iAIH+YH7cyk/1tgdJDg==",
"record": "FAMBAHk0R7vflO5AWnttakVCEBJqz97VdrJK+6zSi9eaR8JsWrffgzIUerZ4Mxp3OJex+78Z8GOhiCC+TQuBhR6gCQE0pHR19Ki+NKd1qqYhSiehaX7qxk5uIZmREgzC5jargRrDj+K4O7+lmGNFb+IAgf5gftzKT/W2B0kO"
}
mtglib/internal/faketls/record/testdata/f083f4501668b759.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 22,
"version": 771,
"payload": "wrXjZrPm3OSyzO0klv6/G+z2PDloR/colS/RlWwQE31Vb2xm8YkEchDDKwlc/KPLD73qMoz3MQOQLtSLc8LhVYp+l7L9jz49yTaVKtBI5UuGbo09snsKxFCgCyYUBETKabATBQtiaEu/D8dmF4Yk/2ww4sEb8DwKLQ==",
"record": "FgMDAHnCteNms+bc5LLM7SSW/r8b7PY8OWhH9yiVL9GVbBATfVVvbGbxiQRyEMMrCVz8o8sPveoyjPcxA5Au1ItzwuFVin6Xsv2PPj3JNpUq0EjlS4ZujT2yewrEUKALJhQERMppsBMFC2JoS78Px2YXhiT/bDDiwRvwPAot"
}
mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json
Vendored
-6
View File
@@ -1,6 +0,0 @@
{
"type": 20,
"version": 770,
"payload": "OU5s8Sa11hpXWEarWzFlX55IZt3Eo+F4AMbQ/2RwB4rfHS/JNl8n63OR4oYs9QXw3RfCrYJuU9n6Xn+I/+7ZzAgZ0PbLSXW1PrLtttdfmhTErK90b49YEWdY9na4g++NMkKykwgXvY1hNxZIHX/qawEWJgxXUR3DdQ==",
"record": "FAMCAHk5TmzxJrXWGldYRqtbMWVfnkhm3cSj4XgAxtD/ZHAHit8dL8k2Xyfrc5Hihiz1BfDdF8Ktgm5T2fpef4j/7tnMCBnQ9stJdbU+su2211+aFMSsr3Rvj1gRZ1j2driD740yQrKTCBe9jWE3Fkgdf+prARYmDFdRHcN1"
}
mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json
Vendored
-8
View File
@@ -1,8 +0,0 @@
{
"time": 1617181365,
"random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=",
"sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCACq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFANAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json
Vendored
-8
View File
@@ -1,8 +0,0 @@
{
"time": 1617181365,
"random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=",
"sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json
Vendored
-8
View File
@@ -1,8 +0,0 @@
{
"time": 1617181352,
"random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=",
"sessionId": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDoYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12osgFGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8gANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAga6CocpFP8Qd4YCFR9pkaCr97po2ALj0P5nI9Nnb3UWMALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json
Vendored
-8
View File
@@ -1,8 +0,0 @@
{
"time": 1617181352,
"random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=",
"sessionId": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMD5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1rogjxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgrulAaqUdKeVYM0F+pu6on/h6LBpOyzOKG4xFIKcoFk4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json
Vendored
-8
View File
@@ -1,8 +0,0 @@
{
"time": 1617181365,
"random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=",
"sessionId": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMD8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQg00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAUANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAg/9P7140NtKzjyDwBf99mOy1+FjRPAPHTNQ9WxHOKpV4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json
Vendored
-8
View File
@@ -1,8 +0,0 @@
{
"time": 1617181352,
"random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=",
"sessionId": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "AQAB/AMDzja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4gqPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnAANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgXviLRAqAYJ8xOLdlcsUhldI4Xl0g/s9+y2Qrd8raPEgALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
}
mtglib/internal/faketls/welcome.go
-91
View File
@@ -1,91 +0,0 @@
package faketls
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"io"
mrand "math/rand/v2"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"golang.org/x/crypto/curve25519"
)
func SendWelcomePacket(writer io.Writer, secret []byte, clientHello ClientHello) error {
buf := &bytes.Buffer{}
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rec.Type = record.TypeHandshake
rec.Version = record.Version12
generateServerHello(&rec.Payload, clientHello)
rec.Dump(buf) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeChangeCipherSpec
rec.Version = record.Version12
rec.Payload.WriteByte(ChangeCipherValue)
rec.Dump(buf) //nolint: errcheck
rec.Reset()
rec.Type = record.TypeApplicationData
rec.Version = record.Version12
if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.IntN(3092))); err != nil {
panic(err)
}
rec.Dump(buf) //nolint: errcheck
packet := buf.Bytes()
mac := hmac.New(sha256.New, secret)
mac.Write(clientHello.Random[:])
mac.Write(packet)
copy(packet[WelcomePacketRandomOffset:], mac.Sum(nil))
if _, err := writer.Write(packet); err != nil {
return err //nolint: wrapcheck
}
return nil
}
func generateServerHello(writer io.Writer, clientHello ClientHello) {
bodyBuf := &bytes.Buffer{}
sliceBuf := [2]byte{}
digest := [RandomLen]byte{}
binary.BigEndian.PutUint16(sliceBuf[:], uint16(record.Version12))
bodyBuf.Write(sliceBuf[:])
bodyBuf.Write(digest[:])
bodyBuf.WriteByte(byte(len(clientHello.SessionID)))
bodyBuf.Write(clientHello.SessionID)
binary.BigEndian.PutUint16(sliceBuf[:], clientHello.CipherSuite)
bodyBuf.Write(sliceBuf[:])
bodyBuf.Write(serverHelloSuffix)
scalar := [32]byte{}
if _, err := rand.Read(scalar[:]); err != nil {
panic(err)
}
curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint)
bodyBuf.Write(curve)
header := [4]byte{0, 0, 0, 0}
binary.BigEndian.PutUint32(header[:], uint32(bodyBuf.Len()))
header[0] = HandshakeTypeServer
writer.Write(header[:]) //nolint: errcheck
bodyBuf.WriteTo(writer) //nolint: errcheck
}
mtglib/internal/faketls/welcome_test.go
-82
View File
@@ -1,82 +0,0 @@
package faketls_test
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"testing"
"time"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/stretchr/testify/suite"
)
type WelcomeTestSuite struct {
suite.Suite
h *faketls.ClientHello
buf *bytes.Buffer
secret mtglib.Secret
}
func (suite *WelcomeTestSuite) SetupTest() {
suite.h = &faketls.ClientHello{
Time: time.Now(),
Host: "google.com",
CipherSuite: 4867,
SessionID: make([]byte, 32),
}
_, err := rand.Read(suite.h.SessionID) //nolint: staticcheck
suite.NoError(err)
_, err = rand.Read(suite.h.Random[:]) //nolint: staticcheck
suite.NoError(err)
suite.buf = &bytes.Buffer{}
suite.secret = mtglib.GenerateSecret("google.com")
}
func (suite *WelcomeTestSuite) TestOk() {
suite.NoError(faketls.SendWelcomePacket(suite.buf, suite.secret.Key[:], *suite.h))
welcomePacket := []byte{}
welcomePacket = append(welcomePacket, suite.buf.Bytes()...)
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
suite.NoError(rec.Read(suite.buf))
suite.Equal(record.TypeHandshake, rec.Type)
suite.Equal(record.Version12, rec.Version)
suite.NoError(rec.Read(suite.buf))
suite.Equal(record.TypeChangeCipherSpec, rec.Type)
suite.Equal(record.Version12, rec.Version)
suite.NoError(rec.Read(suite.buf))
suite.Equal(record.TypeApplicationData, rec.Type)
suite.Equal(record.Version12, rec.Version)
suite.Empty(suite.buf.Bytes())
random := make([]byte, 32)
copy(random, welcomePacket[11:])
empty := make([]byte, 32)
copy(welcomePacket[11:], empty)
mac := hmac.New(sha256.New, suite.secret.Key[:])
mac.Write(suite.h.Random[:])
mac.Write(welcomePacket)
suite.Equal(random, mac.Sum(nil))
}
func TestWelcome(t *testing.T) {
t.Parallel()
suite.Run(t, &WelcomeTestSuite{})
}
mtglib/internal/relay/init.go
-4
View File
@@ -1,9 +1,5 @@
package relay
const (
copyBufferSize = 64 * 1024
)
type Logger interface {
Printf(msg string, args ...any)
}
mtglib/internal/relay/relay.go
+2 -1
View File
@@ -6,6 +6,7 @@ import (
"io"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
func Relay(ctx context.Context, log Logger, telegramConn, clientConn essentials.Conn) {
@@ -35,7 +36,7 @@ func Relay(ctx context.Context, log Logger, telegramConn, clientConn essentials.
}
func pump(log Logger, src, dst essentials.Conn, direction string) {
var buf [copyBufferSize]byte
var buf [tls.MaxRecordPayloadSize]byte
defer src.CloseRead() //nolint: errcheck
defer dst.CloseWrite() //nolint: errcheck
mtglib/internal/tls/conn.go
+86
View File
@@ -0,0 +1,86 @@
package tls
import (
"bufio"
"bytes"
"github.com/9seconds/mtg/v2/essentials"
)
const (
SizeRecordType = 1
SizeVersion = 2
SizeSize = 2
SizeHeader = SizeRecordType + SizeVersion + SizeSize
MaxRecordSize = 16384
MaxRecordPayloadSize = MaxRecordSize - SizeHeader
DefaultBufferSize = 4096
TypeChangeCipherSpec = 0x14
TypeHandshake = 0x16
TypeApplicationData = 0x17
)
// TLS 1.2 is used for both TLS 1.2 and 1.3
var TLSVersion = [SizeVersion]byte{3, 3}
// Conn presents an established TLS 1.3 connection, after handshake
type Conn struct {
essentials.Conn
p *connPayload
}
type connPayload struct {
readBuf bytes.Buffer
writeBuf bytes.Buffer
connBuffered *bufio.Reader
read bool
write bool
}
func (c Conn) Write(p []byte) (int, error) {
if !c.p.write {
return c.Conn.Write(p)
}
return len(p), WriteRecord(c.Conn, p)
}
func (c Conn) Read(p []byte) (int, error) {
if !c.p.read {
return c.Conn.Read(p)
}
for {
if n, err := c.p.readBuf.Read(p); err == nil {
return n, nil
}
recordType, _, err := ReadRecord(c.p.connBuffered, &c.p.readBuf)
if err != nil {
return 0, err
}
if recordType != TypeApplicationData {
c.p.readBuf.Reset()
}
}
}
func New(conn essentials.Conn, read, write bool) Conn {
newConn := Conn{
Conn: conn,
p: &connPayload{
connBuffered: bufio.NewReaderSize(conn, DefaultBufferSize),
read: read,
write: write,
},
}
newConn.p.readBuf.Grow(DefaultBufferSize)
newConn.p.writeBuf.Grow(DefaultBufferSize)
return newConn
}
mtglib/internal/tls/conn_test.go
+160
View File
@@ -0,0 +1,160 @@
package tls
import (
"io"
"testing"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type ConnTestSuite struct {
suite.Suite
connMock *testlib.EssentialsConnMock
}
func (suite *ConnTestSuite) SetupTest() {
suite.connMock = &testlib.EssentialsConnMock{}
}
func (suite *ConnTestSuite) TearDownTest() {
suite.connMock.AssertExpectations(suite.T())
}
func (suite *ConnTestSuite) feedRead(raw []byte) {
suite.connMock.
On("Read", mock.AnythingOfType("[]uint8")).
Run(func(args mock.Arguments) {
copy(args.Get(0).([]byte), raw)
}).
Return(len(raw), nil).
Once()
suite.connMock.
On("Read", mock.AnythingOfType("[]uint8")).
Return(0, io.EOF).
Maybe()
}
func (suite *ConnTestSuite) TestReadTLSEnabled() {
payload := []byte("hello world")
suite.feedRead(MakeTLSRecord(0x17, payload))
conn := New(suite.connMock, true, false)
buf := make([]byte, 128)
n, err := conn.Read(buf)
suite.NoError(err)
suite.Equal(payload, buf[:n])
}
func (suite *ConnTestSuite) TestReadTLSSkipsNonApplicationData() {
raw := append(
MakeTLSRecord(0x14, []byte{1}),
MakeTLSRecord(0x17, []byte("real data"))...,
)
suite.feedRead(raw)
conn := New(suite.connMock, true, false)
buf := make([]byte, 128)
n, err := conn.Read(buf)
suite.NoError(err)
suite.Equal([]byte("real data"), buf[:n])
}
func (suite *ConnTestSuite) TestReadTLSMultipleRecords() {
raw := append(
MakeTLSRecord(0x17, []byte("first")),
MakeTLSRecord(0x17, []byte("second"))...,
)
suite.feedRead(raw)
conn := New(suite.connMock, true, false)
buf := make([]byte, 128)
n, err := conn.Read(buf)
suite.NoError(err)
suite.Equal([]byte("first"), buf[:n])
n, err = conn.Read(buf)
suite.NoError(err)
suite.Equal([]byte("second"), buf[:n])
}
func (suite *ConnTestSuite) TestReadTLSSmallBuffer() {
payload := []byte("hello world, this is a longer payload")
suite.feedRead(MakeTLSRecord(0x17, payload))
conn := New(suite.connMock, true, false)
small := make([]byte, 5)
n, err := conn.Read(small)
suite.NoError(err)
suite.Equal(payload[:5], small[:n])
rest := make([]byte, 128)
n, err = conn.Read(rest)
suite.NoError(err)
suite.Equal(payload[5:], rest[:n])
}
func (suite *ConnTestSuite) TestReadPassthrough() {
data := []byte("raw bytes")
suite.connMock.
On("Read", mock.AnythingOfType("[]uint8")).
Run(func(args mock.Arguments) {
copy(args.Get(0).([]byte), data)
}).
Return(len(data), nil).
Once()
conn := New(suite.connMock, false, false)
buf := make([]byte, 128)
n, err := conn.Read(buf)
suite.NoError(err)
suite.Equal(data, buf[:n])
}
func (suite *ConnTestSuite) TestWritePassthrough() {
data := []byte("outgoing data")
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(len(data), nil).
Once()
conn := New(suite.connMock, false, false)
n, err := conn.Write(data)
suite.NoError(err)
suite.Equal(len(data), n)
}
func (suite *ConnTestSuite) TestWriteTLSEnabled() {
data := []byte("outgoing data")
suite.connMock.
On("Write", mock.AnythingOfType("[]uint8")).
Return(len(data), nil).
Once()
conn := New(suite.connMock, false, true)
n, err := conn.Write(data)
suite.NoError(err)
suite.Equal(len(data), n)
}
func TestConn(t *testing.T) {
t.Parallel()
suite.Run(t, &ConnTestSuite{})
}
mtglib/internal/tls/fake/bytes_pool.go
+21
View File
@@ -0,0 +1,21 @@
package fake
import (
"bytes"
"sync"
)
var bytesPool = sync.Pool{
New: func() any {
return &bytes.Buffer{}
},
}
func acquireBuffer() *bytes.Buffer {
return bytesPool.Get().(*bytes.Buffer)
}
func releaseBuffer(b *bytes.Buffer) {
b.Reset()
bytesPool.Put(b)
}
mtglib/internal/tls/fake/client_side.go
+309
View File
@@ -0,0 +1,309 @@
package fake
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/binary"
"fmt"
"io"
"net"
"slices"
"time"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
)
const (
TypeHandshakeClient = 0x01
RandomLen = 32
// record_type(1) + version(2) + size(2) + handshake_type(1) + uint24_length(3) + client_version(2)
RandomOffset = 1 + 2 + 2 + 1 + 3 + 2
sniDNSNamesListType = 0
)
var (
emptyRandom = [RandomLen]byte{}
extTypeSNI = [2]byte{}
)
type ClientHello struct {
Random [RandomLen]byte
SessionID []byte
CipherSuite uint16
}
func ReadClientHello(
conn net.Conn,
secret []byte,
hostname string,
tolerateTimeSkewness time.Duration,
) (*ClientHello, error) {
if err := conn.SetReadDeadline(time.Now().Add(ClientHelloReadTimeout)); err != nil {
return nil, fmt.Errorf("cannot set read deadline: %w", err)
}
defer conn.SetReadDeadline(resetDeadline) //nolint: errcheck
// This is how FakeTLS is organized:
// 1. We create sha256 HMAC with a given secret
// 2. We dump there a whole TLS frame except of the fact that random
// is filled with all zeroes
// 3. Digest is computed. This digest should be XORed with
// original client random
// 4. New digest should be all 0 except of last 4 bytes
// 5. Last 4 bytes are little endian uint32 of UNIX timestamp when
// this message was created.
handshakeCopyBuf := &bytes.Buffer{}
reader := io.TeeReader(conn, handshakeCopyBuf)
reader, err := parseTLSHeader(reader)
if err != nil {
return nil, fmt.Errorf("cannot parse tls header: %w", err)
}
reader, err = parseHandshakeHeader(reader)
if err != nil {
return nil, fmt.Errorf("cannot parse handshake header: %w", err)
}
hello, err := parseHandshake(reader)
if err != nil {
return nil, fmt.Errorf("cannot parse handshake: %w", err)
}
sniHostnames, err := parseSNI(reader)
if err != nil {
return nil, fmt.Errorf("cannot parse SNI: %w", err)
}
if !slices.Contains(sniHostnames, hostname) {
return nil, fmt.Errorf("cannot find %s in %v", hostname, sniHostnames)
}
digest := hmac.New(sha256.New, secret)
// we write a copy of the handshake with client random all nullified.
digest.Write(handshakeCopyBuf.Next(RandomOffset))
handshakeCopyBuf.Next(RandomLen)
digest.Write(emptyRandom[:])
digest.Write(handshakeCopyBuf.Bytes())
computed := digest.Sum(nil)
for i := range RandomLen {
computed[i] ^= hello.Random[i]
}
if subtle.ConstantTimeCompare(emptyRandom[:RandomLen-4], computed[:RandomLen-4]) != 1 {
return nil, ErrBadDigest
}
timestamp := int64(binary.LittleEndian.Uint32(computed[RandomLen-4:]))
createdAt := time.Unix(timestamp, 0)
if tdiff := time.Since(createdAt).Abs(); tdiff > tolerateTimeSkewness {
return nil, fmt.Errorf("timestamp %q is too old %s", createdAt, tdiff)
}
return hello, nil
}
func parseTLSHeader(r io.Reader) (io.Reader, error) {
// record_type(1) + version(2) + size(2)
// 16 - type is 0x16 (handshake record)
// 03 01 - protocol version is "3,1" (also known as TLS 1.0)
// 00 f8 - 0xF8 (248) bytes of handshake message follows
header := [1 + 2 + 2]byte{}
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, fmt.Errorf("cannot read record header: %w", err)
}
if header[0] != tls.TypeHandshake {
return nil, fmt.Errorf("unexpected record type %#x", header[0])
}
if header[1] != 3 || header[2] != 1 {
return nil, fmt.Errorf("unexpected protocol version %#x %#x", header[1], header[2])
}
length := int64(binary.BigEndian.Uint16(header[3:]))
buf := &bytes.Buffer{}
_, err := io.CopyN(buf, r, length)
return buf, err
}
func parseHandshakeHeader(r io.Reader) (io.Reader, error) {
// type(1) + size(3 / uint24)
// 01 - handshake message type 0x01 (client hello)
// 00 00 f4 - 0xF4 (244) bytes of client hello data follows
header := [1 + 3]byte{}
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, fmt.Errorf("cannot read handshake header: %w", err)
}
if header[0] != TypeHandshakeClient {
return nil, fmt.Errorf("incorrect handshake type: %#x", header[0])
}
// unfortunately there is not uint24 in golang, so we just reust header
header[0] = 0
length := int64(binary.BigEndian.Uint32(header[:]))
buf := &bytes.Buffer{}
_, err := io.CopyN(buf, r, length)
return buf, err
}
func parseHandshake(r io.Reader) (*ClientHello, error) {
// A protocol version of "3,3" (meaning TLS 1.2) is given.
header := [2]byte{}
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, fmt.Errorf("cannot read client version: %w", err)
}
hello := &ClientHello{}
if _, err := io.ReadFull(r, hello.Random[:]); err != nil {
return nil, fmt.Errorf("cannot read client random: %w", err)
}
if _, err := io.ReadFull(r, header[:1]); err != nil {
return nil, fmt.Errorf("cannot read session ID length: %w", err)
}
hello.SessionID = make([]byte, int(header[0]))
if _, err := io.ReadFull(r, hello.SessionID); err != nil {
return nil, fmt.Errorf("cannot read session id: %w", err)
}
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, fmt.Errorf("cannot read cipher suite length: %w", err)
}
cipherSuiteLen := int64(binary.BigEndian.Uint16(header[:]))
// we do not care about picking up any cipher. we pick the first one,
// so it is always should be present.
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, fmt.Errorf("cannot read first cipher suite: %w", err)
}
hello.CipherSuite = binary.BigEndian.Uint16(header[:])
if _, err := io.CopyN(io.Discard, r, cipherSuiteLen-2); err != nil {
return nil, fmt.Errorf("cannot skip remaining cipher suites: %w", err)
}
if _, err := io.ReadFull(r, header[:1]); err != nil {
return nil, fmt.Errorf("cannot read compression methods length: %w", err)
}
if _, err := io.CopyN(io.Discard, r, int64(header[0])); err != nil {
return nil, fmt.Errorf("cannot skip compression methods: %w", err)
}
return hello, nil
}
func parseSNI(r io.Reader) ([]string, error) {
header := [2]byte{}
if _, err := io.ReadFull(r, header[:]); err != nil {
return nil, fmt.Errorf("cannot read length of TLS extensions: %w", err)
}
extensionsLength := int64(binary.BigEndian.Uint16(header[:]))
buf := &bytes.Buffer{}
buf.Grow(int(extensionsLength))
if _, err := io.CopyN(buf, r, extensionsLength); err != nil {
return nil, fmt.Errorf("cannot read extensions: %w", err)
}
for buf.Len() > 0 {
// 00 00 - assigned value for extension "server name"
// 00 18 - 0x18 (24) bytes of "server name" extension data follows
// 00 16 - 0x16 (22) bytes of first (and only) list entry follows
// 00 - list entry is type 0x00 "DNS hostname"
// 00 13 - 0x13 (19) bytes of hostname follows
// 65 78 61 ... 6e 65 74 - "example.ulfheim.net"
// 00 00 - assigned value for extension "server name"
extTypeB := buf.Next(2)
if len(extTypeB) != 2 {
return nil, fmt.Errorf("cannot read extension type: %v", extTypeB)
}
// 00 18 - 0x18 (24) bytes of "server name" extension data follows
lengthB := buf.Next(2)
if len(lengthB) != 2 {
return nil, fmt.Errorf("cannot read extension %v length: %v", extTypeB, lengthB)
}
length := int(binary.BigEndian.Uint16(lengthB))
extDataB := buf.Next(length)
if len(extDataB) != length {
return nil, fmt.Errorf("cannot read extension %v data: len %d != %d", extTypeB, length, len(extDataB))
}
if !bytes.Equal(extTypeB, extTypeSNI[:]) {
continue
}
buf.Reset()
buf.Write(extDataB)
// 00 16 - 0x16 (22) bytes of first (and only) list entry follows
lengthB = buf.Next(2)
if len(lengthB) != 2 {
return nil, fmt.Errorf("cannot read the length of the SNI record: %v", lengthB)
}
length = int(binary.BigEndian.Uint16(lengthB))
if length == 0 {
return nil, nil
}
listType, err := buf.ReadByte()
if err != nil {
return nil, fmt.Errorf("cannot read SNI list type: %w", err)
}
// 00 - list entry is type 0x00 "DNS hostname"
if listType != sniDNSNamesListType {
return nil, fmt.Errorf("incorrect SNI list type %#x", listType)
}
names := []string{}
for buf.Len() > 0 {
// 00 13 - 0x13 (19) bytes of hostname follows
lengthB = buf.Next(2)
if len(lengthB) != 2 {
return nil, fmt.Errorf("incorrect length of the hostname: %v", lengthB)
}
length = int(binary.BigEndian.Uint16(lengthB))
name := buf.Next(length)
if len(name) != length {
return nil, fmt.Errorf("incorrect length of SNI hostname: len %d != %d", length, len(name))
}
names = append(names, string(name))
}
return names, nil
}
return nil, nil
}
mtglib/internal/tls/fake/client_side_fuzz_test.go
+48
View File
@@ -0,0 +1,48 @@
package fake_test
import (
"bytes"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/tls/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
type connMock struct {
testlib.EssentialsConnMock
readBuf *bytes.Buffer
}
func (f *connMock) Read(p []byte) (int, error) {
return f.readBuf.Read(p)
}
func FuzzReadClientHello(f *testing.F) {
seed := [248]byte{}
secret, err := mtglib.ParseSecret(
"ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d",
)
require.NoError(f, err)
f.Add(seed[:])
f.Fuzz(func(t *testing.T, value []byte) {
r := &connMock{
readBuf: bytes.NewBuffer(value),
}
r.
On("SetReadDeadline", mock.AnythingOfType("time.Time")).
Twice().
Return(nil)
_, err := fake.ReadClientHello(r, secret.Key[:], secret.Host, time.Hour)
assert.Error(t, err)
})
}
mtglib/internal/tls/fake/client_side_snapshot_test.go
+153
View File
@@ -0,0 +1,153 @@
package fake_test
import (
"bytes"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/tls/fake"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
type clientHelloSnapshot struct {
Time int `json:"time"`
Random string `json:"random"`
SessionID string `json:"sessionId"`
Host string `json:"host"`
CipherSuite int `json:"cipherSuite"`
Full string `json:"full"`
}
func (c clientHelloSnapshot) GetRandom() []byte {
data, _ := base64.StdEncoding.DecodeString(c.Random)
return data
}
func (c clientHelloSnapshot) GetSessionID() []byte {
data, _ := base64.StdEncoding.DecodeString(c.SessionID)
return data
}
func (c clientHelloSnapshot) GetCipherSuite() uint16 {
return uint16(c.CipherSuite)
}
func (c clientHelloSnapshot) GetFull() []byte {
data, _ := base64.StdEncoding.DecodeString(c.Full)
return data
}
type ParseClientHelloSnapshotTestSuite struct {
suite.Suite
secret mtglib.Secret
}
func (suite *ParseClientHelloSnapshotTestSuite) SetupSuite() {
parsed, err := mtglib.ParseSecret(
"ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d",
)
require.NoError(suite.T(), err)
suite.secret = parsed
}
func (suite *ParseClientHelloSnapshotTestSuite) makeConn(data []byte) *parseClientHelloConnMock {
readBuf := &bytes.Buffer{}
readBuf.Write(data)
connMock := &parseClientHelloConnMock{
readBuf: readBuf,
}
connMock.
On("SetReadDeadline", mock.AnythingOfType("time.Time")).
Twice().
Return(nil)
return connMock
}
func (suite *ParseClientHelloSnapshotTestSuite) TestSnapshotOk() {
files, err := os.ReadDir("testdata")
require.NoError(suite.T(), err)
for _, v := range files {
if !strings.HasPrefix(v.Name(), "client-hello-ok") {
continue
}
path := filepath.Join("testdata", v.Name())
suite.T().Run(v.Name(), func(t *testing.T) {
fileData, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &clientHelloSnapshot{}
assert.NoError(t, json.Unmarshal(fileData, snapshot))
connMock := suite.makeConn(snapshot.GetFull())
defer connMock.AssertExpectations(t)
hello, err := fake.ReadClientHello(
connMock,
suite.secret.Key[:],
suite.secret.Host,
TolerateTime,
)
require.NoError(t, err)
assert.Equal(t, snapshot.GetRandom(), hello.Random[:])
assert.Equal(t, snapshot.GetSessionID(), hello.SessionID)
assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite)
})
}
}
func (suite *ParseClientHelloSnapshotTestSuite) TestSnapshotBad() {
files, err := os.ReadDir("testdata")
require.NoError(suite.T(), err)
for _, v := range files {
if !strings.HasPrefix(v.Name(), "client-hello-bad") {
continue
}
path := filepath.Join("testdata", v.Name())
suite.T().Run(v.Name(), func(t *testing.T) {
fileData, err := os.ReadFile(path)
assert.NoError(t, err)
snapshot := &clientHelloSnapshot{}
assert.NoError(t, json.Unmarshal(fileData, snapshot))
connMock := suite.makeConn(snapshot.GetFull())
defer connMock.AssertExpectations(t)
_, err = fake.ReadClientHello(
connMock,
suite.secret.Key[:],
suite.secret.Host,
TolerateTime,
)
assert.ErrorIs(t, err, fake.ErrBadDigest)
})
}
}
func TestParseClientHelloSnapshot(t *testing.T) {
t.Parallel()
suite.Run(t, &ParseClientHelloSnapshotTestSuite{})
}
mtglib/internal/tls/fake/client_side_test.go
+395
View File
@@ -0,0 +1,395 @@
package fake_test
import (
"bytes"
"encoding/binary"
"errors"
"io"
"testing"
"time"
"github.com/9seconds/mtg/v2/internal/testlib"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/9seconds/mtg/v2/mtglib/internal/tls/fake"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
const (
TolerateTime = 365 * 30 * 24 * time.Hour
)
type parseClientHelloConnMock struct {
testlib.EssentialsConnMock
readBuf *bytes.Buffer
}
func (m *parseClientHelloConnMock) Read(p []byte) (int, error) {
return m.readBuf.Read(p)
}
type ParseClientHelloTestSuite struct {
suite.Suite
secret mtglib.Secret
readBuf *bytes.Buffer
connMock *parseClientHelloConnMock
}
func (suite *ParseClientHelloTestSuite) SetupSuite() {
parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d")
require.NoError(suite.T(), err)
suite.secret = parsed
}
func (suite *ParseClientHelloTestSuite) SetupTest() {
suite.readBuf = &bytes.Buffer{}
suite.connMock = &parseClientHelloConnMock{
readBuf: suite.readBuf,
}
suite.connMock.
On("SetReadDeadline", mock.AnythingOfType("time.Time")).
Twice().
Return(nil)
}
func (suite *ParseClientHelloTestSuite) TearDownTest() {
suite.connMock.AssertExpectations(suite.T())
}
type ParseClientHello_TLSHeaderTestSuite struct {
ParseClientHelloTestSuite
}
func (suite *ParseClientHello_TLSHeaderTestSuite) TestEmpty() {
suite.connMock.ExpectedCalls = []*mock.Call{}
suite.connMock.
On("SetReadDeadline", mock.AnythingOfType("time.Time")).
Once().
Return(errors.New("fail"))
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "fail")
}
func (suite *ParseClientHello_TLSHeaderTestSuite) TestNothing() {
suite.connMock.ExpectedCalls = []*mock.Call{}
suite.connMock.
On("SetReadDeadline", mock.AnythingOfType("time.Time")).
Twice().
Return(nil)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorIs(err, io.EOF)
}
func (suite *ParseClientHello_TLSHeaderTestSuite) TestUnknownRecord() {
suite.readBuf.Write([]byte{
10,
3, 3,
0, 0,
})
suite.readBuf.WriteByte(10)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "unexpected record type 0xa")
}
func (suite *ParseClientHello_TLSHeaderTestSuite) TestUnknownProtocolVersion() {
suite.readBuf.Write([]byte{
tls.TypeHandshake,
3, 3,
0, 0,
})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "unexpected protocol version")
}
func (suite *ParseClientHello_TLSHeaderTestSuite) TestCannotReadRestOfRecord() {
suite.readBuf.Write([]byte{
tls.TypeHandshake,
3, 1,
0, 10,
})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorIs(err, io.EOF)
}
type ParseClientHelloHandshakeTestSuite struct {
ParseClientHelloTestSuite
}
func (suite *ParseClientHelloHandshakeTestSuite) SetupTest() {
suite.ParseClientHelloTestSuite.SetupTest()
suite.readBuf.Write([]byte{
tls.TypeHandshake,
3, 1,
0,
})
}
func (suite *ParseClientHelloHandshakeTestSuite) TestCannotReadHeader() {
suite.readBuf.Write([]byte{
1,
10,
})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read handshake header")
}
func (suite *ParseClientHelloHandshakeTestSuite) TestIncorrectHandshakeType() {
suite.readBuf.Write([]byte{
4,
10, 0, 0, 0,
})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "incorrect handshake type")
}
func (suite *ParseClientHelloHandshakeTestSuite) TestCannotReadHandshake() {
suite.readBuf.Write([]byte{
4 + 3,
10, 0, 0, 0,
})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorIs(err, io.EOF)
}
type ParseClientHelloHandshakeBodyTestSuite struct {
ParseClientHelloTestSuite
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) SetupTest() {
suite.ParseClientHelloTestSuite.SetupTest()
suite.readBuf.Write([]byte{
tls.TypeHandshake,
3, 1,
0,
})
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) writeBody(body []byte) {
suite.readBuf.WriteByte(byte(4 + len(body)))
suite.readBuf.Write([]byte{
fake.TypeHandshakeClient,
0, 0, byte(len(body)),
})
suite.readBuf.Write(body)
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadVersion() {
suite.writeBody(nil)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read client version")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadRandom() {
suite.writeBody([]byte{3, 3})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read client random")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadSessionIDLength() {
body := make([]byte, 2+fake.RandomLen)
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read session ID length")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadSessionID() {
body := make([]byte, 2+fake.RandomLen+1)
body[2+fake.RandomLen] = 32
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read session id")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadCipherSuiteLength() {
body := make([]byte, 2+fake.RandomLen+1)
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read cipher suite length")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadFirstCipherSuite() {
body := make([]byte, 2+fake.RandomLen+1+2)
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read first cipher suite")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotSkipRemainingCipherSuites() {
body := make([]byte, 2+fake.RandomLen+1+2+2)
binary.BigEndian.PutUint16(body[2+fake.RandomLen+1:], 4)
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot skip remaining cipher suites")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotReadCompressionMethodsLength() {
body := make([]byte, 2+fake.RandomLen+1+2+2)
binary.BigEndian.PutUint16(body[2+fake.RandomLen+1:], 2)
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read compression methods length")
}
func (suite *ParseClientHelloHandshakeBodyTestSuite) TestCannotSkipCompressionMethods() {
body := make([]byte, 2+fake.RandomLen+1+2+2+1)
binary.BigEndian.PutUint16(body[2+fake.RandomLen+1:], 2)
body[2+fake.RandomLen+1+2+2] = 1
suite.writeBody(body)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot skip compression methods")
}
type ParseClientHelloSNITestSuite struct {
ParseClientHelloTestSuite
}
func (suite *ParseClientHelloSNITestSuite) SetupTest() {
suite.ParseClientHelloTestSuite.SetupTest()
suite.readBuf.Write([]byte{
tls.TypeHandshake,
3, 1,
0,
})
}
func (suite *ParseClientHelloSNITestSuite) writeExtensions(extensions []byte) {
handshakeBodyLen := 41 + len(extensions)
suite.readBuf.WriteByte(byte(4 + handshakeBodyLen))
suite.readBuf.Write([]byte{
fake.TypeHandshakeClient,
0, 0, byte(handshakeBodyLen),
})
// version(2) + random(32) + sessionIDLen(1) + cipherSuiteLen(2) +
// cipherSuite(2) + compressionLen(1) + compression(1) = 41
body := make([]byte, 41)
binary.BigEndian.PutUint16(body[35:], 2)
body[39] = 1
suite.readBuf.Write(body)
suite.readBuf.Write(extensions)
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionsLength() {
suite.writeExtensions(nil)
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read length of TLS extensions")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensions() {
suite.writeExtensions([]byte{0, 10})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read extensions")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionType() {
suite.writeExtensions([]byte{0, 1, 0xAB})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read extension type")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionLength() {
suite.writeExtensions([]byte{0, 2, 0xFF, 0xFF})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "length:")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadExtensionData() {
suite.writeExtensions([]byte{0, 4, 0xFF, 0xFF, 0, 5})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "data: len")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadSNIRecordLength() {
suite.writeExtensions([]byte{0, 5, 0, 0, 0, 1, 0xAB})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read the length of the SNI record")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadSNIListType() {
suite.writeExtensions([]byte{0, 6, 0, 0, 0, 2, 0, 1})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "cannot read SNI list type")
}
func (suite *ParseClientHelloSNITestSuite) TestIncorrectSNIListType() {
suite.writeExtensions([]byte{0, 7, 0, 0, 0, 3, 0, 1, 5})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "incorrect SNI list type")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadHostnameLength() {
suite.writeExtensions([]byte{0, 8, 0, 0, 0, 4, 0, 2, 0, 0xAB})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "incorrect length of the hostname")
}
func (suite *ParseClientHelloSNITestSuite) TestCannotReadHostname() {
suite.writeExtensions([]byte{0, 9, 0, 0, 0, 5, 0, 3, 0, 0, 5})
_, err := fake.ReadClientHello(suite.connMock, suite.secret.Key[:], suite.secret.Host, TolerateTime)
suite.ErrorContains(err, "incorrect length of SNI hostname")
}
func TestParseClientHelloTLSHeader(t *testing.T) {
t.Parallel()
suite.Run(t, &ParseClientHello_TLSHeaderTestSuite{})
}
func TestParseClientHelloHandshake(t *testing.T) {
t.Parallel()
suite.Run(t, &ParseClientHelloHandshakeTestSuite{})
}
func TestParseClientHelloHandshakeBody(t *testing.T) {
t.Parallel()
suite.Run(t, &ParseClientHelloHandshakeBodyTestSuite{})
}
func TestParseClientHelloSNI(t *testing.T) {
t.Parallel()
suite.Run(t, &ParseClientHelloSNITestSuite{})
}
mtglib/internal/tls/fake/init.go
+16
View File
@@ -0,0 +1,16 @@
package fake
import (
"errors"
"time"
)
const (
ClientHelloReadTimeout = 5 * time.Second
)
var (
resetDeadline time.Time
ErrBadDigest = errors.New("incorrect client random")
)
mtglib/internal/tls/fake/server_side.go
+135
View File
@@ -0,0 +1,135 @@
package fake
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"io"
rnd "math/rand/v2"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"golang.org/x/crypto/curve25519"
)
const (
TypeHandshakeServer = 0x02
ChangeCipherValue = 0x01
EllipticCurveLen = 32
)
var serverHelloSuffix = []byte{
0x00, // no compression
0x00, 0x2e, // 46 bytes of data
0x00, 0x2b, // Extension - Supported Versions
0x00, 0x02, // 2 bytes are following
0x03, 0x04, // TLS 1.3
0x00, 0x33, // Extension - Key Share
0x00, 0x24, // 36 bytes
0x00, 0x1d, // x25519 curve
0x00, 0x20, // 32 bytes of key
}
func SendServerHello(w io.Writer, secret []byte, clientHello *ClientHello) error {
buf := &bytes.Buffer{}
buf.Grow(tls.MaxRecordSize)
generateServerHello(buf, clientHello)
generateChangeCipherValue(buf)
generateNoise(buf)
packet := buf.Bytes()
digest := hmac.New(sha256.New, secret)
digest.Write(clientHello.Random[:])
digest.Write(packet)
copy(packet[RandomOffset:], digest.Sum(nil))
_, err := w.Write(packet)
return err
}
func generateServerHello(buf *bytes.Buffer, hello *ClientHello) {
payload := acquireBuffer()
defer releaseBuffer(payload)
generateServerHelloPayload(payload, hello)
// 16 - type is 0x16 (handshake record)
// 03 03 - legacy protocol version of "3,3" (TLS 1.2)
// 00 7a - 0x7A (122) bytes of handshake message follows
// 16 - type is 0x16 (handshake record)
buf.WriteByte(tls.TypeHandshake)
// 03 03 - legacy protocol version of "3,3" (TLS 1.2)
buf.Write(tls.TLSVersion[:])
// 00 7a - 0x7A (122) bytes of handshake message follows
binary.Write(buf, binary.BigEndian, uint16(payload.Len())) //nolint: errcheck
payload.WriteTo(buf) //nolint: errcheck
}
func generateServerHelloPayload(buf *bytes.Buffer, hello *ClientHello) {
data := [4]byte{}
payload := acquireBuffer()
defer releaseBuffer(payload)
generateServerHelloHandshakePayload(payload, hello)
// 02 - handshake message type 0x02 (server hello)
// 00 00 76 - 0x76 (118) bytes of server hello data follows
buf.WriteByte(TypeHandshakeServer)
// 00 00 76 - 0x76 (118) bytes of server hello data follows
binary.BigEndian.PutUint32(data[:], uint32(payload.Len()))
buf.Write(data[1:])
payload.WriteTo(buf) //nolint: errcheck
}
func generateServerHelloHandshakePayload(buf *bytes.Buffer, hello *ClientHello) {
// The unusual version number ("3,3" representing TLS 1.2) is due to
// TLS 1.0 being a minor revision of the SSL 3.0 protocol. Therefore
// TLS 1.0 is represented by "3,1", TLS 1.1 is "3,2", and so on.
buf.Write(tls.TLSVersion[:])
buf.Write(emptyRandom[:])
// 20 - 0x20 (32) bytes of session ID follow
// e0 e1 ... fe ff - session ID copied from Client Hello
buf.WriteByte(byte(len(hello.SessionID)))
buf.Write(hello.SessionID)
binary.Write(buf, binary.BigEndian, hello.CipherSuite) //nolint: errcheck
buf.Write(serverHelloSuffix)
scalar := [EllipticCurveLen]byte{}
if _, err := rand.Read(scalar[:]); err != nil {
panic(err)
}
curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint)
buf.Write(curve)
}
func generateChangeCipherValue(buf *bytes.Buffer) {
buf.WriteByte(tls.TypeChangeCipherSpec)
buf.Write(tls.TLSVersion[:])
binary.Write(buf, binary.BigEndian, uint16(1)) //nolint: errcheck
buf.WriteByte(ChangeCipherValue)
}
func generateNoise(buf *bytes.Buffer) {
data := make([]byte, int64(1024+rnd.IntN(3092)))
if _, err := rand.Read(data[:]); err != nil {
panic(err)
}
tls.WriteRecord(buf, data[:]) //nolint: errcheck
}
mtglib/internal/tls/fake/server_side_test.go
+130
View File
@@ -0,0 +1,130 @@
package fake_test
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"testing"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/mtglib/internal/doppel"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/9seconds/mtg/v2/mtglib/internal/tls/fake"
"github.com/stretchr/testify/suite"
)
type SendServerHelloTestSuite struct {
suite.Suite
hello *fake.ClientHello
buf *bytes.Buffer
secret mtglib.Secret
}
func (suite *SendServerHelloTestSuite) SetupTest() {
suite.hello = &fake.ClientHello{
CipherSuite: 4867,
SessionID: make([]byte, 32),
}
_, err := rand.Read(suite.hello.SessionID)
suite.NoError(err)
_, err = rand.Read(suite.hello.Random[:])
suite.NoError(err)
suite.buf = &bytes.Buffer{}
suite.secret = mtglib.GenerateSecret("google.com")
}
func (suite *SendServerHelloTestSuite) TestRecordStructure() {
err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
suite.NoError(err)
var rec bytes.Buffer
recordType, _, err := tls.ReadRecord(suite.buf, &rec)
suite.NoError(err)
suite.Equal(byte(tls.TypeHandshake), recordType)
rec.Reset()
recordType, _, err = tls.ReadRecord(suite.buf, &rec)
suite.NoError(err)
suite.Equal(byte(tls.TypeChangeCipherSpec), recordType)
rec.Reset()
recordType, length, err := tls.ReadRecord(suite.buf, &rec)
suite.NoError(err)
suite.Equal(byte(tls.TypeApplicationData), recordType)
suite.Greater(length, int64(doppel.TLSRecordSizeStart))
suite.Empty(suite.buf.Bytes())
}
func (suite *SendServerHelloTestSuite) TestHMAC() {
err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
suite.NoError(err)
packet := make([]byte, suite.buf.Len())
copy(packet, suite.buf.Bytes())
random := make([]byte, fake.RandomLen)
copy(random, packet[fake.RandomOffset:])
copy(packet[fake.RandomOffset:], make([]byte, fake.RandomLen))
mac := hmac.New(sha256.New, suite.secret.Key[:])
mac.Write(suite.hello.Random[:])
mac.Write(packet)
suite.Equal(random, mac.Sum(nil))
}
func (suite *SendServerHelloTestSuite) TestHandshakePayload() {
err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
suite.NoError(err)
packet := suite.buf.Bytes()
// TLS record header: type(1) + version(2) + length(2)
suite.Equal(byte(tls.TypeHandshake), packet[0])
suite.Equal([]byte{3, 3}, packet[1:3])
// Handshake header: type(1) + uint24_length(3)
suite.Equal(byte(fake.TypeHandshakeServer), packet[5])
// ServerHello version
suite.Equal([]byte{3, 3}, packet[9:11])
// Session ID
sessionIDOffset := fake.RandomOffset + fake.RandomLen
suite.Equal(byte(len(suite.hello.SessionID)), packet[sessionIDOffset])
suite.Equal(suite.hello.SessionID, packet[sessionIDOffset+1:sessionIDOffset+1+len(suite.hello.SessionID)])
}
func (suite *SendServerHelloTestSuite) TestChangeCipherSpec() {
err := fake.SendServerHello(suite.buf, suite.secret.Key[:], suite.hello)
suite.NoError(err)
// Skip first record
var rec bytes.Buffer
_, _, err = tls.ReadRecord(suite.buf, &rec)
suite.NoError(err)
// Read ChangeCipherSpec record
rec.Reset()
recordType, length, err := tls.ReadRecord(suite.buf, &rec)
suite.NoError(err)
suite.Equal(byte(tls.TypeChangeCipherSpec), recordType)
suite.Equal(int64(1), length)
suite.Equal([]byte{fake.ChangeCipherValue}, rec.Bytes())
}
func TestSendServerHello(t *testing.T) {
t.Parallel()
suite.Run(t, &SendServerHelloTestSuite{})
}
mtglib/internal/tls/fake/testdata/client-hello-bad-fa2e46cdb33e2a1b.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
{
"time": 1617181365,
"random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=",
"sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "FgMBAgABAAH8AwNe8I9zdoBsduFEu/SRSbLoF89k4a+y6x53n8c2wpcQ+yBK3YFna4cwWfcHa2sPWN922mOgk46DokF4uEVzIIAKrgA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUA0AAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACAH/ugvH0kSUgAuwslL3UfZA3JTUfSiwrAhR6VWd2wvIgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
}
mtglib/internal/tls/fake/testdata/client-hello-ok-19dfe38384b9884b.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
{
"time": 1617181365,
"random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=",
"sessionId": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "FgMBAgABAAH8AwNe8I9zdoBsduFEu/SRSbLoF89k4a+y6x53n8c2wpcQ+yBK3YFna4cwWfcHa2sPWN922mOgk46DokF4uEVzIIwKrgA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACAH/ugvH0kSUgAuwslL3UfZA3JTUfSiwrAhR6VWd2wvIgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
}
mtglib/internal/tls/fake/testdata/client-hello-ok-48f8a72a56f3174a.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
{
"time": 1617181352,
"random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=",
"sessionId": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "FgMBAgABAAH8AwOhgS7feOX7NBtQoy1BtXU4cHSBcgzbLZqr2JjRBfXaiyAUaoDdkVitKWP//GXuVqaY2frgr38wraZDcclQa8/7yAA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACBroKhykU/xB3hgIVH2mRoKv3umjYAuPQ/mcj02dvdRYwAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
}
mtglib/internal/tls/fake/testdata/client-hello-ok-651054256093c6cd.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
{
"time": 1617181352,
"random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=",
"sessionId": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "FgMBAgABAAH8AwPlXmxKmuT+0UiDL7HUF4o0aEuUWSp+CAumA3sZg4DWuiCPGvh3o9c8OT4vDHdZSn3C+PxMaU7ER2tEnTJ3In34fwA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACCu6UBqpR0p5VgzQX6m7qif+HosGk7LM4objEUgpygWTgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
}
mtglib/internal/tls/fake/testdata/client-hello-ok-79d01ef18a9d2621.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
{
"time": 1617181365,
"random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=",
"sessionId": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "FgMBAgABAAH8AwPzGWOU6GQOWRp8QXm+7d7Wvd9a+HwBfnjzALeEtnezNCDTS68NgqcXJkUrJ/ceUvBYYI7J4ew8WJTlRn4WzmkMBQA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACD/0/vXjQ20rOPIPAF/32Y7LX4WNE8A8dM1D1bEc4qlXgAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
}
mtglib/internal/tls/fake/testdata/client-hello-ok-7a5569f05b118145.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
{
"time": 1617181352,
"random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=",
"sessionId": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=",
"host": "storage.googleapis.com",
"cipherSuite": 4867,
"full": "FgMBAgABAAH8AwPONrcwtnxYZJ+xBG09Xvn61jqBsrfMo+LVLLtIEFp+DiCo+63bIypdr3MYsgj/zvZJDd3hN+qjS6gNvu2mRF8GcAA0EwMTARMCwCzAK8AkwCPACsAJzKnAMMAvwCjAJ8AUwBPMqACdAJwAPQA8ADUAL8AIwBIACgEAAX//AQABAAAAABsAGQAAFnN0b3JhZ2UuZ29vZ2xlYXBpcy5jb20AFwAAAA0AGAAWBAMIBAQBBQMCAwgFCAUFAQgGBgECAQAFAAUBAAAAADN0AAAAEgAAABAAMAAuAmgyBWgyLTE2BWgyLTE1BWgyLTE0CHNwZHkvMy4xBnNwZHkvMwhodHRwLzEuMQALAAIBAAAzACYAJAAdACBe+ItECoBgnzE4t2VyxSGV0jheXSD+z37LZCt3yto8SAAtAAIBAQArAAkIAwQDAwMCAwEACgAKAAgAHQAXABgAGQAVAKEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=="
}
mtglib/internal/tls/init_test.go
+30
View File
@@ -0,0 +1,30 @@
package tls
import (
"encoding/binary"
"github.com/stretchr/testify/mock"
)
type WriterMock struct {
mock.Mock
}
func (m *WriterMock) Write(p []byte) (int, error) {
args := m.Called(p)
return args.Int(0), args.Error(1)
}
// makeTLSRecord builds a raw TLS record from hardcoded offsets:
// type(1) + version(2, {3,3}) + length(2, big-endian) + payload.
func MakeTLSRecord(recordType byte, payload []byte) []byte {
buf := make([]byte, 5+len(payload))
buf[0] = recordType
buf[1] = 3
buf[2] = 3
binary.BigEndian.PutUint16(buf[3:5], uint16(len(payload)))
copy(buf[5:], payload)
return buf
}
mtglib/internal/tls/utils.go
+48
View File
@@ -0,0 +1,48 @@
package tls
import (
"bytes"
"encoding/binary"
"fmt"
"io"
)
func ReadRecord(r io.Reader, w io.Writer) (byte, int64, error) {
buf := [SizeHeader]byte{}
if _, err := io.ReadFull(r, buf[:]); err != nil {
return 0, 0, err
}
pVer := buf[SizeRecordType:]
pLen := pVer[SizeVersion:]
if !bytes.Equal(TLSVersion[:], pVer[:SizeVersion]) {
return 0, 0, fmt.Errorf("incorrect tls version %v", pVer)
}
length := int64(binary.BigEndian.Uint16(pLen[:SizeSize]))
_, err := io.CopyN(w, r, length)
return buf[0], length, err
}
func WriteRecord(w io.Writer, payload []byte) error {
buf := [MaxRecordSize]byte{}
buf[0] = TypeApplicationData
bufV := buf[SizeRecordType:]
copy(bufV[:SizeVersion], TLSVersion[:])
bufS := bufV[SizeVersion:]
binary.BigEndian.PutUint16(bufS[:SizeSize], uint16(len(payload)))
bufP := buf[SizeHeader:]
if n := copy(bufP, payload); n != len(payload) {
return fmt.Errorf("copied %d bytes of payload instead of %d", n, len(payload))
}
_, err := w.Write(buf[:SizeHeader+len(payload)])
return err
}
mtglib/internal/tls/utils_test.go
+125
View File
@@ -0,0 +1,125 @@
package tls
import (
"bytes"
"encoding/binary"
"errors"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
)
type UtilsTestSuite struct {
suite.Suite
dst *bytes.Buffer
}
func (suite *UtilsTestSuite) SetupTest() {
suite.dst = &bytes.Buffer{}
}
func (suite *UtilsTestSuite) TestReadRecord() {
payload := []byte("hello world")
raw := MakeTLSRecord(0x17, payload)
recordType, length, err := ReadRecord(bytes.NewReader(raw), suite.dst)
suite.NoError(err)
suite.Equal(byte(0x17), recordType)
suite.Equal(int64(len(payload)), length)
suite.Equal(payload, suite.dst.Bytes())
}
func (suite *UtilsTestSuite) TestReadRecordChangeCipherSpec() {
payload := []byte{1}
raw := MakeTLSRecord(0x14, payload)
recordType, length, err := ReadRecord(bytes.NewReader(raw), suite.dst)
suite.NoError(err)
suite.Equal(byte(0x14), recordType)
suite.Equal(int64(1), length)
}
func (suite *UtilsTestSuite) TestReadRecordRejectsWrongVersion() {
record := []byte{0x17, 3, 1, 0, 5, 0, 0, 0, 0, 0}
_, _, err := ReadRecord(bytes.NewReader(record), suite.dst)
suite.ErrorContains(err, "incorrect tls version")
}
func (suite *UtilsTestSuite) TestReadRecordEmptyReader() {
_, _, err := ReadRecord(bytes.NewReader(nil), suite.dst)
suite.Error(err)
}
func (suite *UtilsTestSuite) TestReadRecordTruncatedHeader() {
_, _, err := ReadRecord(bytes.NewReader([]byte{0x17, 3}), suite.dst)
suite.Error(err)
}
func (suite *UtilsTestSuite) TestReadRecordTruncatedPayload() {
raw := MakeTLSRecord(0x17, []byte("full payload"))
truncated := raw[:5+3]
_, _, err := ReadRecord(bytes.NewReader(truncated), suite.dst)
suite.Error(err)
}
func (suite *UtilsTestSuite) TestWriteRecord() {
payload := []byte("hello world")
err := WriteRecord(suite.dst, payload)
suite.NoError(err)
written := suite.dst.Bytes()
suite.Equal(byte(0x17), written[0])
suite.Equal([]byte{3, 3}, written[1:3])
length := binary.BigEndian.Uint16(written[3:5])
suite.Equal(uint16(len(payload)), length)
suite.Equal(payload, written[5:])
}
func (suite *UtilsTestSuite) TestWriteRecordRoundTrip() {
payload := []byte("round trip test")
var wire bytes.Buffer
err := WriteRecord(&wire, payload)
suite.NoError(err)
var recovered bytes.Buffer
recordType, length, err := ReadRecord(&wire, &recovered)
suite.NoError(err)
suite.Equal(byte(0x17), recordType)
suite.Equal(int64(len(payload)), length)
suite.Equal(payload, recovered.Bytes())
}
func (suite *UtilsTestSuite) TestWriteRecordPropagatesError() {
m := &WriterMock{}
m.
On("Write", mock.AnythingOfType("[]uint8")).
Once().
Return(0, errors.New("dist full"))
err := WriteRecord(m, []byte("data"))
suite.Error(err)
m.AssertExpectations(suite.T())
}
func (suite *UtilsTestSuite) TestWriteRecordPayloadTooLarge() {
err := WriteRecord(suite.dst, make([]byte, MaxRecordPayloadSize+1))
suite.Error(err)
}
func TestUtils(t *testing.T) {
t.Parallel()
suite.Run(t, &UtilsTestSuite{})
}
mtglib/proxy.go
+41 -37
View File
@@ -11,10 +11,11 @@ import (
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib/internal/dc"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls"
"github.com/9seconds/mtg/v2/mtglib/internal/faketls/record"
"github.com/9seconds/mtg/v2/mtglib/internal/doppel"
"github.com/9seconds/mtg/v2/mtglib/internal/obfuscation"
"github.com/9seconds/mtg/v2/mtglib/internal/relay"
"github.com/9seconds/mtg/v2/mtglib/internal/tls"
"github.com/9seconds/mtg/v2/mtglib/internal/tls/fake"
"github.com/panjf2000/ants/v2"
)
@@ -32,6 +33,7 @@ type Proxy struct {
workerPool *ants.PoolWithFunc
telegram *dc.Telegram
configUpdater *dc.PublicConfigUpdater
doppelGanger *doppel.Ganger
clientObfuscatror obfuscation.Obfuscator
secret Secret
@@ -80,15 +82,22 @@ func (p *Proxy) ServeConn(conn essentials.Conn) {
return
}
if err := p.doObfuscatedHandshake(ctx); err != nil {
p.logger.InfoError("obfuscated handshake is failed", err)
clientConn, err := p.doppelGanger.NewConn(ctx.clientConn)
if err != nil {
ctx.logger.InfoError("cannot wrap into doppelganger connection", err)
return
}
defer clientConn.Stop()
ctx.clientConn = clientConn
if err := p.doObfuscatedHandshake(ctx); err != nil {
ctx.logger.InfoError("obfuscated handshake is failed", err)
return
}
if err := p.doTelegramCall(ctx); err != nil {
p.logger.WarningError("cannot dial to telegram", err)
ctx.logger.WarningError("cannot dial to telegram", err)
return
}
@@ -155,59 +164,40 @@ func (p *Proxy) Shutdown() {
p.streamWaitGroup.Wait()
p.workerPool.Release()
p.configUpdater.Wait()
p.doppelGanger.Shutdown()
p.allowlist.Shutdown()
p.blocklist.Shutdown()
}
func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool {
rec := record.AcquireRecord()
defer record.ReleaseRecord(rec)
rewind := newConnRewind(ctx.clientConn)
if err := rec.Read(rewind); err != nil {
clientHello, err := fake.ReadClientHello(
rewind,
p.secret.Key[:],
p.secret.Host,
p.tolerateTimeSkewness,
)
if err != nil {
p.logger.InfoError("cannot read client hello", err)
p.doDomainFronting(ctx, rewind)
return false
}
hello, err := faketls.ParseClientHello(p.secret.Key[:], rec.Payload.Bytes())
if err != nil {
p.logger.InfoError("cannot parse client hello", err)
p.doDomainFronting(ctx, rewind)
return false
}
if err := hello.Valid(p.secret.Host, p.tolerateTimeSkewness); err != nil {
p.logger.
BindStr("hostname", hello.Host).
BindStr("hello-time", hello.Time.String()).
InfoError("invalid faketls client hello", err)
p.doDomainFronting(ctx, rewind)
return false
}
if p.antiReplayCache.SeenBefore(hello.SessionID) {
if p.antiReplayCache.SeenBefore(clientHello.SessionID) {
p.logger.Warning("replay attack has been detected!")
p.eventStream.Send(p.ctx, NewEventReplayAttack(ctx.streamID))
p.doDomainFronting(ctx, rewind)
return false
}
if err := faketls.SendWelcomePacket(rewind, p.secret.Key[:], hello); err != nil {
if err := fake.SendServerHello(ctx.clientConn, p.secret.Key[:], clientHello); err != nil {
p.logger.InfoError("cannot send welcome packet", err)
return false
}
ctx.clientConn = &faketls.Conn{
Conn: ctx.clientConn,
}
ctx.clientConn = tls.New(ctx.clientConn, true, false)
return true
}
@@ -282,13 +272,16 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) {
p.eventStream.Send(p.ctx, NewEventDomainFronting(ctx.streamID))
conn.Rewind()
frontConn, err := p.network.DialContext(ctx, "tcp", p.DomainFrontingAddress())
nativeDialer := p.network.NativeDialer()
fConn, err := nativeDialer.DialContext(ctx, "tcp", p.DomainFrontingAddress())
if err != nil {
p.logger.WarningError("cannot dial to the fronting domain", err)
return
}
frontConn := essentials.WrapNetConn(fConn)
if p.domainFrontingProxyProtocol {
frontConn = newConnProxyProtocol(ctx.clientConn, frontConn)
}
@@ -338,6 +331,15 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {
tolerateTimeSkewness: opts.getTolerateTimeSkewness(),
allowFallbackOnUnknownDC: opts.AllowFallbackOnUnknownDC,
telegram: tg,
doppelGanger: doppel.NewGanger(
ctx,
opts.Network,
logger.Named("doppelganger"),
opts.DoppelGangerEach,
int(opts.DoppelGangerPerRaid),
opts.DoppelGangerURLs,
opts.DoppelGangerDRS,
),
configUpdater: dc.NewPublicConfigUpdater(
tg,
updatersLogger.Named("public-config"),
@@ -349,6 +351,8 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) {
domainFrontingProxyProtocol: opts.DomainFrontingProxyProtocol,
}
proxy.doppelGanger.Run()
if opts.AutoUpdate {
proxy.configUpdater.Run(ctx, dc.PublicConfigUpdateURLv4, "tcp4")
proxy.configUpdater.Run(ctx, dc.PublicConfigUpdateURLv6, "tcp6")
mtglib/proxy_opts.go
+18
View File
@@ -142,6 +142,24 @@ type ProxyOpts struct {
//
// OBSOLETE and DEPRECATED. Ignored.
DCOverrides map[int][]string
// DoppelGangerURLs is a list of URLs that should be crawled by
// mtg to calculate parameters for statistical distribution of a
// traffic for fronting domains. If nothing is given, then predefined
// statistics is going to be used.
DoppelGangerURLs []string
// DoppelGangerPerRaid defines how many time each URL from
// DoppelGangerURLs list should be crawled per raid. We recommend to
// have this number ~10.
DoppelGangerPerRaid uint
// DoppelGangerEach defines a time period between each raid. We recommend
// to use hours here.
DoppelGangerEach time.Duration
// DoppelGangerDRS defines if TLS Dynamic Record Sizing is active.
DoppelGangerDRS bool
}
func (p ProxyOpts) valid() error {
network/network.go
+4
View File
@@ -60,6 +60,10 @@ func (n *network) DialContext(ctx context.Context, protocol, address string) (es
return nil, fmt.Errorf("cannot dial to %s:%s: %w", protocol, address, err)
}
func (n *network) NativeDialer() *net.Dialer {
return &net.Dialer{}
}
func (n *network) MakeHTTPClient(dialFunc func(ctx context.Context,
network, address string) (essentials.Conn, error),
) *http.Client {
network/v2/base_network_test.go
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"testing"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/network/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
@@ -12,7 +13,7 @@ import (
type BaseNetworkTestSuite struct {
EchoServerTestSuite
net network.Network
net mtglib.Network
}
func (suite *BaseNetworkTestSuite) SetupSuite() {
network/v2/init.go
+3 -9
View File
@@ -11,10 +11,7 @@ package network
import (
"errors"
"net"
"time"
"github.com/9seconds/mtg/v2/mtglib"
)
const (
@@ -31,15 +28,12 @@ const (
// probes.
DefaultTCPKeepAlivePeriod = 10 * time.Second
// User Agent to use in HTTP client.
UserAgent = "curl/8.5.0"
// tcpLingerTimeout defines a number of seconds to wait for sending
// unacknowledged data.
tcpLingerTimeout = 1
)
var ErrCannotDial = errors.New("cannot dial to any address")
type Network interface {
mtglib.Network
NativeDialer() *net.Dialer
}
network/v2/multi_network.go
+4 -3
View File
@@ -8,10 +8,11 @@ import (
"net/http"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib"
)
type multiNetwork struct {
networks []Network
networks []mtglib.Network
}
func (m multiNetwork) Dial(network, address string) (essentials.Conn, error) {
@@ -22,7 +23,7 @@ func (m multiNetwork) DialContext(ctx context.Context, network, address string)
networks := m.networks
if len(networks) > 1 {
networks = make([]Network, len(m.networks))
networks = make([]mtglib.Network, len(m.networks))
copy(networks, m.networks)
rand.Shuffle(len(m.networks), func(i, j int) {
@@ -59,7 +60,7 @@ func (m multiNetwork) MakeHTTPClient(
return m.networks[0].MakeHTTPClient(dialFunc)
}
func Join(networks ...Network) (Network, error) {
func Join(networks ...mtglib.Network) (mtglib.Network, error) {
if len(networks) == 0 {
return nil, errors.New("cannot join no networks")
}
network/v2/network.go
+6 -1
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib"
)
type network struct {
@@ -70,11 +71,15 @@ func New(
tcpTimeout,
httpTimeout,
idleTimeout time.Duration,
) Network {
) mtglib.Network {
if dnsResolver == nil {
dnsResolver = net.DefaultResolver
}
if userAgent == "" {
userAgent = UserAgent
}
return &network{
Dialer: net.Dialer{
Timeout: tcpTimeout,
network/v2/proxy_network.go
+3 -2
View File
@@ -6,11 +6,12 @@ import (
"net/url"
"github.com/9seconds/mtg/v2/essentials"
"github.com/9seconds/mtg/v2/mtglib"
"golang.org/x/net/proxy"
)
type proxyNetwork struct {
Network
mtglib.Network
client proxy.ContextDialer
}
@@ -23,7 +24,7 @@ func (p proxyNetwork) DialContext(ctx context.Context, network, address string)
return essentials.WrapNetConn(conn), nil
}
func NewProxyNetwork(base Network, proxyURL *url.URL) (*proxyNetwork, error) {
func NewProxyNetwork(base mtglib.Network, proxyURL *url.URL) (*proxyNetwork, error) {
socks, err := proxy.FromURL(proxyURL, base.NativeDialer())
if err != nil {
return nil, fmt.Errorf("cannot build proxy dialer: %w", err)
network/v2/socks_proxy_test.go
+3 -2
View File
@@ -6,6 +6,7 @@ import (
"sync"
"testing"
"github.com/9seconds/mtg/v2/mtglib"
"github.com/9seconds/mtg/v2/network/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -17,7 +18,7 @@ type SocksProxyTestSuite struct {
EchoServerTestSuite
wg sync.WaitGroup
baseNetwork network.Network
baseNetwork mtglib.Network
noAuthURL *url.URL
authURL *url.URL
@@ -85,7 +86,7 @@ func (suite *SocksProxyTestSuite) TestRead() {
for name, proxies := range testData {
suite.T().Run(name, func(t *testing.T) {
proxyNetworks := []network.Network{}
proxyNetworks := []mtglib.Network{}
for _, u := range proxies {
value, err := network.NewProxyNetwork(suite.baseNetwork, u)