diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..d0f97cb --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,4 @@ +--- + +fixes: + - "github.com/9seconds/mtg/v2/::" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4bd3b39..c122a34 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,8 +30,6 @@ jobs: strategy: matrix: go_version: - - ~1.14 - - ~1.15 - ^1.16 steps: - name: Checkout diff --git a/.gitignore b/.gitignore index cb7ca4d..ef266e3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ mtg vendor/ ccbuilds/ .bin/ +coverage.txt diff --git a/.golangci.toml b/.golangci.toml index 04dcc43..40560ac 100644 --- a/.golangci.toml +++ b/.golangci.toml @@ -10,4 +10,4 @@ format = "colored-line-number" [linters] enable-all = true -disable = ["gochecknoglobals", "gas", "gomnd", "goerr113", "exhaustivestruct"] +disable = ["gochecknoglobals", "gas", "goerr113", "exhaustivestruct"] diff --git a/Dockerfile b/Dockerfile index 506f222..4a71943 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,8 @@ RUN set -x \ FROM scratch ENTRYPOINT ["/mtg"] -ENV MTG_BIND=0.0.0.0:3128 \ - MTG_STATS_BIND=0.0.0.0:3129 -EXPOSE 3128 3129 +CMD ["run", "/config.toml"] COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=build /app/mtg /mtg +COPY --from=build /app/example.config.toml /config.toml diff --git a/Makefile b/Makefile index 3f84fba..fff4788 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ GOLANGCI_LINT_VERSION := v1.37.1 VERSION_GO := $(shell go version) VERSION_DATE := $(shell date -Ru) VERSION_TAG := $(shell git describe --tags --always) -COMMON_BUILD_FLAGS := -mod=readonly -ldflags="-s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'" +COMMON_BUILD_FLAGS := -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'" GOBIN := $(ROOT_DIR)/.bin GOTOOL := env "GOBIN=$(GOBIN)" "PATH=$(ROOT_DIR)/.bin:$(PATH)" @@ -27,7 +27,11 @@ $(APP_NAME): build .PHONY: static static: - @env CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo $(COMMON_BUILD_FLAGS) -o "$(APP_NAME)" + @env CGO_ENABLED=0 GOOS=linux go build \ + $(COMMON_BUILD_FLAGS) \ + -tags netgo \ + -a \ + -o "$(APP_NAME)" $(APP_NAME)-%: GOOS=$(shell echo -n "$@" | sed 's?$(APP_NAME)-??' | cut -f1 -d-) $(APP_NAME)-%: GOARCH=$(shell echo -n "$@" | sed 's?$(APP_NAME)-??' | cut -f2 -d-) @@ -35,6 +39,8 @@ $(APP_NAME)-%: ccbuilds @env "GOOS=$(GOOS)" "GOARCH=$(GOARCH)" \ go build \ $(COMMON_BUILD_FLAGS) \ + -tags netgo \ + -a \ -o "./ccbuilds/$(APP_NAME)-$(GOOS)-$(GOARCH)" .PHONY: ccbuilds @@ -44,13 +50,17 @@ ccbuilds: vendor: go.mod go.sum @$(MOD_ON) go mod vendor +.PHONY: fmt +fmt: + @$(GOTOOL) gofumpt -w -s -extra "$(ROOT_DIR)" + .PHONY: test test: @go test -v ./... .PHONY: citest citest: - @go test -coverprofile=coverage.txt -covermode=atomic -race -v ./... + @go test -coverprofile=coverage.txt -covermode=atomic -parallel 2 -race -v ./... .PHONY: crosscompile crosscompile: $(CC_BINARIES) @@ -74,7 +84,7 @@ doc: @$(GOTOOL) godoc -http 0.0.0.0:10000 .PHONY: install-tools -install-tools: install-tools-lint install-tools-godoc +install-tools: install-tools-lint install-tools-godoc install-tools-gofumpt .PHONY: install-tools-lint install-tools-lint: @@ -87,6 +97,11 @@ install-tools-godoc: @mkdir -p "$(GOBIN)" || true && \ $(GOTOOL) go get -u golang.org/x/tools/cmd/godoc +.PHONY: install-tools-gofumpt +install-tools-gofumpt: + @mkdir -p "$(GOBIN)" || true && \ + $(GOTOOL) go get -u mvdan.cc/gofumpt + .PHONY: update-deps upgrade-deps: $go get -u && go mod tidy diff --git a/README.md b/README.md index 10f0b16..d9cd8a4 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # mtg -Bullshit-free MTPROTO proxy for Telegram +Highly-opionated (ex-bullshit-free) MTPROTO proxy for +[Telegram](https://telegram.org/). -[![Build Status](https://travis-ci.org/9seconds/mtg.svg?branch=master)](https://travis-ci.org/9seconds/mtg) -[![Go Report Card](https://goreportcard.com/badge/github.com/9seconds/mtg)](https://goreportcard.com/report/github.com/9seconds/mtg) -[![Docker Build Status](https://img.shields.io/docker/build/nineseconds/mtg.svg)](https://hub.docker.com/r/nineseconds/mtg/) +[![CI](https://github.com/9seconds/mtg/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/9seconds/mtg/actions/workflows/ci.yaml) +[![codecov](https://codecov.io/gh/9seconds/mtg/branch/master/graph/badge.svg?token=JfdDyGVpT4)](https://codecov.io/gh/9seconds/mtg) +[![Go Reference](https://pkg.go.dev/badge/github.com/9seconds/mtg.svg)](https://pkg.go.dev/github.com/9seconds/mtg) -**Please see a guide on upgrading to 1.0 at the end of this README.** +**If you use v1.0 or upgrade broke you proxy, please read the chapter +[Version 2](#version-2)** -# Rationale +## Rationale There are several available proxies for Telegram MTPROTO available. Here are the most notable: @@ -17,267 +19,322 @@ are the most notable: * [Python](https://github.com/alexbers/mtprotoproxy) * [Erlang](https://github.com/seriyps/mtproto_proxy) -Almost all of them follow the way how official proxy was built. This -includes support of multiple secrets, support of promoted channels, etc. +You can use any of these. They work great and all implementations have +feature parity now. This includes support of adtag, replay attack +protection, domain fronting, faketls, and so on. mtg has a similar +goal: to give a possibility to connect to Telegram in a restricted, +censored environment. But it does it slightly differently in details +that probably matter. -mtg is an implementation in golang which is intended to be: +* **Resource-efficient** + + It has to be resource-efficient. It does not mean that you will see + the smallest memory usage. It means that it will try to use allocated + resources in zero-waste mode, reusing as much memory as possible and + so on. -* **Lightweight** - It has to consume as few resources as possible but not by losing - maintainability. * **Easily deployable** + I strongly believe that Telegram proxies should follow the way of - ShadowSocks: promoted channels is a strange way of doing business - I suppose. I think the only viable way is to have a proxy with - minimum configuration which should work everywhere. + [ShadowSocks](https://shadowsocks.org): promoted channels is a strange + way of doing business I suppose. I think the only viable way is to + have a proxy that can be restored anywhere easily. + * **A single secret** - I think that multiple secrets solve no problems and just complexify + + I think that multiple secrets solve no problems and just complex software. I also believe that in the case of throwout proxies, this - feature is a useless luxury. -* **Minimum docker image size** - Official image is less than 3.5 megabytes. Literally. + the feature is a useless luxury. + +* **No adtag support** + + Please read [Version 2](#version-2) chapter. + * **No management WebUI** + This is an implementation of a simple lightweight proxy. I won't do that. -This proxy supports 2 modes of work: direct connection to Telegram and -promoted channel mode. If you do not need promoted channels, I would -recommend you to go with direct mode: this way is more robust. +* **Proxy chaining** -To run a proxy in direct mode, all you need to do is just provide a -secret. If you do not provide ADTag as a second parameter, promoted -channels mode won't be activated. + mtg has the support of [SOCKS5](https://en.wikipedia.org/wiki/SOCKS) + proxies. So, in theory, you can run this proxy as a frontend + and route traffic via [v2ray](https://www.v2ray.com/), + [Gost](https://docs.ginuerzh.xyz/gost/), + [Trojan](https://trojan-gfw.github.io/trojan/), or any other project + you like. -To get promoted channel, please contact -[@MTProxybot](https://t.me/MTProxybot) and provide generated adtag as a -second parameter. +* **Native blocklist support** + Previously, this was delegated to the [FireHOL](https://firehol.org/) + project or similar ones which track attacks and publish a list of + potentially dangerous IPs. mtg has native support of such blocklists. -# Source code organization +* **Can be used as a library** -There are 2 main branches: + mtg v2 was redesigned in a way so it can be embedded into your + software (written in Golang) with a minimum effort + you can replace + some parts with those you want. -1. `master` branch contains potentially unstable features -2. `stable` branch contains stable version. Usually you want to use this branch. +### Version 2 -# How to build +If you use version 1.x before, you are probably noticed some major +backward non-compatible details: + +1. Configuration file +2. Removed support of adtag + +For the configuration file, please check out the full example in this +repository. It has a lot of comments and most of the options are +optional. We do have only `secret` and `bind-to` sections mandatory. +Other sections in the example configuration file are filled with default +values. + +Adtag support was removed completely. This was done to debloat mtg and +keep it simple and obvious. Hopefully, this goal is achieved and the +source code is clean and straightforward enough. + +I always was quite skeptical about adtag. In my POV, a proxy as a fat +big connectivity point for hundreds of clients is an illusion. If you +work in a censored environment, the first thing that authority does is +IP blocking. For us, it means, those big proxies that can benefit from +having a pinned channel are going to be blocked in a minute. + +Proxy has to be intimate. It has to be shared within a small group as +a family or maybe your college friends. It has to have a small number +of connections and never publicly announced its presence. It has to fly +under the radar. If the proxy is detected, you need to be able to give +a rebirth on a new IP address as soon as possible. I do no think that +having some special channel for such a use case makes any sense. + +But other details like replay attack protection, domain fronting, +accurate FakeTLS implementation, IP blacklisting, and proxy +chaining matter here. If you work in censored perimeter like +[GFW](https://en.wikipedia.org/wiki/Great_Firewall)-protected +country, you probably want to have an MTPROTO proxy as +a frontend that transports traffic via cloaked tunnels +made by [Trojan](https://trojan-gfw.github.io/trojan/), +[Shadowsocks](https://shadowsocks.org), [v2ray](https://www.v2ray.com/), +or [Gost](https://docs.ginuerzh.xyz/gost/). That's why you have to have +the support of chaining as a first-class citizen. + +Yes, this is possible and doable with optional adtag support. But the +truth is that the MTPROTO proxy for Telegram is just a thing that either +work as a normal client (direct mode) or doing some RPC calls in [TL +language](https://core.telegram.org/mtproto/TL) (adtag support). I +understand the intention of the developers and I understand that they +were under high pressure fighting with [RKN](https://rkn.gov.ru/) and +doing TON after that. Nothing is ideal. But for the proxy, it means that +source code is full of complex non-trivial code which is required only +to support a feature that we barely need. + +So, to have a reasonable MTPROTO proxy, adtag support was removed. This +is a rare chance in my career where software v2 debloats a previous +version. It feels so good :) + +### Version 1 and 2 + +I do continue to support both versions 1 and 2. But in a different mode. + +Version 1 is now officially in maintenance mode. It means that I won't +make any new features or improvements there. You can consider a feature +freeze there. No bugs are going to be fixed there except for critical +ones. PRs are welcome though. The goal is to keep it working. It will +get some periodical updates like updates to the new Golang version of +dependencies version bump, but that's mostly it. + +**If you want to have mtg with _adtag support_, please use version 1**. + +Version 2 is going to have all my love, active support, bug fixing, etc. +It is under active development and maintenance. + +This project has several main branches + +1. [`master`](https://github.com/9seconds/mtg/tree/master) branch + contains a bleeding edge. It may potentially have some features + which will break your source code. +2. [`stable`](https://github.com/9seconds/mtg/tree/stable) branch contains + dumps of a master branch when we consider it 'stable'. This is a + branch you probably want to pick. +3. [`v2`](https://github.com/9seconds/mtg/tree/v2) has a development + of the v2.x version. In theory, it is the same as `master` but this + will change when we have v3.x. +4. [`v1`](https://github.com/9seconds/mtg/tree/v1) has a version 1.x. + +## Getting started + +### Download a tool + +#### Download binaries + +Binaries can be downloaded from the release page. Also, you can download +docker image. + +For the current version, please download like ```console -make +docker pull nineseconds/mtg:2 ``` -If you want to build for another platform: +For version 1: ```console -make crosscompile +docker pull nineseconds/mtg:1 ``` -If you want to build Docker image (called `mtg`): +You may also check both [Docker +Hub](https://hub.docker.com/r/nineseconds/mtg/tags) and [Github +Registry](https://github.com/users/9seconds/packages/container/package/mtg). +Please do not choose `latest` or `stable` if you want to avoid +surprises. Always choose some version tag. + +Also, if you have `go` installed, you can always download this tool with `go get`: + +```console +go get github.com/9seconds/mtg/v2 +``` + +#### Build from sources + +```console +git clone https://github.com:9seconds/mtg.git +cd mtg +make static +``` + +or for the docker image: ```console make docker ``` -# Docker image +### Generate secret -Docker follows the same policy as the source code organization: - -- `latest` mirrors the master branch -- `stable` mirrors the stable branch -- tags are for tagged releases +If you already have a secret in Base64 format or that, which starts with `ee`, +you can skip this chapter. Otherwise: ```console -docker pull nineseconds/mtg:latest +$ mtg generate-secret google.com +7ibaERuTSGPH1RdztfYnN4tnb29nbGUuY29t ``` +or + ```console -docker pull nineseconds/mtg:stable +$ mtg generate-secret --hex google.com +ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d ``` +This secret is a keystone for a proxy and your password for a client. +You need to keep it secured. + +We recommend choosing a hostname wisely. Here we have a _google.com_ +but in reality, all providers can easily detect that this is not a +Google. Google has a list of networks it officially uses and your IP +address won't probably belong to it. It is a great idea to hide behind +some domain that has some relation to this IP address. + +For example, you've bought a VPS from [Digital +Ocean](https://www.digitalocean.com/). Then it might be a good idea to +generate a secret for _digitalocean.com_ then. + +### Prepare a configuration file + +Please checkout an example configuration file. All options except of +`secret` and `bind-to` are optional. You can safely have this minimal +configuration file: + +```toml +secret = "ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d" +bind-to = "0.0.0.0:443" +``` + +This is enough to run the whole application. All other +options already have sensible defaults for the app at almost any scale. + +Oh, the configuration is done in [TOML format](https://toml.io/en/). + +### Run a proxy + +Put a binary and a config into your webserver. Just for example, +a binary goes to `/usr/local/bin/mtg` and configuration to `/etc/mtg.toml`. + +Now you can create a systemd unit: + ```console -docker pull nineseconds/mtg:0.10 +$ cat /etc/systemd/system/mtg.service +[Unit] +Description=mtg + +[Service] +ExecStart=/usr/local/bin/mtg run /etc/mtg.toml +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target +$ sudo systemctl daemon-reload +$ sudo systemctl enable mtg +$ sudo systemctl start mtg ``` -# Ansible role - -You can find unofficial Ansible role for mtg here: https://github.com/rlex/ansible-role-mtg -Also, there is another project on Ansible Galaxy: https://galaxy.ansible.com/ivansible/lin_mtproxy - -# Configuration - -To run this tool you need to configure as less as possible. Telegram -clients support 3 different secret types: - -* Simple - basically, it is just a flow of frames ciphered by AES-CTR stream - cipher. -* Secured - the same stream as simple but with some random noise to prevent - statistical analysis of traffic flow. -* FakeTLS - this mode envelops telegram stream in TLS so it looks (in theory) - the same as any TLS1.3 traffic from DPI point of view. - -If you do not have preferences, go with FakeTLS or at least secured. -Simple mode is a little bit naive and traffic flow can be easily -identified as Telegram one. - -Unlike the rest of implementation, mtg is quite strict about the -execution mode: if you run a proxy instance with FakeTLS secret, you -can't connect to it with simple or secured clients. You can't connect -to the proxy with secured secret with FakeTLS key. It forces one mode -of working. So, unfortunately, there is no way how to connect to the -deployed proxy with another secret (if you know how to construct and -convert them). But at the same time, old clients can't connect so they -won't expose the type of the service. - -First, you need to generate a secret: +or you can run a docker image ```console -$ mtg generate-secret simple -52a493bdfb90eea55739eabff2d92a14 +docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --restart=unless-stopped nineseconds/mtg:2 ``` +where _443_ is a host port (a port you want to connect to from a +client), and _3128_ is the one you have in your config in the `bind-to` +section. + +### Access a proxy + +Now you can generate some useful links: + ```console -$ mtg generate-secret secured -ddf05fb7acb549be047a7c585116581418 +$ mtg access /etc/mtg.toml +{ + "ipv4": { + "ip": "x.y.z.a", + "port": 3128, + "tg_url": "tg://proxy?...", + "tg_qrcode": "https://api.qrserver.com/v1/create-qr-code?data...", + "tme_url": "https://t.me/proxy?...", + "tme_qrcode": "https://api.qrserver.com/v1/create-qr-code?data..." + }, + "secret": { + "hex": "...", + "base64": "..." + } +} ``` -```console -$ mtg generate-secret -c google.com tls -ee852380f362a09343efb4690c4e17862e676f6f676c652e636f6d -``` +## Metrics -Or, if you prefer docker: +Out of the box, mtg works with +[statsd](https://github.com/statsd/statsd) and +[Prometheus](https://prometheus.io/). Please check configuration file +example to get how to set this integration up. -```console -$ docker run --rm nineseconds/mtg generate-secret tls -c bing.com -eedf71035a8ed48a623d8e83e66aec4d0562696e672e636f6d -``` +Here goes a list of metrics with their types but without a prefix. -## Antireplay cache +| Name | Type | Tags | Description | +|-----------------------------|---------|----------------------------------|--------------------------------------------------------------------------------------------| +| client_connections | gauge | `ip_family` | Count of processing client connections. | +| telegram_connections | gauge | `telegram_ip`, `dc` | Count of connections to Telegram servers. | +| domain_fronting_connections | gauge | `ip_family` | Count of connections to fronting domain. | +| telegram_traffic | counter | `telegram_ip`, `dc`, `direction` | Count of bytes, transmitted to/from Telegram. | +| domain_fronting_traffic | counter | `direction` | Count of bytes, transmitted to/from fronting domain. | +| domain_fronting | counter | – | Count of domain fronting events. | +| concurrency_limited | counter | – | Count of events, when client connection was rejected due to concurrency limit. | +| ip_blocklisted | counter | – | Count of events when client connection was rejected because IP was found in the blacklist. | +| replay_attacks | counter | – | Count of detected replay attacks. | -To prevent replay attacks, we have internal storage of first frames -messages for connected clients. These frames are generated randomly -by design and we have the negligible possibility of duplication -(probability is 1/(2^64)) but it could be quite effective to prevent -replays. +Tag meaning: -It is possible to disable this cache. To do that, please explicitly set -its size to 0. - - -## FakeTLS - -If you run this a proxy in faketls mode, this proxy will try to hide -itself cloaking a host provided as a part of the generated secret. It -means that if you cloak google.com then you can curl this proxy and -you'll get a google.com response back. - -mtg proxies L3 traffic. In other words, only TCP, without interfering in -TLS, HTTP or any other high-level protocol. - - -## Environment variables - -It is possible to configure this tool using environment variables. You -can configure any flag but not secret or adtag. Here is the list of -supported environment variables: - -| Environment variable | Corresponding flags | Default value | Description | -|-------------------------------|------------------------------|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `MTG_DEBUG` | `-d`, `--debug` | `false` | Run in debug mode. Usually, you need to run in this mode only if you develop this tool or its maintainer is asking you to provide logs with such verbosity. | -| `MTG_VERBOSE` | `-v`, `--verbose` | `false` | Run in verbose mode. This is way less chatty than debug mode. | -| `MTG_BIND` | `-b`, `--bind` | `0.0.0.0:3128` | Which host/port pair should we bind to (listen on). | -| `MTG_IPV4` | `-4`, `--public-ipv4` | [Autodetect](https://ifconfig.co) | IPv4 address:port of this proxy. This is required if you NAT your proxy or run it in a docker container. In that case, you absolutely need to specify public IPv4 address of the proxy, otherwise either URLs will be broken or proxy could not access Telegram middle proxies. | -| `MTG_IPV6` | `-6`, `--public-ipv6` | [Autodetect](https://ifconfig.co) | IPv6 address:port of this proxy. This is required if you NAT your proxy or run it in a docker container. In that case, you absolutely need to specify public IPv6 address of the proxy, otherwise either URLs will be broken or proxy could not access Telegram middle proxies. | -| `MTG_STATS_BIND` | `-t`, `--stats-bind` | `127.0.0.1:3129` | Which hist:port should we bind the internal statistics HTTP server (Prometheus). | -| `MTG_STATS_NAMESPACE` | `--stats-namespace` | `mtg` | Which namespace should be used for prometheus metrics. | -| `MTG_STATSD_ADDR` | `--statsd-addr` | | host:port of statsd service. No defaults, by default we do not send anything there. | -| `MTG_STATSD_PREFIX` | `--statsd-prefix` | `mtg` | Which bucket prefix we should use. For example, if you set `mtg`, then metric `traffic.ingress` would be send as `mtg.traffic.ingress`. | -| `MTG_STATSD_TAGS_FORMAT` | `--statsd-tags-format` | | Which tags format we should use. By default, we are using default vanilla statsd tags format but if you want to send directly to InfluxDB or Datadog, please specify it there. Possible options are `influxdb` and `datadog`. | -| `MTG_STATSD_TAGS` | `--statsd-tags` | | Which tags should we send to statsd with our metrics. Please specify them as `key=value` pairs. | -| `MTG_BUFFER_WRITE` | `-w`, `--write-buffer` | `32KB` | The size of TCP write buffer in bytes. Write buffer is the buffer for messages which are going from client to Telegram. | -| `MTG_BUFFER_READ` | `-r`, `--read-buffer` | `32KB` | The size of TCP read buffer in bytes. Read buffer is the buffer for messages from Telegram to client. | -| `MTG_ANTIREPLAY_MAXSIZE` | `--anti-replay-max-size` | `128MB` | Max size of antireplay cache. | -| `MTG_CLOAK_PORT` | `--cloak-port` | `443` | Which port we should use to connect to cloaked host in FakeTLS mode. | -| `MTG_MULTIPLEX_PERCONNECTION` | `--multiplex-per-connection` | `50` | How many client connections can share a single Telegram connection in adtag mode | -| `MTG_NTP_SERVERS` | `--ntp-server` | default pool | A list of NTP servers to use. | -| `MTG_PREFER_DIRECT_IP` | `--prefer-ip` | `ipv6` | Which IP protocol to prefer if possible. Works mostly in direct mode. | - - -Usually you want to modify only read/write buffer sizes. If you feel -that proxy is slow, try to increase both sizes giving more priority to -read buffer. - -Unfortunately, MTPROTO proxy protocol does not allow us to use splice -or any other neat tricks how to eliminate the need of copying data into -userspace. - -# How to run the tool - -Now run the tool: - -```console -$ mtg run -``` - -How to run the tool with ADTag: - -```console -$ mtg run -``` - -This tool will listen on port 3128 by default with the given secret. - - -# oneliner to run this proxy - -Please ensure that docker is installed. After that just execute - -```console -curl -sfL --compressed https://raw.githubusercontent.com/9seconds/mtg/master/run.sh | bash -``` - - -# statsd integration - -mtg provides an integration with statsd, you can enable it with command -line interface. To enable it, you have to provide IP address of statsd -service. - -Out of the box, mtg supports 2 additional dialects: [InfluxDB](https://www.influxdata.com/blog/getting-started-with-sending-statsd-metrics-to-telegraf-influxdb/) -and [Datadog](https://docs.datadoghq.com/developers/dogstatsd/). - -All metrics are gauges. Here is the list of metrics and their meaning: - -| Metric name | Unit | Description | -|------------------------|---------|--------------------------------------------| -| `connections` | number | The number of active connections. | -| `telegram_connections` | number | The number of active telegram connections. | -| `crashes` | number | An amount of crashes in client handlers. | -| `traffic.egress` | bytes | Traffic from the start of application. | -| `replay_attacks` | number | The number of prevented replay attacks. | - -All metrics are prefixed with given prefix. Default prefix is `mtg`. -Also, metrics provide tags (ipv4/ipv6, dc indexes etc). - - -# Prometheus integration - -[Prometheus](https://prometheus.io) integration comes out of -the box, you do not need to setup anything special. - - -# Upgrade to 1.0 - -Version 1.0 breaks compatibility with previous versions so please read -this chapter carefully: - -1. mtg now uses subcommands. Please use `mtg run` instead of just - `mtg` to run a proxy. -2. Options which set host and port separately were removed in a - favor of fused `host:port` options. -3. Own stats server was removed. Prometheus endpoint is moved to - default stats endpoint. -4. It is possible to connect to this proxy only with a secret which - was used to run it. So, no backward compatibility of clients. -5. Multiplexing involves connectivity with middle proxies and involves - the most complex code path of this proxy. To avoid potential bugs, - we still recommend using direct mode. +| Name | Values | Description | +|-------------|------------------------|-----------------------------------------------| +| ip_family | ipv4, ipv6 | A version of the IP protocol. | +| dc | | A number of the Telegram DC for a connection. | +| telegram_ip | | IP address of the Telegram server. | +| direction | to_client, from_client | A direction of the traffic flow. | diff --git a/antireplay/cache.go b/antireplay/cache.go deleted file mode 100644 index dab3af4..0000000 --- a/antireplay/cache.go +++ /dev/null @@ -1,36 +0,0 @@ -package antireplay - -import "github.com/VictoriaMetrics/fastcache" - -var ( - prefixObfuscated2 = []byte{0x00} - prefixTLS = []byte{0x01} -) - -type cache struct { - data *fastcache.Cache -} - -func (c cache) AddObfuscated2(data []byte) { - c.data.Set(keyObfuscated2(data), nil) -} - -func (c cache) AddTLS(data []byte) { - c.data.Set(keyTLS(data), nil) -} - -func (c cache) HasObfuscated2(data []byte) bool { - return c.data.Has(keyObfuscated2(data)) -} - -func (c cache) HasTLS(data []byte) bool { - return c.data.Has(keyTLS(data)) -} - -func keyObfuscated2(data []byte) []byte { - return append(prefixObfuscated2, data...) -} - -func keyTLS(data []byte) []byte { - return append(prefixTLS, data...) -} diff --git a/antireplay/init.go b/antireplay/init.go index 4ad9950..eb4eeb1 100644 --- a/antireplay/init.go +++ b/antireplay/init.go @@ -1,30 +1,17 @@ +// Antireplay package has cache implementations that are effective +// against replay attacks. +// +// To understand more about replay attacks, please read documentation +// for mtglib.AntiReplayCache interface. This package has a list of some +// implementations of this interface. package antireplay -import ( - "sync" +const ( + // DefaultStableBloomFilterMaxSize is a recommended byte size for a + // stable bloom filter. + DefaultStableBloomFilterMaxSize = 1024 * 1024 // 1MiB - "github.com/9seconds/mtg/config" - "github.com/VictoriaMetrics/fastcache" + // DefaultStableBloomFilterErrorRate is a recommended default error + // rate for a stable bloom filter. + DefaultStableBloomFilterErrorRate = 0.001 ) - -type CacheInterface interface { - AddObfuscated2([]byte) - AddTLS([]byte) - HasObfuscated2([]byte) bool - HasTLS([]byte) bool -} - -var ( - Cache CacheInterface - initOnce sync.Once -) - -func Init() { - initOnce.Do(func() { - if config.C.AntiReplayMaxSize == 0 { - Cache = nilCache{} - } else { - Cache = cache{fastcache.New(config.C.AntiReplayMaxSize)} - } - }) -} diff --git a/antireplay/nilcache.go b/antireplay/nilcache.go deleted file mode 100644 index baa0947..0000000 --- a/antireplay/nilcache.go +++ /dev/null @@ -1,8 +0,0 @@ -package antireplay - -type nilCache struct{} - -func (n nilCache) AddObfuscated2(_ []byte) {} -func (n nilCache) AddTLS(_ []byte) {} -func (n nilCache) HasObfuscated2(_ []byte) bool { return false } -func (n nilCache) HasTLS(_ []byte) bool { return false } diff --git a/antireplay/noop.go b/antireplay/noop.go new file mode 100644 index 0000000..e4e7cf4 --- /dev/null +++ b/antireplay/noop.go @@ -0,0 +1,14 @@ +package antireplay + +import "github.com/9seconds/mtg/v2/mtglib" + +type noop struct{} + +func (n noop) SeenBefore(_ []byte) bool { return false } + +// NewNoop returns an implementation that does nothing. A corresponding +// method always returns false, so this cache accepts everything you +// pass to it. +func NewNoop() mtglib.AntiReplayCache { + return noop{} +} diff --git a/antireplay/noop_test.go b/antireplay/noop_test.go new file mode 100644 index 0000000..b78c48d --- /dev/null +++ b/antireplay/noop_test.go @@ -0,0 +1,26 @@ +package antireplay_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite +} + +func (suite *NoopTestSuite) TestOp() { + filter := antireplay.NewNoop() + + suite.False(filter.SeenBefore([]byte{1, 2, 3})) + suite.False(filter.SeenBefore([]byte{4, 5, 6})) + suite.False(filter.SeenBefore([]byte{1, 2, 3})) + suite.False(filter.SeenBefore([]byte{4, 5, 6})) +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go new file mode 100644 index 0000000..61f9934 --- /dev/null +++ b/antireplay/stable_bloom_filter.go @@ -0,0 +1,51 @@ +package antireplay + +import ( + "sync" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/OneOfOne/xxhash" + boom "github.com/tylertreat/BoomFilters" +) + +type stableBloomFilter struct { + filter boom.StableBloomFilter + mutex sync.Mutex +} + +func (s *stableBloomFilter) SeenBefore(digest []byte) bool { + s.mutex.Lock() + defer s.mutex.Unlock() + + return s.filter.TestAndAdd(digest) +} + +// NewStableBloomFilter returns an implementation of AntiReplayCache +// based on stable bloom filter. +// +// http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf +// +// The basic idea of a stable bloom filter is quite simple: each time +// when you set a new element, you randomly reset P elements. There is a +// hardcore math which proves that if you choose this P correctly, you +// can maintain the same error rate for a stream of elements. +// +// byteSize is the number of bytes you want to give to a bloom filter. +// errorRate is desired false-positive error rate. If you want to use +// default values, please pass 0 for byteSize and <0 for errorRate. +func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { + if byteSize == 0 { + byteSize = DefaultStableBloomFilterMaxSize + } + + if errorRate < 0 { + errorRate = DefaultStableBloomFilterErrorRate + } + + sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd + sf.SetHash(xxhash.New64()) + + return &stableBloomFilter{ + filter: *sf, + } +} diff --git a/antireplay/stable_bloom_filter_test.go b/antireplay/stable_bloom_filter_test.go new file mode 100644 index 0000000..accb590 --- /dev/null +++ b/antireplay/stable_bloom_filter_test.go @@ -0,0 +1,26 @@ +package antireplay_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/stretchr/testify/suite" +) + +type StableBloomFilterTestSuite struct { + suite.Suite +} + +func (suite *StableBloomFilterTestSuite) TestOp() { + filter := antireplay.NewStableBloomFilter(500, 0.001) + + suite.False(filter.SeenBefore([]byte{1, 2, 3})) + suite.False(filter.SeenBefore([]byte{4, 5, 6})) + suite.True(filter.SeenBefore([]byte{1, 2, 3})) + suite.True(filter.SeenBefore([]byte{4, 5, 6})) +} + +func TestStableBloomFilter(t *testing.T) { + t.Parallel() + suite.Run(t, &StableBloomFilterTestSuite{}) +} diff --git a/cli/generate.go b/cli/generate.go deleted file mode 100644 index 5811798..0000000 --- a/cli/generate.go +++ /dev/null @@ -1,26 +0,0 @@ -package cli - -import ( - "crypto/rand" - "encoding/hex" - - "github.com/9seconds/mtg/config" -) - -func Generate(secretType, hostname string) { - data := make([]byte, config.SimpleSecretLength) - if _, err := rand.Read(data); err != nil { - panic(err) - } - - secret := hex.EncodeToString(data) - - switch secretType { - case "simple": - PrintStdout(secret) - case "secured": - PrintStdout("dd" + secret) - default: - PrintStdout("ee" + secret + hex.EncodeToString([]byte(hostname))) - } -} diff --git a/cli/proxy.go b/cli/proxy.go deleted file mode 100644 index 038239e..0000000 --- a/cli/proxy.go +++ /dev/null @@ -1,101 +0,0 @@ -package cli - -import ( - "net" - "os" - "time" - - "github.com/9seconds/mtg/antireplay" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/faketls" - "github.com/9seconds/mtg/hub" - "github.com/9seconds/mtg/ntp" - "github.com/9seconds/mtg/obfuscated2" - "github.com/9seconds/mtg/proxy" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/utils" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func Proxy() error { // nolint: funlen,cyclop - ctx := utils.GetSignalContext() - - atom := zap.NewAtomicLevel() - - switch { - case config.C.Debug: - atom.SetLevel(zapcore.DebugLevel) - case config.C.Verbose: - atom.SetLevel(zapcore.InfoLevel) - default: - atom.SetLevel(zapcore.ErrorLevel) - } - - encoderCfg := zap.NewProductionEncoderConfig() - logger := zap.New(zapcore.NewCore( - zapcore.NewJSONEncoder(encoderCfg), - zapcore.Lock(os.Stderr), - atom, - )) - - zap.ReplaceGlobals(logger) - defer logger.Sync() // nolint: errcheck - - if err := config.InitPublicAddress(ctx); err != nil { - Fatal(err) - } - - zap.S().Debugw("Configuration", "config", config.Printable()) - - if config.C.MiddleProxyMode() { - zap.S().Infow("Use middle proxy connection to Telegram") - - diff, err := ntp.Fetch() - if err != nil { - Fatal("Cannot fetch time data from NTP") - } - - if diff > time.Second { - Fatal("Your local time is skewed and drift is bigger than a second. Please sync your time.") - } - - go ntp.AutoUpdate() - } else { - zap.S().Infow("Use direct connection to Telegram") - } - - PrintJSONStdout(config.GetURLs()) - - if err := stats.Init(ctx); err != nil { - Fatal(err) - } - - antireplay.Init() - telegram.Init() - hub.Init(ctx) - - proxyListener, err := net.Listen("tcp", config.C.Bind.String()) - if err != nil { - Fatal(err) - } - - go func() { - <-ctx.Done() - proxyListener.Close() - }() - - app := &proxy.Proxy{ - Logger: zap.S().Named("proxy"), - Context: ctx, - ClientProtocolMaker: obfuscated2.MakeClientProtocol, - } - if config.C.SecretMode == config.SecretModeTLS { - app.ClientProtocolMaker = faketls.MakeClientProtocol - } - - app.Serve(proxyListener) - - return nil -} diff --git a/cli/utils.go b/cli/utils.go deleted file mode 100644 index 948d875..0000000 --- a/cli/utils.go +++ /dev/null @@ -1,43 +0,0 @@ -package cli - -import ( - "encoding/json" - "fmt" - "io" - "os" -) - -func Fatal(arg interface{}) { - if value, ok := arg.(error); ok { - arg = fmt.Errorf("fatal error: %+v", value) // nolint: errorlint - } - - PrintStderr(arg) - os.Exit(1) -} - -func PrintStderr(args ...interface{}) { - fmt.Fprintln(os.Stderr, args...) -} - -func PrintStdout(args ...interface{}) { - fmt.Println(args...) // nolint: forbidigo -} - -func PrintJSONStderr(data interface{}) { - printJSON(os.Stderr, data) -} - -func PrintJSONStdout(data interface{}) { - printJSON(os.Stdout, data) -} - -func printJSON(writer io.Writer, data interface{}) { - encoder := json.NewEncoder(writer) - encoder.SetEscapeHTML(false) - encoder.SetIndent("", " ") - - if err := encoder.Encode(data); err != nil { - panic(err) - } -} diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 5e97f86..0000000 --- a/config/config.go +++ /dev/null @@ -1,312 +0,0 @@ -package config - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "math" - "net" - - "github.com/alecthomas/units" - statsd "github.com/smira/go-statsd" - "go.uber.org/zap" -) - -type SecretMode uint8 - -func (s SecretMode) String() string { - switch s { - case SecretModeSimple: - return "simple" - case SecretModeSecured: - return "secured" - case SecretModeTLS: - return "tls" - } - - return "tls" -} - -const ( - SecretModeSimple SecretMode = iota - SecretModeSecured - SecretModeTLS -) - -type PreferIP uint8 - -const ( - PreferIPv4 PreferIP = iota - PreferIPv6 -) - -const SimpleSecretLength = 16 - -type OptionType uint8 - -const ( - OptionTypeDebug OptionType = iota - OptionTypeVerbose - - OptionTypePreferIP - - OptionTypeBind - OptionTypePublicIPv4 - OptionTypePublicIPv6 - - OptionTypeStatsBind - OptionTypeStatsNamespace - OptionTypeStatsdAddress - OptionTypeStatsdTagsFormat - OptionTypeStatsdTags - - OptionTypeWriteBufferSize - OptionTypeReadBufferSize - - OptionTypeCloakPort - - OptionTypeAntiReplayMaxSize - - OptionTypeMultiplexPerConnection - - OptionTypeNTPServers - - OptionTypeSecret - OptionTypeAdtag -) - -type Config struct { - Bind *net.TCPAddr `json:"bind"` - PublicIPv4 *net.TCPAddr `json:"public_ipv4"` - PublicIPv6 *net.TCPAddr `json:"public_ipv6"` - StatsBind *net.TCPAddr `json:"stats_bind"` - StatsdAddr *net.TCPAddr `json:"stats_addr"` - StatsdTagsFormat *statsd.TagFormat `json:"statsd_tags_format"` - - StatsNamespace string `json:"stats_namespace"` - CloakHost string `json:"cloak_host"` - StatsdTags map[string]string `json:"statsd_tags"` - - WriteBuffer int `json:"write_buffer"` - ReadBuffer int `json:"read_buffer"` - CloakPort int `json:"cloak_port"` - - AntiReplayMaxSize int `json:"anti_replay_max_size"` - - MultiplexPerConnection int `json:"multiplex_per_connection"` - - Debug bool `json:"debug"` - Verbose bool `json:"verbose"` - SecretMode SecretMode `json:"secret_mode"` - PreferIP PreferIP `json:"prefer_ip"` - NTPServers []string `json:"ntp_servers"` - - Secret []byte `json:"secret"` - AdTag []byte `json:"adtag"` -} - -func (c *Config) ClientReadBuffer() int { - return c.ReadBuffer -} - -func (c *Config) ClientWriteBuffer() int { - return c.WriteBuffer -} - -func (c *Config) MiddleProxyMode() bool { - return len(c.AdTag) > 0 -} - -func (c *Config) ProxyReadBuffer() int { - value := c.ReadBuffer - - if c.MiddleProxyMode() { - value = c.adjustProxyValue(value) - } - - return value -} - -func (c *Config) ProxyWriteBuffer() int { - value := c.WriteBuffer - - if c.MiddleProxyMode() { - value = c.adjustProxyValue(value) - } - - return value -} - -func (c *Config) adjustProxyValue(value int) int { - if c.MultiplexPerConnection == 0 { - return value - } - - fvalue := float64(value) - - newValue := fvalue * 2 * math.Log(float64(c.MultiplexPerConnection)) - newValue = math.Ceil(newValue) - newValue = math.Max(fvalue, newValue) - - return int(newValue) -} - -type Opt struct { - Option OptionType - Value interface{} -} - -var C = Config{} - -func Init(options ...Opt) error { // nolint: gocyclo, funlen, cyclop - for _, opt := range options { - switch opt.Option { - case OptionTypeDebug: - C.Debug = opt.Value.(bool) - case OptionTypeVerbose: - C.Verbose = opt.Value.(bool) - case OptionTypePreferIP: - value := opt.Value.(string) - switch value { - case "ipv4": - C.PreferIP = PreferIPv4 - case "ipv6": - C.PreferIP = PreferIPv6 - default: - return fmt.Errorf("incorrect direct IP mode %s", value) - } - case OptionTypeBind: - C.Bind = opt.Value.(*net.TCPAddr) - case OptionTypePublicIPv4: - C.PublicIPv4 = opt.Value.(*net.TCPAddr) - if C.PublicIPv4 == nil { - C.PublicIPv4 = &net.TCPAddr{} - } - case OptionTypePublicIPv6: - C.PublicIPv6 = opt.Value.(*net.TCPAddr) - if C.PublicIPv6 == nil { - C.PublicIPv6 = &net.TCPAddr{} - } - case OptionTypeStatsBind: - C.StatsBind = opt.Value.(*net.TCPAddr) - case OptionTypeStatsNamespace: - C.StatsNamespace = opt.Value.(string) - case OptionTypeStatsdAddress: - C.StatsdAddr = opt.Value.(*net.TCPAddr) - case OptionTypeStatsdTagsFormat: - value := opt.Value.(string) - switch value { - case "datadog": - C.StatsdTagsFormat = statsd.TagFormatDatadog - case "influxdb": - C.StatsdTagsFormat = statsd.TagFormatInfluxDB - default: - return fmt.Errorf("incorrect statsd tag %s", value) - } - case OptionTypeStatsdTags: - C.StatsdTags = opt.Value.(map[string]string) - case OptionTypeWriteBufferSize: - C.WriteBuffer = int(opt.Value.(units.Base2Bytes)) - case OptionTypeReadBufferSize: - C.ReadBuffer = int(opt.Value.(units.Base2Bytes)) - case OptionTypeCloakPort: - C.CloakPort = int(opt.Value.(uint16)) - case OptionTypeAntiReplayMaxSize: - C.AntiReplayMaxSize = int(opt.Value.(units.Base2Bytes)) - case OptionTypeMultiplexPerConnection: - C.MultiplexPerConnection = int(opt.Value.(uint)) - case OptionTypeNTPServers: - C.NTPServers = opt.Value.([]string) - if len(C.NTPServers) == 0 { - return errors.New("ntp server list is empty") - } - case OptionTypeSecret: - C.Secret = opt.Value.([]byte) - case OptionTypeAdtag: - C.AdTag = opt.Value.([]byte) - default: - return fmt.Errorf("unknown tag %v", opt.Option) - } - } - - switch { - case len(C.Secret) == 1+SimpleSecretLength && bytes.HasPrefix(C.Secret, []byte{0xdd}): - C.SecretMode = SecretModeSecured - C.Secret = bytes.TrimPrefix(C.Secret, []byte{0xdd}) - case len(C.Secret) > SimpleSecretLength && bytes.HasPrefix(C.Secret, []byte{0xee}): - C.SecretMode = SecretModeTLS - secret := bytes.TrimPrefix(C.Secret, []byte{0xee}) - C.Secret = secret[:SimpleSecretLength] - C.CloakHost = string(secret[SimpleSecretLength:]) - case len(C.Secret) == SimpleSecretLength: - C.SecretMode = SecretModeSimple - default: - return errors.New("incorrect secret") - } - - if C.MultiplexPerConnection == 0 { - return errors.New("cannot use 0 clients per connection for multiplexing") - } - - if C.CloakHost != "" { - if _, err := net.LookupHost(C.CloakHost); err != nil { - zap.S().Warnw("Cannot resolve address of host", "hostname", C.CloakHost, "error", err) - } - } - - return nil -} - -func InitPublicAddress(ctx context.Context) error { - if C.PublicIPv4.Port == 0 { - C.PublicIPv4.Port = C.Bind.Port - } - - if C.PublicIPv6.Port == 0 { - C.PublicIPv6.Port = C.Bind.Port - } - - foundAddress := C.PublicIPv4.IP != nil || C.PublicIPv6.IP != nil - - if C.PublicIPv4.IP == nil { - ip, err := getGlobalIPv4(ctx) - if err != nil { - zap.S().Warnw("Cannot resolve public address", "error", err) - } else { - C.PublicIPv4.IP = ip - foundAddress = true - } - } - - if C.PublicIPv6.IP == nil { - ip, err := getGlobalIPv6(ctx) - if err != nil { - zap.S().Warnw("Cannot resolve public address", "error", err) - } else { - C.PublicIPv6.IP = ip - foundAddress = true - } - } - - if !foundAddress { - return errors.New("cannot resolve any public address") - } - - return nil -} - -func Printable() interface{} { - data, err := json.Marshal(C) - if err != nil { - panic(err) - } - - rv := map[string]interface{}{} - if err := json.Unmarshal(data, &rv); err != nil { - panic(err) - } - - return rv -} diff --git a/config/global_ips.go b/config/global_ips.go deleted file mode 100644 index 0dd78da..0000000 --- a/config/global_ips.go +++ /dev/null @@ -1,78 +0,0 @@ -package config - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "strings" - "time" -) - -const ( - ifconfigAddress = "https://ifconfig.co/ip" - ifconfigTimeout = 10 * time.Second -) - -func getGlobalIPv4(ctx context.Context) (net.IP, error) { - ip, err := fetchIP(ctx, "tcp4") - if err != nil || ip.To4() == nil { - return nil, fmt.Errorf("cannot find public ipv4 address: %w", err) - } - - return ip, nil -} - -func getGlobalIPv6(ctx context.Context) (net.IP, error) { - ip, err := fetchIP(ctx, "tcp6") - if err != nil || ip.To4() != nil { - return nil, fmt.Errorf("cannot find public ipv6 address: %w", err) - } - - return ip, nil -} - -func fetchIP(ctx context.Context, network string) (net.IP, error) { - dialer := &net.Dialer{FallbackDelay: -1} - client := &http.Client{ - Jar: nil, - Timeout: ifconfigTimeout, - Transport: &http.Transport{ - DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { - return dialer.DialContext(ctx, network, addr) - }, - }, - } - - req, err := http.NewRequest("GET", ifconfigAddress, nil) - if err != nil { - return nil, fmt.Errorf("cannot create a request: %w", err) - } - - resp, err := client.Do(req.WithContext(ctx)) - if err != nil { - if resp != nil { - io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck - } - - return nil, fmt.Errorf("cannot perform a request: %w", err) - } - - defer resp.Body.Close() - - respDataBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("cannot read response body: %w", err) - } - - respData := strings.TrimSpace(string(respDataBytes)) - - ip := net.ParseIP(respData) - if ip == nil { - return nil, fmt.Errorf("ifconfig.co returns incorrect IP %s", respData) - } - - return ip, nil -} diff --git a/config/urls.go b/config/urls.go deleted file mode 100644 index f657e82..0000000 --- a/config/urls.go +++ /dev/null @@ -1,99 +0,0 @@ -package config - -import ( - "encoding/hex" - "net" - "net/url" - "strconv" -) - -type URLs struct { - TG string `json:"tg_url"` - TMe string `json:"tme_url"` - TGQRCode string `json:"tg_qrcode"` - TMeQRCode string `json:"tme_qrcode"` -} - -type IPURLs struct { - IPv4 *URLs `json:"ipv4,omitempty"` - IPv6 *URLs `json:"ipv6,omitempty"` - BotSecret string `json:"secret_for_mtproxybot"` -} - -func GetURLs() (urls IPURLs) { - secret := "" - - switch C.SecretMode { - case SecretModeSimple: - secret = hex.EncodeToString(C.Secret) - case SecretModeSecured: - secret = "dd" + hex.EncodeToString(C.Secret) - case SecretModeTLS: - secret = "ee" + hex.EncodeToString(C.Secret) + hex.EncodeToString([]byte(C.CloakHost)) - } - - if C.PublicIPv4.IP != nil { - urls.IPv4 = makeURLs(C.PublicIPv4, secret) - } - - if C.PublicIPv6.IP != nil { - urls.IPv6 = makeURLs(C.PublicIPv6, secret) - } - - urls.BotSecret = hex.EncodeToString(C.Secret) - - return urls -} - -func makeURLs(addr *net.TCPAddr, secret string) *URLs { - urls := &URLs{} - - values := url.Values{} - values.Set("server", addr.IP.String()) - values.Set("port", strconv.Itoa(addr.Port)) - values.Set("secret", secret) - - return &URLs{ - TG: makeTGURL(values), - TMe: makeTMeURL(values), - TGQRCode: makeQRCodeURL(urls.TG), - TMeQRCode: makeQRCodeURL(urls.TMe), - } -} - -func makeTGURL(values url.Values) string { - tgURL := url.URL{ - Scheme: "tg", - Host: "proxy", - RawQuery: values.Encode(), - } - - return tgURL.String() -} - -func makeTMeURL(values url.Values) string { - tMeURL := url.URL{ - Scheme: "https", - Host: "t.me", - Path: "proxy", - RawQuery: values.Encode(), - } - - return tMeURL.String() -} - -func makeQRCodeURL(data string) string { - qr := url.URL{ - Scheme: "https", - Host: "api.qrserver.com", - Path: "v1/create-qr-code", - } - - values := url.Values{} - values.Set("qzone", "4") - values.Set("format", "svg") - values.Set("data", data) - qr.RawQuery = values.Encode() - - return qr.String() -} diff --git a/conntypes/acks.go b/conntypes/acks.go deleted file mode 100644 index d3dbd8d..0000000 --- a/conntypes/acks.go +++ /dev/null @@ -1,6 +0,0 @@ -package conntypes - -type ConnectionAcks struct { - Simple bool - Quick bool -} diff --git a/conntypes/dc.go b/conntypes/dc.go deleted file mode 100644 index 3cf2850..0000000 --- a/conntypes/dc.go +++ /dev/null @@ -1,5 +0,0 @@ -package conntypes - -type DC int16 - -const DCDefaultIdx DC = 1 diff --git a/conntypes/id.go b/conntypes/id.go deleted file mode 100644 index 1aa482c..0000000 --- a/conntypes/id.go +++ /dev/null @@ -1,24 +0,0 @@ -package conntypes - -import ( - "crypto/rand" - "encoding/hex" -) - -const ConnIDLength = 8 - -type ConnID [ConnIDLength]byte - -func (c ConnID) String() string { - return hex.EncodeToString(c[:]) -} - -func NewConnID() ConnID { - var id ConnID - - if _, err := rand.Read(id[:]); err != nil { - panic(err) - } - - return id -} diff --git a/conntypes/packet.go b/conntypes/packet.go deleted file mode 100644 index fa72022..0000000 --- a/conntypes/packet.go +++ /dev/null @@ -1,3 +0,0 @@ -package conntypes - -type Packet []byte diff --git a/conntypes/protocol.go b/conntypes/protocol.go deleted file mode 100644 index b89ac65..0000000 --- a/conntypes/protocol.go +++ /dev/null @@ -1,22 +0,0 @@ -package conntypes - -type ConnectionProtocol uint8 - -func (c ConnectionProtocol) String() string { - switch c { - case ConnectionProtocolAny: - return "any" - case ConnectionProtocolIPv4: - return "ipv4" - case ConnectionProtocolIPv6: - return "ipv6" - } - - return "ipv6" -} - -const ( - ConnectionProtocolIPv4 ConnectionProtocol = 1 - ConnectionProtocolIPv6 = ConnectionProtocolIPv4 << 1 - ConnectionProtocolAny = ConnectionProtocolIPv4 | ConnectionProtocolIPv6 -) diff --git a/conntypes/type.go b/conntypes/type.go deleted file mode 100644 index c41d42f..0000000 --- a/conntypes/type.go +++ /dev/null @@ -1,29 +0,0 @@ -package conntypes - -type ConnectionType uint8 - -const ( - ConnectionTypeUnknown ConnectionType = iota - ConnectionTypeAbridged - ConnectionTypeIntermediate - ConnectionTypeSecure -) - -var ( - ConnectionTagAbridged = []byte{0xef, 0xef, 0xef, 0xef} - ConnectionTagIntermediate = []byte{0xee, 0xee, 0xee, 0xee} - ConnectionTagSecure = []byte{0xdd, 0xdd, 0xdd, 0xdd} -) - -func (t ConnectionType) Tag() []byte { - switch t { - case ConnectionTypeAbridged: - return ConnectionTagAbridged - case ConnectionTypeIntermediate: - return ConnectionTagIntermediate - case ConnectionTypeSecure, ConnectionTypeUnknown: - return ConnectionTagSecure - } - - return ConnectionTagSecure -} diff --git a/conntypes/wrap_interfaces.go b/conntypes/wrap_interfaces.go deleted file mode 100644 index f21eb91..0000000 --- a/conntypes/wrap_interfaces.go +++ /dev/null @@ -1,14 +0,0 @@ -package conntypes - -import ( - "net" - - "go.uber.org/zap" -) - -type Wrap interface { - Conn() net.Conn - Logger() *zap.SugaredLogger - LocalAddr() *net.TCPAddr - RemoteAddr() *net.TCPAddr -} diff --git a/conntypes/wrap_packet_ack_interfaces.go b/conntypes/wrap_packet_ack_interfaces.go deleted file mode 100644 index 6388f0c..0000000 --- a/conntypes/wrap_packet_ack_interfaces.go +++ /dev/null @@ -1,41 +0,0 @@ -package conntypes - -import "io" - -type PacketAckReader interface { - Read(*ConnectionAcks) (Packet, error) -} - -type PacketAckWriter interface { - Write(Packet, *ConnectionAcks) error -} - -type PacketAckCloser interface { - io.Closer -} - -type PacketAckReadCloser interface { - PacketAckReader - PacketAckCloser -} - -type PacketAckWriteCloser interface { - PacketAckWriter - PacketAckCloser -} - -type PacketAckReadWriter interface { - PacketAckReader - PacketAckWriter -} - -type PacketAckReadWriteCloser interface { - PacketAckReader - PacketAckWriter - PacketAckCloser -} - -type PacketAckFullReadWriteCloser interface { - Wrap - PacketAckReadWriteCloser -} diff --git a/conntypes/wrap_packet_interfaces.go b/conntypes/wrap_packet_interfaces.go deleted file mode 100644 index 6c573d4..0000000 --- a/conntypes/wrap_packet_interfaces.go +++ /dev/null @@ -1,51 +0,0 @@ -package conntypes - -import "io" - -type BasePacketReader interface { - Read() (Packet, error) -} - -type BasePacketWriter interface { - Write(Packet) error -} - -type PacketReader interface { - Wrap - BasePacketReader -} - -type PacketWriter interface { - Wrap - BasePacketWriter -} - -type PacketCloser interface { - Wrap - io.Closer -} - -type PacketReadCloser interface { - Wrap - BasePacketReader - io.Closer -} - -type PacketWriteCloser interface { - Wrap - BasePacketWriter - io.Closer -} - -type PacketReadWriter interface { - Wrap - BasePacketWriter - BasePacketReader -} - -type PacketReadWriteCloser interface { - Wrap - BasePacketWriter - BasePacketReader - io.Closer -} diff --git a/conntypes/wrap_stream_interfaces.go b/conntypes/wrap_stream_interfaces.go deleted file mode 100644 index 5924d66..0000000 --- a/conntypes/wrap_stream_interfaces.go +++ /dev/null @@ -1,56 +0,0 @@ -package conntypes - -import ( - "io" - "time" -) - -type BaseStreamReaderWithTimeout interface { - ReadTimeout([]byte, time.Duration) (int, error) -} - -type BaseStreamWriterWithTimeout interface { - WriteTimeout([]byte, time.Duration) (int, error) -} - -type StreamReader interface { - Wrap - io.Reader - BaseStreamReaderWithTimeout -} - -type StreamWriter interface { - Wrap - io.Writer - BaseStreamWriterWithTimeout -} - -type StreamCloser interface { - Wrap - io.Closer -} - -type StreamReadCloser interface { - Wrap - io.ReadCloser - BaseStreamReaderWithTimeout -} - -type StreamWriteCloser interface { - Wrap - io.WriteCloser - BaseStreamWriterWithTimeout -} - -type StreamReadWriter interface { - Wrap - io.ReadWriter - BaseStreamReaderWithTimeout -} - -type StreamReadWriteCloser interface { - Wrap - io.ReadWriteCloser - BaseStreamReaderWithTimeout - BaseStreamWriterWithTimeout -} diff --git a/events/event_stream.go b/events/event_stream.go new file mode 100644 index 0000000..cc971a6 --- /dev/null +++ b/events/event_stream.go @@ -0,0 +1,108 @@ +package events + +import ( + "context" + "math/rand" + "runtime" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/OneOfOne/xxhash" +) + +// EventStream is a default implementation of the mtglib.EventStream +// interface. +// +// EventStream manages a set of goroutines, observers. Main +// responsibility of the event stream is to route an event to relevant +// observer based on some hash so each observer will have all events +// which belong to some stream id. +// +// Thus, EventStream can spawn many observers. +type EventStream struct { + ctx context.Context + ctxCancel context.CancelFunc + chans []chan mtglib.Event +} + +// Send starts delivering of the message to observer with respect to a +// given context If context is closed, message could be not delivered. +func (e EventStream) Send(ctx context.Context, evt mtglib.Event) { + var chanNo uint32 + + if streamID := evt.StreamID(); streamID != "" { + chanNo = xxhash.ChecksumString32(streamID) + } else { + chanNo = rand.Uint32() + } + + select { + case <-ctx.Done(): + case <-e.ctx.Done(): + case e.chans[int(chanNo)%len(e.chans)] <- evt: + } +} + +// Shutdown stops an event stream pipeline. +func (e EventStream) Shutdown() { + e.ctxCancel() +} + +// NewEventStream builds a new default event stream. +// +// If you give an empty array of observers, then NoopObserver is going +// to be used. If you give many observers, then they will process a +// message concurrently. +func NewEventStream(observerFactories []ObserverFactory) EventStream { + if len(observerFactories) == 0 { + observerFactories = append(observerFactories, NewNoopObserver) + } + + ctx, cancel := context.WithCancel(context.Background()) + rv := EventStream{ + ctx: ctx, + ctxCancel: cancel, + chans: make([]chan mtglib.Event, runtime.NumCPU()), + } + + for i := 0; i < runtime.NumCPU(); i++ { + rv.chans[i] = make(chan mtglib.Event, 1) + + if len(observerFactories) == 1 { + go eventStreamProcessor(ctx, rv.chans[i], observerFactories[0]()) + } else { + go eventStreamProcessor(ctx, rv.chans[i], newMultiObserver(observerFactories)) + } + } + + return rv +} + +func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, observer Observer) { // nolint: cyclop + defer observer.Shutdown() + + for { + select { + case <-ctx.Done(): + return + case evt := <-eventChan: + switch typedEvt := evt.(type) { + case mtglib.EventTraffic: + observer.EventTraffic(typedEvt) + case mtglib.EventStart: + observer.EventStart(typedEvt) + case mtglib.EventFinish: + observer.EventFinish(typedEvt) + case mtglib.EventConnectedToDC: + observer.EventConnectedToDC(typedEvt) + case mtglib.EventDomainFronting: + observer.EventDomainFronting(typedEvt) + case mtglib.EventIPBlocklisted: + observer.EventIPBlocklisted(typedEvt) + case mtglib.EventConcurrencyLimited: + observer.EventConcurrencyLimited(typedEvt) + case mtglib.EventReplayAttack: + observer.EventReplayAttack(typedEvt) + } + } + } +} diff --git a/events/event_stream_test.go b/events/event_stream_test.go new file mode 100644 index 0000000..93152c6 --- /dev/null +++ b/events/event_stream_test.go @@ -0,0 +1,212 @@ +package events_test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type EventStreamTestSuite struct { + suite.Suite + + ctx context.Context + ctxCancel context.CancelFunc + observerMock1 *ObserverMock + observerMock2 *ObserverMock + stream events.EventStream +} + +func (suite *EventStreamTestSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) + + suite.observerMock1 = &ObserverMock{} + suite.observerMock2 = &ObserverMock{} + + suite.observerMock1.On("Shutdown") + suite.observerMock2.On("Shutdown") + + factories := make([]events.ObserverFactory, 2) + factories[0] = func() events.Observer { return suite.observerMock1 } + factories[1] = func() events.Observer { return suite.observerMock2 } + + suite.stream = events.NewEventStream(factories) +} + +func (suite *EventStreamTestSuite) TestEventStart() { + evt := mtglib.NewEventStart("connID", net.ParseIP("10.0.0.1")) + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventStart", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventStart) + + suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventConnectedToDC() { + evt := mtglib.NewEventConnectedToDC("connID", net.ParseIP("10.0.0.1"), 3) + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventConnectedToDC", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventConnectedToDC) + + suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.DC, caught.DC) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventDomainFronting() { + evt := mtglib.NewEventDomainFronting("connID") + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventDomainFronting", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventDomainFronting) + + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventTraffic() { + evt := mtglib.NewEventTraffic("connID", 1024, true) + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventTraffic", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventTraffic) + + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + suite.Equal(evt.Traffic, caught.Traffic) + suite.Equal(evt.IsRead, caught.IsRead) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventFinish() { + evt := mtglib.NewEventFinish("connID") + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventFinish", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventFinish) + + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventConcurrencyLimited() { + evt := mtglib.NewEventConcurrencyLimited() + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventConcurrencyLimited", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventConcurrencyLimited) + + suite.Equal(evt.Timestamp(), caught.Timestamp()) + suite.Empty(evt.StreamID()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventIPBlocklisted() { + evt := mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")) + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventIPBlocklisted", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventIPBlocklisted) + + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventReplayAttack() { + evt := mtglib.NewEventReplayAttack("CONNID") + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventReplayAttack", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventReplayAttack) + + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TearDownTest() { + suite.stream.Shutdown() + suite.ctxCancel() + + time.Sleep(100 * time.Millisecond) + + suite.observerMock1.AssertExpectations(suite.T()) + suite.observerMock2.AssertExpectations(suite.T()) +} + +func TestEventStream(t *testing.T) { + t.Parallel() + suite.Run(t, &EventStreamTestSuite{}) +} diff --git a/events/init.go b/events/init.go new file mode 100644 index 0000000..f754ede --- /dev/null +++ b/events/init.go @@ -0,0 +1,69 @@ +// Events has a default implementations of EventStream for mtglib. +// +// Please see documentation for mtglib.EventStream interface to get an +// idea of such an abstraction. This package has implementations for the +// default event stream. +// +// Default event stream has a list of its own concepts. First, all it +// does is a routing of messages to known observers. It takes an event, +// defines its type and pass this message to a method of the observer. +// +// There might be many observers, but default event stream has a +// guarantee though. It uses StreamID as a sharding key and guarantees +// that a message with the same StreamID will be devlivered to the same +// observer instance. So, each producer is guarateed to get all relevant +// messages related to the same session. It is not possible that it will +// get EventFinish if it has not seen EventStart for that session yet. +package events + +import "github.com/9seconds/mtg/v2/mtglib" + +// Observer is an instance that listens for the incoming events. +// +// As it is said in the package description, the default event stream +// guarantees that all events with the same StreamID are going to be +// routed to the same instance of the observer. So, there is no need +// to synchronize information about streams between many observers +// instances, they can have their local storage. +type Observer interface { + // EventStart reacts on incoming mtglib.EventStart event. + EventStart(mtglib.EventStart) + + // EventFinish reacts on incoming mtglib.EventFinish event. + EventFinish(mtglib.EventFinish) + + // EventConnectedToDC reacts on incoming mtglib.EventConnectedToDC + // event. + EventConnectedToDC(mtglib.EventConnectedToDC) + + // EventDomainFronting reacts on incoming mtglib.EventDomainFronting + // event. + EventDomainFronting(mtglib.EventDomainFronting) + + // EventTraffic reacts on incoming mtglib.EventTraffic event. + EventTraffic(mtglib.EventTraffic) + + // EventConcurrencyLimited reacts on incoming + // mtglib.EventConcurrencyLimited event. + EventConcurrencyLimited(mtglib.EventConcurrencyLimited) + + // EventIPBlocklisted reacts on incoming mtglib.EventIPBlocklisted event. + EventIPBlocklisted(mtglib.EventIPBlocklisted) + + // EventReplayAttack reacts on incoming mtglib.EventReplayAttack event. + EventReplayAttack(mtglib.EventReplayAttack) + + // Shutdown stop observer. Default event stream guarantees: + // 1. If shutdown is executed, it is executed only once + // 2. Observer won't receieve any new message after this + // function call. + Shutdown() +} + +// ObserverFactory creates a new instance of the observer. +// +// Default event stream creates a small set of goroutines to manage +// incoming messages. Each message is routed to an appropriate observer +// based on a sharding key, stream id. So, it is possible that an +// instance of mtg will have many observer instances, not a single one. +type ObserverFactory func() Observer diff --git a/events/init_test.go b/events/init_test.go new file mode 100644 index 0000000..1b2e0cd --- /dev/null +++ b/events/init_test.go @@ -0,0 +1,46 @@ +package events_test + +import ( + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/mock" +) + +type ObserverMock struct { + mock.Mock +} + +func (o *ObserverMock) EventStart(evt mtglib.EventStart) { + o.Called(evt) +} + +func (o *ObserverMock) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + o.Called(evt) +} + +func (o *ObserverMock) EventDomainFronting(evt mtglib.EventDomainFronting) { + o.Called(evt) +} + +func (o *ObserverMock) EventTraffic(evt mtglib.EventTraffic) { + o.Called(evt) +} + +func (o *ObserverMock) EventFinish(evt mtglib.EventFinish) { + o.Called(evt) +} + +func (o *ObserverMock) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimited) { + o.Called(evt) +} + +func (o *ObserverMock) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { + o.Called(evt) +} + +func (o *ObserverMock) EventReplayAttack(evt mtglib.EventReplayAttack) { + o.Called(evt) +} + +func (o *ObserverMock) Shutdown() { + o.Called() +} diff --git a/events/multi_observer.go b/events/multi_observer.go new file mode 100644 index 0000000..eeeacae --- /dev/null +++ b/events/multi_observer.go @@ -0,0 +1,149 @@ +package events + +import ( + "sync" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type multiObserver struct { + observers []Observer +} + +func (m multiObserver) EventStart(evt mtglib.EventStart) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventStart(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventConnectedToDC(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventDomainFronting(evt mtglib.EventDomainFronting) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventDomainFronting(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventTraffic(evt mtglib.EventTraffic) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventTraffic(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventFinish(evt mtglib.EventFinish) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventFinish(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimited) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventConcurrencyLimited(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventIPBlocklisted(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventReplayAttack(evt mtglib.EventReplayAttack) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventReplayAttack(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) Shutdown() { + for _, v := range m.observers { + v.Shutdown() + } +} + +func newMultiObserver(factories []ObserverFactory) Observer { + observers := make([]Observer, len(factories)) + + for i, v := range factories { + observers[i] = v() + } + + return multiObserver{ + observers: observers, + } +} diff --git a/events/noop.go b/events/noop.go new file mode 100644 index 0000000..0582d9f --- /dev/null +++ b/events/noop.go @@ -0,0 +1,33 @@ +package events + +import ( + "context" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type noop struct{} + +func (n noop) Send(ctx context.Context, evt mtglib.Event) {} + +// NewNoopStream creates a stream which discards each message. +func NewNoopStream() mtglib.EventStream { + return noop{} +} + +type noopObserver struct{} + +func (n noopObserver) EventStart(_ mtglib.EventStart) {} +func (n noopObserver) EventConnectedToDC(_ mtglib.EventConnectedToDC) {} +func (n noopObserver) EventDomainFronting(_ mtglib.EventDomainFronting) {} +func (n noopObserver) EventTraffic(_ mtglib.EventTraffic) {} +func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} +func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} +func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} +func (n noopObserver) EventReplayAttack(_ mtglib.EventReplayAttack) {} +func (n noopObserver) Shutdown() {} + +// NewNoopObserver creates an observer which discards each message. +func NewNoopObserver() Observer { + return noopObserver{} +} diff --git a/events/noop_test.go b/events/noop_test.go new file mode 100644 index 0000000..369a061 --- /dev/null +++ b/events/noop_test.go @@ -0,0 +1,78 @@ +package events_test + +import ( + "context" + "net" + "testing" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite + + testData map[string]mtglib.Event + ctx context.Context +} + +func (suite *NoopTestSuite) SetupSuite() { + suite.testData = map[string]mtglib.Event{ + "start": mtglib.NewEventStart("connID", net.ParseIP("127.0.0.1")), + "connected-to-dc": mtglib.NewEventConnectedToDC("connID", net.ParseIP("127.1.0.1"), 2), + "domain-fronting": mtglib.NewEventDomainFronting("connID"), + "traffic": mtglib.NewEventTraffic("connID", 1000, true), + "finish": mtglib.NewEventFinish("connID"), + "concurrency-limited": mtglib.NewEventConcurrencyLimited(), + "ip-blacklisted": mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")), + "replay-attack": mtglib.NewEventReplayAttack("connID"), + } + suite.ctx = context.Background() +} + +func (suite *NoopTestSuite) TestStream() { + stream := events.NewNoopStream() + + for name, v := range suite.testData { + value := v + + suite.T().Run(name, func(t *testing.T) { + stream.Send(suite.ctx, value) + }) + } +} + +func (suite *NoopTestSuite) TestObserver() { + observer := events.NewNoopObserver() + + for name, v := range suite.testData { + value := v + + suite.T().Run(name, func(t *testing.T) { + switch typedEvt := value.(type) { + case mtglib.EventStart: + observer.EventStart(typedEvt) + case mtglib.EventConnectedToDC: + observer.EventConnectedToDC(typedEvt) + case mtglib.EventDomainFronting: + observer.EventDomainFronting(typedEvt) + case mtglib.EventFinish: + observer.EventFinish(typedEvt) + case mtglib.EventConcurrencyLimited: + observer.EventConcurrencyLimited(typedEvt) + case mtglib.EventIPBlocklisted: + observer.EventIPBlocklisted(typedEvt) + case mtglib.EventReplayAttack: + observer.EventReplayAttack(typedEvt) + } + }) + } + + observer.Shutdown() +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/example.config.toml b/example.config.toml new file mode 100644 index 0000000..c96e354 --- /dev/null +++ b/example.config.toml @@ -0,0 +1,186 @@ +# This is an example of the configuration file for mtg. You actually can +# run mtg with it. It starts a proxy on all interfaces with a secret +# ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d +# +# It has all possible options with default values. So, a real world +# configuration file should contain only those options you are going to +# use. You do not need to enumerate all of them. In other words, each +# option here has a default value. If you comment a key-value pair, it +# should not make any effect. +# +# stats is the only exception. + +# Debug starts application in debug mode. It starts to be quite verbose +# in output. Actually, the idea is that you run it in debug mode only if +# you have any issue. +debug = true + +# A secret. Please remember that mtg supports only FakeTLS mode, legacy +# simple and secured mode are prohibited. For you it means that secret +# should either be base64-encoded or starts with ee. +secret = "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d" + +# Host:port pair to run proxy on. +bind-to = "0.0.0.0:3128" + +# Defines how many concurrent connections are allowed to this proxy. +# All other incoming connections are going to be dropped. +concurrency = 8192 + +# A size of user-space buffer for TCP to use. Since we do 2 connections, +# then we have tcp-buffer * (4 + 2) per each connection: read/write for +# each connection + 2 copy buffers to pump the data between sockets. +tcp-buffer = "4kb" + +# Sometimes you want to enforce mtg to use some types of +# IP connectivity to Telegram. We have 4 modes: +# - prefer-ipv6: +# We can use both ipv4 and ipv6 but ipv6 has a preference +# - prefer-ipv4: +# We can use both ipv4 and ipv6 but ipv4 has a preference +# - only-ipv6: +# Only ipv6 connectivity is used +# - only-ipv4: +# Only ipv4 connectivity is used +prefer-ip = "prefer-ipv6" + +# FakeTLS uses domain fronting protection. So it needs to know a port to +# access. +domain-fronting-port = 443 + +# FakeTLS can compare timestamps to prevent probes. Each message has +# encrypted timestamp. So, mtg can compare this timestamp and decide if +# we need to proceed with connection or not. +# +# Sometimes time can be skewed so we accept all messages within a +# time range of this parameter. +tolerate-time-skewness = "5s" + +# network defines different network-related settings +[network] +# please be aware that mtg needs to do some external requests. For +# example, if you do not pass public ips, it will request your public ip +# address from some external service. +# +# As for 2.0, if you set a public-ip on your own, mtg won't issue any +# network requests except of those required for Telegram. +# +# so, in order of doing them, it needs to do DNS lookup. mtg ignores DNS +# resolver of the operating system and uses DOH instead. This is a host +# it has to access. +# +# By default we use Quad9. +doh-ip = "9.9.9.9" + +# mtg can work via proxies (for now, we support only socks5). Proxy +# configuration is done via list. So, you can specify many proxies +# there. +# +# Actually, if you supply an empty list, then no proxies are going to be +# used. If you supply a single proxy, then mtg will use it exclusively. +# If you supply >= 2, then mtg will load balance between them. +# +# If you add an empty string here, this is an equivalent of 'plain network', +# with no proxy usage. +# +# Proxy configuration is done via ordinary URI schema: +# +# socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s +# +# Only socks5 proxy is used. user/password is optional. As you can +# see, you can specify some parameters in GET query. These parameters +# configure circuit breaker. +# +# open_threshold means a number of errors which should happen so we stop +# use a proxy. +# +# half_open_timeout means a time period (in Golang duration notation) +# after which we can retry with this proxy +# +# reset_failures_timeout means a time period when we flush out errors +# when circuit breaker in closed state. +# +# Please see https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker +# on details about circuit breakers. +proxies = [ + # "socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s" +] + +# network timeouts define different settings for timeouts. tcp timeout +# define a global timeout on establishing of network connections. idle +# means a timeout on pumping data between sockset when nothing is +# happening. +# +# please be noticed that handshakes have no timeouts intentionally. You can +# find a reasoning here: +# https://www.ndss-symposium.org/wp-content/uploads/2020/02/23087-paper.pdf +[network.timeout] +tcp = "5s" +http = "10s" +idle = "1m" + +# Some countries do active probing on Telegram connections. This technique +# allows to protect from such effort. +# +# mtg has a cache of some connection fingerprints. Actually, first bytes +# of each connection. So, it stores them in some in-memory LRU+TTL cache. +# You can configure this cache here. +[defense.anti-replay] +# You can enable/disable this feature. +enabled = true +# max size of such a cache. Please be aware that this number is +# approximate we try hard to store data quite dense but it is possible +# that we can go over this limit for 10-20% under some conditions and +# architectures. +max-size = "1mib" +# we use stable bloom filters for anti-replay cache. This helps +# to maintain a desired error ratio. +error-rate = 0.001 + +# You can protect proxies by using different blocklists. If client has +# ip from the given range, we do not try to do a proper handshake. We +# actually route it to fronting domain. So, this client will never ever +# have a chance to use mtg to access Telegram. +# +# Please remember that blocklists are initialized in async way. So, +# when you start a proxy, blocklists are empty, they are populated and +# processed in backgrounds. An error in any URL is ignored. +[defense.blocklist] +# You can enable/disable this feature. +enabled = true +# This is a limiter for concurrency. In order to protect website +# from overloading, we download files in this number of threads. +download-concurrency = 2 +# A list of URLs in FireHOL format (https://iplists.firehol.org/) +# You can provider links here (starts with https:// or http://) or +# path to a local file, but in this case it should be absolute. +urls = [ + # "https://iplists.firehol.org/files/firehol_level1.netset", + # "/local.file" +] +# How often do we need to update a blocklist set. +update-each = "24h" + +# statsd statistics integration. +[stats.statsd] +# enabled/disabled +enabled = false +# host:port for UDP endpoint of statsd +address = "127.0.0.1:8888" +# prefix of metric for statsd +metric-prefix = "mtg" +# tag format to use +# supported values are 'datadog', 'influxdb' and 'graphite' +# default format is graphite. +tag-format = "datadog" + +# prometheus metrics integration. +[stats.prometheus] +# enabled/disabled +enabled = true +# host:port where to start http server for endpoint +bind-to = "127.0.0.1:3129" +# prefix of http path +http-path = "/" +# prefix for metrics for prometheus +metric-prefix = "mtg" diff --git a/faketls/client_protocol.go b/faketls/client_protocol.go deleted file mode 100644 index 8208ac0..0000000 --- a/faketls/client_protocol.go +++ /dev/null @@ -1,122 +0,0 @@ -package faketls - -import ( - "bufio" - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - "net" - "strconv" - "time" - - "github.com/9seconds/mtg/antireplay" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/obfuscated2" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/tlstypes" - "github.com/9seconds/mtg/wrappers/stream" -) - -type ClientProtocol struct { - obfuscated2.ClientProtocol -} - -func (c *ClientProtocol) Handshake(socket conntypes.StreamReadWriteCloser) (conntypes.StreamReadWriteCloser, error) { - rewinded := stream.NewRewind(socket) - bufferedReader := bufio.NewReader(rewinded) - - for _, expected := range faketlsStartBytes { - if actual, err := bufferedReader.ReadByte(); err != nil || actual != expected { - rewinded.Rewind() - c.cloakHost(rewinded) - - return nil, errors.New("failed first bytes of tls handshake") - } - } - - rewinded.Rewind() - rewinded = stream.NewRewind(rewinded) - - if err := c.tlsHandshake(rewinded); err != nil { - rewinded.Rewind() - c.cloakHost(rewinded) - - return nil, fmt.Errorf("failed tls handshake: %w", err) - } - - conn := stream.NewFakeTLS(socket) - - conn, err := c.ClientProtocol.Handshake(conn) - if err != nil { - return nil, err // nolint: wrapcheck - } - - return conn, err // nolint: wrapcheck -} - -func (c *ClientProtocol) tlsHandshake(conn io.ReadWriter) error { - helloRecord, err := tlstypes.ReadRecord(conn) - if err != nil { - return fmt.Errorf("cannot read initial record: %w", err) - } - - buf := &bytes.Buffer{} - helloRecord.Data.WriteBytes(buf) - - clientHello, err := tlstypes.ParseClientHello(buf.Bytes()) - if err != nil { - return fmt.Errorf("cannot parse client hello: %w", err) - } - - digest := clientHello.Digest() - for i := 0; i < len(digest)-4; i++ { - if digest[i] != 0 { - return errBadDigest - } - } - - timestamp := int64(binary.LittleEndian.Uint32(digest[len(digest)-4:])) - createdAt := time.Unix(timestamp, 0) - timeDiff := time.Since(createdAt) - - if (timeDiff > TimeSkew || timeDiff < -TimeSkew) && timestamp > TimeFromBoot { - return errBadTime - } - - if antireplay.Cache.HasTLS(clientHello.Random[:]) { - stats.Stats.ReplayDetected() - - return errors.New("replay attack is detected") - } - - antireplay.Cache.AddTLS(clientHello.Random[:]) - serverHello := tlstypes.NewServerHello(clientHello) - serverHelloPacket := serverHello.WelcomePacket() - - if _, err := conn.Write(serverHelloPacket); err != nil { - return fmt.Errorf("cannot send welcome packet: %w", err) - } - - return nil -} - -func (c *ClientProtocol) cloakHost(clientConn io.ReadWriteCloser) { - stats.Stats.CloakedRequest() - - addr := net.JoinHostPort(config.C.CloakHost, strconv.Itoa(config.C.CloakPort)) - - hostConn, err := net.Dial("tcp", addr) - if err != nil { - return - } - - cloak(clientConn, hostConn) -} - -func MakeClientProtocol() protocol.ClientProtocol { - return &ClientProtocol{} -} diff --git a/faketls/cloak.go b/faketls/cloak.go deleted file mode 100644 index b134de8..0000000 --- a/faketls/cloak.go +++ /dev/null @@ -1,73 +0,0 @@ -package faketls - -import ( - "context" - "io" - "sync" - "time" - - "github.com/9seconds/mtg/wrappers/rwc" -) - -const ( - cloakLastActivityTimeout = 5 * time.Second - cloakMaxTimeout = 30 * time.Second -) - -func cloak(one, another io.ReadWriteCloser) { - defer func() { - one.Close() - another.Close() - }() - - channelPing := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) - one = rwc.NewPing(ctx, one, channelPing) - another = rwc.NewPing(ctx, another, channelPing) - wg := &sync.WaitGroup{} - - wg.Add(2) - - go cloakPipe(one, another, wg) - - go cloakPipe(another, one, wg) - - go func() { - wg.Wait() - cancel() - }() - - go func() { - lastActivityTimer := time.NewTimer(cloakLastActivityTimeout) - defer lastActivityTimer.Stop() - - maxTimer := time.NewTimer(cloakMaxTimeout) - defer maxTimer.Stop() - - for { - select { - case <-channelPing: - lastActivityTimer.Stop() - lastActivityTimer = time.NewTimer(cloakLastActivityTimeout) - case <-ctx.Done(): - return - case <-lastActivityTimer.C: - cancel() - - return - case <-maxTimer.C: - cancel() - - return - } - } - }() - - <-ctx.Done() -} - -func cloakPipe(one io.Writer, another io.Reader, wg *sync.WaitGroup) { - defer wg.Done() - - io.Copy(one, another) // nolint: errcheck -} diff --git a/faketls/consts.go b/faketls/consts.go deleted file mode 100644 index 9bd2353..0000000 --- a/faketls/consts.go +++ /dev/null @@ -1,30 +0,0 @@ -package faketls - -import ( - "errors" - "time" -) - -const ( - TimeSkew = 5 * time.Second - TimeFromBoot = 24 * 60 * 60 -) - -var ( - errBadDigest = errors.New("bad digest") - errBadTime = errors.New("bad time") - - faketlsStartBytes = [...]byte{ - 0x16, - 0x03, - 0x01, - 0x02, - 0x00, - 0x01, - 0x00, - 0x01, - 0xfc, - 0x03, - 0x03, - } -) diff --git a/go.mod b/go.mod index a2c7190..b9f9f50 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,28 @@ -module github.com/9seconds/mtg +module github.com/9seconds/mtg/v2 -go 1.13 +go 1.16 require ( - github.com/VictoriaMetrics/fastcache v1.5.7 + github.com/OneOfOne/xxhash v1.2.8 + github.com/alecthomas/kong v0.2.16 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 - github.com/beevik/ntp v0.3.0 - github.com/golang/snappy v0.0.3 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 + github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 + github.com/d4l3k/messagediff v1.2.1 // indirect + github.com/jarcoal/httpmock v1.0.8 + github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 + github.com/libp2p/go-reuseport v0.0.2 + github.com/mccutchen/go-httpbin v1.1.1 + github.com/panjf2000/ants/v2 v2.4.3 + github.com/pelletier/go-toml v1.8.1 github.com/prometheus/client_golang v1.9.0 - github.com/prometheus/common v0.18.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/rs/zerolog v1.20.0 github.com/smira/go-statsd v1.3.2 - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.16.0 - golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 - golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect - golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 - golang.org/x/tools v0.1.0 // indirect - google.golang.org/protobuf v1.25.0 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 - honnef.co/go/tools v0.0.1-2020.1.3 // indirect + github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/testify v1.7.0 + github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c + github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 ) diff --git a/go.sum b/go.sum index 2433d93..0cf083c 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,35 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/alecthomas/kong v0.2.16 h1:F232CiYSn54Tnl1sJGTeHmx4vJDNLVP2b9yCVMOQwHQ= +github.com/alecthomas/kong v0.2.16/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= -github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= +github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= +github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -46,9 +46,12 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= +github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -90,23 +93,17 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -142,6 +139,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= +github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -152,6 +151,8 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 h1:D7qhJP3R49ZjUzpzKQ6B2H3lgejPs6DTO5gRomhhOpE= +github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -162,6 +163,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= +github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -171,6 +174,8 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE= +github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -208,8 +213,12 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/panjf2000/ants/v2 v2.4.3 h1:wHghL17YKFanB62QjPQ9o+DuM4q7WrQ7zAhoX8+eBXU= +github.com/panjf2000/ants/v2 v2.4.3/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -239,20 +248,21 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= +github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -274,15 +284,27 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c h1:pGEq55pv/5i+G/Dy+kDsUugzvg6R02jzegxtrwOhE7A= +github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -290,19 +312,11 @@ go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -310,21 +324,16 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -342,7 +351,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -353,8 +361,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -364,25 +370,21 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -399,19 +401,14 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -421,7 +418,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -431,18 +427,13 @@ google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -460,13 +451,13 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/hub/connection.go b/hub/connection.go deleted file mode 100644 index e96cae9..0000000 --- a/hub/connection.go +++ /dev/null @@ -1,172 +0,0 @@ -package hub - -import ( - "fmt" - "math/rand" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" - "go.uber.org/zap" -) - -const connectionTTL = time.Hour - -type connection struct { - conn conntypes.PacketReadWriteCloser - proxyConns map[string]*ProxyConn - closeOnce sync.Once - proxyConnsMutex sync.RWMutex - id int - logger *zap.SugaredLogger - - channelDone chan struct{} - channelWrite chan conntypes.Packet - channelRead chan *rpc.ProxyResponse - channelConnAttach chan *ProxyConn - channelConnDetach chan conntypes.ConnID -} - -func (c *connection) run() { // nolint: cyclop - defer c.Close() - - ttl := time.NewTimer(connectionTTL) - defer ttl.Stop() - - for { - select { - case <-c.channelDone: - for _, v := range c.proxyConns { - v.Close() - } - - return - case <-ttl.C: - c.logger.Debugw("Closing connection by TTL") - c.Close() - case resp := <-c.channelRead: - if channel, ok := c.proxyConns[string(resp.ConnID[:])]; ok { - if resp.Type == rpc.ProxyResponseTypeCloseExt { - channel.Close() - } else { - channel.put(resp) - } - } - case packet := <-c.channelWrite: - if err := c.conn.Write(packet); err != nil { - c.logger.Debugw("Cannot write packet", "error", err) - c.Close() - } - case conn := <-c.channelConnAttach: - c.proxyConnsMutex.Lock() - c.proxyConns[string(conn.req.ConnID[:])] = conn - c.proxyConnsMutex.Unlock() - conn.channelWrite = c.channelWrite - case connID := <-c.channelConnDetach: - if conn, ok := c.proxyConns[string(connID[:])]; ok { - c.proxyConnsMutex.Lock() - delete(c.proxyConns, string(connID[:])) - c.proxyConnsMutex.Unlock() - conn.Close() - } - } - } -} - -func (c *connection) readLoop() { - for { - packet, err := c.conn.Read() - if err != nil { - c.logger.Debugw("Cannot read packet", "error", err) - c.Close() - - return - } - - response, err := rpc.ParseProxyResponse(packet) - if err != nil { - c.logger.Debugw("Failed response", "error", err) - - continue - } - - select { - case <-c.channelDone: - return - case c.channelRead <- response: - } - } -} - -func (c *connection) Close() { - c.closeOnce.Do(func() { - c.logger.Debugw("Closing connection") - - close(c.channelDone) - c.conn.Close() - }) -} - -func (c *connection) Done() bool { - select { - case <-c.channelDone: - return true - default: - return c.Len() == 0 - } -} - -func (c *connection) Len() int { - c.proxyConnsMutex.RLock() - defer c.proxyConnsMutex.RUnlock() - - return len(c.proxyConns) -} - -func (c *connection) Attach(conn *ProxyConn) error { - select { - case <-c.channelDone: - return ErrClosed - case c.channelConnAttach <- conn: - return nil - } -} - -func (c *connection) Detach(connID conntypes.ConnID) { - select { - case <-c.channelDone: - case c.channelConnDetach <- connID: - } -} - -func newConnection(req *protocol.TelegramRequest) (*connection, error) { - conn, err := mtproto.TelegramProtocol(req) - if err != nil { - return nil, fmt.Errorf("cannot create a new connection: %w", err) - } - - id := rand.Int() // nolint: gosec - rv := &connection{ - conn: conn, - id: id, - logger: zap.S().Named("hub-connection").With("id", id, - "dc", req.ClientProtocol.DC(), - "protocol", req.ClientProtocol.ConnectionProtocol()), - proxyConns: make(map[string]*ProxyConn), - - channelRead: make(chan *rpc.ProxyResponse, 1), - channelDone: make(chan struct{}), - channelWrite: make(chan conntypes.Packet), - channelConnAttach: make(chan *ProxyConn), - channelConnDetach: make(chan conntypes.ConnID), - } - - go rv.readLoop() - - go rv.run() - - return rv, nil -} diff --git a/hub/connection_list.go b/hub/connection_list.go deleted file mode 100644 index d1d6a61..0000000 --- a/hub/connection_list.go +++ /dev/null @@ -1,70 +0,0 @@ -package hub - -import ( - "fmt" - "sort" - - "github.com/9seconds/mtg/config" -) - -type connectionList struct { - connections []*connection -} - -func (c *connectionList) get(conn *ProxyConn) (*connection, error) { - if len(c.connections) > 0 && c.connections[0].Len() < config.C.MultiplexPerConnection { - if err := c.connections[0].Attach(conn); err == nil { - return c.connections[0], nil - } - } - - newConn, err := newConnection(conn.req) - if err != nil { - return nil, fmt.Errorf("cannot allocate a new connection: %w", err) - } - - if err = newConn.Attach(conn); err != nil { - newConn.Close() - - return nil, fmt.Errorf("cannot attach to the newly created connection: %w", err) - } - - c.connections = append(c.connections, newConn) - lastIndex := len(c.connections) - 1 - c.connections[0], c.connections[lastIndex] = c.connections[lastIndex], c.connections[0] - - return newConn, nil -} - -func (c *connectionList) gc() { - prevLen := len(c.connections) - if prevLen == 0 { - return - } - - for i := len(c.connections) - 1; i >= 0; i-- { - lastIndex := len(c.connections) - 1 - - if c.connections[i].Done() { - c.connections[i].Close() - - if len(c.connections)-1 == i { - c.connections = c.connections[:lastIndex] - } else { - c.connections[i], c.connections[lastIndex] = c.connections[lastIndex], c.connections[i] - } - } - } - - if prevLen != len(c.connections) { - c.sort() - } -} - -func (c *connectionList) sort() { - if len(c.connections) > 1 { - sort.Slice(c.connections, func(i, j int) bool { - return c.connections[i].Len() < c.connections[j].Len() - }) - } -} diff --git a/hub/hub.go b/hub/hub.go deleted file mode 100644 index edfc442..0000000 --- a/hub/hub.go +++ /dev/null @@ -1,40 +0,0 @@ -package hub - -import ( - "context" - "sync" - - "github.com/9seconds/mtg/protocol" -) - -type hub struct { - muxes map[int32]*mux - mutex sync.RWMutex - ctx context.Context -} - -func (h *hub) Register(req *protocol.TelegramRequest) (*ProxyConn, error) { - return h.getMux(req).Get(req) -} - -func (h *hub) getMux(req *protocol.TelegramRequest) *mux { - var key int32 = 32767 + int32(req.ClientProtocol.DC()) + 100000*int32(req.ClientProtocol.ConnectionProtocol()) - - h.mutex.RLock() - m, ok := h.muxes[key] - h.mutex.RUnlock() - - if !ok { - h.mutex.Lock() - m, ok = h.muxes[key] - - if !ok { - m = newMux(h.ctx) - h.muxes[key] = m - } - - h.mutex.Unlock() - } - - return m -} diff --git a/hub/init.go b/hub/init.go deleted file mode 100644 index 19f5289..0000000 --- a/hub/init.go +++ /dev/null @@ -1,24 +0,0 @@ -package hub - -import ( - "context" - "errors" - "sync" -) - -var ( - ErrTimeout = errors.New("timeout") - ErrClosed = errors.New("context is closed") - - Hub Interface - initOnce sync.Once -) - -func Init(ctx context.Context) { - initOnce.Do(func() { - Hub = &hub{ - muxes: make(map[int32]*mux), - ctx: ctx, - } - }) -} diff --git a/hub/interface.go b/hub/interface.go deleted file mode 100644 index a1efcdc..0000000 --- a/hub/interface.go +++ /dev/null @@ -1,7 +0,0 @@ -package hub - -import "github.com/9seconds/mtg/protocol" - -type Interface interface { - Register(*protocol.TelegramRequest) (*ProxyConn, error) -} diff --git a/hub/mux.go b/hub/mux.go deleted file mode 100644 index f812b07..0000000 --- a/hub/mux.go +++ /dev/null @@ -1,90 +0,0 @@ -package hub - -import ( - "context" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" -) - -const muxGCEvery = time.Minute - -type muxNewRequest struct { - req *protocol.TelegramRequest - resp chan<- muxNewResponse -} - -type muxNewResponse struct { - conn *ProxyConn - err error -} - -type mux struct { - connections connectionList - clients map[string]*connection - ctx context.Context - channelClosed chan conntypes.ConnID - channelNew chan muxNewRequest -} - -func (m *mux) run() { - gcTicker := time.NewTicker(muxGCEvery) - defer gcTicker.Stop() - - for { - select { - case <-m.ctx.Done(): - for _, v := range m.clients { - v.Close() - } - - return - case <-gcTicker.C: - m.connections.gc() - case req := <-m.channelNew: - m.connections.gc() - proxyConn := newProxyConn(req.req, m.channelClosed) - conn, err := m.connections.get(proxyConn) - - if err == nil { - m.clients[string(req.req.ConnID[:])] = conn - } - - req.resp <- muxNewResponse{ - conn: proxyConn, - err: err, - } - close(req.resp) - case connID := <-m.channelClosed: - if conn, ok := m.clients[string(connID[:])]; ok { - conn.Detach(connID) - delete(m.clients, string(connID[:])) - } - } - } -} - -func (m *mux) Get(req *protocol.TelegramRequest) (*ProxyConn, error) { - resp := make(chan muxNewResponse) - m.channelNew <- muxNewRequest{ - req: req, - resp: resp, - } - - rv := <-resp - - return rv.conn, rv.err -} - -func newMux(ctx context.Context) *mux { - m := &mux{ - ctx: ctx, - clients: make(map[string]*connection), - channelClosed: make(chan conntypes.ConnID, 1), - channelNew: make(chan muxNewRequest), - } - go m.run() - - return m -} diff --git a/hub/proxy_conn.go b/hub/proxy_conn.go deleted file mode 100644 index 331d4c8..0000000 --- a/hub/proxy_conn.go +++ /dev/null @@ -1,79 +0,0 @@ -package hub - -import ( - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" -) - -const ( - proxyConnWriteTimeout = 2 * time.Minute - proxyConnReadTimeout = 2 * time.Minute - - proxyConnBackpressureAfter = 10 -) - -type ProxyConn struct { - closeOnce sync.Once - req *protocol.TelegramRequest - channelResponse chan *rpc.ProxyResponse - channelClosed chan<- conntypes.ConnID - channelWrite chan<- conntypes.Packet - channelDone chan struct{} -} - -func (p *ProxyConn) Read() (*rpc.ProxyResponse, error) { - timer := time.NewTimer(proxyConnReadTimeout) - defer timer.Stop() - - select { - case <-timer.C: - return nil, ErrTimeout - case <-p.channelDone: - return nil, ErrClosed - case packet := <-p.channelResponse: - return packet, nil - } -} - -func (p *ProxyConn) Write(packet conntypes.Packet) error { - timer := time.NewTimer(proxyConnWriteTimeout) - defer timer.Stop() - - select { - case <-timer.C: - return ErrTimeout - case <-p.channelDone: - return ErrClosed - case p.channelWrite <- packet: - return nil - } -} - -func (p *ProxyConn) put(response *rpc.ProxyResponse) { - select { - case <-p.channelDone: - case p.channelResponse <- response: - } -} - -func (p *ProxyConn) Close() { - p.closeOnce.Do(func() { - close(p.channelDone) - go func() { - p.channelClosed <- p.req.ConnID - }() - }) -} - -func newProxyConn(req *protocol.TelegramRequest, channelClosed chan<- conntypes.ConnID) *ProxyConn { - return &ProxyConn{ - channelResponse: make(chan *rpc.ProxyResponse, proxyConnBackpressureAfter), - channelDone: make(chan struct{}), - channelClosed: channelClosed, - req: req, - } -} diff --git a/internal/cli/access.go b/internal/cli/access.go new file mode 100644 index 0000000..d443f87 --- /dev/null +++ b/internal/cli/access.go @@ -0,0 +1,192 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" +) + +type accessResponse struct { + IPv4 *accessResponseURLs `json:"ipv4,omitempty"` + IPv6 *accessResponseURLs `json:"ipv6,omitempty"` + Secret struct { + Hex string `json:"hex"` + Base64 string `json:"base64"` + } `json:"secret"` +} + +type accessResponseURLs struct { + IP net.IP `json:"ip"` + Port uint `json:"port"` + TgURL string `json:"tg_url"` + TgQrCode string `json:"tg_qrcode"` + TmeURL string `json:"tme_url"` + TmeQrCode string `json:"tme_qrcode"` +} + +type Access struct { + base + + PublicIPv4 net.IP `kong:"help='Public IPv4 address for proxy. By default it is resolved via remote website',name='ipv4',short='i'"` // nolint: lll + PublicIPv6 net.IP `kong:"help='Public IPv6 address for proxy. By default it is resolved via remote website',name='ipv6',short='I'"` // nolint: lll + Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint',short='p'"` // nolint: lll + Hex bool `kong:"help='Print secret in hex encoding.',short='x'"` +} + +func (c *Access) Run(cli *CLI, version string) error { + if err := c.ReadConfig(version); err != nil { + return fmt.Errorf("cannot init config: %w", err) + } + + return c.Execute(cli) +} + +func (c *Access) Execute(cli *CLI) error { + resp := &accessResponse{} + resp.Secret.Base64 = c.Config.Secret.Base64() + resp.Secret.Hex = c.Config.Secret.Hex() + + wg := &sync.WaitGroup{} + wg.Add(2) // nolint: gomnd + + go func() { + defer wg.Done() + + ip := cli.Access.PublicIPv4 + if ip == nil { + ip = c.getIP("tcp4") + } + + if ip != nil { + ip = ip.To4() + } + + resp.IPv4 = c.makeURLs(ip, cli) + }() + + go func() { + defer wg.Done() + + ip := cli.Access.PublicIPv6 + if ip == nil { + ip = c.getIP("tcp6") + } + + if ip != nil { + ip = ip.To16() + } + + resp.IPv6 = c.makeURLs(ip, cli) + }() + + wg.Wait() + + encoder := json.NewEncoder(os.Stdout) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + + if err := encoder.Encode(resp); err != nil { + return fmt.Errorf("cannot dump access json: %w", err) + } + + return nil +} + +func (c *Access) getIP(protocol string) net.IP { + client := c.Network.MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error) { + return c.Network.DialContext(ctx, protocol, address) + }) + + req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) // nolint: noctx + if err != nil { + panic(err) + } + + req.Header.Add("Accept", "text/plain") + + resp, err := client.Do(req) + if err != nil { + return nil + } + + if resp.StatusCode != http.StatusOK { + return nil + } + + defer func() { + io.Copy(io.Discard, resp.Body) // nolint: errcheck + resp.Body.Close() + }() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil + } + + return net.ParseIP(strings.TrimSpace(string(data))) +} + +func (c *Access) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { + if ip == nil { + return nil + } + + portNo := cli.Access.Port + if portNo == 0 { + portNo = c.Config.BindTo.PortValue(0) + } + + values := url.Values{} + values.Set("server", ip.String()) + values.Set("port", strconv.Itoa(int(portNo))) + + if cli.Access.Hex { + values.Set("secret", c.Config.Secret.Hex()) + } else { + values.Set("secret", c.Config.Secret.Base64()) + } + + urlQuery := values.Encode() + + rv := &accessResponseURLs{ + IP: ip, + Port: portNo, + TgURL: (&url.URL{ + Scheme: "tg", + Host: "proxy", + RawQuery: urlQuery, + }).String(), + TmeURL: (&url.URL{ + Scheme: "https", + Host: "t.me", + Path: "proxy", + RawQuery: urlQuery, + }).String(), + } + rv.TgQrCode = c.makeQRCode(rv.TgURL) + rv.TmeQrCode = c.makeQRCode(rv.TmeURL) + + return rv +} + +func (c *Access) makeQRCode(data string) string { + values := url.Values{} + values.Set("qzone", "4") + values.Set("format", "svg") + values.Set("data", data) + + return (&url.URL{ + Scheme: "https", + Host: "api.qrserver.com", + Path: "v1/create-qr-code", + RawQuery: values.Encode(), + }).String() +} diff --git a/internal/cli/access_test.go b/internal/cli/access_test.go new file mode 100644 index 0000000..b55d40a --- /dev/null +++ b/internal/cli/access_test.go @@ -0,0 +1,197 @@ +package cli_test + +import ( + "net" + "net/http" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/suite" + "github.com/xeipuuv/gojsonschema" +) + +var accressResponseJSONSchema = func() *gojsonschema.Schema { + schema, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(` +{ + "type": "object", + "required": ["secret"], + "additionalProperties": true, + "properties": { + "secret": { + "type": "object", + "required": [ + "hex", + "base64" + ], + "additionalProperties": false, + "properties": { + "hex": { + "type": "string", + "minLength": 34 + }, + "base64": { + "type": "string", + "minLength": 10 + } + } + }, + "ipv4": { + "$ref": "#/definitions/ip" + }, + "ipv6": { + "$ref": "#/definitions/ip" + } + }, + "definitions": { + "ip": { + "type": "object", + "required": [ + "ip", + "port", + "tg_url", + "tg_qrcode", + "tme_url", + "tme_qrcode" + ], + "additionalProperties": false, + "properties": { + "ip": { + "type": "string", + "minLength": 1, + "anyOf": [ + { + "format": "ipv4" + }, + { + "format": "ipv6" + } + ] + }, + "port": { + "type": "integer", + "multipleOf": 1.0, + "exclusiveMinimum": 0, + "exclusiveMaximum": 65536 + }, + "tg_url": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "tg_qrcode": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "tme_url": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "tme_qrcode": { + "type": "string", + "minLength": 1, + "format": "uri" + } + } + } + } +} + `)) + if err != nil { + panic(err) + } + + return schema +}() + +type AccessTestSuite struct { + CommonTestSuite +} + +func (suite *AccessTestSuite) SetupTest() { + suite.CommonTestSuite.SetupTest() + + suite.cli.Access.Config = &config.Config{} + suite.cli.Access.Config.Secret = mtglib.GenerateSecret("google.com") + suite.cli.Access.Network = suite.networkMock + + suite.NoError( + suite.cli.Access.Config.BindTo.UnmarshalText([]byte("0.0.0.0:80"))) +} + +func (suite *AccessTestSuite) TestGenerateNoCalls() { + suite.cli.Access.PublicIPv4 = net.ParseIP("10.0.0.10") + suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") + + output := testlib.CaptureStdout(func() { + suite.NoError(suite.cli.Access.Execute(suite.cli)) + }) + + validated, err := accressResponseJSONSchema.Validate( + gojsonschema.NewStringLoader(output)) + suite.NoError(err) + suite.Empty(validated.Errors()) + suite.True(validated.Valid()) + + suite.Contains(output, "10.0.0.10") + suite.Contains(output, "2001:db8:85a3::8a2e:370:7334") + suite.Contains(output, "ipv4") + suite.Contains(output, "ipv6") + suite.Contains(output, suite.cli.Access.Config.Secret.Base64()) + suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) +} + +func (suite *AccessTestSuite) TestGenerateIPv4Call() { + suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") + + httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", + httpmock.NewStringResponder(http.StatusOK, "10.11.12.13")) + + output := testlib.CaptureStdout(func() { + suite.NoError(suite.cli.Access.Execute(suite.cli)) + }) + + validated, err := accressResponseJSONSchema.Validate( + gojsonschema.NewStringLoader(output)) + suite.NoError(err) + suite.Empty(validated.Errors()) + suite.True(validated.Valid()) + + suite.Contains(output, "10.11.12.13") + suite.Contains(output, "2001:db8:85a3::8a2e:370:7334") + suite.Contains(output, "ipv4") + suite.Contains(output, "ipv6") + suite.Contains(output, suite.cli.Access.Config.Secret.Base64()) + suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) +} + +func (suite *AccessTestSuite) TestIPv4CallFail() { + suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") + + httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", + httpmock.NewStringResponder(http.StatusForbidden, "")) + + output := testlib.CaptureStdout(func() { + suite.NoError(suite.cli.Access.Execute(suite.cli)) + }) + + validated, err := accressResponseJSONSchema.Validate( + gojsonschema.NewStringLoader(output)) + suite.NoError(err) + suite.Empty(validated.Errors()) + suite.True(validated.Valid()) + + suite.Contains(output, "2001:db8:85a3::8a2e:370:7334") + suite.NotContains(output, "ipv4") + suite.Contains(output, "ipv6") + suite.Contains(output, suite.cli.Access.Config.Secret.Base64()) + suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) +} + +func TestAccess(t *testing.T) { // nolint: paralleltest + suite.Run(t, &AccessTestSuite{}) +} diff --git a/internal/cli/base.go b/internal/cli/base.go new file mode 100644 index 0000000..95c4410 --- /dev/null +++ b/internal/cli/base.go @@ -0,0 +1,81 @@ +package cli + +import ( + "fmt" + "net" + "net/url" + "os" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/network" +) + +type base struct { + ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll + + Network mtglib.Network `kong:"-"` + Config *config.Config `kong:"-"` +} + +func (b *base) ReadConfig(version string) error { + content, err := os.ReadFile(b.ConfigPath) + if err != nil { + return fmt.Errorf("cannot read config file: %w", err) + } + + conf, err := config.Parse(content) + if err != nil { + return fmt.Errorf("cannot parse config: %w", err) + } + + ntw, err := b.makeNetwork(conf, version) + if err != nil { + return fmt.Errorf("cannot build a network: %w", err) + } + + b.Config = conf + b.Network = ntw + + return nil +} + +func (b *base) makeNetwork(conf *config.Config, version string) (mtglib.Network, error) { + tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) + httpTimeout := conf.Network.Timeout.HTTP.Value(network.DefaultHTTPTimeout) + dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() + bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) + userAgent := "mtg/" + version + + baseDialer, err := network.NewDefaultDialer(tcpTimeout, int(bufferSize)) + if err != nil { + return nil, fmt.Errorf("cannot build a default dialer: %w", err) + } + + proxyURLs := make([]*url.URL, 0, len(conf.Network.Proxies)) + + for _, v := range conf.Network.Proxies { + if value := v.Value(nil); value != nil { + proxyURLs = append(proxyURLs, v.Value(nil)) + } + } + + switch len(proxyURLs) { + case 0: + return network.NewNetwork(baseDialer, userAgent, dohIP, httpTimeout) + case 1: + socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) + if err != nil { + return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) + } + + return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout) + } + + socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) + if err != nil { + return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) + } + + return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout) +} diff --git a/internal/cli/base_internal_test.go b/internal/cli/base_internal_test.go new file mode 100644 index 0000000..51909d4 --- /dev/null +++ b/internal/cli/base_internal_test.go @@ -0,0 +1,33 @@ +package cli + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/suite" +) + +type BaseTestSuite struct { + suite.Suite + + b base +} + +func (suite *BaseTestSuite) SetupTest() { + suite.b = base{} +} + +func (suite *BaseTestSuite) TestReadConfigNok() { + suite.b.ConfigPath = filepath.Join("testdata", "unknown") + suite.Error(suite.b.ReadConfig("dev")) +} + +func (suite *BaseTestSuite) TestReadConfig() { + suite.b.ConfigPath = filepath.Join("testdata", "minimal.toml") + suite.NoError(suite.b.ReadConfig("dev")) +} + +func TestBase(t *testing.T) { + t.Parallel() + suite.Run(t, &BaseTestSuite{}) +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 0000000..d2d813c --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,10 @@ +package cli + +import "github.com/alecthomas/kong" + +type CLI struct { + GenerateSecret GenerateSecret `kong:"cmd,help='Generate new proxy secret'"` + Access Access `kong:"cmd,help='Print access information.'"` + Run Proxy `kong:"cmd,help='Run proxy.'"` + Version kong.VersionFlag `kong:"help='Print version.',short='v'"` +} diff --git a/internal/cli/generate_secret.go b/internal/cli/generate_secret.go new file mode 100644 index 0000000..e3fb0e8 --- /dev/null +++ b/internal/cli/generate_secret.go @@ -0,0 +1,24 @@ +package cli + +import ( + "fmt" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type GenerateSecret struct { + HostName string `kong:"arg,required,help='Hostname to use for domain fronting.',name='hostname'"` + Hex bool `kong:"help='Print secret in hex encoding.',short='x'"` +} + +func (c *GenerateSecret) Run(cli *CLI, _ string) error { + secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) + + if cli.GenerateSecret.Hex { + fmt.Println(secret.Hex()) // nolint: forbidigo + } else { + fmt.Println(secret.Base64()) // nolint: forbidigo + } + + return nil +} diff --git a/internal/cli/generate_secret_test.go b/internal/cli/generate_secret_test.go new file mode 100644 index 0000000..64c6002 --- /dev/null +++ b/internal/cli/generate_secret_test.go @@ -0,0 +1,51 @@ +package cli_test + +import ( + "strings" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type GenerateSecretTestSuite struct { + CommonTestSuite +} + +func (suite *GenerateSecretTestSuite) SetupTest() { + suite.CommonTestSuite.SetupTest() + + suite.cli.GenerateSecret.HostName = "google.com" +} + +func (suite *GenerateSecretTestSuite) TestDefault() { + output := testlib.CaptureStdout(func() { + suite.NoError(suite.cli.GenerateSecret.Run(suite.cli, "dev")) + }) + suite.True(strings.HasPrefix(output, "7")) + + secret, err := mtglib.ParseSecret(output) + suite.NoError(err) + suite.True(secret.Valid()) + suite.Equal("google.com", secret.Host) +} + +func (suite *GenerateSecretTestSuite) TestHex() { + suite.cli.GenerateSecret.Hex = true + + output := testlib.CaptureStdout(func() { + suite.NoError(suite.cli.GenerateSecret.Run(suite.cli, "dev")) + }) + suite.True(strings.HasPrefix(output, "ee")) + + secret, err := mtglib.ParseSecret(output) + suite.NoError(err) + suite.True(secret.Valid()) + suite.Equal("google.com", secret.Host) +} + +func TestGenerateSecret(t *testing.T) { + t.Parallel() + suite.Run(t, &GenerateSecretTestSuite{}) +} diff --git a/internal/cli/init_test.go b/internal/cli/init_test.go new file mode 100644 index 0000000..e94e829 --- /dev/null +++ b/internal/cli/init_test.go @@ -0,0 +1,37 @@ +package cli_test + +import ( + "net/http" + + "github.com/9seconds/mtg/v2/internal/cli" + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type CommonTestSuite struct { + suite.Suite + + cli *cli.CLI + networkMock *testlib.MtglibNetworkMock + httpClient *http.Client +} + +func (suite *CommonTestSuite) SetupTest() { + suite.networkMock = &testlib.MtglibNetworkMock{} + suite.httpClient = &http.Client{} + suite.cli = &cli.CLI{} + + httpmock.ActivateNonDefault(suite.httpClient) + + suite.networkMock. + On("MakeHTTPClient", mock.Anything). + Maybe(). + Return(suite.httpClient) +} + +func (suite *CommonTestSuite) TearDownTest() { + suite.networkMock.AssertExpectations(suite.T()) + httpmock.DeactivateAndReset() +} diff --git a/internal/cli/proxy.go b/internal/cli/proxy.go new file mode 100644 index 0000000..b20b39d --- /dev/null +++ b/internal/cli/proxy.go @@ -0,0 +1,167 @@ +package cli + +import ( + "fmt" + "net" + "os" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/internal/utils" + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/stats" + "github.com/rs/zerolog" +) + +type Proxy struct { + base +} + +func (c *Proxy) Run(cli *CLI, version string) error { + if err := c.ReadConfig(version); err != nil { + return fmt.Errorf("cannot init config: %w", err) + } + + return c.Execute() +} + +func (c *Proxy) Execute() error { + zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs + zerolog.TimestampFieldName = "timestamp" + zerolog.LevelFieldName = "level" + + if c.Config.Debug { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + } else { + zerolog.SetGlobalLevel(zerolog.WarnLevel) + } + + ctx := utils.RootContext() + opts := mtglib.ProxyOpts{ + Logger: logger.NewZeroLogger(zerolog.New(os.Stdout).With().Timestamp().Logger()), + Network: c.Network, + AntiReplayCache: antireplay.NewNoop(), + IPBlocklist: ipblocklist.NewNoop(), + EventStream: events.NewNoopStream(), + + Secret: c.Config.Secret, + BufferSize: c.Config.TCPBuffer.Value(mtglib.DefaultBufferSize), + DomainFrontingPort: c.Config.DomainFrontingPort.Value(mtglib.DefaultDomainFrontingPort), + IdleTimeout: c.Config.Network.Timeout.Idle.Value(mtglib.DefaultIdleTimeout), + PreferIP: c.Config.PreferIP.Value(mtglib.DefaultPreferIP), + } + + opts.Logger.BindStr("configuration", c.Config.String()).Debug("configuration") + + c.setupAntiReplayCache(&opts) + + if err := c.setupIPBlocklist(&opts); err != nil { + return fmt.Errorf("cannot setup ipblocklist: %w", err) + } + + if err := c.setupEventStream(&opts); err != nil { + return fmt.Errorf("cannot setup event stream: %w", err) + } + + proxy, err := mtglib.NewProxy(opts) + if err != nil { + return fmt.Errorf("cannot create a proxy: %w", err) + } + + listener, err := net.Listen("tcp", c.Config.BindTo.String()) + if err != nil { + return fmt.Errorf("cannot start proxy: %w", err) + } + + go proxy.Serve(listener) // nolint: errcheck + + <-ctx.Done() + listener.Close() + proxy.Shutdown() + + return nil +} + +func (c *Proxy) setupAntiReplayCache(opts *mtglib.ProxyOpts) { + if !c.Config.Defense.AntiReplay.Enabled { + return + } + + opts.AntiReplayCache = antireplay.NewStableBloomFilter( + c.Config.Defense.AntiReplay.MaxSize.Value(antireplay.DefaultStableBloomFilterMaxSize), + c.Config.Defense.AntiReplay.ErrorRate.Value(antireplay.DefaultStableBloomFilterErrorRate), + ) +} + +func (c *Proxy) setupIPBlocklist(opts *mtglib.ProxyOpts) error { + if !c.Config.Defense.Blocklist.Enabled { + return nil + } + + remoteURLs := []string{} + localFiles := []string{} + + for _, v := range c.Config.Defense.Blocklist.URLs { + if v.IsRemote() { + remoteURLs = append(remoteURLs, v.String()) + } else { + localFiles = append(localFiles, v.String()) + } + } + + firehol, err := ipblocklist.NewFirehol(opts.Logger.Named("ipblockist"), + c.Network, + c.Config.Defense.Blocklist.DownloadConcurrency, + remoteURLs, + localFiles) + if err != nil { + return err // nolint: wrapcheck + } + + go firehol.Run(c.Config.Defense.Blocklist.UpdateEach.Value(ipblocklist.DefaultFireholUpdateEach)) + + opts.IPBlocklist = firehol + + return nil +} + +func (c *Proxy) setupEventStream(opts *mtglib.ProxyOpts) error { + factories := make([]events.ObserverFactory, 0, 2) + + if c.Config.Stats.StatsD.Enabled { + statsdFactory, err := stats.NewStatsd( + c.Config.Stats.StatsD.Address.String(), + opts.Logger.Named("statsd"), + c.Config.Stats.StatsD.MetricPrefix.Value(stats.DefaultStatsdMetricPrefix), + c.Config.Stats.StatsD.TagFormat.Value(stats.DefaultStatsdTagFormat)) + if err != nil { + return fmt.Errorf("cannot build statsd observer: %w", err) + } + + factories = append(factories, statsdFactory.Make) + } + + if c.Config.Stats.Prometheus.Enabled { + prometheus := stats.NewPrometheus( + c.Config.Stats.Prometheus.MetricPrefix.Value(stats.DefaultMetricPrefix), + c.Config.Stats.Prometheus.HTTPPath.Value("/"), + ) + + listener, err := net.Listen("tcp", c.Config.Stats.Prometheus.BindTo.String()) + if err != nil { + return fmt.Errorf("cannot start a listener for prometheus: %w", err) + } + + go prometheus.Serve(listener) // nolint: errcheck + + factories = append(factories, prometheus.Make) + } + + if len(factories) > 0 { + opts.EventStream = events.NewEventStream(factories) + } + + return nil +} diff --git a/internal/cli/testdata/minimal.toml b/internal/cli/testdata/minimal.toml new file mode 100644 index 0000000..ea35f78 --- /dev/null +++ b/internal/cli/testdata/minimal.toml @@ -0,0 +1,2 @@ +secret = "7mqFMMq3P2Tvvt_rPx5qhmFnb29nbGUuY29t" +bind-to = "0.0.0.0:80" diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..a634c7b --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,157 @@ +package config + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/pelletier/go-toml" +) + +type Config struct { + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` + BindTo TypeHostPort `json:"bind-to"` + TCPBuffer TypeBytes `json:"tcp-buffer"` + PreferIP TypePreferIP `json:"prefer-ip"` + DomainFrontingPort TypePort `json:"domain-fronting-port"` + TolerateTimeSkewness TypeDuration `json:"tolerate-time-skewness"` + Concurrency uint `json:"concurrency"` + Defense struct { + AntiReplay struct { + Enabled bool `json:"enabled"` + MaxSize TypeBytes `json:"max-size"` + ErrorRate TypeErrorRate `json:"error-rate"` + } `json:"anti-replay"` + Blocklist struct { + Enabled bool `json:"enabled"` + DownloadConcurrency uint `json:"download-concurrency"` + URLs []TypeBlocklistURI `json:"urls"` + UpdateEach TypeDuration `json:"update-each"` + } `json:"blocklist"` + } `json:"defense"` + Network struct { + Timeout struct { + TCP TypeDuration `json:"tcp"` + HTTP TypeDuration `json:"http"` + Idle TypeDuration `json:"idle"` + } `json:"timeout"` + DOHIP TypeIP `json:"doh-ip"` + Proxies []TypeURL `json:"proxies"` + } `json:"network"` + Stats struct { + StatsD struct { + Enabled bool `json:"enabled"` + Address TypeHostPort `json:"address"` + MetricPrefix TypeMetricPrefix `json:"metric-prefix"` + TagFormat TypeStatsdTagFormat `json:"tag-format"` + } `json:"statsd"` + Prometheus struct { + Enabled bool `json:"enabled"` + BindTo TypeHostPort `json:"bind-to"` + HTTPPath TypeHTTPPath `json:"http-path"` + MetricPrefix TypeMetricPrefix `json:"metric-prefix"` + } `json:"prometheus"` + } `json:"stats"` +} + +func (c *Config) Validate() error { + if !c.Secret.Valid() { + return fmt.Errorf("invalid secret %s", c.Secret.String()) + } + + if len(c.BindTo.HostValue(nil)) == 0 || c.BindTo.PortValue(0) == 0 { + return fmt.Errorf("incorrect bind-to parameter %s", c.BindTo.String()) + } + + return nil +} + +func (c *Config) String() string { + buf := &bytes.Buffer{} + encoder := json.NewEncoder(buf) + + encoder.SetEscapeHTML(false) + + if err := encoder.Encode(c); err != nil { + panic(err) + } + + return buf.String() +} + +type configRaw struct { + Debug bool `toml:"debug" json:"debug,omitempty"` + Secret string `toml:"secret" json:"secret"` + BindTo string `toml:"bind-to" json:"bind-to"` + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` + DomainFrontingPort uint `toml:"domain-fronting-port" json:"domain-fronting-port,omitempty"` + TolerateTimeSkewness string `toml:"tolerate-time-skewness" json:"tolerate-time-skewness,omitempty"` + Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"` + Defense struct { + AntiReplay struct { + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + MaxSize string `toml:"max-size" json:"max-size,omitempty"` + ErrorRate float64 `toml:"error-rate" json:"error-rate,omitempty"` + } `toml:"anti-replay" json:"anti-replay,omitempty"` + Blocklist struct { + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + DownloadConcurrency uint `toml:"download-concurrency" json:"download-concurrency,omitempty"` + URLs []string `toml:"urls" json:"urls,omitempty"` + UpdateEach string `toml:"update-each" json:"update-each,omitempty"` + } `toml:"blocklist" json:"blocklist,omitempty"` + } `toml:"defense" json:"defense,omitempty"` + Network struct { + Timeout struct { + TCP string `toml:"tcp" json:"tcp,omitempty"` + HTTP string `toml:"http" json:"http,omitempty"` + Idle string `toml:"idle" json:"idle,omitempty"` + } `toml:"timeout" json:"timeout,omitempty"` + DOHIP string `toml:"doh-ip" json:"doh-ip,omitempty"` + Proxies []string `toml:"proxies" json:"proxies,omitempty"` + } `toml:"network" json:"network,omitempty"` + Stats struct { + StatsD struct { + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + Address string `toml:"address" json:"address,omitempty"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix,omitempty"` + TagFormat string `toml:"tag-format" json:"tag-format,omitempty"` + } `toml:"statsd" json:"statsd,omitempty"` + Prometheus struct { + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + BindTo string `toml:"bind-to" json:"bind-to,omitempty"` + HTTPPath string `toml:"http-path" json:"http-path,omitempty"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix,omitempty"` + } `toml:"prometheus" json:"prometheus,omitempty"` + } `toml:"stats" json:"stats,omitempty"` +} + +func Parse(rawData []byte) (*Config, error) { + rawConf := &configRaw{} + jsonBuf := &bytes.Buffer{} + conf := &Config{} + + jsonEncoder := json.NewEncoder(jsonBuf) + jsonEncoder.SetEscapeHTML(false) + jsonEncoder.SetIndent("", "") + + if err := toml.Unmarshal(rawData, rawConf); err != nil { + return nil, fmt.Errorf("cannot parse toml config: %w", err) + } + + if err := jsonEncoder.Encode(rawConf); err != nil { + panic(err) + } + + if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { + return nil, fmt.Errorf("cannot parse a config: %w", err) + } + + if err := conf.Validate(); err != nil { + return nil, fmt.Errorf("cannot validate config: %w", err) + } + + return conf, nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..e6b19e0 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,54 @@ +package config_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/suite" +) + +type ConfigTestSuite struct { + suite.Suite +} + +func (suite *ConfigTestSuite) ReadConfig(filename string) []byte { + data, err := os.ReadFile(filepath.Join("testdata", filename)) + suite.NoError(err) + + return data +} + +func (suite *ConfigTestSuite) TestParseEmpty() { + _, err := config.Parse([]byte{}) + suite.Error(err) +} + +func (suite *ConfigTestSuite) TestParseBrokenToml() { + _, err := config.Parse(suite.ReadConfig("broken.toml")) + suite.Error(err) +} + +func (suite *ConfigTestSuite) TestParseOnlySecret() { + _, err := config.Parse(suite.ReadConfig("only_secret.toml")) + suite.Error(err) +} + +func (suite *ConfigTestSuite) TestParseMinimalConfig() { + conf, err := config.Parse(suite.ReadConfig("minimal.toml")) + suite.NoError(err) + suite.Equal("7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t", conf.Secret.Base64()) + suite.Equal("0.0.0.0:3128", conf.BindTo.String()) +} + +func (suite *ConfigTestSuite) TestString() { + conf, err := config.Parse(suite.ReadConfig("minimal.toml")) + suite.NoError(err) + suite.NotEmpty(conf.String()) +} + +func TestConfig(t *testing.T) { + t.Parallel() + suite.Run(t, &ConfigTestSuite{}) +} diff --git a/internal/config/testdata/broken.toml b/internal/config/testdata/broken.toml new file mode 100644 index 0000000..d95f791 --- /dev/null +++ b/internal/config/testdata/broken.toml @@ -0,0 +1 @@ +s = sdfsdfds diff --git a/internal/config/testdata/minimal.toml b/internal/config/testdata/minimal.toml new file mode 100644 index 0000000..9d0961a --- /dev/null +++ b/internal/config/testdata/minimal.toml @@ -0,0 +1,2 @@ +secret = "7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t" +bind-to = "0.0.0.0:3128" diff --git a/internal/config/testdata/only_secret.toml b/internal/config/testdata/only_secret.toml new file mode 100644 index 0000000..f6b0bee --- /dev/null +++ b/internal/config/testdata/only_secret.toml @@ -0,0 +1 @@ +secret = "7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t" diff --git a/internal/config/type_blocklist_uri.go b/internal/config/type_blocklist_uri.go new file mode 100644 index 0000000..6bcf6b5 --- /dev/null +++ b/internal/config/type_blocklist_uri.go @@ -0,0 +1,68 @@ +package config + +import ( + "fmt" + "net/url" + "os" + "path/filepath" +) + +type TypeBlocklistURI struct { + value string +} + +func (c *TypeBlocklistURI) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := string(data) + if filepath.IsAbs(text) { + if _, err := os.Stat(text); os.IsNotExist(err) { + return fmt.Errorf("filepath %s does not exist", text) + } + + c.value = text + + return nil + } + + parsedURL, err := url.Parse(text) + if err != nil { + return fmt.Errorf("incorrect url: %w", err) + } + + switch parsedURL.Scheme { + case "http", "https": // nolint: goconst + default: + return fmt.Errorf("unknown schema %s", parsedURL.Scheme) + } + + if parsedURL.Host == "" { + return fmt.Errorf("incorrect url %s", text) + } + + c.value = parsedURL.String() + + return nil +} + +func (c TypeBlocklistURI) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + +func (c TypeBlocklistURI) String() string { + return c.value +} + +func (c TypeBlocklistURI) IsRemote() bool { + return !filepath.IsAbs(c.value) +} + +func (c TypeBlocklistURI) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_blocklist_uri_test.go b/internal/config/type_blocklist_uri_test.go new file mode 100644 index 0000000..cdba55a --- /dev/null +++ b/internal/config/type_blocklist_uri_test.go @@ -0,0 +1,177 @@ +package config_test + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeBlocklistURITestStruct struct { + Value config.TypeBlocklistURI `json:"value"` +} + +type TypeBlocklistURITestSuite struct { + suite.Suite +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalNil() { + typ := &config.TypeBlocklistURI{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + +func (suite *TypeBlocklistURITestSuite) TestUnknownSchema() { + typ := &config.TypeBlocklistURI{} + suite.Error(typ.UnmarshalText([]byte("gopher://lalala"))) +} + +func (suite *TypeBlocklistURITestSuite) TestEmptyHost() { + typ := &config.TypeBlocklistURI{} + suite.Error(typ.UnmarshalText([]byte("https:///path"))) +} + +func (suite *TypeBlocklistURITestSuite) TestIncorrectURL() { + typ := &config.TypeBlocklistURI{} + suite.Error(typ.UnmarshalText([]byte("h:/--"))) +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() { + rnd := make([]byte, 48) + + rand.Read(rnd) // nolint: errcheck + + unknownPath := base64.StdEncoding.EncodeToString(rnd) + + testData := []string{ + "1", + unknownPath, + "/" + unknownPath, + "http:/", + "gopher://lalalal", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeBlocklistURITestStruct{})) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalOk() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := []string{ + "http://lalala", + filepath.Join(dir, "config.go"), + "https://lalala", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestMarshalOk() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := []string{ + "http://lalalal", + filepath.Join(dir, "config.go"), + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestValue() { + testStruct := &typeBlocklistURITestStruct{} + + suite.Equal("http://lalala", testStruct.Value.Value("http://lalala")) + + data, err := json.Marshal(map[string]string{ + "value": "http://blablabla", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("http://blablabla", testStruct.Value.Value("")) +} + +func (suite *TypeBlocklistURITestSuite) TestIsRemote() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := map[bool]string{ + true: "http://lalalal", + false: filepath.Join(dir, "config.go"), + } + + for k, v := range testData { + ok := k + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatBool(ok), func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + assert.NoError(t, json.Unmarshal(data, testStruct)) + + if ok { + assert.True(t, testStruct.Value.IsRemote()) + } else { + assert.False(t, testStruct.Value.IsRemote()) + } + }) + } +} + +func TestTypeBlocklistURI(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeBlocklistURITestSuite{}) +} diff --git a/internal/config/type_bytes.go b/internal/config/type_bytes.go new file mode 100644 index 0000000..4c7d7d6 --- /dev/null +++ b/internal/config/type_bytes.go @@ -0,0 +1,54 @@ +package config + +import ( + "fmt" + "strings" + + "github.com/alecthomas/units" +) + +type TypeBytes struct { + value units.Base2Bytes +} + +func (c *TypeBytes) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + normalizedData := strings.ToUpper(string(data)) + normalizedData = strings.ReplaceAll(normalizedData, "IB", "iB") + + value, err := units.ParseBase2Bytes(normalizedData) + if err != nil { + return fmt.Errorf("incorrect bytes value: %w", err) + } + + if value < 0 { + return fmt.Errorf("%d should be positive number", value) + } + + c.value = value + + return nil +} + +func (c TypeBytes) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeBytes) String() string { + if c.value == 0 { + return "" + } + + return strings.ToLower(c.value.String()) +} + +func (c TypeBytes) Value(defaultValue uint) uint { + if c.value == 0 { + return defaultValue + } + + return uint(c.value) +} diff --git a/internal/config/type_bytes_test.go b/internal/config/type_bytes_test.go new file mode 100644 index 0000000..e395274 --- /dev/null +++ b/internal/config/type_bytes_test.go @@ -0,0 +1,120 @@ +package config_test + +import ( + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeBytesTestStruct struct { + Value config.TypeBytes `json:"value"` +} + +type TypeBytesTestSuite struct { + suite.Suite +} + +func (suite *TypeBytesTestSuite) TestUnmarshalNil() { + typ := &config.TypeBytes{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + +func (suite *TypeBytesTestSuite) TestUnmarshalFail() { + testData := []string{ + "1m", + "1", + "-1kb", + "-1kib", + "-1QB", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeBytesTestStruct{})) + }) + } +} + +func (suite *TypeBytesTestSuite) TestUnmarshalOk() { + testData := map[string]uint{ + "1b": 1, + "1kb": 1024, + "1kib": 1024, + "2mb": 2 * 1024 * 1024, + "2mib": 2 * 1024 * 1024, + } + + for k, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": k, + }) + suite.NoError(err) + + suite.T().Run(k, func(t *testing.T) { + testStruct := &typeBytesTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value(0)) + }) + } +} + +func (suite *TypeBytesTestSuite) TestMarshalOk() { + testData := []string{ + "1b", + "1kib", + "2mib", + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeBytesTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypeBytesTestSuite) TestValue() { + testStruct := &typeBytesTestStruct{} + + suite.EqualValues(0, testStruct.Value.Value(0)) + suite.EqualValues(1, testStruct.Value.Value(1)) + + data, err := json.Marshal(map[string]string{ + "value": "1kb", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(1024, testStruct.Value.Value(0)) + suite.EqualValues(1024, testStruct.Value.Value(1)) +} + +func TestTypeBytes(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeBytesTestSuite{}) +} diff --git a/internal/config/type_duration.go b/internal/config/type_duration.go new file mode 100644 index 0000000..8971a25 --- /dev/null +++ b/internal/config/type_duration.go @@ -0,0 +1,46 @@ +package config + +import ( + "fmt" + "strings" + "time" +) + +type TypeDuration struct { + value time.Duration +} + +func (c *TypeDuration) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + dur, err := time.ParseDuration(strings.ToLower(string(data))) + if err != nil { + return fmt.Errorf("incorrect duration: %w", err) + } + + if dur < 0 { + return fmt.Errorf("%s should be positive duration", dur) + } + + c.value = dur + + return nil +} + +func (c TypeDuration) MarshalText() ([]byte, error) { + return []byte(c.value.String()), nil +} + +func (c TypeDuration) String() string { + return c.value.String() +} + +func (c TypeDuration) Value(defaultValue time.Duration) time.Duration { + if c.value == 0 { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_duration_test.go b/internal/config/type_duration_test.go new file mode 100644 index 0000000..ad2dd4e --- /dev/null +++ b/internal/config/type_duration_test.go @@ -0,0 +1,118 @@ +package config_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeDurationTestStruct struct { + Value config.TypeDuration `json:"value"` +} + +type TypeDurationTestSuite struct { + suite.Suite +} + +func (suite *TypeDurationTestSuite) TestUnmarshalNil() { + typ := &config.TypeDuration{} + suite.NoError(typ.UnmarshalText(nil)) + suite.EqualValues(0, typ.Value(0)) +} + +func (suite *TypeDurationTestSuite) TestUnmarshalFail() { + testData := []string{ + "1t", + "1", + "-1s", + "-1h", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeDurationTestStruct{})) + }) + } +} + +func (suite *TypeDurationTestSuite) TestUnmarshalOk() { + testData := map[string]time.Duration{ + "1s": time.Second, + "1m": time.Minute, + "2h1s": 2*time.Hour + time.Second, + } + + for k, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": k, + }) + suite.NoError(err) + + suite.T().Run(k, func(t *testing.T) { + testStruct := &typeDurationTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.Value(0)) + }) + } +} + +func (suite *TypeDurationTestSuite) TestMarshalOk() { + testData := []string{ + "1s", + "1m0s", + "2h0m1s", + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeDurationTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypeDurationTestSuite) TestValue() { + testStruct := &typeDurationTestStruct{} + + suite.EqualValues(0, testStruct.Value.Value(0)) + suite.Equal(time.Second, testStruct.Value.Value(time.Second)) + + data, err := json.Marshal(map[string]string{ + "value": "1s", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal(time.Second, testStruct.Value.Value(0)) + suite.Equal(time.Second, testStruct.Value.Value(time.Minute)) +} + +func TestTypeDuration(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeDurationTestSuite{}) +} diff --git a/internal/config/type_error_rate.go b/internal/config/type_error_rate.go new file mode 100644 index 0000000..0214956 --- /dev/null +++ b/internal/config/type_error_rate.go @@ -0,0 +1,43 @@ +package config + +import ( + "fmt" + "strconv" +) + +const typeErrorRateIgnoreLess = 1e-8 + +type TypeErrorRate struct { + value float64 +} + +func (c *TypeErrorRate) UnmarshalJSON(data []byte) error { + value, err := strconv.ParseFloat(string(data), 64) + if err != nil { + return fmt.Errorf("incorrect float value: %w", err) + } + + if value <= 0 || value >= 100 { + return fmt.Errorf("%f should be 0 < x < 100", value) + } + + c.value = value + + return nil +} + +func (c *TypeErrorRate) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeErrorRate) String() string { + return strconv.FormatFloat(c.value, 'f', -1, 64) +} + +func (c TypeErrorRate) Value(defaultValue float64) float64 { + if c.value < typeErrorRateIgnoreLess { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_error_rate_test.go b/internal/config/type_error_rate_test.go new file mode 100644 index 0000000..de99f09 --- /dev/null +++ b/internal/config/type_error_rate_test.go @@ -0,0 +1,125 @@ +package config_test + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeErrorRateTestStruct struct { + Value config.TypeErrorRate `json:"value"` +} + +type TypeErrorRateTestSuite struct { + suite.Suite +} + +func (suite *TypeErrorRateTestSuite) TestUnmarshalFail() { + testData := []float64{ + 1000, + -100, + -0.0001, + } + + for _, v := range testData { + data, err := json.Marshal(map[string]float64{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatFloat(v, 'f', -1, 64), func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeErrorRateTestStruct{})) + }) + } + + data, err := json.Marshal(map[string]string{ + "value": "hello", + }) + suite.NoError(err) + suite.Error(json.Unmarshal(data, &typeErrorRateTestStruct{})) +} + +func (suite *TypeErrorRateTestSuite) TestUnmarshalOk() { + testData := []float64{ + 1, + 55.5, + 0.0001, + 1e-6, + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]float64{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatFloat(v, 'f', -1, 64), func(t *testing.T) { + testStruct := &typeErrorRateTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.InEpsilon(t, value, testStruct.Value.Value(0), 1e-10) + }) + } +} + +func (suite *TypeErrorRateTestSuite) TestMarshalOk() { + testData := []float64{ + 1, + 55.5, + 0.0001, + 1e-6, + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]float64{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatFloat(v, 'f', -1, 64), func(t *testing.T) { + testStruct := &typeErrorRateTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + + parsed, err := strconv.ParseFloat(testStruct.Value.String(), 64) + assert.NoError(t, err) + assert.InEpsilon(t, value, parsed, 1e-10) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + + parsed, err = strconv.ParseFloat(string(marshalled), 64) + assert.NoError(t, err) + assert.InEpsilon(t, value, parsed, 1e-10) + }) + } +} + +func (suite *TypeErrorRateTestSuite) TestValue() { + testStruct := &typeErrorRateTestStruct{} + + suite.InEpsilon(1, testStruct.Value.Value(1), 1e-10) + suite.InEpsilon(2, testStruct.Value.Value(2), 1e-10) + + data, err := json.Marshal(map[string]float64{ + "value": 1, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.InEpsilon(1, testStruct.Value.Value(2), 1e-10) + suite.InEpsilon(1, testStruct.Value.Value(3), 1e-10) +} + +func TestTypeErrorRate(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeErrorRateTestSuite{}) +} diff --git a/internal/config/type_hostport.go b/internal/config/type_hostport.go new file mode 100644 index 0000000..4bc58f1 --- /dev/null +++ b/internal/config/type_hostport.go @@ -0,0 +1,67 @@ +package config + +import ( + "fmt" + "net" + "strconv" +) + +type TypeHostPort struct { + host TypeIP + port TypePort +} + +func (c *TypeHostPort) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := string(data) + + host, port, err := net.SplitHostPort(text) + if err != nil { + return fmt.Errorf("incorrect host:port syntax: %w", err) + } + + if port == "" { + return fmt.Errorf("port in %s host:port pair cannot be empty", text) + } + + if err := c.port.UnmarshalJSON([]byte(port)); err != nil { + return fmt.Errorf("incorrect port in host:port: %w", err) + } + + if err := c.host.UnmarshalText([]byte(host)); err != nil { + return fmt.Errorf("incorrect host: %w", err) + } + + return nil +} + +func (c TypeHostPort) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeHostPort) String() string { + return c.Value(net.IP{}, 0) +} + +func (c TypeHostPort) HostValue(defaultValue net.IP) net.IP { + return c.host.Value(defaultValue) +} + +func (c TypeHostPort) PortValue(defaultValue uint) uint { + return c.port.Value(defaultValue) +} + +func (c TypeHostPort) Value(defaultHostValue net.IP, defaultPortValue uint) string { + host := c.HostValue(defaultHostValue) + port := c.PortValue(defaultPortValue) + + hostStr := "" + if len(host) > 0 { + hostStr = host.String() + } + + return net.JoinHostPort(hostStr, strconv.Itoa(int(port))) +} diff --git a/internal/config/type_hostport_test.go b/internal/config/type_hostport_test.go new file mode 100644 index 0000000..6972e1b --- /dev/null +++ b/internal/config/type_hostport_test.go @@ -0,0 +1,115 @@ +package config_test + +import ( + "encoding/json" + "net" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeHostPortTestStruct struct { + Value config.TypeHostPort `json:"value"` +} + +type TypeHostPortTestSuite struct { + suite.Suite +} + +func (suite *TypeHostPortTestSuite) TestUnmarshalFail() { + testData := []string{ + "10.0.0.10:aaa", + "10.0.0.10:", + ":", + "xxx", + "xxx:80", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeHostPortTestStruct{})) + }) + } +} + +func (suite *TypeHostPortTestSuite) TestUnmarshalOk() { + testData := []string{ + "10.0.0.10:80", + "0.0.0.0:80", + ":8000", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeHostPortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value(nil, 0)) + }) + } +} + +func (suite *TypeHostPortTestSuite) TestMarshalOk() { + testData := []string{ + "10.0.0.10:80", + "0.0.0.0:80", + ":8000", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeHostPortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, value, string(marshalled)) + }) + } +} + +func (suite *TypeHostPortTestSuite) TestValue() { + testStruct := &typeHostPortTestStruct{} + + suite.EqualValues("127.0.0.1:80", + testStruct.Value.Value(net.ParseIP("127.0.0.1"), 80)) + suite.EqualValues("127.1.0.1:80", + testStruct.Value.Value(net.ParseIP("127.1.0.1"), 80)) + + data, err := json.Marshal(map[string]string{ + "value": "127.0.0.1:80", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues("127.0.0.1:80", testStruct.Value.Value(nil, 0)) + suite.EqualValues("127.0.0.1:80", testStruct.Value.Value(net.ParseIP("10.0.0.10"), 3000)) +} + +func TestTypeHostPort(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeHostPortTestSuite{}) +} diff --git a/internal/config/type_http_path.go b/internal/config/type_http_path.go new file mode 100644 index 0000000..56a7a4e --- /dev/null +++ b/internal/config/type_http_path.go @@ -0,0 +1,35 @@ +package config + +import "strings" + +type TypeHTTPPath struct { + value string +} + +func (c *TypeHTTPPath) UnmarshalText(data []byte) error { + if len(data) > 0 { + c.value = "/" + strings.Trim(string(data), "/") + } + + return nil +} + +func (c TypeHTTPPath) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeHTTPPath) String() string { + if c.value == "" { + return "/" + } + + return c.value +} + +func (c TypeHTTPPath) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_http_path_test.go b/internal/config/type_http_path_test.go new file mode 100644 index 0000000..01843db --- /dev/null +++ b/internal/config/type_http_path_test.go @@ -0,0 +1,91 @@ +package config_test + +import ( + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeHTTPPathTestStruct struct { + Value config.TypeHTTPPath `json:"value"` +} + +type TypeHTTPPathTestSuite struct { + suite.Suite +} + +func (suite *TypeHTTPPathTestSuite) TestUnmarshal() { + testData := []string{ + "/hello", + "hello", + "hello/", + "/hello/", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeHTTPPathTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, "/hello", testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeHTTPPathTestSuite) TestMarshalOk() { + testData := map[string]string{ + "": "/", + "/hello": "/hello", + "/hello/": "/hello", + "hello/": "/hello", + "hello": "/hello", + } + + for k, v := range testData { + toPass := k + compareWith := v + + data, err := json.Marshal(map[string]string{ + "value": toPass, + }) + suite.NoError(err) + + suite.T().Run(toPass, func(t *testing.T) { + testStruct := &typeHTTPPathTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, compareWith, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, compareWith, string(marshalled)) + }) + } +} + +func (suite *TypeHTTPPathTestSuite) TestValue() { + testStruct := &typeHTTPPathTestStruct{} + + suite.Equal("/hello", testStruct.Value.Value("/hello")) + + data, err := json.Marshal(map[string]string{ + "value": "/map", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("/map", testStruct.Value.Value("/hello")) +} + +func TestTypeHTTPPath(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeHTTPPathTestSuite{}) +} diff --git a/internal/config/type_ip.go b/internal/config/type_ip.go new file mode 100644 index 0000000..0fe61e7 --- /dev/null +++ b/internal/config/type_ip.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "net" +) + +type TypeIP struct { + value net.IP +} + +func (c *TypeIP) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + ip := net.ParseIP(string(data)) + if ip == nil { + return fmt.Errorf("incorrect ip address: %s", string(data)) + } + + c.value = ip + + return nil +} + +func (c *TypeIP) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeIP) String() string { + if len(c.value) > 0 { + return c.value.String() + } + + return "" +} + +func (c TypeIP) Value(defaultValue net.IP) net.IP { + if c.value == nil { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_ip_test.go b/internal/config/type_ip_test.go new file mode 100644 index 0000000..86f14ed --- /dev/null +++ b/internal/config/type_ip_test.go @@ -0,0 +1,115 @@ +package config_test + +import ( + "encoding/json" + "net" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeIPTestStruct struct { + Value config.TypeIP `json:"value"` +} + +type TypeIPTestSuite struct { + suite.Suite +} + +func (suite *TypeIPTestSuite) TestUnmarshalFail() { + testData := []string{ + "0.0.10", + "10.0.0.10:", + "xxx:80", + "2001:0db8:85a3:0000:0000:8a2e:4", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeIPTestStruct{})) + }) + } +} + +func (suite *TypeIPTestSuite) TestUnmarshalOk() { + testData := []string{ + "0.0.0.0", + "10.0.0.10", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, + net.ParseIP(value).String(), + testStruct.Value.Value(nil).String()) + }) + } +} + +func (suite *TypeIPTestSuite) TestMarshalOk() { + testData := []string{ + "0.0.0.0", + "10.0.0.10", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + } + + for _, v := range testData { + value := net.ParseIP(v).String() + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, value, string(marshalled)) + }) + } +} + +func (suite *TypeIPTestSuite) TestValue() { + testStruct := &typeIPTestStruct{} + suite.Empty(testStruct.Value.String()) + + suite.Nil(testStruct.Value.Value(nil)) + suite.Equal("127.1.0.1", testStruct.Value.Value(net.ParseIP("127.1.0.1")).String()) + + data, err := json.Marshal(map[string]string{ + "value": "127.0.0.1", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("127.0.0.1", testStruct.Value.Value(nil).String()) + suite.Equal("127.0.0.1", testStruct.Value.Value(net.ParseIP("10.0.0.10")).String()) +} + +func TestTypeIP(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeIPTestSuite{}) +} diff --git a/internal/config/type_metric_prefix.go b/internal/config/type_metric_prefix.go new file mode 100644 index 0000000..64b5f5b --- /dev/null +++ b/internal/config/type_metric_prefix.go @@ -0,0 +1,41 @@ +package config + +import ( + "fmt" + "regexp" +) + +type TypeMetricPrefix struct { + value string +} + +func (c *TypeMetricPrefix) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + prefix := string(data) + if ok, err := regexp.MatchString("^[a-z0-9]+$", prefix); !ok || err != nil { + return fmt.Errorf("incorrect metric prefix: %s", prefix) + } + + c.value = prefix + + return nil +} + +func (c TypeMetricPrefix) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeMetricPrefix) String() string { + return c.value +} + +func (c TypeMetricPrefix) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_metric_prefix_test.go b/internal/config/type_metric_prefix_test.go new file mode 100644 index 0000000..0c3727d --- /dev/null +++ b/internal/config/type_metric_prefix_test.go @@ -0,0 +1,115 @@ +package config_test + +import ( + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeMetricPrefixTestStruct struct { + Value config.TypeMetricPrefix `json:"value"` +} + +type TypeMetricPrefixTestSuite struct { + suite.Suite +} + +func (suite *TypeMetricPrefixTestSuite) TestUnmarshalNil() { + typ := &config.TypeMetricPrefix{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + +func (suite *TypeMetricPrefixTestSuite) TestUnmarshalFail() { + testData := []string{ + "aaa.aaa", + "aaa-bbb", + "aaa:ccc", + "metric prefix", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeMetricPrefixTestStruct{})) + }) + } +} + +func (suite *TypeMetricPrefixTestSuite) TestUnmarshalOk() { + testData := []string{ + "mtg", + "mtg111", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeMetricPrefixTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeMetricPrefixTestSuite) TestMarshalOk() { + testData := []string{ + "mtg", + "mtg111", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeMetricPrefixTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, value, string(marshalled)) + }) + } +} + +func (suite *TypeMetricPrefixTestSuite) TestValue() { + testStruct := &typeMetricPrefixTestStruct{} + + suite.Equal("mtg", testStruct.Value.Value("mtg")) + suite.Equal("vvv", testStruct.Value.Value("vvv")) + + data, err := json.Marshal(map[string]string{ + "value": "aaa", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("aaa", testStruct.Value.Value("mtg")) + suite.Equal("aaa", testStruct.Value.Value("vvv")) +} + +func TestTypeMetricPrefix(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeMetricPrefixTestSuite{}) +} diff --git a/internal/config/type_port.go b/internal/config/type_port.go new file mode 100644 index 0000000..892558e --- /dev/null +++ b/internal/config/type_port.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "strconv" +) + +type TypePort struct { + value uint +} + +func (c *TypePort) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + return nil + } + + intValue, err := strconv.ParseUint(string(data), 10, 64) + if err != nil { + return fmt.Errorf("port number is not a number: %w", err) + } + + if intValue == 0 || intValue >= 65536 { + return fmt.Errorf("port number should be 0 < portNo < 65536: %d", intValue) + } + + c.value = uint(intValue) + + return nil +} + +func (c *TypePort) MarshalJSON() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypePort) String() string { + return strconv.Itoa(int(c.value)) +} + +func (c TypePort) Value(defaultValue uint) uint { + if c.value == 0 { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_port_test.go b/internal/config/type_port_test.go new file mode 100644 index 0000000..75fc920 --- /dev/null +++ b/internal/config/type_port_test.go @@ -0,0 +1,117 @@ +package config_test + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typePortTestStruct struct { + Value config.TypePort `json:"value"` +} + +type TypePortTestSuite struct { + suite.Suite +} + +func (suite *TypePortTestSuite) TestUnmarshalNil() { + typ := &config.TypePort{} + suite.NoError(typ.UnmarshalJSON(nil)) + suite.Equal("0", typ.String()) +} + +func (suite *TypePortTestSuite) TestUnmarshalFail() { + testData := []int{ + -1, + 1_000_000, + } + + for _, v := range testData { + data, err := json.Marshal(map[string]int{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.Itoa(v), func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typePortTestStruct{})) + }) + } +} + +func (suite *TypePortTestSuite) TestUnmarshalOk() { + testData := []int{ + 1, + 1_000, + 65535, + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]int{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.Itoa(v), func(t *testing.T) { + testStruct := &typePortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value(0)) + }) + } +} + +func (suite *TypePortTestSuite) TestMarshalOk() { + testData := map[string]int{ + "1": 1, + "1000": 1000, + "65535": 65535, + } + + for k, v := range testData { + name := k + value := v + + data, err := json.Marshal(map[string]int{ + "value": value, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typePortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalJSON() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypePortTestSuite) TestValue() { + testStruct := &typePortTestStruct{} + + suite.EqualValues(0, testStruct.Value.Value(0)) + suite.EqualValues(1, testStruct.Value.Value(1)) + + data, err := json.Marshal(map[string]int{ + "value": 5, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(5, testStruct.Value.Value(0)) + suite.EqualValues(5, testStruct.Value.Value(1)) +} + +func TestTypePort(t *testing.T) { + t.Parallel() + suite.Run(t, &TypePortTestSuite{}) +} diff --git a/internal/config/type_prefer_ip.go b/internal/config/type_prefer_ip.go new file mode 100644 index 0000000..3d150d4 --- /dev/null +++ b/internal/config/type_prefer_ip.go @@ -0,0 +1,50 @@ +package config + +import ( + "fmt" + "strings" +) + +const ( + TypePreferIPPreferIPv4 = "prefer-ipv4" + TypePreferIPPreferIPv6 = "prefer-ipv6" + TypePreferOnlyIPv4 = "only-ipv4" + TypePreferOnlyIPv6 = "only-ipv6" +) + +type TypePreferIP struct { + value string +} + +func (c *TypePreferIP) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := strings.ToLower(string(data)) + + switch text { + case TypePreferIPPreferIPv4, TypePreferIPPreferIPv6, TypePreferOnlyIPv4, TypePreferOnlyIPv6: + c.value = text + default: + return fmt.Errorf("incorrect prefer-ip value: %s", string(data)) + } + + return nil +} + +func (c TypePreferIP) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + +func (c *TypePreferIP) String() string { + return c.value +} + +func (c *TypePreferIP) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_prefer_ip_test.go b/internal/config/type_prefer_ip_test.go new file mode 100644 index 0000000..917b918 --- /dev/null +++ b/internal/config/type_prefer_ip_test.go @@ -0,0 +1,142 @@ +package config_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typePreferIPTestStruct struct { + Value config.TypePreferIP `json:"value"` +} + +type TypePreferIPTestSuite struct { + suite.Suite +} + +func (suite *TypePreferIPTestSuite) TestUnmarshalNil() { + typ := &config.TypePreferIP{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + +func (suite *TypePreferIPTestSuite) TestUnmarshalFail() { + testData := []string{ + "p", + "ipv4", + "onlyipv4", + "ipv6prefer", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typePreferIPTestStruct{})) + }) + } +} + +func (suite *TypePreferIPTestSuite) TestUnmarshalOk() { + testData := []string{ + config.TypePreferIPPreferIPv4, + config.TypePreferIPPreferIPv6, + config.TypePreferOnlyIPv4, + config.TypePreferOnlyIPv6, + strings.ToUpper(config.TypePreferIPPreferIPv4), + strings.ToUpper(config.TypePreferIPPreferIPv6), + strings.ToUpper(config.TypePreferOnlyIPv4), + strings.ToUpper(config.TypePreferOnlyIPv6), + strings.ToLower(config.TypePreferIPPreferIPv4), + strings.ToLower(config.TypePreferIPPreferIPv6), + strings.ToLower(config.TypePreferOnlyIPv4), + strings.ToLower(config.TypePreferOnlyIPv6), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typePreferIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, + strings.ToLower(value), + testStruct.Value.Value(config.TypePreferIPPreferIPv4)) + }) + } +} + +func (suite *TypePreferIPTestSuite) TestMarshalOk() { + testData := []string{ + config.TypePreferIPPreferIPv4, + config.TypePreferIPPreferIPv6, + config.TypePreferOnlyIPv4, + config.TypePreferOnlyIPv6, + strings.ToUpper(config.TypePreferIPPreferIPv4), + strings.ToUpper(config.TypePreferIPPreferIPv6), + strings.ToUpper(config.TypePreferOnlyIPv4), + strings.ToUpper(config.TypePreferOnlyIPv6), + strings.ToLower(config.TypePreferIPPreferIPv4), + strings.ToLower(config.TypePreferIPPreferIPv6), + strings.ToLower(config.TypePreferOnlyIPv4), + strings.ToLower(config.TypePreferOnlyIPv6), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typePreferIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, strings.ToLower(value), testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, strings.ToLower(value), string(marshalled)) + }) + } +} + +func (suite *TypePreferIPTestSuite) TestValue() { + testStruct := &typePreferIPTestStruct{} + + suite.EqualValues(config.TypePreferIPPreferIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv4)) + suite.EqualValues(config.TypePreferIPPreferIPv6, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) + + data, err := json.Marshal(map[string]string{ + "value": config.TypePreferOnlyIPv4, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferOnlyIPv6)) + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) +} + +func TestTypePreferIP(t *testing.T) { + t.Parallel() + suite.Run(t, &TypePreferIPTestSuite{}) +} diff --git a/internal/config/type_statsd_tag_format.go b/internal/config/type_statsd_tag_format.go new file mode 100644 index 0000000..7903b0c --- /dev/null +++ b/internal/config/type_statsd_tag_format.go @@ -0,0 +1,49 @@ +package config + +import ( + "fmt" + "strings" +) + +const ( + TypeStatsdTagFormatInfluxdb = "influxdb" + TypeStatsdTagFormatDatadog = "datadog" + TypeStatsdTagFormatGraphite = "graphite" +) + +type TypeStatsdTagFormat struct { + value string +} + +func (c *TypeStatsdTagFormat) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := strings.ToLower(string(data)) + + switch text { + case TypeStatsdTagFormatInfluxdb, TypeStatsdTagFormatDatadog, TypeStatsdTagFormatGraphite: + c.value = text + default: + return fmt.Errorf("incorrect tag format value: %s", string(data)) + } + + return nil +} + +func (c TypeStatsdTagFormat) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + +func (c *TypeStatsdTagFormat) String() string { + return c.value +} + +func (c *TypeStatsdTagFormat) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_statsd_tag_format_test.go b/internal/config/type_statsd_tag_format_test.go new file mode 100644 index 0000000..b639ed4 --- /dev/null +++ b/internal/config/type_statsd_tag_format_test.go @@ -0,0 +1,136 @@ +package config_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeStatsdTagFormatTestStruct struct { + Value config.TypeStatsdTagFormat `json:"value"` +} + +type TypeStatsdTagFormatTestSuite struct { + suite.Suite +} + +func (suite *TypeStatsdTagFormatTestSuite) TestUnmarshalNil() { + typ := &config.TypeStatsdTagFormat{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Equal("lalala", typ.Value("lalala")) +} + +func (suite *TypeStatsdTagFormatTestSuite) TestUnmarshalFail() { + testData := []string{ + "p", + "ipv4", + "onlyipv4", + "ipv6prefer", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeStatsdTagFormatTestStruct{})) + }) + } +} + +func (suite *TypeStatsdTagFormatTestSuite) TestUnmarshalOk() { + testData := []string{ + config.TypeStatsdTagFormatDatadog, + config.TypeStatsdTagFormatInfluxdb, + config.TypeStatsdTagFormatGraphite, + strings.ToUpper(config.TypeStatsdTagFormatDatadog), + strings.ToUpper(config.TypeStatsdTagFormatInfluxdb), + strings.ToUpper(config.TypeStatsdTagFormatGraphite), + strings.ToLower(config.TypeStatsdTagFormatDatadog), + strings.ToLower(config.TypeStatsdTagFormatInfluxdb), + strings.ToLower(config.TypeStatsdTagFormatGraphite), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeStatsdTagFormatTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, + strings.ToLower(value), + testStruct.Value.Value(config.TypeStatsdTagFormatDatadog)) + }) + } +} + +func (suite *TypeStatsdTagFormatTestSuite) TestMarshalOk() { + testData := []string{ + config.TypeStatsdTagFormatDatadog, + config.TypeStatsdTagFormatInfluxdb, + config.TypeStatsdTagFormatGraphite, + strings.ToUpper(config.TypeStatsdTagFormatDatadog), + strings.ToUpper(config.TypeStatsdTagFormatInfluxdb), + strings.ToUpper(config.TypeStatsdTagFormatGraphite), + strings.ToLower(config.TypeStatsdTagFormatDatadog), + strings.ToLower(config.TypeStatsdTagFormatInfluxdb), + strings.ToLower(config.TypeStatsdTagFormatGraphite), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeStatsdTagFormatTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, strings.ToLower(value), testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, strings.ToLower(value), string(marshalled)) + }) + } +} + +func (suite *TypeStatsdTagFormatTestSuite) TestValue() { + testStruct := &typePreferIPTestStruct{} + + suite.EqualValues(config.TypePreferIPPreferIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv4)) + suite.EqualValues(config.TypePreferIPPreferIPv6, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) + + data, err := json.Marshal(map[string]string{ + "value": config.TypePreferOnlyIPv4, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferOnlyIPv6)) + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) +} + +func TestTypeStatsdTagFormat(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeStatsdTagFormatTestSuite{}) +} diff --git a/internal/config/type_url.go b/internal/config/type_url.go new file mode 100644 index 0000000..9250492 --- /dev/null +++ b/internal/config/type_url.go @@ -0,0 +1,71 @@ +package config + +import ( + "fmt" + "net" + "net/url" +) + +type TypeURL struct { + value *url.URL +} + +func (c *TypeURL) UnmarshalText(data []byte) error { // nolint: cyclop + if len(data) == 0 { + return nil + } + + value, err := url.Parse(string(data)) + if err != nil { + return fmt.Errorf("incorrect URL: %w", err) + } + + switch value.Scheme { + case "http", "https", "socks5": + case "": + return fmt.Errorf("url %s has to have a schema", value) + default: + return fmt.Errorf("unsupported schema %s", value.Scheme) + } + + if value.Host == "" { + return fmt.Errorf("url %s has to have a host", value) + } + + if _, _, err := net.SplitHostPort(value.Host); err != nil { + switch value.Scheme { + case "http": + value.Host = net.JoinHostPort(value.Host, "80") + case "https": + value.Host = net.JoinHostPort(value.Host, "443") + case "socks5": + value.Host = net.JoinHostPort(value.Host, "1080") + default: + return fmt.Errorf("cannot set a default port for %s", value) + } + } + + c.value = value + + return nil +} + +func (c *TypeURL) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c TypeURL) String() string { + if c.value == nil { + return "" + } + + return c.value.String() +} + +func (c TypeURL) Value(defaultValue *url.URL) *url.URL { + if c.value == nil { + return defaultValue + } + + return c.value +} diff --git a/internal/config/type_url_test.go b/internal/config/type_url_test.go new file mode 100644 index 0000000..4288172 --- /dev/null +++ b/internal/config/type_url_test.go @@ -0,0 +1,107 @@ +package config_test + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeURLTestStruct struct { + Value config.TypeURL `json:"value"` +} + +type TypeURLTestSuite struct { + suite.Suite +} + +func (suite *TypeURLTestSuite) TestUnmarshalNil() { + u, _ := url.Parse("https://google.com") + + typ := &config.TypeURL{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) + suite.Equal("https://google.com", typ.Value(u).String()) +} + +func (suite *TypeURLTestSuite) TestUnmarshalFail() { + testData := []string{ + "http:/aaa.com", + "ipv4", + "111", + "://111", + "http://aaa.com:xxx", + "gopher://aaa.com:888", + "gopher://aaa.com", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeURLTestStruct{})) + }) + } +} + +func (suite *TypeURLTestSuite) TestUnmarshalOk() { + testData := map[string]string{ + "https://10.0.0.10:80": "https://10.0.0.10:80", + "https://10.0.0.10:443": "https://10.0.0.10", + "http://10.0.0.10:8": "http://10.0.0.10:8", + "http://10.0.0.10:80": "http://10.0.0.10", + "socks5://10.0.0.10:1080": "socks5://10.0.0.10", + "socks5://10.0.0.10:888": "socks5://10.0.0.10:888", + } + + for k, v := range testData { + expected := k + actual := v + + data, err := json.Marshal(map[string]string{ + "value": actual, + }) + suite.NoError(err) + + suite.T().Run(actual, func(t *testing.T) { + testStruct := &typeURLTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, expected, testStruct.Value.Value(nil).String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, expected, string(marshalled)) + }) + } +} + +func (suite *TypeURLTestSuite) TestValue() { + testStruct := &typeURLTestStruct{} + + u1, _ := url.Parse("https://10.0.0.10:80") + u2, _ := url.Parse("https://10.1.0.10:80") + + suite.Equal("https://10.0.0.10:80", testStruct.Value.Value(u1).String()) + suite.Equal("https://10.1.0.10:80", testStruct.Value.Value(u2).String()) + + data, err := json.Marshal(map[string]string{ + "value": "http://127.0.0.1:80", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("http://127.0.0.1:80", testStruct.Value.Value(u1).String()) + suite.Equal("http://127.0.0.1:80", testStruct.Value.Value(u2).String()) +} + +func TestTypeURL(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeURLTestSuite{}) +} diff --git a/internal/testlib/capture_output.go b/internal/testlib/capture_output.go new file mode 100644 index 0000000..aa538c2 --- /dev/null +++ b/internal/testlib/capture_output.go @@ -0,0 +1,42 @@ +package testlib + +import ( + "bytes" + "io" + "os" + "strings" +) + +func CaptureStdout(callback func()) string { + return captureOutput(&os.Stdout, callback) +} + +func CaptureStderr(callback func()) string { + return captureOutput(&os.Stderr, callback) +} + +func captureOutput(filefp **os.File, callback func()) string { + oldFp := *filefp + + defer func() { + *filefp = oldFp + }() + + reader, writer, _ := os.Pipe() + buf := &bytes.Buffer{} + closeChan := make(chan bool) + + go func() { + io.Copy(buf, reader) // nolint: errcheck + close(closeChan) + }() + + *filefp = writer + + callback() + + writer.Close() + <-closeChan + + return strings.TrimSpace(buf.String()) +} diff --git a/internal/testlib/events_observer_mock.go b/internal/testlib/events_observer_mock.go new file mode 100644 index 0000000..e3d8979 --- /dev/null +++ b/internal/testlib/events_observer_mock.go @@ -0,0 +1,7 @@ +package testlib + +import "github.com/stretchr/testify/mock" + +type EventsObserverMock struct { + mock.Mock +} diff --git a/internal/testlib/mtglib_antireplay_cache_mock.go b/internal/testlib/mtglib_antireplay_cache_mock.go new file mode 100644 index 0000000..20ac993 --- /dev/null +++ b/internal/testlib/mtglib_antireplay_cache_mock.go @@ -0,0 +1,11 @@ +package testlib + +import "github.com/stretchr/testify/mock" + +type MtglibAntiReplayCacheMock struct { + mock.Mock +} + +func (m *MtglibAntiReplayCacheMock) SeenBefore(data []byte) bool { + return m.Called(data).Bool(0) +} diff --git a/internal/testlib/mtglib_network_mock.go b/internal/testlib/mtglib_network_mock.go new file mode 100644 index 0000000..092bd28 --- /dev/null +++ b/internal/testlib/mtglib_network_mock.go @@ -0,0 +1,30 @@ +package testlib + +import ( + "context" + "net" + "net/http" + + "github.com/stretchr/testify/mock" +) + +type MtglibNetworkMock struct { + mock.Mock +} + +func (m *MtglibNetworkMock) Dial(network, address string) (net.Conn, error) { + args := m.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := m.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, + network, address string) (net.Conn, error)) *http.Client { + return m.Called(dialFunc).Get(0).(*http.Client) +} diff --git a/internal/testlib/net_conn_mock.go b/internal/testlib/net_conn_mock.go new file mode 100644 index 0000000..b2fc1a3 --- /dev/null +++ b/internal/testlib/net_conn_mock.go @@ -0,0 +1,48 @@ +package testlib + +import ( + "net" + "time" + + "github.com/stretchr/testify/mock" +) + +type NetConnMock struct { + mock.Mock +} + +func (n *NetConnMock) Read(b []byte) (int, error) { + args := n.Called(b) + + return args.Int(0), args.Error(1) +} + +func (n *NetConnMock) Write(b []byte) (int, error) { + args := n.Called(b) + + return args.Int(0), args.Error(1) +} + +func (n *NetConnMock) Close() error { + return n.Called().Error(0) +} + +func (n *NetConnMock) LocalAddr() net.Addr { + return n.Called().Get(0).(net.Addr) +} + +func (n *NetConnMock) RemoteAddr() net.Addr { + return n.Called().Get(0).(net.Addr) +} + +func (n *NetConnMock) SetDeadline(t time.Time) error { + return n.Called(t).Error(0) +} + +func (n *NetConnMock) SetReadDeadline(t time.Time) error { + return n.Called(t).Error(0) +} + +func (n *NetConnMock) SetWriteDeadline(t time.Time) error { + return n.Called(t).Error(0) +} diff --git a/utils/rlimit.go b/internal/utils/rlimit.go similarity index 100% rename from utils/rlimit.go rename to internal/utils/rlimit.go diff --git a/utils/rlimit_windows.go b/internal/utils/rlimit_windows.go similarity index 100% rename from utils/rlimit_windows.go rename to internal/utils/rlimit_windows.go diff --git a/utils/signal_context.go b/internal/utils/root_context.go similarity index 88% rename from utils/signal_context.go rename to internal/utils/root_context.go index 6d3951f..9fe4b49 100644 --- a/utils/signal_context.go +++ b/internal/utils/root_context.go @@ -9,7 +9,7 @@ import ( "syscall" ) -func GetSignalContext() context.Context { +func RootContext() context.Context { ctx, cancel := context.WithCancel(context.Background()) sigChan := make(chan os.Signal, 1) diff --git a/utils/signal_context_windows.go b/internal/utils/root_context_windows.go similarity index 86% rename from utils/signal_context_windows.go rename to internal/utils/root_context_windows.go index d18ce71..9a5b6e0 100644 --- a/utils/signal_context_windows.go +++ b/internal/utils/root_context_windows.go @@ -8,7 +8,7 @@ import ( "os/signal" ) -func GetSignalContext() context.Context { +func RootContext() context.Context { ctx, cancel := context.WithCancel(context.Background()) sigChan := make(chan os.Signal, 1) diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go new file mode 100644 index 0000000..947134b --- /dev/null +++ b/ipblocklist/firehol.go @@ -0,0 +1,373 @@ +package ipblocklist + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/kentik/patricia" + "github.com/kentik/patricia/bool_tree" + "github.com/panjf2000/ants/v2" +) + +const ( + fireholIPv4DefaultCIDR = 32 + fireholIPv6DefaultCIDR = 128 +) + +var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) + +// Firehol is IPBlocklist which uses lists from FireHOL: +// https://iplists.firehol.org/ +// +// It can use both local files and remote URLs. This is not necessary +// that blocklists should be taken from this website, we expect only +// compatible formats here. +// +// Example of the format: +// +// # this is a comment +// # to ignore +// 127.0.0.1 # you can specify an IP +// 10.0.0.0/8 # or cidr +type Firehol struct { + ctx context.Context + ctxCancel context.CancelFunc + logger mtglib.Logger + + rwMutex sync.RWMutex + + remoteURLs []string + localFiles []string + + httpClient *http.Client + workerPool *ants.Pool + + treeV4 *bool_tree.TreeV4 + treeV6 *bool_tree.TreeV6 +} + +// Shutdown stop a background update process. +func (f *Firehol) Shutdown() { + f.ctxCancel() +} + +// Contains is given IP list can be found in FireHOL blocklists. +func (f *Firehol) Contains(ip net.IP) bool { + if ip == nil { + return true + } + + ip4 := ip.To4() + + f.rwMutex.RLock() + defer f.rwMutex.RUnlock() + + if ip4 != nil { + return f.containsIPv4(ip4) + } + + return f.containsIPv6(ip.To16()) +} + +// Run starts a background update process. +// +// This is a blocking method so you probably want to run it in a +// goroutine. +func (f *Firehol) Run(updateEach time.Duration) { + if updateEach == 0 { + updateEach = DefaultFireholUpdateEach + } + + ticker := time.NewTicker(updateEach) + + defer func() { + ticker.Stop() + + select { + case <-ticker.C: + default: + } + }() + + if err := f.update(); err != nil { + f.logger.WarningError("cannot update blocklist", err) + } else { + f.logger.Info("blocklist was updated") + } + + for { + select { + case <-f.ctx.Done(): + return + case <-ticker.C: + if err := f.update(); err != nil { + f.logger.WarningError("cannot update blocklist", err) + } else { + f.logger.Info("blocklist was updated") + } + } + } +} + +func (f *Firehol) containsIPv4(addr net.IP) bool { + ip := patricia.NewIPv4AddressFromBytes(addr, 32) + + if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) containsIPv6(addr net.IP) bool { + ip := patricia.NewIPv6Address(addr, 128) + + if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) update() error { // nolint: funlen, cyclop + ctx, cancel := context.WithCancel(f.ctx) + defer cancel() + + wg := &sync.WaitGroup{} + wg.Add(len(f.remoteURLs) + len(f.localFiles)) + + treeMutex := &sync.Mutex{} + v4tree := bool_tree.NewTreeV4() + v6tree := bool_tree.NewTreeV6() + + errorChan := make(chan error, 1) + defer close(errorChan) + + for _, v := range f.localFiles { + go func(filename string) { + defer wg.Done() + + if err := f.updateLocalFile(ctx, filename, treeMutex, v4tree, v6tree); err != nil { + cancel() + f.logger.BindStr("filename", filename).WarningError("cannot update", err) + + select { + case errorChan <- err: + default: + } + } + }(v) + } + + for _, v := range f.remoteURLs { + value := v + + f.workerPool.Submit(func() { // nolint: errcheck + defer wg.Done() + + if err := f.updateRemoteURL(ctx, value, treeMutex, v4tree, v6tree); err != nil { + cancel() + f.logger.BindStr("url", value).WarningError("cannot update", err) + + select { + case errorChan <- err: + default: + } + } + }) + } + + wg.Wait() + + select { + case err := <-errorChan: + return fmt.Errorf("cannot update trees: %w", err) + default: + } + + f.rwMutex.Lock() + defer f.rwMutex.Unlock() + + f.treeV4 = v4tree + f.treeV6 = v6tree + + return nil +} + +func (f *Firehol) updateLocalFile(ctx context.Context, filename string, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + filefp, err := os.Open(filename) + if err != nil { + return fmt.Errorf("cannot open file: %w", err) + } + + go func(ctx context.Context, closer io.Closer) { + <-ctx.Done() + closer.Close() + }(ctx, filefp) + + defer filefp.Close() + + return f.updateTrees(mutex, filefp, v4tree, v6tree) +} + +func (f *Firehol) updateRemoteURL(ctx context.Context, url string, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("cannot build a request: %w", err) + } + + resp, err := f.httpClient.Do(req) // nolint: bodyclose + if err != nil { + return fmt.Errorf("cannot request a remote URL %s: %w", url, err) + } + + go func(ctx context.Context, closer io.Closer) { + <-ctx.Done() + closer.Close() + }(ctx, resp.Body) + + defer func(rc io.ReadCloser) { + io.Copy(io.Discard, rc) // nolint: errcheck + rc.Close() + }(resp.Body) + + return f.updateTrees(mutex, resp.Body, v4tree, v6tree) +} + +func (f *Firehol) updateTrees(mutex sync.Locker, + reader io.Reader, + v4tree *bool_tree.TreeV4, + v6tree *bool_tree.TreeV6) error { + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + text := scanner.Text() + text = fireholRegexpComment.ReplaceAllLiteralString(text, "") + text = strings.TrimSpace(text) + + if text == "" { + continue + } + + ip, cidr, err := f.updateParseLine(text) + if err != nil { + return fmt.Errorf("cannot parse a line: %w", err) + } + + if err := f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree); err != nil { + return fmt.Errorf("cannot add a node to the tree: %w", err) + } + } + + if scanner.Err() != nil { + return fmt.Errorf("cannot parse a response: %w", scanner.Err()) + } + + return nil +} + +func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) { + _, ipnet, err := net.ParseCIDR(text) + if err != nil { + ipaddr := net.ParseIP(text) + if ipaddr == nil { + return nil, 0, fmt.Errorf("incorrect ip address %s", text) + } + + ip4 := ipaddr.To4() + if ip4 != nil { + return ip4, fireholIPv4DefaultCIDR, nil + } + + return ipaddr.To16(), fireholIPv6DefaultCIDR, nil + } + + ones, _ := ipnet.Mask.Size() + + return ipnet.IP, uint(ones), nil +} + +func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + mutex.Lock() + defer mutex.Unlock() + + if ip.To4() != nil { + addr := patricia.NewIPv4AddressFromBytes(ip, cidr) + + if _, _, err := v4tree.Set(addr, true); err != nil { + return err // nolint: wrapcheck + } + } else { + addr := patricia.NewIPv6Address(ip, cidr) + + if _, _, err := v6tree.Set(addr, true); err != nil { + return err // nolint: wrapcheck + } + } + + return nil +} + +// NewFirehol creates a new instance of FireHOL IP blocklist. +// +// This method does not start an update process so please execute Run +// when it is necessary. +func NewFirehol(logger mtglib.Logger, network mtglib.Network, + downloadConcurrency uint, + remoteURLs []string, + localFiles []string) (*Firehol, error) { + for _, v := range remoteURLs { + parsed, err := url.Parse(v) + if err != nil { + return nil, fmt.Errorf("incorrect url %s: %w", v, err) + } + + switch parsed.Scheme { + case "http", "https": + default: + return nil, fmt.Errorf("unsupported url %s", v) + } + } + + for _, v := range localFiles { + if stat, err := os.Stat(v); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 { + return nil, fmt.Errorf("%s is not a readable file", v) + } + } + + if downloadConcurrency == 0 { + downloadConcurrency = DefaultFireholDownloadConcurrency + } + + workerPool, _ := ants.NewPool(int(downloadConcurrency)) + ctx, cancel := context.WithCancel(context.Background()) + + return &Firehol{ + ctx: ctx, + ctxCancel: cancel, + logger: logger.Named("firehol"), + httpClient: network.MakeHTTPClient(nil), + treeV4: bool_tree.NewTreeV4(), + treeV6: bool_tree.NewTreeV6(), + workerPool: workerPool, + remoteURLs: remoteURLs, + localFiles: localFiles, + }, nil +} diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go new file mode 100644 index 0000000..9d26c6d --- /dev/null +++ b/ipblocklist/firehol_test.go @@ -0,0 +1,148 @@ +package ipblocklist_test + +import ( + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/network" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type FireholTestSuite struct { + suite.Suite + + networkMock *testlib.MtglibNetworkMock + httpServer *httptest.Server +} + +func (suite *FireholTestSuite) SetupSuite() { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + filefp, err := os.Open(filepath.Join("testdata", "remote_ipset.ipset")) + if err != nil { + panic(err) + } + + defer filefp.Close() + + io.Copy(w, filefp) // nolint: errcheck + }) + + suite.httpServer = httptest.NewServer(mux) +} + +func (suite *FireholTestSuite) SetupTest() { + httpClient := &http.Client{} + suite.networkMock = &testlib.MtglibNetworkMock{} + + httpmock.ActivateNonDefault(httpClient) + + suite.networkMock. + On("MakeHTTPClient", mock.Anything). + Maybe(). + Return(httpClient) +} + +func (suite *FireholTestSuite) TearDownTest() { + suite.networkMock.AssertExpectations(suite.T()) + httpmock.DeactivateAndReset() +} + +func (suite *FireholTestSuite) TearDownSuite() { + suite.httpServer.Close() +} + +func (suite *FireholTestSuite) TestLocalFail() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + nil, []string{filepath.Join("testdata", "broken_ipset.ipset")}) + + suite.NoError(err) + + go blocklist.Run(time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.False(blocklist.Contains(net.ParseIP("10.0.0.10"))) + suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) +} + +func (suite *FireholTestSuite) TestLocalOk() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + nil, []string{filepath.Join("testdata", "good_ipset.ipset")}) + + suite.NoError(err) + + go blocklist.Run(time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.True(blocklist.Contains(net.ParseIP("10.0.0.10"))) + suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) +} + +func (suite *FireholTestSuite) TestRemoteFail() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + []string{"https://google.com"}, nil) + + suite.NoError(err) + + go blocklist.Run(time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.False(blocklist.Contains(net.ParseIP("10.2.2.2"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) +} + +func (suite *FireholTestSuite) TestMixed() { + dialer, _ := network.NewDefaultDialer(0, 0) + ntw, _ := network.NewNetwork(dialer, "mtg", "1.1.1.1", 0) + + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + ntw, 2, + []string{ + suite.httpServer.URL, + }, []string{ + filepath.Join("testdata", "good_ipset.ipset"), + }) + + suite.NoError(err) + + go blocklist.Run(time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.True(blocklist.Contains(net.ParseIP("10.2.2.2"))) + suite.True(blocklist.Contains(net.ParseIP("10.1.0.100"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) +} + +func TestFirehol(t *testing.T) { + t.Parallel() + suite.Run(t, &FireholTestSuite{}) +} diff --git a/ipblocklist/init.go b/ipblocklist/init.go new file mode 100644 index 0000000..f5a06d3 --- /dev/null +++ b/ipblocklist/init.go @@ -0,0 +1,18 @@ +// Package ipblocklist contains default implementation of the +// IPBlocklist for mtg. +// +// Please check documentation for mtglib.IPBlocklist interface to get an +// idea of this abstraction. +package ipblocklist + +import "time" + +const ( + // DefaultFireholDownloadConcurrency defines a default max number of + // concurrent downloads of ip blocklists for Firehol. + DefaultFireholDownloadConcurrency = 1 + + // DefaultFireholUpdateEach defines a default time period when + // Firehol requests updates of the blocklists. + DefaultFireholUpdateEach = 6 * time.Hour +) diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go new file mode 100644 index 0000000..2df31cb --- /dev/null +++ b/ipblocklist/noop.go @@ -0,0 +1,17 @@ +package ipblocklist + +import ( + "net" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type noop struct{} + +func (n noop) Contains(ip net.IP) bool { return false } + +// NewNoop returns a dummy ipblocklist which allows all incoming +// connections. +func NewNoop() mtglib.IPBlocklist { + return noop{} +} diff --git a/ipblocklist/noop_test.go b/ipblocklist/noop_test.go new file mode 100644 index 0000000..f5bb346 --- /dev/null +++ b/ipblocklist/noop_test.go @@ -0,0 +1,23 @@ +package ipblocklist_test + +import ( + "net" + "testing" + + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite +} + +func (suite *NoopTestSuite) TestOp() { + suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) + suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/ipblocklist/testdata/broken_ipset.ipset b/ipblocklist/testdata/broken_ipset.ipset new file mode 100644 index 0000000..2758925 --- /dev/null +++ b/ipblocklist/testdata/broken_ipset.ipset @@ -0,0 +1,5 @@ +# +# This is an intentionally broken ipset. +# + +ajsdkfbd diff --git a/ipblocklist/testdata/good_ipset.ipset b/ipblocklist/testdata/good_ipset.ipset new file mode 100644 index 0000000..411ba01 --- /dev/null +++ b/ipblocklist/testdata/good_ipset.ipset @@ -0,0 +1,7 @@ +# +# This is very good ipset +# + +10.0.0.10 # just an example +10.1.0.0/24 +2001:0db8:85a3:0000:0000:8a2e:0370:7334 diff --git a/ipblocklist/testdata/remote_ipset.ipset b/ipblocklist/testdata/remote_ipset.ipset new file mode 100644 index 0000000..80d49e4 --- /dev/null +++ b/ipblocklist/testdata/remote_ipset.ipset @@ -0,0 +1 @@ +10.2.2.2 diff --git a/logger/init.go b/logger/init.go new file mode 100644 index 0000000..8ff8ddc --- /dev/null +++ b/logger/init.go @@ -0,0 +1,14 @@ +// Package logger has implementation of loggers for mtglib.Logger +// interface. +// +// Please see a description of that interface to get some agreements +// which are used by mtglib. +package logger + +// StdLikeLogger is an interface which is close to log.Logger. This is +// commonly used by many 3pp tools. While mtglib itself does not need +// it, it is always a good idea to support it and have a transient end +// to end logging. +type StdLikeLogger interface { + Printf(format string, args ...interface{}) +} diff --git a/logger/noop.go b/logger/noop.go new file mode 100644 index 0000000..61470ac --- /dev/null +++ b/logger/noop.go @@ -0,0 +1,21 @@ +package logger + +import "github.com/9seconds/mtg/v2/mtglib" + +type noopLogger struct{} + +func (n noopLogger) Named(_ string) mtglib.Logger { return n } +func (n noopLogger) BindInt(_ string, _ int) mtglib.Logger { return n } +func (n noopLogger) BindStr(_, _ string) mtglib.Logger { return n } +func (n noopLogger) Printf(_ string, _ ...interface{}) {} +func (n noopLogger) Info(_ string) {} +func (n noopLogger) Warning(_ string) {} +func (n noopLogger) Debug(_ string) {} +func (n noopLogger) InfoError(_ string, _ error) {} +func (n noopLogger) WarningError(_ string, _ error) {} +func (n noopLogger) DebugError(_ string, _ error) {} + +// NewNoopLogger returns a logger which discards all events. +func NewNoopLogger() mtglib.Logger { + return noopLogger{} +} diff --git a/logger/noop_test.go b/logger/noop_test.go new file mode 100644 index 0000000..dca71a8 --- /dev/null +++ b/logger/noop_test.go @@ -0,0 +1,35 @@ +package logger_test + +import ( + "io" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/logger" + "github.com/stretchr/testify/suite" +) + +type NoopLoggerTestSuite struct { + suite.Suite +} + +func (suite *NoopLoggerTestSuite) TestLog() { + suite.Empty(testlib.CaptureStdout(func() { + suite.Empty(testlib.CaptureStderr(func() { + log := logger.NewNoopLogger().Named("name") + + log.BindInt("int", 1).BindStr("str", "1").Printf("info", 1, 2) + log.BindInt("int", 1).BindStr("str", "1").Info("info") + log.BindInt("int", 1).BindStr("str", "1").Warning("info") + log.BindInt("int", 1).BindStr("str", "1").Debug("info") + log.BindInt("int", 1).BindStr("str", "1").InfoError("info", io.EOF) + log.BindInt("int", 1).BindStr("str", "1").WarningError("info", io.EOF) + log.BindInt("int", 1).BindStr("str", "1").DebugError("info", io.EOF) + })) + })) +} + +func TestNoopLogger(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopLoggerTestSuite{}) +} diff --git a/logger/zerolog.go b/logger/zerolog.go new file mode 100644 index 0000000..b0dc0b6 --- /dev/null +++ b/logger/zerolog.go @@ -0,0 +1,122 @@ +package logger + +import ( + "fmt" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/rs/zerolog" +) + +const loggerFieldName = "logger" + +type zeroLogContextVarType uint8 + +const ( + zeroLogContextVarTypeUnknown zeroLogContextVarType = iota + zeroLogContextVarTypeStr + zeroLogContextVarTypeInt +) + +type zeroLogContext struct { + name string + log *zerolog.Logger + + ctxVarType zeroLogContextVarType + ctxVarName string + ctxVarStr string + ctxVarInt int + + parent *zeroLogContext +} + +func (z *zeroLogContext) Named(name string) mtglib.Logger { + loggerName := z.name + if loggerName == "" { + loggerName = name + } else { + loggerName += "." + name + } + + return &zeroLogContext{ + name: loggerName, + log: z.log, + parent: z, + } +} + +func (z *zeroLogContext) BindInt(name string, value int) mtglib.Logger { + return &zeroLogContext{ + name: z.name, + log: z.log, + ctxVarType: zeroLogContextVarTypeInt, + ctxVarInt: value, + ctxVarName: name, + parent: z, + } +} + +func (z *zeroLogContext) BindStr(name, value string) mtglib.Logger { + return &zeroLogContext{ + name: z.name, + log: z.log, + ctxVarType: zeroLogContextVarTypeStr, + ctxVarStr: value, + ctxVarName: name, + parent: z, + } +} + +func (z *zeroLogContext) Printf(format string, args ...interface{}) { + z.Debug(fmt.Sprintf(format, args...)) +} + +func (z *zeroLogContext) Info(msg string) { + z.InfoError(msg, nil) +} + +func (z *zeroLogContext) Warning(msg string) { + z.WarningError(msg, nil) +} + +func (z *zeroLogContext) Debug(msg string) { + z.DebugError(msg, nil) +} + +func (z *zeroLogContext) InfoError(msg string, err error) { + z.emitLog(z.log.Info(), msg, err) +} + +func (z *zeroLogContext) WarningError(msg string, err error) { + z.emitLog(z.log.Warn(), msg, err) +} + +func (z *zeroLogContext) DebugError(msg string, err error) { + z.emitLog(z.log.Debug(), msg, err) +} + +func (z *zeroLogContext) emitLog(evt *zerolog.Event, msg string, err error) { + z.attachCtx(evt) + + for current := z.parent; current != nil; current = current.parent { + current.attachCtx(evt) + } + + evt.Str(loggerFieldName, z.name).Err(err).Msg(msg) +} + +func (z *zeroLogContext) attachCtx(evt *zerolog.Event) { + switch z.ctxVarType { + case zeroLogContextVarTypeStr: + evt.Str(z.ctxVarName, z.ctxVarStr) + case zeroLogContextVarTypeInt: + evt.Int(z.ctxVarName, z.ctxVarInt) + case zeroLogContextVarTypeUnknown: + } +} + +// NewZeroLogger returns a logger which is using rs/zerolog library. +func NewZeroLogger(log zerolog.Logger) mtglib.Logger { + return &zeroLogContext{ + log: &log, + } +} diff --git a/logger/zerolog_test.go b/logger/zerolog_test.go new file mode 100644 index 0000000..a3e9ae2 --- /dev/null +++ b/logger/zerolog_test.go @@ -0,0 +1,121 @@ +package logger_test + +import ( + "bytes" + "encoding/json" + "io" + "strings" + "testing" + "time" + + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type zeroLoggerLogMessage struct { + Timestamp int64 `json:"timestamp"` + Level string `json:"level"` + StrParam string `json:"strparam"` + IntParam int `json:"intparam"` + Logger string `json:"logger"` + Error string `json:"error"` + Message string `json:"message"` +} + +type ZeroLoggerTestSuite struct { + suite.Suite +} + +func (suite *ZeroLoggerTestSuite) SetupSuite() { + zerolog.SetGlobalLevel(zerolog.TraceLevel) + + zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs + zerolog.TimestampFieldName = "timestamp" + zerolog.LevelFieldName = "level" +} + +func (suite *ZeroLoggerTestSuite) TestLog() { + testData := map[string]func(mtglib.Logger){ + "info": func(l mtglib.Logger) { l.Info("hello") }, + "warn": func(l mtglib.Logger) { l.Warning("hello") }, + "printf": func(l mtglib.Logger) { l.Printf("hello") }, + "debug": func(l mtglib.Logger) { l.Debug("hello") }, + "info-error": func(l mtglib.Logger) { l.InfoError("hello", io.EOF) }, + "warn-error": func(l mtglib.Logger) { l.WarningError("hello", io.EOF) }, + "debug-error": func(l mtglib.Logger) { l.DebugError("hello", io.EOF) }, + } + + for k, v := range testData { + name := k + callback := v + level := strings.TrimSuffix(name, "-error") + + suite.T().Run(name, func(t *testing.T) { + buf := &bytes.Buffer{} + log := logger.NewZeroLogger(zerolog.New(buf).With().Timestamp().Logger()) + + callback(log.Named("name").BindInt("intparam", 1).BindStr("strparam", name)) + + msg := &zeroLoggerLogMessage{} + assert.NoError(t, json.Unmarshal(buf.Bytes(), msg)) + + timestamp := time.Unix(msg.Timestamp/1000, (msg.Timestamp%1000)*1_000_000) + assert.WithinDuration(t, time.Now(), timestamp, 100*time.Millisecond) + + if level == "printf" { + level = "debug" + } + + assert.Equal(t, level, msg.Level) + assert.Equal(t, name, msg.StrParam) + assert.EqualValues(t, 1, msg.IntParam) + assert.Equal(t, "name", msg.Logger) + assert.Equal(t, "hello", msg.Message) + + if level != name && name != "printf" { + assert.Equal(t, io.EOF.Error(), msg.Error) + } else { + assert.Empty(t, msg.Error) + } + }) + } +} + +func (suite *ZeroLoggerTestSuite) TestIndependence() { + buf := &bytes.Buffer{} + log := logger.NewZeroLogger(zerolog.New(buf).With().Timestamp().Logger()) + + log1 := log.Named("1") + log2 := log.Named("2") + log12 := log1.Named("2") + + log1.BindInt("param", 1).Info("hello") + + log1Output := buf.String() + + buf.Reset() + + log2.BindInt("lalala", 2).Info("hello") + + log2Output := buf.String() + + buf.Reset() + + log12.BindStr("tttt", "qqq").Info("hello") + + log12Output := buf.String() + + suite.NotContains("lalala", log1Output) + suite.NotContains("tttt", log1Output) + suite.NotContains("param", log2Output) + suite.NotContains("tttt", log1Output) + suite.NotContains("param", log12Output) + suite.NotContains("lalala", log12Output) +} + +func TestZeroLogger(t *testing.T) { // nolint: paralleltest + suite.Run(t, &ZeroLoggerTestSuite{}) +} diff --git a/main.go b/main.go index b16f040..b081d8c 100644 --- a/main.go +++ b/main.go @@ -2,189 +2,26 @@ package main import ( "math/rand" - "os" - "runtime/debug" - "strings" "time" - "github.com/9seconds/mtg/cli" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/utils" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "github.com/9seconds/mtg/v2/internal/cli" + "github.com/9seconds/mtg/v2/internal/utils" + "github.com/alecthomas/kong" ) var version = "dev" // has to be set by ldflags -var ( - app = kingpin.New("mtg", "Simple MTPROTO proxy.") - - generateSecretCommand = app.Command("generate-secret", - "Generate new secret") - generateCloakHost = generateSecretCommand.Flag("cloak-host", - "A host to use for TLS cloaking."). - Short('c'). - Default("storage.googleapis.com"). - String() - generateSecretType = generateSecretCommand.Arg("type", - "A type of secret to generate. Valid options are 'simple', 'secured' and 'tls'"). - Required(). - Enum("simple", "secured", "tls") - - runCommand = app.Command("run", - "Run new proxy instance") - runDebug = runCommand.Flag("debug", - "Run in debug mode."). - Short('d'). - Envar("MTG_DEBUG"). - Bool() - runVerbose = runCommand.Flag("verbose", - "Run in verbose mode."). - Short('v'). - Envar("MTG_VERBOSE"). - Bool() - runPreferIP = runCommand.Flag("prefer-ip", - "Prefer this IP protocol if possible. Valid options are 'ipv4' and 'ipv6'"). - Envar("MTG_PREFER_DIRECT_IP"). - Default("ipv6"). - Enum("ipv4", "ipv6") - runBind = runCommand.Flag("bind", - "Host:Port to bind proxy to."). - Short('b'). - Envar("MTG_BIND"). - Default("0.0.0.0:3128"). - TCP() - runPublicIPv4 = runCommand.Flag("public-ipv4", - "Which IPv4 host:port to use."). - Short('4'). - Envar("MTG_IPV4"). - TCP() - runPublicIPv6 = runCommand.Flag("public-ipv6", - "Which IPv6 host:port to use."). - Short('6'). - Envar("MTG_IPV6"). - TCP() - runStatsBind = runCommand.Flag("stats-bind", - "Which Host:Port to bind stats server to."). - Short('t'). - Envar("MTG_STATS_BIND"). - Default("127.0.0.1:3129"). - TCP() - runStatsNamespace = runCommand.Flag("stats-namespace", - "Which namespace to use for Prometheus."). - Envar("MTG_STATS_NAMESPACE"). - Default("mtg"). - String() - runStatsdAddress = runCommand.Flag("statsd-addr", - "Host:port of statsd server"). - Envar("MTG_STATSD_ADDR"). - TCP() - runStatsdTagsFormat = runCommand.Flag("statsd-tags-format", - "Which tag format should we use to send stats metrics. Valid options are 'datadog' and 'influxdb'."). - Envar("MTG_STATSD_TAGS_FORMAT"). - Default("influxdb"). - Enum("datadog", "influxdb") - runStatsdTags = runCommand.Flag("statsd-tags", - "Tags to use for working with statsd (specified as 'key=value')."). - Envar("MTG_STATSD_TAGS"). - StringMap() - runWriteBufferSize = runCommand.Flag("write-buffer", - "Write buffer size. You can think about it as a buffer from client to Telegram."). - Short('w'). - Envar("MTG_BUFFER_WRITE"). - Default("32KB"). - Bytes() - runReadBufferSize = runCommand.Flag("read-buffer", - "Read buffer size. You can think about it as a buffer from Telegram to client."). - Short('r'). - Envar("MTG_BUFFER_READ"). - Default("32KB"). - Bytes() - runTLSCloakPort = runCommand.Flag("cloak-port", - "Port which should be used for host cloaking."). - Envar("MTG_CLOAK_PORT"). - Default("443"). - Uint16() - runAntiReplayMaxSize = runCommand.Flag("anti-replay-max-size", - "Max size of antireplay cache."). - Envar("MTG_ANTIREPLAY_MAXSIZE"). - Default("128MB"). - Bytes() - runMultiplexPerConnection = runCommand.Flag("multiplex-per-connection", - "How many clients can share a single connection to Telegram."). - Envar("MTG_MULTIPLEX_PERCONNECTION"). - Default("50"). - Uint() - runNTPServers = runCommand.Flag("ntp-server", - "A list of NTP servers to use."). - Envar("MTG_NTP_SERVERS"). - Default("0.pool.ntp.org", "1.pool.ntp.org", "2.pool.ntp.org", "3.pool.ntp.org"). - Strings() - runSecret = runCommand.Arg("secret", "Secret of this proxy.").Required().HexBytes() - runAdtag = runCommand.Arg("adtag", "ADTag of the proxy.").HexBytes() -) - func main() { rand.Seed(time.Now().UTC().UnixNano()) - app.Version(getVersion()) - app.HelpFlag.Short('h') if err := utils.SetLimits(); err != nil { - cli.Fatal(err) + panic(err) } - switch kingpin.MustParse(app.Parse(os.Args[1:])) { - case generateSecretCommand.FullCommand(): - cli.Generate(*generateSecretType, *generateCloakHost) - case runCommand.FullCommand(): - err := config.Init( - config.Opt{Option: config.OptionTypeDebug, Value: *runDebug}, - config.Opt{Option: config.OptionTypeVerbose, Value: *runVerbose}, - config.Opt{Option: config.OptionTypePreferIP, Value: *runPreferIP}, - config.Opt{Option: config.OptionTypeBind, Value: *runBind}, - config.Opt{Option: config.OptionTypePublicIPv4, Value: *runPublicIPv4}, - config.Opt{Option: config.OptionTypePublicIPv6, Value: *runPublicIPv6}, - config.Opt{Option: config.OptionTypeStatsBind, Value: *runStatsBind}, - config.Opt{Option: config.OptionTypeStatsNamespace, Value: *runStatsNamespace}, - config.Opt{Option: config.OptionTypeStatsdAddress, Value: *runStatsdAddress}, - config.Opt{Option: config.OptionTypeStatsdTagsFormat, Value: *runStatsdTagsFormat}, - config.Opt{Option: config.OptionTypeStatsdTags, Value: *runStatsdTags}, - config.Opt{Option: config.OptionTypeWriteBufferSize, Value: *runWriteBufferSize}, - config.Opt{Option: config.OptionTypeReadBufferSize, Value: *runReadBufferSize}, - config.Opt{Option: config.OptionTypeCloakPort, Value: *runTLSCloakPort}, - config.Opt{Option: config.OptionTypeAntiReplayMaxSize, Value: *runAntiReplayMaxSize}, - config.Opt{Option: config.OptionTypeMultiplexPerConnection, Value: *runMultiplexPerConnection}, - config.Opt{Option: config.OptionTypeNTPServers, Value: *runNTPServers}, - config.Opt{Option: config.OptionTypeSecret, Value: *runSecret}, - config.Opt{Option: config.OptionTypeAdtag, Value: *runAdtag}, - ) - if err != nil { - cli.Fatal(err) - } + cli := &cli.CLI{} + ctx := kong.Parse(cli, kong.Vars{ + "version": version, + }) - if err := cli.Proxy(); err != nil { - cli.Fatal(err) - } - } -} - -func getVersion() string { - if version != "dev" { - return version - } - - info, ok := debug.ReadBuildInfo() - if !ok { - return version - } - - builder := strings.Builder{} - builder.WriteString(info.Main.Version) - - if info.Main.Sum != "" { - builder.WriteString(" (checksum: ") - builder.WriteString(info.Main.Sum) - builder.WriteRune(')') - } - - return builder.String() + ctx.FatalIfErrorf(ctx.Run(cli, version)) } diff --git a/mtglib/conns.go b/mtglib/conns.go new file mode 100644 index 0000000..da0b046 --- /dev/null +++ b/mtglib/conns.go @@ -0,0 +1,68 @@ +package mtglib + +import ( + "bytes" + "context" + "io" + "net" + "sync" +) + +type connTraffic struct { + net.Conn + + streamID string + stream EventStream + ctx context.Context +} + +func (c connTraffic) Read(b []byte) (int, error) { + n, err := c.Conn.Read(b) + + if n > 0 { + c.stream.Send(c.ctx, NewEventTraffic(c.streamID, uint(n), true)) + } + + return n, err // nolint: wrapcheck +} + +func (c connTraffic) Write(b []byte) (int, error) { + n, err := c.Conn.Write(b) + + if n > 0 { + c.stream.Send(c.ctx, NewEventTraffic(c.streamID, uint(n), false)) + } + + return n, err // nolint: wrapcheck +} + +type connRewind struct { + net.Conn + + active io.Reader + buf bytes.Buffer + mutex sync.RWMutex +} + +func (c *connRewind) Read(p []byte) (int, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.active.Read(p) +} + +func (c *connRewind) Rewind() { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.active = io.MultiReader(&c.buf, c.Conn) +} + +func newConnRewind(conn net.Conn) *connRewind { + rv := &connRewind{ + Conn: conn, + } + rv.active = io.TeeReader(conn, &rv.buf) + + return rv +} diff --git a/mtglib/conns_internal_test.go b/mtglib/conns_internal_test.go new file mode 100644 index 0000000..0c73759 --- /dev/null +++ b/mtglib/conns_internal_test.go @@ -0,0 +1,207 @@ +package mtglib + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnRewindBaseConn struct { + testlib.NetConnMock + + readBuffer bytes.Buffer +} + +func (c *ConnRewindBaseConn) Read(p []byte) (int, error) { + c.Called(p) + + return c.readBuffer.Read(p) +} + +type ConnTrafficTestSuite struct { + suite.Suite + + eventStreamMock *EventStreamMock + connMock *testlib.NetConnMock + conn io.ReadWriter +} + +func (suite *ConnTrafficTestSuite) SetupTest() { + suite.eventStreamMock = &EventStreamMock{} + suite.connMock = &testlib.NetConnMock{} + suite.conn = connTraffic{ + Conn: suite.connMock, + streamID: "CONNID", + ctx: context.Background(), + stream: suite.eventStreamMock, + } +} + +func (suite *ConnTrafficTestSuite) TearDownTest() { + suite.eventStreamMock.AssertExpectations(suite.T()) + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnTrafficTestSuite) TestReadOk() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.True(evt.IsRead) + }) + suite.connMock.On("Read", mock.Anything).Once().Return(10, nil) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.NoError(err) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestReadErr() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.True(evt.IsRead) + }) + suite.connMock.On("Read", mock.Anything).Once().Return(10, io.EOF) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestReadNothingOk() { + suite.connMock.On("Read", mock.Anything).Once().Return(0, nil) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.NoError(err) + suite.Equal(0, n) +} + +func (suite *ConnTrafficTestSuite) TestReadNothingErr() { + suite.connMock.On("Read", mock.Anything).Once().Return(0, io.EOF) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(0, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteOk() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.False(evt.IsRead) + }) + suite.connMock.On("Write", mock.Anything).Once().Return(10, nil) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.NoError(err) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteErr() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.False(evt.IsRead) + }) + suite.connMock.On("Write", mock.Anything).Once().Return(10, io.EOF) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteNothingOk() { + suite.connMock.On("Write", mock.Anything).Once().Return(0, nil) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.NoError(err) + suite.Equal(0, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteNothingErr() { + suite.connMock.On("Write", mock.Anything).Once().Return(0, io.EOF) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(0, n) +} + +type ConnRewindTestSuite struct { + suite.Suite + + connMock *ConnRewindBaseConn + conn *connRewind +} + +func (suite *ConnRewindTestSuite) SetupTest() { + suite.connMock = &ConnRewindBaseConn{} + suite.conn = newConnRewind(suite.connMock) +} + +func (suite *ConnRewindTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnRewindTestSuite) TestRead() { + suite.connMock.On("Read", mock.Anything) + suite.connMock.readBuffer.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + buf := make([]byte, 2) + + n, err := suite.conn.Read(buf) + suite.NoError(err) + suite.Equal(2, n) + suite.Equal([]byte{1, 2}, buf) + + n, err = suite.conn.Read(buf) + suite.NoError(err) + suite.Equal(2, n) + suite.Equal([]byte{3, 4}, buf) + + suite.conn.Rewind() + + data, err := io.ReadAll(suite.conn) + suite.NoError(err) + suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, data) +} + +func TestConnTraffic(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTrafficTestSuite{}) +} + +func TestConnRewind(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnRewindTestSuite{}) +} diff --git a/mtglib/events.go b/mtglib/events.go new file mode 100644 index 0000000..1bc6fb7 --- /dev/null +++ b/mtglib/events.go @@ -0,0 +1,178 @@ +package mtglib + +import ( + "net" + "time" +) + +type eventBase struct { + streamID string + timestamp time.Time +} + +// StreamID returns a ID of the stream this event belongs to. +func (e eventBase) StreamID() string { + return e.streamID +} + +// Timestamp return a time when this event was generated. +func (e eventBase) Timestamp() time.Time { + return e.timestamp +} + +// EventStart is emitted when mtg proxy starts to process a new +// connection. +type EventStart struct { + eventBase + + // RemoteIP is an IP address of the client. + RemoteIP net.IP +} + +// EventConnectedToDC is emitted when mtg proxy has connected to a +// Telegram server. +type EventConnectedToDC struct { + eventBase + + // RemoteIP is an IP address of the Telegram server proxy has been + // connected to. + RemoteIP net.IP + + // DC is an index of the datacenter proxy has been connected to. + DC int +} + +// EventTraffic is emitted when we read/write some bytes on a connection. +type EventTraffic struct { + eventBase + + // Traffic is a count of bytes which were transmitted. + Traffic uint + + // IsRead defines if we _read_ or _write_ to connection. A rule of + // thumb is simple: EventTraffic is bound to a remote connection. Not + // to a client one, but either to Telegram or front domain one. + // + // In the case of Telegram, isRead means that we've fetched some bytes + // from Telegram to send it to a client. + // + // In the case of the front domain, it means that we've fetched some + // bytes from this domain to send it to a client. + IsRead bool +} + +// EventFinish is emitted when we stop to manage a connection. +type EventFinish struct { + eventBase +} + +// EventDomainFronting is emitted when we connect to a front domain +// instead of Telegram server. +type EventDomainFronting struct { + eventBase +} + +// EventConcurrencyLimited is emitted when connection was declined +// because of the concurrency limit of the worker pool. +type EventConcurrencyLimited struct { + eventBase +} + +// EventIPBlocklisted is emitted when connection was declined because +// IP address was found in IP blocklist. +type EventIPBlocklisted struct { + eventBase + + RemoteIP net.IP +} + +// EventReplayAttack is emitted when mtg detects a replay attack on a +// connection. +type EventReplayAttack struct { + eventBase +} + +// NewEventStart creates a new EventStart event. +func NewEventStart(streamID string, remoteIP net.IP) EventStart { + return EventStart{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + RemoteIP: remoteIP, + } +} + +// NewEventConnectedToDC creates a new EventConnectedToDC event. +func NewEventConnectedToDC(streamID string, remoteIP net.IP, dc int) EventConnectedToDC { + return EventConnectedToDC{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + RemoteIP: remoteIP, + DC: dc, + } +} + +// NewEventTraffic creates a new EventTraffic event. +func NewEventTraffic(streamID string, traffic uint, isRead bool) EventTraffic { + return EventTraffic{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + Traffic: traffic, + IsRead: isRead, + } +} + +// NewEventFinish creates a new EventFinish event. +func NewEventFinish(streamID string) EventFinish { + return EventFinish{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + } +} + +// NewEventDomainFronting creates a new EventDomainFronting event. +func NewEventDomainFronting(streamID string) EventDomainFronting { + return EventDomainFronting{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + } +} + +// NewEventConcurrencyLimited creates a new EventConcurrencyLimited +// event. +func NewEventConcurrencyLimited() EventConcurrencyLimited { + return EventConcurrencyLimited{ + eventBase: eventBase{ + timestamp: time.Now(), + }, + } +} + +// NewEventIPBlocklisted creates a new EventIPBlocklisted event. +func NewEventIPBlocklisted(remoteIP net.IP) EventIPBlocklisted { + return EventIPBlocklisted{ + eventBase: eventBase{ + timestamp: time.Now(), + }, + RemoteIP: remoteIP, + } +} + +// NewEventReplayAttack creates a new EventReplayAttack event. +func NewEventReplayAttack(streamID string) EventReplayAttack { + return EventReplayAttack{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + } +} diff --git a/mtglib/events_test.go b/mtglib/events_test.go new file mode 100644 index 0000000..bb88b0a --- /dev/null +++ b/mtglib/events_test.go @@ -0,0 +1,75 @@ +package mtglib_test + +import ( + "net" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type EventsTestSuite struct { + suite.Suite +} + +func (suite *EventsTestSuite) TestEventStart() { + evt := mtglib.NewEventStart("CONNID", net.ParseIP("10.0.0.10")) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventFinish() { + evt := mtglib.NewEventFinish("CONNID") + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventConnectedToDC() { + evt := mtglib.NewEventConnectedToDC("CONNID", net.ParseIP("10.0.0.10"), 3) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventTraffic() { + evt := mtglib.NewEventTraffic("CONNID", 1000, true) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventDomainFronting() { + evt := mtglib.NewEventDomainFronting("CONNID") + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventConcurrencyLimited() { + evt := mtglib.NewEventConcurrencyLimited() + + suite.Empty(evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventIPBlocklisted() { + evt := mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")) + + suite.Empty(evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func (suite *EventsTestSuite) TestEventReplayAttack() { + evt := mtglib.NewEventReplayAttack("CONNID") + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + +func TestEvents(t *testing.T) { + t.Parallel() + suite.Run(t, &EventsTestSuite{}) +} diff --git a/mtglib/init.go b/mtglib/init.go new file mode 100644 index 0000000..d4f265b --- /dev/null +++ b/mtglib/init.go @@ -0,0 +1,264 @@ +// mtglib defines a package with MTPROTO proxy. +// +// Since mtg itself is build as an example of how to work with mtglib, +// it worth to telling a couple of words about a project organization. +// +// A core object of the project is mtglib.Proxy. This is a proxy you +// expect: that one which you configure, set to serve on a listener +// and/or shutdown on application termination. +// +// But it also has a core logic unrelated to Telegram per se: anti +// replay cache, network connectivity (who knows, maybe you want to have +// a native VMESS integration) and so on. +// +// You can supply such parts to a proxy with interfaces. The rest of +// the packages in mtg define some default implementations of these +// interfaces. But if you want to integrate it with, let say, influxdb, +// you can do it easily. +package mtglib + +import ( + "context" + "errors" + "net" + "net/http" + "time" +) + +var ( + // ErrSecretEmpty is returned if you are trying to create a proxy + // but do not provide a secret. + ErrSecretEmpty = errors.New("secret is empty") + + // ErrSecretInvalid is returned if you are trying to create a proxy + // but secret value is invalid (no host or payload are zeroes). + ErrSecretInvalid = errors.New("secret is invalid") + + // ErrNetworkIsNotDefined is returned if you are trying to create a + // proxy but network value is undefined. + ErrNetworkIsNotDefined = errors.New("network is not defined") + + // ErrAntiReplayCacheIsNotDefined is returned if you are trying to + // create a proxy but anti replay cache value is undefined. + ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") + + // ErrIPBlocklistIsNotDefined is returned if you are trying to + // create a proxy but ip blocklist instance is not defined. + ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") + + // ErrEventStreamIsNotDefined is returned if you are trying to create a + // proxy but event stream instance is not defined. + ErrEventStreamIsNotDefined = errors.New("event stream is not defined") + + // ErrLoggerIsNotDefined is returned if you are trying to + // create a proxy but logger is not defined. + ErrLoggerIsNotDefined = errors.New("logger is not defined") +) + +const ( + // DefaultConcurrency is a default max count of simultaneously + // connected clients. + DefaultConcurrency = 4096 + + // DefaultBufferSize is a default size of a copy buffer. + DefaultBufferSize = 16 * 1024 // 16 kib + + // DefaultDomainFrontingPort is a default port (HTTPS) to connect to in + // case of probe-resistance activity. + DefaultDomainFrontingPort = 443 + + // DefaultIdleTimeout is a default timeout for closing a connection + // in case of idling. + DefaultIdleTimeout = time.Minute + + // DefaultTolerateTimeSkewness is a default timeout for time + // skewness on a faketls timeout verification. + DefaultTolerateTimeSkewness = 3 * time.Second + + // DefaultPreferIP is a default value for Telegram IP connectivity + // preference. + DefaultPreferIP = "prefer-ipv6" + + // SecretKeyLength defines a length of the secret bytes used + // by Telegram and a proxy. + SecretKeyLength = 16 + + // ConnectionIDBytesLength defines a count of random bytes + // used to generate a stream/connection ids. + ConnectionIDBytesLength = 16 +) + +// Network defines a knowledge how to work with a network. It may sound +// fun but it encapsulates all the knowledge how to properly establish +// connections to remote hosts and configure HTTP clients. +// +// For example, if you want to use SOCKS5 proxy, you probably want to +// have all traffic routed to this proxy: telegram connections, http +// requests and so on. This knowledge is encapsulated into instances of +// such interface. +// +// mtglib uses Network for: +// +// 1. Dialing to Telegram +// +// 2. Dialing to front domain +// +// 3. Doing HTTP requests (for example, for FireHOL ipblocklist). +type Network interface { + // Dial establishes context-free TCP connections. + Dial(network, address string) (net.Conn, error) + + // DialContext dials using a context. This is a preferrable + // way of establishing TCP connections. + DialContext(ctx context.Context, network, address string) (net.Conn, error) + + // MakeHTTPClient build an HTTP client with given dial function. If + // nothing is provided, then DialContext of this interface is going + // to be used. + MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client +} + +// AntiReplayCache is an interface that is used to detect replay attacks +// based on some traffic fingerprints. +// +// Replay attacks are probe attacks whose main goal is to identify if +// server software can be classified in some way. For example, if you +// send some HTTP request to a web server, then you can expect that this +// server will respond with HTTP response back. +// +// There is a problem though. Let's imagine, that connection is +// encrypted. Let's imagine, that it is encrypted with some static key +// like ShadowSocks (https://shadowsocks.org/assets/whitepaper.pdf). +// In that case, in theory, if you repeat the same bytes, you can get +// the same responses. Let's imagine, that you've cracked the key. then +// if you send the same bytes, you can decrypt a response and see its +// structure. Based on its structure you can identify if this server is +// SOCKS5, MTPROTO proxy etc. +// +// This is just one example, maybe not the best or not the most +// relevant. In real life, different organizations use such replay +// attacks to perform some reverse engineering of the proxy, do some +// statical analysis to identify server software. +// +// There are many ways how to protect your proxy against them. One +// is domain fronting which is a core part of mtg. Another one is to +// collect some 'handshake fingerprints' and forbid duplication. +// +// So, it one is sending the same byte flow right after you (or a couple +// of hours after), mtg should detect that and reject this connection +// (or redirect to fronting domain). +type AntiReplayCache interface { + // Seen before checks if this set of bytes was observed before or + // not. If it is required to store this information somewhere else, + // then it has to do that. + SeenBefore(data []byte) bool +} + +// IPBlocklist filters requests based on IP address. +// +// If this filter has an IP address, then mtg closes a request without +// reading anything from a socket. It also does not give such request to +// a worker pool, so in worst cases you can expect that you invoke this +// object more frequent than defined proxy concurrency. +type IPBlocklist interface { + // Contains checks if given IP address belongs to this blocklist If. + // it is, a connection is terminated . + Contains(net.IP) bool +} + +// Event is a data structure which is populated during mtg request +// processing lifecycle. Each request popluates many events: +// +// 1. Client connected +// +// 2. Request is finished +// +// 3. Connection to Telegram server is established +// +// and so on. All these events are data structures but all of them +// must conform the same interface. +type Event interface { + // StreamID returns an identifier of the stream, connection, + // request, you name it. All events within the same stream returns + // the same stream id. + StreamID() string + + // Timestamp returns a timestamp when this event was generated. + Timestamp() time.Time +} + +// EventStream is an abstraction that accepts a set of events produced +// by mtg. Its main goal is to inject your logging or monitoring system. +// +// The idea is simple. When mtg works, it emits a set of events during +// a lifecycle of the requestor: EventStart, EventFinish etc. mtg is a +// producer which puts these events into a stream. Responsibility of +// the stream is to deliver this event to consumers/observers. There +// might be many different observers (for example, you want to have both +// statsd and prometheus), mtg should know nothing about them. +type EventStream interface { + // Send delivers an event to observers. Given context has to be + // respected. If the context is closed, all blocking operations should + // be released ASAP. + // + // It is possible that context is closed but the message is delivered. + // EventStream implementations should solve this issue somehow. + Send(context.Context, Event) +} + +// Logger defines an interface of the logger used by mtglib. +// +// Each logger has a name. It is possible to stack names to organize +// poor-man namespaces. Also, each logger must be able to bind +// parameters to avoid pushing them all the time. +// +// Example +// +// logger := SomeLogger{} +// logger = logger.BindStr("ip", net.IP{127, 0, 0, 1}) +// logger.Info("Hello") +// +// In that case, ip is bound as a parameter. It is a great idea to +// put this parameter somewhere in a log message. +// +// logger1 = logger.BindStr("param1", "11") +// logger2 = logger.BindInt("param2", 11) +// +// logger1 should see no param2 and vice versa, logger2 should not see param1 +// If you attach a parameter to a logger, parents should not know about that. +type Logger interface { + // Named returns a new logger with a bound name. Name chaining is + // allowed and appreciated. + Named(name string) Logger + + // BindInt binds new integer parameter to a new logger instance. + BindInt(name string, value int) Logger + + // BindStr binds new string parameter to a new logger instance. + BindStr(name, value string) Logger + + // Printf is to support log.Logger behavior. + Printf(format string, args ...interface{}) + + // Info puts a message about some normal situation. + Info(msg string) + + // InfoError puts a message about some normal situation but this + // situation is related to a given error. + InfoError(msg string, err error) + + // Warning puts a message about some extraordinary situation + // worth to look at. + Warning(msg string) + + // WarningError puts a message about some extraordinary situation + // worth to look at. This situation is related to a given error. + WarningError(msg string, err error) + + // Debug puts a message useful for debugging only. + Debug(msg string) + + // Debug puts a message useful for debugging only. This message is + // related to a given error. + DebugError(msg string, err error) +} diff --git a/mtglib/init_internal_test.go b/mtglib/init_internal_test.go new file mode 100644 index 0000000..98127dd --- /dev/null +++ b/mtglib/init_internal_test.go @@ -0,0 +1,28 @@ +package mtglib + +import ( + "context" + + "github.com/stretchr/testify/mock" +) + +type NoopLogger struct{} + +func (n NoopLogger) Named(_ string) Logger { return n } +func (n NoopLogger) BindInt(_ string, _ int) Logger { return n } +func (n NoopLogger) BindStr(_, _ string) Logger { return n } +func (n NoopLogger) Printf(_ string, _ ...interface{}) {} +func (n NoopLogger) Info(_ string) {} +func (n NoopLogger) Warning(_ string) {} +func (n NoopLogger) Debug(_ string) {} +func (n NoopLogger) InfoError(_ string, _ error) {} +func (n NoopLogger) WarningError(_ string, _ error) {} +func (n NoopLogger) DebugError(_ string, _ error) {} + +type EventStreamMock struct { + mock.Mock +} + +func (e *EventStreamMock) Send(ctx context.Context, evt Event) { + e.Called(ctx, evt) +} diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go new file mode 100644 index 0000000..2062a8a --- /dev/null +++ b/mtglib/internal/faketls/client_hello.go @@ -0,0 +1,134 @@ +package faketls + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/binary" + "fmt" + "time" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" +) + +type ClientHello struct { + Time time.Time + Random [RandomLen]byte + SessionID []byte + Host string + CipherSuite uint16 +} + +func (c ClientHello) Valid(hostname string, tolerateTimeSkewness time.Duration) error { + if c.Host != "" && c.Host != hostname { + return fmt.Errorf("incorrect hostname %s", hostname) + } + + now := time.Now() + + timeDiff := now.Sub(c.Time) + if timeDiff < 0 { + timeDiff = -timeDiff + } + + if timeDiff > tolerateTimeSkewness { + return fmt.Errorf("incorrect timestamp. got=%d, now=%d, diff=%s", + c.Time.Unix(), now.Unix(), timeDiff.String()) + } + + return nil +} + +func ParseClientHello(secret, handshake []byte) (ClientHello, error) { + hello := ClientHello{} + + if len(handshake) < ClientHelloMinLen { + return hello, fmt.Errorf("lengh of handshake is too small: %d", len(handshake)) + } + + if handshake[0] != HandshakeTypeClient { + return hello, fmt.Errorf("unknown handshake type %#x", handshake[0]) + } + + handshakeSizeBytes := [4]byte{0, handshake[1], handshake[2], handshake[3]} + handshakeLength := binary.BigEndian.Uint32(handshakeSizeBytes[:]) + + if len(handshake)-4 != int(handshakeLength) { + return hello, + fmt.Errorf("incorrect handshake size. manifested=%d, real=%d", + handshakeLength, len(handshake)-4) // nolint: gomnd + } + + copy(hello.Random[:], handshake[ClientHelloRandomOffset:]) + copy(handshake[ClientHelloRandomOffset:], clientHelloEmptyRandom) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeHandshake + rec.Version = record.Version10 + rec.Payload.Write(handshake) + + // mac is calculated for the whole record, not only + // for the payload part + mac := hmac.New(sha256.New, secret) + rec.Dump(mac) // nolint: errcheck + + computedRandom := mac.Sum(nil) + + for i := 0; i < RandomLen; i++ { + computedRandom[i] ^= hello.Random[i] + } + + if subtle.ConstantTimeCompare(clientHelloEmptyRandom[:RandomLen-4], computedRandom[:RandomLen-4]) != 1 { + return hello, ErrBadDigest + } + + timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:])) + hello.Time = time.Unix(timestamp, 0) + + parseSessionID(&hello, handshake) + parseCipherSuite(&hello, handshake) + parseSNI(&hello, handshake) + + return hello, nil +} + +func parseSessionID(hello *ClientHello, handshake []byte) { + hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) + copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) +} + +func parseCipherSuite(hello *ClientHello, handshake []byte) { + cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3 // nolint: gomnd + hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2]) +} + +func parseSNI(hello *ClientHello, handshake []byte) { + cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 1 + handshake = handshake[cipherSuiteOffset:] + + cipherSuiteLength := binary.BigEndian.Uint16(handshake[:2]) + handshake = handshake[2+cipherSuiteLength:] + + compressionMethodsLength := int(handshake[0]) + handshake = handshake[1+compressionMethodsLength:] + + extensionsLength := binary.BigEndian.Uint16(handshake[:2]) + handshake = handshake[2 : 2+extensionsLength] + + for len(handshake) > 0 { + if binary.BigEndian.Uint16(handshake[:2]) != ExtensionSNI { + extensionsLength := binary.BigEndian.Uint16(handshake[2:4]) + handshake = handshake[4+extensionsLength:] + + continue + } + + hostnameLength := binary.BigEndian.Uint16(handshake[7:9]) + handshake = handshake[9:] + hello.Host = string(handshake[:int(hostnameLength)]) + + return + } +} diff --git a/mtglib/internal/faketls/client_hello_test.go b/mtglib/internal/faketls/client_hello_test.go new file mode 100644 index 0000000..4d5ad53 --- /dev/null +++ b/mtglib/internal/faketls/client_hello_test.go @@ -0,0 +1,191 @@ +package faketls_test + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type ClientHelloSnapshot struct { + Time int `json:"time"` + Random string `json:"random"` + SessionID string `json:"session-id"` + Host string `json:"host"` + CipherSuite int `json:"cipher-suite"` + Full string `json:"full"` +} + +func (c ClientHelloSnapshot) GetTime() time.Time { + return time.Unix(int64(c.Time), 0) +} + +func (c ClientHelloSnapshot) GetRandom() []byte { + data, _ := base64.StdEncoding.DecodeString(c.Random) + + return data +} + +func (c ClientHelloSnapshot) GetSessionID() []byte { + data, _ := base64.StdEncoding.DecodeString(c.SessionID) + + return data +} + +func (c ClientHelloSnapshot) GetHost() string { + return c.Host +} + +func (c ClientHelloSnapshot) GetCipherSuite() uint16 { + return uint16(c.CipherSuite) +} + +func (c ClientHelloSnapshot) GetFull() []byte { + data, _ := base64.StdEncoding.DecodeString(c.Full) + + return data +} + +type ClientHelloTestSuite struct { + suite.Suite + + secret mtglib.Secret +} + +func (suite *ClientHelloTestSuite) SetupSuite() { + parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d") + if err != nil { + panic(err) + } + + suite.secret = parsed +} + +func (suite *ClientHelloTestSuite) TestEmptyHandshake() { + _, err := faketls.ParseClientHello(suite.secret.Key[:], nil) + suite.Error(err) +} + +func (suite *ClientHelloTestSuite) TestIncorrectHandshakeType() { + data := make([]byte, 1024) + data[0] = 0x02 + + _, err := faketls.ParseClientHello(suite.secret.Key[:], data) + suite.Error(err) +} + +func (suite *ClientHelloTestSuite) TestIncorrectLength() { + data := make([]byte, 1024) + data[0] = 0x01 + data[1] = 0xff + data[2] = 0xff + + _, err := faketls.ParseClientHello(suite.secret.Key[:], data) + suite.Error(err) +} + +func (suite *ClientHelloTestSuite) TestSnapshotOk() { + files, err := os.ReadDir("testdata") + suite.NoError(err) + + testData := []string{} + + for _, v := range files { + if strings.HasPrefix(v.Name(), "client-hello-ok") { + testData = append(testData, v.Name()) + } + } + + for _, name := range testData { + path := filepath.Join("testdata", name) + + suite.T().Run(name, func(t *testing.T) { + fileData, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &ClientHelloSnapshot{} + assert.NoError(t, json.Unmarshal(fileData, snapshot)) + + hello, err := faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull()) + assert.NoError(t, err) + assert.WithinDuration(t, snapshot.GetTime(), hello.Time, time.Second) + assert.Equal(t, snapshot.GetRandom(), hello.Random[:]) + assert.Equal(t, snapshot.GetSessionID(), hello.SessionID) + assert.Equal(t, snapshot.GetHost(), hello.Host) + assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite) + }) + } +} + +func (suite *ClientHelloTestSuite) TestSnapshotBad() { + files, err := os.ReadDir("testdata") + suite.NoError(err) + + testData := []string{} + + for _, v := range files { + if strings.HasPrefix(v.Name(), "client-hello-bad") { + testData = append(testData, v.Name()) + } + } + + for _, name := range testData { + path := filepath.Join("testdata", name) + + suite.T().Run(name, func(t *testing.T) { + fileData, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &ClientHelloSnapshot{} + assert.NoError(t, json.Unmarshal(fileData, snapshot)) + + _, err = faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull()) + assert.Error(t, err) + }) + } +} + +func (suite *ClientHelloTestSuite) TestValidateHostname() { + hello := faketls.ClientHello{ + Time: time.Now(), + } + suite.NoError(hello.Valid("hostname", time.Second)) + + hello.Host = "hostname" + suite.Error(hello.Valid("hostname2", time.Second)) + suite.NoError(hello.Valid("hostname", time.Second)) +} + +func (suite *ClientHelloTestSuite) TestValidateTime() { + testData := []time.Duration{ + -2 * time.Second, + 2 * time.Second, + } + + for _, v := range testData { + value := v + + suite.T().Run(value.String(), func(t *testing.T) { + hello := faketls.ClientHello{ + Host: "hostname", + Time: time.Now().Add(value), + } + suite.Error(hello.Valid("hostname", 500*time.Millisecond)) + suite.Error(hello.Valid("hostname", time.Second)) + suite.NoError(hello.Valid("hostname", 3*time.Second)) + }) + } +} + +func TestClientHello(t *testing.T) { + t.Parallel() + suite.Run(t, &ClientHelloTestSuite{}) +} diff --git a/mtglib/internal/faketls/conn.go b/mtglib/internal/faketls/conn.go new file mode 100644 index 0000000..06592ff --- /dev/null +++ b/mtglib/internal/faketls/conn.go @@ -0,0 +1,69 @@ +package faketls + +import ( + "bytes" + "fmt" + "math/rand" + "net" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" +) + +type Conn struct { + net.Conn + + readBuffer bytes.Buffer +} + +func (c *Conn) Read(p []byte) (int, error) { + if n, _ := c.readBuffer.Read(p); n > 0 { + return n, nil + } + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + for { + if err := rec.Read(c.Conn); err != nil { + return 0, err // nolint: wrapcheck + } + + switch rec.Type { // nolint: exhaustive + case record.TypeApplicationData: + rec.Payload.WriteTo(&c.readBuffer) // nolint: errcheck + + return c.readBuffer.Read(p) + case record.TypeChangeCipherSpec: + default: + return 0, fmt.Errorf("unsupported record type %v", rec.Type) + } + } +} + +func (c *Conn) Write(p []byte) (int, error) { + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeApplicationData + rec.Version = record.Version12 + written := 0 + + for len(p) > 0 { + chunkSize := rand.Intn(record.TLSMaxRecordSize) + if chunkSize > len(p) || chunkSize == 0 { + chunkSize = len(p) + } + + rec.Payload.Reset() + rec.Payload.Write(p[:chunkSize]) + + if err := rec.Dump(c.Conn); err != nil { + return written, err // nolint: wrapcheck + } + + written += chunkSize + p = p[chunkSize:] + } + + return written, nil +} diff --git a/mtglib/internal/faketls/conn_test.go b/mtglib/internal/faketls/conn_test.go new file mode 100644 index 0000000..73c0e65 --- /dev/null +++ b/mtglib/internal/faketls/conn_test.go @@ -0,0 +1,153 @@ +package faketls_test + +import ( + "bytes" + "errors" + "io" + "math/rand" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnMock struct { + testlib.NetConnMock + + readBuffer bytes.Buffer + writeBuffer bytes.Buffer +} + +func (m *ConnMock) Read(p []byte) (int, error) { + m.Called(p) + + return m.readBuffer.Read(p) +} + +func (m *ConnMock) Write(p []byte) (int, error) { + m.Called(p) + + return m.writeBuffer.Write(p) +} + +type ConnTestSuite struct { + suite.Suite + + connMock *ConnMock + c *faketls.Conn +} + +func (suite *ConnTestSuite) SetupTest() { + suite.connMock = &ConnMock{} + suite.c = &faketls.Conn{ + Conn: suite.connMock, + } +} + +func (suite *ConnTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnTestSuite) TestRead() { + suite.connMock.On("Read", mock.Anything).Return(0, nil) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeChangeCipherSpec + rec.Version = record.Version12 + + rec.Payload.WriteByte(0x01) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + rec.Reset() + + rec.Type = record.TypeApplicationData + rec.Version = record.Version12 + + rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + + resultBuffer := &bytes.Buffer{} + buf := make([]byte, 2) + + for { + n, err := suite.c.Read(buf) + if errors.Is(err, io.EOF) { + break + } + + resultBuffer.Write(buf[:n]) + } + + suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, resultBuffer.Bytes()) +} + +func (suite *ConnTestSuite) TestReadUnexpected() { + suite.connMock.On("Read", mock.Anything).Return(0, nil) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeChangeCipherSpec + rec.Version = record.Version12 + + rec.Payload.WriteByte(0x01) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + rec.Reset() + + rec.Type = record.TypeHandshake + rec.Version = record.Version12 + + rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + + buf := make([]byte, 2) + + for { + _, err := suite.c.Read(buf) + + switch { + case err == nil: + case errors.Is(err, io.EOF): + suite.FailNow("unexpected to finish") + default: + return + } + } +} + +func (suite *ConnTestSuite) TestWrite() { + suite.connMock.On("Write", mock.Anything).Return(0, nil) + + dataToRec := make([]byte, record.TLSMaxRecordSize*2) + rand.Read(dataToRec) + + n, err := suite.c.Write(dataToRec) + suite.NoError(err) + suite.Equal(len(dataToRec), n) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + buf := &bytes.Buffer{} + + for { + if err := rec.Read(&suite.connMock.writeBuffer); err != nil { + break + } + + suite.Equal(record.TypeApplicationData, rec.Type) + suite.Equal(record.Version12, rec.Version) + rec.Payload.WriteTo(buf) // nolint: errcheck + } + + suite.Equal(dataToRec, buf.Bytes()) +} + +func TestConn(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTestSuite{}) +} diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go new file mode 100644 index 0000000..5bbf6f3 --- /dev/null +++ b/mtglib/internal/faketls/init.go @@ -0,0 +1,41 @@ +package faketls + +import ( + "bytes" + "errors" +) + +const ( + RandomLen = 32 + + ClientHelloRandomOffset = 6 + ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen + ClientHelloMinLen = 4 + + WelcomePacketRandomOffset = 11 + + HandshakeTypeClient = 0x01 + HandshakeTypeServer = 0x02 + + ChangeCipherValue = 0x01 + + ExtensionSNI = 0x00 +) + +var ( + ErrBadDigest = errors.New("bad digest") + ErrAntiReplayAttack = errors.New("antireplay attack was detected") + + serverHelloSuffix = []byte{ + 0x00, // no compression + 0x00, 0x2e, // 46 bytes of data + 0x00, 0x2b, // Extension - Supported Versions + 0x00, 0x02, // 2 bytes are following + 0x03, 0x04, // TLS 1.3 + 0x00, 0x33, // Extension - Key Share + 0x00, 0x24, // 36 bytes + 0x00, 0x1d, // x25519 curve + 0x00, 0x20, // 32 bytes of key + } + clientHelloEmptyRandom = bytes.Repeat([]byte{0}, RandomLen) +) diff --git a/mtglib/internal/faketls/pools.go b/mtglib/internal/faketls/pools.go new file mode 100644 index 0000000..52288b1 --- /dev/null +++ b/mtglib/internal/faketls/pools.go @@ -0,0 +1,21 @@ +package faketls + +import ( + "bytes" + "sync" +) + +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, +} + +func acquireBytesBuffer() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func releaseBytesBuffer(b *bytes.Buffer) { + b.Reset() + bytesBufferPool.Put(b) +} diff --git a/mtglib/internal/faketls/record/init.go b/mtglib/internal/faketls/record/init.go new file mode 100644 index 0000000..0ea3b5f --- /dev/null +++ b/mtglib/internal/faketls/record/init.go @@ -0,0 +1,68 @@ +package record + +import "fmt" + +const TLSMaxRecordSize = 65535 // max uint16 + +type Type uint8 + +const ( + TypeChangeCipherSpec Type = 0x14 + TypeHandshake Type = 0x16 + TypeApplicationData Type = 0x17 +) + +func (t Type) String() string { + switch t { + case TypeChangeCipherSpec: + return "changeCipher(0x14)" + case TypeHandshake: + return "handshake(0x16)" + case TypeApplicationData: + return "applicationData(0x17)" + } + + return fmt.Sprintf("unknown(%#x)", byte(t)) +} + +func (t Type) Valid() error { + switch t { + case TypeChangeCipherSpec, TypeHandshake, TypeApplicationData: + return nil + } + + return fmt.Errorf("unknown type %#x", byte(t)) +} + +type Version uint16 + +const ( + Version10 Version = 769 // 0x03 0x01 + Version11 Version = 770 // 0x03 0x02 + Version12 Version = 771 // 0x03 0x03 + Version13 Version = 772 // 0x03 0x04 +) + +func (v Version) String() string { + switch v { + case Version10: + return "tls1.0" + case Version11: + return "tls1.1" + case Version12: + return "tls1.2" + case Version13: + return "tls1.3" + } + + return fmt.Sprintf("tls?(%d)", uint16(v)) +} + +func (v Version) Valid() error { + switch v { + case Version10, Version11, Version12, Version13: + return nil + } + + return fmt.Errorf("unknown version %d", uint16(v)) +} diff --git a/mtglib/internal/faketls/record/init_test.go b/mtglib/internal/faketls/record/init_test.go new file mode 100644 index 0000000..7922777 --- /dev/null +++ b/mtglib/internal/faketls/record/init_test.go @@ -0,0 +1,79 @@ +package record_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/suite" +) + +type TypeTestSuite struct { + suite.Suite +} + +func (suite *TypeTestSuite) TestChangeCipherSpec() { + suite.Contains(record.TypeChangeCipherSpec.String(), "changeCipher") + suite.Contains(record.TypeChangeCipherSpec.String(), "0x14") + suite.NoError(record.TypeChangeCipherSpec.Valid()) +} + +func (suite *TypeTestSuite) TestHandshake() { + suite.Contains(record.TypeHandshake.String(), "handshake") + suite.Contains(record.TypeHandshake.String(), "0x16") + suite.NoError(record.TypeHandshake.Valid()) +} + +func (suite *TypeTestSuite) TestApplicationData() { + suite.Contains(record.TypeApplicationData.String(), "applicationData") + suite.Contains(record.TypeApplicationData.String(), "0x17") + suite.NoError(record.TypeApplicationData.Valid()) +} + +func (suite *TypeTestSuite) TestUnknown() { + value := record.Type(0x20) + + suite.Contains(value.String(), "unknown") + suite.Contains(value.String(), "0x20") + suite.Error(value.Valid()) +} + +type VersionTestSuite struct { + suite.Suite +} + +func (suite *VersionTestSuite) Test10() { + suite.Equal("tls1.0", record.Version10.String()) + suite.NoError(record.Version10.Valid()) +} + +func (suite *VersionTestSuite) Test11() { + suite.Equal("tls1.1", record.Version11.String()) + suite.NoError(record.Version11.Valid()) +} + +func (suite *VersionTestSuite) Test12() { + suite.Equal("tls1.2", record.Version12.String()) + suite.NoError(record.Version12.Valid()) +} + +func (suite *VersionTestSuite) Test13() { + suite.Equal("tls1.3", record.Version13.String()) + suite.NoError(record.Version13.Valid()) +} + +func (suite *VersionTestSuite) TestUnknown() { + value := record.Version(900) + + suite.Equal("tls?(900)", value.String()) + suite.Error(value.Valid()) +} + +func TestType(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeTestSuite{}) +} + +func TestVersion(t *testing.T) { + t.Parallel() + suite.Run(t, &VersionTestSuite{}) +} diff --git a/mtglib/internal/faketls/record/pools.go b/mtglib/internal/faketls/record/pools.go new file mode 100644 index 0000000..16fa81f --- /dev/null +++ b/mtglib/internal/faketls/record/pools.go @@ -0,0 +1,20 @@ +package record + +import ( + "sync" +) + +var recordPool = sync.Pool{ + New: func() interface{} { + return &Record{} + }, +} + +func AcquireRecord() *Record { + return recordPool.Get().(*Record) +} + +func ReleaseRecord(r *Record) { + r.Reset() + recordPool.Put(r) +} diff --git a/mtglib/internal/faketls/record/record.go b/mtglib/internal/faketls/record/record.go new file mode 100644 index 0000000..2c8b23e --- /dev/null +++ b/mtglib/internal/faketls/record/record.go @@ -0,0 +1,86 @@ +package record + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "fmt" + "io" +) + +type Record struct { + Type Type + Version Version + Payload bytes.Buffer +} + +func (r *Record) String() string { + return fmt.Sprintf("", + r.Type, + r.Version, + base64.StdEncoding.EncodeToString(r.Payload.Bytes())) +} + +func (r *Record) Reset() { + r.Payload.Reset() +} + +func (r *Record) Read(reader io.Reader) error { + r.Reset() + + buf := [2]byte{} + + if _, err := io.ReadFull(reader, buf[:1]); err != nil { + return fmt.Errorf("cannot read type: %w", err) + } + + r.Type = Type(buf[0]) + if err := r.Type.Valid(); err != nil { + return fmt.Errorf("invalid type: %w", err) + } + + if _, err := io.ReadFull(reader, buf[:]); err != nil { + return fmt.Errorf("cannot read version: %w", err) + } + + r.Version = Version(binary.BigEndian.Uint16(buf[:])) + if err := r.Version.Valid(); err != nil { + return fmt.Errorf("invalid version: %w", err) + } + + if _, err := io.ReadFull(reader, buf[:]); err != nil { + return fmt.Errorf("cannot read payload length: %w", err) + } + + length := int64(binary.BigEndian.Uint16(buf[:])) + if _, err := io.CopyN(&r.Payload, reader, length); err != nil { + return fmt.Errorf("cannot read payload: %w", err) + } + + return nil +} + +func (r *Record) Dump(writer io.Writer) error { + buf := [2]byte{byte(r.Type), 0} + if _, err := writer.Write(buf[:1]); err != nil { + return fmt.Errorf("cannot dump record type: %w", err) + } + + binary.BigEndian.PutUint16(buf[:], uint16(r.Version)) + + if _, err := writer.Write(buf[:]); err != nil { + return fmt.Errorf("cannot dump version: %w", err) + } + + binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len())) + + if _, err := writer.Write(buf[:]); err != nil { + return fmt.Errorf("cannot dump payload length: %w", err) + } + + if _, err := writer.Write(r.Payload.Bytes()); err != nil { + return fmt.Errorf("cannot dump record: %w", err) + } + + return nil +} diff --git a/mtglib/internal/faketls/record/record_test.go b/mtglib/internal/faketls/record/record_test.go new file mode 100644 index 0000000..e29b5ba --- /dev/null +++ b/mtglib/internal/faketls/record/record_test.go @@ -0,0 +1,110 @@ +package record_test + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type RecordTestSnapshot struct { + Type int `json:"type"` + Version int `json:"version"` + Payload string `json:"payload"` + Record string `json:"record"` +} + +func (r RecordTestSnapshot) RecordBytes() []byte { + data, _ := base64.StdEncoding.DecodeString(r.Record) + + return data +} + +func (r RecordTestSnapshot) PayloadBytes() []byte { + data, _ := base64.StdEncoding.DecodeString(r.Payload) + + return data +} + +type RecordTestSuite struct { + suite.Suite + + r *record.Record + buf *bytes.Buffer +} + +func (suite *RecordTestSuite) SetupTest() { + suite.r = record.AcquireRecord() + suite.buf = &bytes.Buffer{} +} + +func (suite *RecordTestSuite) TearDownTest() { + record.ReleaseRecord(suite.r) + suite.buf.Reset() +} + +func (suite *RecordTestSuite) TestIdempotent() { + suite.r.Type = record.TypeApplicationData + suite.r.Version = record.Version13 + + suite.r.Payload.Write([]byte{1, 2, 3}) + suite.NoError(suite.r.Dump(suite.buf)) + + suite.r.Reset() + suite.NoError(suite.r.Read(suite.buf)) + + suite.Equal(0, suite.buf.Len()) + suite.Equal(record.TypeApplicationData, suite.r.Type) + suite.Equal(record.Version13, suite.r.Version) + suite.Equal([]byte{1, 2, 3}, suite.r.Payload.Bytes()) +} + +func (suite *RecordTestSuite) TestString() { + _ = suite.r.String() +} + +func (suite *RecordTestSuite) TestSnapshot() { + files, err := os.ReadDir("testdata") + suite.NoError(err) + + testData := map[string]string{} + + for _, f := range files { + testData[f.Name()] = filepath.Join("testdata", f.Name()) + } + + for name, pathV := range testData { + path := pathV + + suite.T().Run(name, func(t *testing.T) { + data, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &RecordTestSnapshot{} + assert.NoError(t, json.Unmarshal(data, snapshot)) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + assert.NoError(t, rec.Read(bytes.NewReader(snapshot.RecordBytes()))) + assert.Equal(t, snapshot.Type, int(rec.Type)) + assert.Equal(t, snapshot.Version, int(rec.Version)) + assert.Equal(t, snapshot.PayloadBytes(), rec.Payload.Bytes()) + + buf := &bytes.Buffer{} + assert.NoError(t, rec.Dump(buf)) + assert.Equal(t, snapshot.RecordBytes(), buf.Bytes()) + }) + } +} + +func TestRecord(t *testing.T) { + t.Parallel() + suite.Run(t, &RecordTestSuite{}) +} diff --git a/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json b/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json new file mode 100644 index 0000000..6cac5ae --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 772, + "payload": "sxS+0oAyk+NBv0LLVtQOp9WSx4CweyUZPz01tQ0o4oyp8aaBl6/kMFvLq3q52KE8lCiKejLw2NxVBUkE+4izCf2gLx9qfr81opWnqJTChWzcDijvttbq9cmtDFNL+odKsS3v1/TfYEFtPsoRPrJRmOHRAnqnf49Y5Q==", + "record": "FAMEAHmzFL7SgDKT40G/QstW1A6n1ZLHgLB7JRk/PTW1DSjijKnxpoGXr+QwW8urernYoTyUKIp6MvDY3FUFSQT7iLMJ/aAvH2p+vzWilaeolMKFbNwOKO+21ur1ya0MU0v6h0qxLe/X9N9gQW0+yhE+slGY4dECeqd/j1jl" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json b/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json new file mode 100644 index 0000000..875d64f --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 772, + "payload": "waNH223htyxCBKAb6hm0u/SK/9mhI8Ck91nfWob7QMOaIREogrDYREJH4Djcp47XrpAlEaUIDiCvoFLVJ/LK1nYs4swzfHSSl/+Aj1eqPA63XqPa8EG4FAbf0DwjwXxV9qVIhvP9b2TafKbzr4Yb5GCygzFRb/zawA==", + "record": "FgMEAHnBo0fbbeG3LEIEoBvqGbS79Ir/2aEjwKT3Wd9ahvtAw5ohESiCsNhEQkfgONynjteukCURpQgOIK+gUtUn8srWdizizDN8dJKX/4CPV6o8Drdeo9rwQbgUBt/QPCPBfFX2pUiG8/1vZNp8pvOvhhvkYLKDMVFv/NrA" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/736f358216afe91f.json b/mtglib/internal/faketls/record/testdata/736f358216afe91f.json new file mode 100644 index 0000000..5b322d4 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/736f358216afe91f.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 769, + "payload": "jmJ0o1E5+ehAHHYAbCo4AMV03X7RSivYl250s06nD9CO44fyjaoGELz0N7IeCg1jFKcRVSCRmYYmiIY9wydn2fXOJhKif8B0BlM3qhbethYgyP+l1S8hyyETpIiOtiiiOnAJwl1D1j9OryFiJFSdRRXReIMZ4CPqPg==", + "record": "FwMBAHmOYnSjUTn56EAcdgBsKjgAxXTdftFKK9iXbnSzTqcP0I7jh/KNqgYQvPQ3sh4KDWMUpxFVIJGZhiaIhj3DJ2fZ9c4mEqJ/wHQGUzeqFt62FiDI/6XVLyHLIROkiI62KKI6cAnCXUPWP06vIWIkVJ1FFdF4gxngI+o+" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json b/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json new file mode 100644 index 0000000..16a4ba7 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 769, + "payload": "hBnpBnNUdlqe/rKXa7Judcz79u7AkUgSGOycn8EqvbkZpVxnI31rNOvAsPZqG+GF7DWJ3R7H2ETmFmrpnyyng32MjSs1jptmV1oAs63zTADD7sVipgid9AJHwfl4CrC3FIQr43IPMYd29JPOl5bqu/SfrgI16PBiJw==", + "record": "FgMBAHmEGekGc1R2Wp7+spdrsm51zPv27sCRSBIY7JyfwSq9uRmlXGcjfWs068Cw9mob4YXsNYndHsfYROYWaumfLKeDfYyNKzWOm2ZXWgCzrfNMAMPuxWKmCJ30AkfB+XgKsLcUhCvjcg8xh3b0k86Xluq79J+uAjXo8GIn" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json b/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json new file mode 100644 index 0000000..5d60155 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 770, + "payload": "Vm/C+DO56czlbtR915aHzsugSyDtp8CtojF9w1jKY0efyyfcLrNuhNg/pZm3gQ7v2BBbL1UJ97v/RIjST+5gRIfg3bBN1BE9hkf+N2AYY2lHLi0yeInHB0zFWPeHscsDopDFadIi5KtC8HvbEMuK+kK8POVk5tN9UQ==", + "record": "FwMCAHlWb8L4M7npzOVu1H3XlofOy6BLIO2nwK2iMX3DWMpjR5/LJ9wus26E2D+lmbeBDu/YEFsvVQn3u/9EiNJP7mBEh+DdsE3UET2GR/43YBhjaUcuLTJ4iccHTMVY94exywOikMVp0iLkq0Lwe9sQy4r6Qrw85WTm031R" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json b/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json new file mode 100644 index 0000000..459986b --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 770, + "payload": "ajPzpsgk4gwm2stRQKbllvKRLdI7vmyaj1uxEJ/kKoQnQSPumdDNKD618U2Cq6PVd0/b+9YtH67Uzx1QxtpKuby5fUXqw06WUuDAQsmjq7F26EkE5FND6rQUjUPC+e1U0dF4TQzOUSS4IAkFQPAaVehUVTRxVWa/0g==", + "record": "FgMCAHlqM/OmyCTiDCbay1FApuWW8pEt0ju+bJqPW7EQn+QqhCdBI+6Z0M0oPrXxTYKro9V3T9v71i0frtTPHVDG2kq5vLl9RerDTpZS4MBCyaOrsXboSQTkU0PqtBSNQ8L57VTR0XhNDM5RJLggCQVA8BpV6FRVNHFVZr/S" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json b/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json new file mode 100644 index 0000000..e49ea0a --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 771, + "payload": "d1Hiv1NYVgEDR9mtJyv9j8mg3dWqfUpeKfOsL+jzSDfVIxeDiJZFLDT50TjNW44/yEOVEX/Y/pk+wnc7E8aCEiwGwAvB+Insw1UCJ2ejt689VWLo2u4klGVKTHuOpUvdGVTc7Lo4FAt91KQSPLYB5iqxomjEv5e3Vg==", + "record": "FAMDAHl3UeK/U1hWAQNH2a0nK/2PyaDd1ap9Sl4p86wv6PNIN9UjF4OIlkUsNPnROM1bjj/IQ5URf9j+mT7CdzsTxoISLAbAC8H4iezDVQInZ6O3rz1VYuja7iSUZUpMe46lS90ZVNzsujgUC33UpBI8tgHmKrGiaMS/l7dW" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json b/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json new file mode 100644 index 0000000..7eac0da --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 771, + "payload": "wbdU1CbrzuAJDsh6CFjGyE+AFArJj/Wmsa2wtDyW0kRuE2vUO8gg+nXkg0kkoz0WnvQEOdaswfJIaVrloD78yoyeQVfBB+VUP/63vqn60v5ccaQEn0jLdxgLjiTAxKDQDxCTMRoLnFE2ZZf28zw+HfqpIxiOZs8LhQ==", + "record": "FwMDAHnBt1TUJuvO4AkOyHoIWMbIT4AUCsmP9aaxrbC0PJbSRG4Ta9Q7yCD6deSDSSSjPRae9AQ51qzB8khpWuWgPvzKjJ5BV8EH5VQ//re+qfrS/lxxpASfSMt3GAuOJMDEoNAPEJMxGgucUTZll/bzPD4d+qkjGI5mzwuF" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json b/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json new file mode 100644 index 0000000..3fffda7 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 772, + "payload": "qqnBMb1Af3zZt4DPHpVRuIiON9ODGJUNFicFjranORh67L/HI4D6HnHyycZFUSBOw2FjMBF6UialY8snOYaRKrQmQzuUNg1Ztq7yAZ+Lgj3TBarR6OMlYhEAY0Px9Xv1UuJ0YcvQx33gdM1skJ5HBR3yZvEKNJV1LA==", + "record": "FwMEAHmqqcExvUB/fNm3gM8elVG4iI4304MYlQ0WJwWOtqc5GHrsv8cjgPoecfLJxkVRIE7DYWMwEXpSJqVjyyc5hpEqtCZDO5Q2DVm2rvIBn4uCPdMFqtHo4yViEQBjQ/H1e/VS4nRhy9DHfeB0zWyQnkcFHfJm8Qo0lXUs" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json b/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json new file mode 100644 index 0000000..aa11362 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 769, + "payload": "NEe735TuQFp7bWpFQhASas/e1XaySvus0ovXmkfCbFq334MyFHq2eDMadziXsfu/GfBjoYggvk0LgYUeoAkBNKR0dfSovjSndaqmIUonoWl+6sZObiGZkRIMwuY2q4Eaw4/iuDu/pZhjRW/iAIH+YH7cyk/1tgdJDg==", + "record": "FAMBAHk0R7vflO5AWnttakVCEBJqz97VdrJK+6zSi9eaR8JsWrffgzIUerZ4Mxp3OJex+78Z8GOhiCC+TQuBhR6gCQE0pHR19Ki+NKd1qqYhSiehaX7qxk5uIZmREgzC5jargRrDj+K4O7+lmGNFb+IAgf5gftzKT/W2B0kO" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/f083f4501668b759.json b/mtglib/internal/faketls/record/testdata/f083f4501668b759.json new file mode 100644 index 0000000..2518193 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/f083f4501668b759.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 771, + "payload": "wrXjZrPm3OSyzO0klv6/G+z2PDloR/colS/RlWwQE31Vb2xm8YkEchDDKwlc/KPLD73qMoz3MQOQLtSLc8LhVYp+l7L9jz49yTaVKtBI5UuGbo09snsKxFCgCyYUBETKabATBQtiaEu/D8dmF4Yk/2ww4sEb8DwKLQ==", + "record": "FgMDAHnCteNms+bc5LLM7SSW/r8b7PY8OWhH9yiVL9GVbBATfVVvbGbxiQRyEMMrCVz8o8sPveoyjPcxA5Au1ItzwuFVin6Xsv2PPj3JNpUq0EjlS4ZujT2yewrEUKALJhQERMppsBMFC2JoS78Px2YXhiT/bDDiwRvwPAot" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json b/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json new file mode 100644 index 0000000..055a7a6 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 770, + "payload": "OU5s8Sa11hpXWEarWzFlX55IZt3Eo+F4AMbQ/2RwB4rfHS/JNl8n63OR4oYs9QXw3RfCrYJuU9n6Xn+I/+7ZzAgZ0PbLSXW1PrLtttdfmhTErK90b49YEWdY9na4g++NMkKykwgXvY1hNxZIHX/qawEWJgxXUR3DdQ==", + "record": "FAMCAHk5TmzxJrXWGldYRqtbMWVfnkhm3cSj4XgAxtD/ZHAHit8dL8k2Xyfrc5Hihiz1BfDdF8Ktgm5T2fpef4j/7tnMCBnQ9stJdbU+su2211+aFMSsr3Rvj1gRZ1j2driD740yQrKTCBe9jWE3Fkgdf+prARYmDFdRHcN1" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json b/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json new file mode 100644 index 0000000..046a122 --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", + "session-id": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCACq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFANAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json b/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json new file mode 100644 index 0000000..0a47b92 --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", + "session-id": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json b/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json new file mode 100644 index 0000000..adc7cc7 --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=", + "session-id": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDoYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12osgFGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8gANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAga6CocpFP8Qd4YCFR9pkaCr97po2ALj0P5nI9Nnb3UWMALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json b/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json new file mode 100644 index 0000000..c5165fb --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=", + "session-id": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMD5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1rogjxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgrulAaqUdKeVYM0F+pu6on/h6LBpOyzOKG4xFIKcoFk4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json b/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json new file mode 100644 index 0000000..a642fec --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=", + "session-id": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMD8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQg00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAUANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAg/9P7140NtKzjyDwBf99mOy1+FjRPAPHTNQ9WxHOKpV4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json b/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json new file mode 100644 index 0000000..5b803da --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=", + "session-id": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDzja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4gqPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnAANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgXviLRAqAYJ8xOLdlcsUhldI4Xl0g/s9+y2Qrd8raPEgALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/welcome.go b/mtglib/internal/faketls/welcome.go new file mode 100644 index 0000000..d0b48d2 --- /dev/null +++ b/mtglib/internal/faketls/welcome.go @@ -0,0 +1,92 @@ +package faketls + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "io" + mrand "math/rand" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "golang.org/x/crypto/curve25519" +) + +func SendWelcomePacket(writer io.Writer, secret []byte, clientHello ClientHello) error { + buf := acquireBytesBuffer() + defer releaseBytesBuffer(buf) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeHandshake + rec.Version = record.Version12 + + generateServerHello(&rec.Payload, clientHello) + rec.Dump(buf) // nolint: errcheck + rec.Reset() + + rec.Type = record.TypeChangeCipherSpec + rec.Version = record.Version12 + rec.Payload.WriteByte(ChangeCipherValue) + + rec.Dump(buf) // nolint: errcheck + rec.Reset() + + rec.Type = record.TypeApplicationData + rec.Version = record.Version12 + + if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.Intn(3092))); err != nil { // nolint: gomnd + panic(err) + } + + rec.Dump(buf) // nolint: errcheck + + packet := buf.Bytes() + mac := hmac.New(sha256.New, secret) + + mac.Write(clientHello.Random[:]) // nolint: errcheck + mac.Write(packet) // nolint: errcheck + + copy(packet[WelcomePacketRandomOffset:], mac.Sum(nil)) + + if _, err := writer.Write(packet); err != nil { + return err // nolint: wrapcheck + } + + return nil +} + +func generateServerHello(writer io.Writer, clientHello ClientHello) { + bodyBuf := acquireBytesBuffer() + defer releaseBytesBuffer(bodyBuf) + + sliceBuf := [2]byte{} + digest := [RandomLen]byte{} + + binary.BigEndian.PutUint16(sliceBuf[:], uint16(record.Version12)) + bodyBuf.Write(sliceBuf[:]) + bodyBuf.Write(digest[:]) + bodyBuf.WriteByte(byte(len(clientHello.SessionID))) + bodyBuf.Write(clientHello.SessionID) + + binary.BigEndian.PutUint16(sliceBuf[:], clientHello.CipherSuite) + bodyBuf.Write(sliceBuf[:]) + bodyBuf.Write(serverHelloSuffix) + + scalar := [32]byte{} + + if _, err := rand.Read(scalar[:]); err != nil { + panic(err) + } + + curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) + bodyBuf.Write(curve) + + header := [4]byte{0, 0, 0, 0} + binary.BigEndian.PutUint32(header[:], uint32(bodyBuf.Len())) + header[0] = HandshakeTypeServer + + writer.Write(header[:]) // nolint: errcheck + bodyBuf.WriteTo(writer) // nolint: errcheck +} diff --git a/mtglib/internal/faketls/welcome_test.go b/mtglib/internal/faketls/welcome_test.go new file mode 100644 index 0000000..4f1b990 --- /dev/null +++ b/mtglib/internal/faketls/welcome_test.go @@ -0,0 +1,82 @@ +package faketls_test + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "math/rand" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/suite" +) + +type WelcomeTestSuite struct { + suite.Suite + + h *faketls.ClientHello + buf *bytes.Buffer + secret mtglib.Secret +} + +func (suite *WelcomeTestSuite) SetupTest() { + suite.h = &faketls.ClientHello{ + Time: time.Now(), + Host: "google.com", + CipherSuite: 4867, + SessionID: make([]byte, 32), + } + + _, err := rand.Read(suite.h.SessionID) + suite.NoError(err) + + _, err = rand.Read(suite.h.Random[:]) + suite.NoError(err) + + suite.buf = &bytes.Buffer{} + + suite.secret = mtglib.GenerateSecret("google.com") +} + +func (suite *WelcomeTestSuite) TestOk() { + suite.NoError(faketls.SendWelcomePacket(suite.buf, suite.secret.Key[:], *suite.h)) + + welcomePacket := []byte{} + welcomePacket = append(welcomePacket, suite.buf.Bytes()...) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + suite.NoError(rec.Read(suite.buf)) + suite.Equal(record.TypeHandshake, rec.Type) + suite.Equal(record.Version12, rec.Version) + + suite.NoError(rec.Read(suite.buf)) + suite.Equal(record.TypeChangeCipherSpec, rec.Type) + suite.Equal(record.Version12, rec.Version) + + suite.NoError(rec.Read(suite.buf)) + suite.Equal(record.TypeApplicationData, rec.Type) + suite.Equal(record.Version12, rec.Version) + suite.Empty(suite.buf.Bytes()) + + random := make([]byte, 32) + copy(random, welcomePacket[11:]) + + empty := make([]byte, 32) + copy(welcomePacket[11:], empty) + + mac := hmac.New(sha256.New, suite.secret.Key[:]) + mac.Write(suite.h.Random[:]) // nolint: errcheck + mac.Write(welcomePacket) // nolint: errcheck + + suite.Equal(random, mac.Sum(nil)) +} + +func TestWelcome(t *testing.T) { + t.Parallel() + suite.Run(t, &WelcomeTestSuite{}) +} diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go new file mode 100644 index 0000000..89cc396 --- /dev/null +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -0,0 +1,54 @@ +package obfuscated2 + +import ( + "crypto/cipher" + "crypto/subtle" + "encoding/hex" + "fmt" + "io" +) + +type clientHandhakeFrame struct { + handshakeFrame +} + +func (c *clientHandhakeFrame) decryptor(secret []byte) cipher.Stream { + hasher := acquireSha256Hasher() + defer releaseSha256Hasher(hasher) + + hasher.Write(c.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck + + return makeAesCtr(hasher.Sum(nil), c.iv()) +} + +func (c *clientHandhakeFrame) encryptor(secret []byte) cipher.Stream { + invertedHandshake := c.invert() + + hasher := acquireSha256Hasher() + defer releaseSha256Hasher(hasher) + + hasher.Write(invertedHandshake.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck + + return makeAesCtr(hasher.Sum(nil), invertedHandshake.iv()) +} + +func ClientHandshake(secret []byte, reader io.Reader) (int, cipher.Stream, cipher.Stream, error) { + handshake := clientHandhakeFrame{} + + if _, err := io.ReadFull(reader, handshake.data[:]); err != nil { + return 0, nil, nil, fmt.Errorf("cannot read frame: %w", err) + } + + decryptor := handshake.decryptor(secret) + encryptor := handshake.encryptor(secret) + + decryptor.XORKeyStream(handshake.data[:], handshake.data[:]) + + if val := handshake.connectionType(); subtle.ConstantTimeCompare(handshakeConnectionType, val) != 1 { + return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(val)) + } + + return handshake.dc(), encryptor, decryptor, nil +} diff --git a/mtglib/internal/obfuscated2/client_handshake_test.go b/mtglib/internal/obfuscated2/client_handshake_test.go new file mode 100644 index 0000000..6d4b95a --- /dev/null +++ b/mtglib/internal/obfuscated2/client_handshake_test.go @@ -0,0 +1,85 @@ +package obfuscated2_test + +import ( + "bytes" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ClientHandshakeTestSuite struct { + suite.Suite + SnapshotTestSuite +} + +func (suite *ClientHandshakeTestSuite) SetupSuite() { + suite.NoError(suite.IngestSnapshots(".", "client-handshake-snapshot-")) +} + +func (suite *ClientHandshakeTestSuite) TestCannotRead() { + buf := bytes.NewBuffer([]byte{1, 2, 3}) + _, _, _, err := obfuscated2.ClientHandshake([]byte{1, 2, 3}, buf) // nolint: dogsled + + suite.Error(err) +} + +func (suite *ClientHandshakeTestSuite) TestOk() { + for nameV, snapshotV := range suite.snapshots { + snapshot := snapshotV + + suite.T().Run(nameV, func(t *testing.T) { + buf := bytes.NewBuffer(snapshot.Frame.data) + + dc, encryptor, decryptor, err := obfuscated2.ClientHandshake( + snapshot.Secret.data, buf) + assert.NoError(t, err) + assert.EqualValues(t, snapshot.DC, dc) + + writeData := make([]byte, len(snapshot.Encrypted.Text.data)) + readData := make([]byte, len(snapshot.Decrypted.Text.data)) + + connMock := &testlib.NetConnMock{} + connMock.On("Read", mock.Anything). + Once(). + Return(len(snapshot.Decrypted.Text.data), nil). + Run(func(args mock.Arguments) { + arr := args.Get(0).([]byte) + copy(arr, snapshot.Decrypted.Cipher.data) + }) + connMock.On("Write", mock.Anything). + Once(). + Return(len(snapshot.Encrypted.Text.data), nil). + Run(func(args mock.Arguments) { + arr := args.Get(0).([]byte) + copy(writeData, arr) + }) + + conn := obfuscated2.Conn{ + Conn: connMock, + Encryptor: encryptor, + Decryptor: decryptor, + } + + n, err := conn.Read(readData) + assert.Equal(t, len(readData), n) + assert.NoError(t, err) + assert.Equal(t, snapshot.Decrypted.Text.data, readData) + + n, err = conn.Write(snapshot.Encrypted.Text.data) + assert.Equal(t, len(writeData), n) + assert.NoError(t, err) + assert.Equal(t, snapshot.Encrypted.Cipher.data, writeData) + + connMock.AssertExpectations(t) + }) + } +} + +func TestClientHandshake(t *testing.T) { + t.Parallel() + suite.Run(t, &ClientHandshakeTestSuite{}) +} diff --git a/mtglib/internal/obfuscated2/conn.go b/mtglib/internal/obfuscated2/conn.go new file mode 100644 index 0000000..87221c3 --- /dev/null +++ b/mtglib/internal/obfuscated2/conn.go @@ -0,0 +1,36 @@ +package obfuscated2 + +import ( + "crypto/cipher" + "net" +) + +type Conn struct { + net.Conn + + Encryptor cipher.Stream + Decryptor cipher.Stream +} + +func (c Conn) Read(p []byte) (int, error) { + n, err := c.Conn.Read(p) + if err != nil { + return n, err // nolint: wrapcheck + } + + c.Decryptor.XORKeyStream(p, p[:n]) + + return n, nil +} + +func (c Conn) Write(p []byte) (int, error) { + buf := acquireBytesBuffer() + defer releaseBytesBuffer(buf) + + buf.Write(p) + + payload := buf.Bytes() + c.Encryptor.XORKeyStream(payload, payload) + + return c.Conn.Write(payload) +} diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go new file mode 100644 index 0000000..fff548f --- /dev/null +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -0,0 +1,69 @@ +package obfuscated2 + +const ( + DefaultDC = 2 + + handshakeFrameLen = 64 + + handshakeFrameLenKey = 32 + handshakeFrameLenIV = 16 + handshakeFrameLenConnectionType = 4 + + handshakeFrameOffsetStart = 8 + handshakeFrameOffsetKey = handshakeFrameOffsetStart + handshakeFrameOffsetIV = handshakeFrameOffsetKey + handshakeFrameLenKey + handshakeFrameOffsetConnectionType = handshakeFrameOffsetIV + handshakeFrameLenIV + handshakeFrameOffsetDC = handshakeFrameOffsetConnectionType + handshakeFrameLenConnectionType +) + +// Connection-Type: Secure. We support only fake tls. +var handshakeConnectionType = []byte{0xdd, 0xdd, 0xdd, 0xdd} + +// A structure of obfuscated2 handshake frame is following: +// +// [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd]. +// +// - 8 bytes of noise +// - 32 bytes of AES Key +// - 16 bytes of AES IV +// - 4 bytes of 'connection type' - this has some setting like a connection type +// - 2 bytes of 'DC'. DC is little endian int16 +// - 2 bytes of noise +type handshakeFrame struct { + data [handshakeFrameLen]byte +} + +func (h *handshakeFrame) dc() int { + idx := int16(h.data[handshakeFrameOffsetDC]) | int16(h.data[handshakeFrameOffsetDC+1])<<8 // nolint: gomnd, lll // little endian for int16 is here + + switch { + case idx > 0: + return int(idx) + case idx < 0: + return -int(idx) + default: + return DefaultDC + } +} + +func (h *handshakeFrame) key() []byte { + return h.data[handshakeFrameOffsetKey:handshakeFrameOffsetIV] +} + +func (h *handshakeFrame) iv() []byte { + return h.data[handshakeFrameOffsetIV:handshakeFrameOffsetConnectionType] +} + +func (h *handshakeFrame) connectionType() []byte { + return h.data[handshakeFrameOffsetConnectionType:handshakeFrameOffsetDC] +} + +func (h *handshakeFrame) invert() handshakeFrame { + copyFrame := *h + + for i := 0; i < handshakeFrameLenKey+handshakeFrameLenIV; i++ { + copyFrame.data[handshakeFrameOffsetKey+i] = h.data[handshakeFrameOffsetConnectionType-1-i] + } + + return copyFrame +} diff --git a/mtglib/internal/obfuscated2/handshake_frame_internal_test.go b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go new file mode 100644 index 0000000..7b197db --- /dev/null +++ b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go @@ -0,0 +1,73 @@ +package obfuscated2 + +import ( + "crypto/rand" + "encoding/base64" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type HandshakeFrameTestSuite struct { + suite.Suite +} + +func (suite *HandshakeFrameTestSuite) Decode(value string) []byte { + v, err := base64.RawStdEncoding.DecodeString(value) + suite.NoError(err) + + return v +} + +func (suite *HandshakeFrameTestSuite) Encode(value []byte) string { + return base64.RawStdEncoding.EncodeToString(value) +} + +func (suite *HandshakeFrameTestSuite) TestOk() { + hf := handshakeFrame{} + testFrame := suite.Decode( + "L9TmCzzxl9bPKODBpZeVM/qqNUxQ/axxBup1S2ymbIfUd6f7YSyzzM9EmTFv2/XzGqJGEHuj2zofmUGBLghu5g") + copy(hf.data[:], testFrame) + + suite.Equal("zyjgwaWXlTP6qjVMUP2scQbqdUtspmyH1Hen+2Ess8w", suite.Encode(hf.key())) + suite.Equal("z0SZMW/b9fMaokYQe6PbOg", suite.Encode(hf.iv())) + suite.Equal("H5lBgQ", suite.Encode(hf.connectionType())) + suite.EqualValues(2094, hf.dc()) + + inverted := hf.invert() + suite.Equal("OtujexBGohrz9dtvMZlEz8yzLGH7p3fUh2ymbEt16gY", suite.Encode(inverted.key())) + suite.Equal("caz9UEw1qvozlZelweAozw", suite.Encode(inverted.iv())) + suite.Equal("H5lBgQ", suite.Encode(inverted.connectionType())) + suite.EqualValues(2094, inverted.dc()) +} + +func (suite *HandshakeFrameTestSuite) TestDC() { + testData := map[int16]int{ + 1: 1, + -1: 1, + 0: DefaultDC, + } + + for k, v := range testData { + incoming := k + expected := v + + suite.T().Run(strconv.Itoa(int(incoming)), func(t *testing.T) { + frame := handshakeFrame{} + + rand.Read(frame.data[:]) // nolint: errcheck + + frame.data[handshakeFrameOffsetDC] = byte(incoming) + frame.data[handshakeFrameOffsetDC+1] = byte(incoming >> 8) + + assert.Equal(t, expected, frame.dc()) + }) + } +} + +func TestHandshakeFrame(t *testing.T) { + t.Parallel() + suite.Run(t, &HandshakeFrameTestSuite{}) +} diff --git a/mtglib/internal/obfuscated2/init_test.go b/mtglib/internal/obfuscated2/init_test.go new file mode 100644 index 0000000..001c84b --- /dev/null +++ b/mtglib/internal/obfuscated2/init_test.go @@ -0,0 +1,83 @@ +package obfuscated2_test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +type snapshotBytes struct { + data []byte +} + +func (s snapshotBytes) MarshalText() ([]byte, error) { + if len(s.data) == 0 { + return nil, nil + } + + return []byte(base64.RawStdEncoding.EncodeToString(s.data)), nil +} + +func (s *snapshotBytes) UnmarshalText(data []byte) error { + val, err := base64.RawStdEncoding.DecodeString(string(data)) + if err != nil { + return fmt.Errorf("cannot unmarshal %v: %w", len(val), err) + } + + s.data = val + + return nil +} + +type Obfuscated2Snapshot struct { + Secret snapshotBytes `json:"secret"` + Frame snapshotBytes `json:"frame"` + DC int16 `json:"dc"` + Encrypted struct { + Text snapshotBytes `json:"text"` + Cipher snapshotBytes `json:"cipher"` + } `json:"encrypted"` + Decrypted struct { + Text snapshotBytes `json:"text"` + Cipher snapshotBytes `json:"cipher"` + } `json:"decrypted"` +} + +type SnapshotTestSuite struct { + snapshots map[string]*Obfuscated2Snapshot +} + +func (suite *SnapshotTestSuite) IngestSnapshots(dirname, namePrefix string) error { + suite.snapshots = map[string]*Obfuscated2Snapshot{} + + files, err := os.ReadDir(filepath.Join("testdata", dirname)) + if err != nil { + return fmt.Errorf("cannot ingest snapshots: %w", err) + } + + for _, v := range files { + if !strings.HasPrefix(v.Name(), namePrefix) { + continue + } + + filename := filepath.Join("testdata", dirname, v.Name()) + + contents, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("cannot read %s: %w", filename, err) + } + + value := &Obfuscated2Snapshot{} + + if err := json.Unmarshal(contents, value); err != nil { + return fmt.Errorf("cannot unmarshal %s: %w", filename, err) + } + + suite.snapshots[v.Name()] = value + } + + return nil +} diff --git a/mtglib/internal/obfuscated2/pools.go b/mtglib/internal/obfuscated2/pools.go new file mode 100644 index 0000000..811c540 --- /dev/null +++ b/mtglib/internal/obfuscated2/pools.go @@ -0,0 +1,39 @@ +package obfuscated2 + +import ( + "bytes" + "crypto/sha256" + "hash" + "sync" +) + +var ( + sha256HasherPool = sync.Pool{ + New: func() interface{} { + return sha256.New() + }, + } + bytesBufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, + } +) + +func acquireSha256Hasher() hash.Hash { + return sha256HasherPool.Get().(hash.Hash) +} + +func releaseSha256Hasher(h hash.Hash) { + h.Reset() + sha256HasherPool.Put(h) +} + +func acquireBytesBuffer() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func releaseBytesBuffer(buf *bytes.Buffer) { + buf.Reset() + bytesBufferPool.Put(buf) +} diff --git a/mtglib/internal/obfuscated2/server_handshake.go b/mtglib/internal/obfuscated2/server_handshake.go new file mode 100644 index 0000000..9712fde --- /dev/null +++ b/mtglib/internal/obfuscated2/server_handshake.go @@ -0,0 +1,67 @@ +package obfuscated2 + +import ( + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "fmt" + "io" +) + +type serverHandshakeFrame struct { + handshakeFrame +} + +func (s *serverHandshakeFrame) decryptor() cipher.Stream { + invertedHandshake := s.invert() + + return makeAesCtr(invertedHandshake.key(), invertedHandshake.iv()) +} + +func (s *serverHandshakeFrame) encryptor() cipher.Stream { + return makeAesCtr(s.key(), s.iv()) +} + +func ServerHandshake(writer io.Writer) (cipher.Stream, cipher.Stream, error) { + handshake := generateServerHanshakeFrame() + copyHandshake := handshake + encryptor := handshake.encryptor() + decryptor := handshake.decryptor() + + encryptor.XORKeyStream(handshake.data[:], handshake.data[:]) + copy(handshake.key(), copyHandshake.key()) + copy(handshake.iv(), copyHandshake.iv()) + + if _, err := writer.Write(handshake.data[:]); err != nil { + return nil, nil, fmt.Errorf("cannot send a handshake frame to telegram: %w", err) + } + + return encryptor, decryptor, nil +} + +func generateServerHanshakeFrame() serverHandshakeFrame { + frame := serverHandshakeFrame{} + + for { + if _, err := rand.Read(frame.data[:]); err != nil { + panic(err) + } + + if frame.data[0] == 0xef { // nolint: gomnd // taken from tg sources + continue + } + + switch binary.LittleEndian.Uint32(frame.data[:4]) { + case 0x44414548, 0x54534f50, 0x20544547, 0x4954504f, 0xeeeeeeee: // nolint: gomnd // taken from tg sources + continue + } + + if frame.data[4]|frame.data[5]|frame.data[6]|frame.data[7] == 0 { + continue + } + + copy(frame.connectionType(), handshakeConnectionType) + + return frame + } +} diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go new file mode 100644 index 0000000..418af0c --- /dev/null +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -0,0 +1,104 @@ +package obfuscated2_test + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ServerHandshakeTestSuite struct { + suite.Suite + + connMock *testlib.NetConnMock + proxyConn obfuscated2.Conn + encryptor cipher.Stream + decryptor cipher.Stream +} + +func (suite *ServerHandshakeTestSuite) SetupTest() { + buf := &bytes.Buffer{} + suite.connMock = &testlib.NetConnMock{} + + encryptor, decryptor, err := obfuscated2.ServerHandshake(buf) + suite.NoError(err) + + suite.proxyConn = obfuscated2.Conn{ + Conn: suite.connMock, + Encryptor: encryptor, + Decryptor: decryptor, + } + + serverEncrypted := buf.Bytes() + + decBlock, _ := aes.NewCipher(serverEncrypted[8 : 8+32]) + suite.decryptor = cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16]) + + serverDecrypted := make([]byte, len(serverEncrypted)) + suite.decryptor.XORKeyStream(serverDecrypted, serverEncrypted) + + suite.Equal("3d3d3Q", + base64.RawStdEncoding.EncodeToString(serverDecrypted[8+32+16:8+32+16+4])) + + serverEncryptedReverted := make([]byte, len(serverEncrypted)) + + for i := 0; i < 32+16; i++ { + serverEncryptedReverted[8+i] = serverEncrypted[8+32+16-1-i] + } + + encBlock, _ := aes.NewCipher(serverEncryptedReverted[8 : 8+32]) + suite.encryptor = cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16]) +} + +func (suite *ServerHandshakeTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ServerHandshakeTestSuite) TestSendToTelegram() { + messageToTelegram := []byte{10, 11, 12, 13, 14, 'a'} + + suite.connMock. + On("Write", mock.Anything). + Return(len(messageToTelegram), nil). + Once(). + Run(func(args mock.Arguments) { + message := make([]byte, len(messageToTelegram)) + suite.decryptor.XORKeyStream(message, args.Get(0).([]byte)) + suite.Equal(messageToTelegram, message) + }) + + n, err := suite.proxyConn.Write(messageToTelegram) + suite.EqualValues(len(messageToTelegram), n) + suite.NoError(err) +} + +func (suite *ServerHandshakeTestSuite) TestRecieveFromTelegram() { + messageFromTelegram := []byte{10, 11, 12, 13, 14, 'a'} + buffer := make([]byte, len(messageFromTelegram)) + + suite.connMock. + On("Read", mock.Anything). + Return(len(messageFromTelegram), nil). + Once(). + Run(func(args mock.Arguments) { + message := make([]byte, len(messageFromTelegram)) + suite.encryptor.XORKeyStream(message, messageFromTelegram) + copy(args.Get(0).([]byte), message) + }) + + n, err := suite.proxyConn.Read(buffer) + suite.EqualValues(len(messageFromTelegram), n) + suite.NoError(err) + suite.Equal(messageFromTelegram, buffer) +} + +func TestServerHandshake(t *testing.T) { + t.Parallel() + suite.Run(t, &ServerHandshakeTestSuite{}) +} diff --git a/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json new file mode 100644 index 0000000..f34dc15 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "gDcXwaMY4RwlR+nJw+ILDr123UJHHjjE/U5pF4m/Y04AmH7lEpEL6UYRnIYDbDlOHSDxc1ToziPvNlJJh8RMow", + "dc": 2, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "wZV3TR39l9nRoQ" + }, + "decrypted": { + "text": "4wZj6mUUew", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json new file mode 100644 index 0000000..a59adb4 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "M2WyxeiwIQB+ZOFxNzSNHtu9OdESkfxv3JkKFimCxUoYA3BD/Ql9nXB/OIonCKLUKCcS0VzZ2P6/+5oQ9GI8YA", + "dc": 2, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "tzAwrCz00odERg" + }, + "decrypted": { + "text": "QkIvwGQDgA", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/utils.go b/mtglib/internal/obfuscated2/utils.go new file mode 100644 index 0000000..a3fdcf9 --- /dev/null +++ b/mtglib/internal/obfuscated2/utils.go @@ -0,0 +1,15 @@ +package obfuscated2 + +import ( + "crypto/aes" + "crypto/cipher" +) + +func makeAesCtr(key, iv []byte) cipher.Stream { + block, err := aes.NewCipher(key) + if err != nil { + panic(err) + } + + return cipher.NewCTR(block, iv) +} diff --git a/mtglib/internal/relay/conn.go b/mtglib/internal/relay/conn.go new file mode 100644 index 0000000..31f8bb2 --- /dev/null +++ b/mtglib/internal/relay/conn.go @@ -0,0 +1,35 @@ +package relay + +import ( + "context" + "io" +) + +type conn struct { + io.ReadWriteCloser + + ctx context.Context + tickChannel chan struct{} +} + +func (c conn) Read(p []byte) (int, error) { + n, err := c.ReadWriteCloser.Read(p) + + select { + case <-c.ctx.Done(): + case c.tickChannel <- struct{}{}: + } + + return n, err // nolint: wrapcheck +} + +func (c conn) Write(p []byte) (int, error) { + n, err := c.ReadWriteCloser.Write(p) + + select { + case <-c.ctx.Done(): + case c.tickChannel <- struct{}{}: + } + + return n, err // nolint: wrapcheck +} diff --git a/mtglib/internal/relay/conn_internal_test.go b/mtglib/internal/relay/conn_internal_test.go new file mode 100644 index 0000000..86bd81b --- /dev/null +++ b/mtglib/internal/relay/conn_internal_test.go @@ -0,0 +1,125 @@ +package relay + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnTestSuite struct { + suite.Suite + + ctxCancel context.CancelFunc + connMock *testlib.NetConnMock + tickChannel chan struct{} + buf []byte + c conn +} + +func (suite *ConnTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + + suite.tickChannel = make(chan struct{}, 1) + suite.connMock = &testlib.NetConnMock{} + suite.ctxCancel = cancel + suite.buf = make([]byte, 5) + + suite.c = conn{ + ReadWriteCloser: suite.connMock, + ctx: ctx, + tickChannel: suite.tickChannel, + } +} + +func (suite *ConnTestSuite) TestReadOk() { + suite.connMock.On("Read", mock.Anything).Once().Return(len(suite.buf), nil) + + n, err := suite.c.Read(suite.buf) + suite.NoError(err) + suite.Equal(len(suite.buf), n) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestReadErr() { + suite.connMock.On("Read", mock.Anything).Once().Return(0, io.EOF) + + _, err := suite.c.Read(suite.buf) + suite.True(errors.Is(err, io.EOF)) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestReadContextDone() { + suite.connMock.On("Read", mock.Anything).Once().Return(len(suite.buf), nil) + suite.ctxCancel() + + suite.tickChannel <- struct{}{} + + suite.c.Read(suite.buf) // nolint: errcheck +} + +func (suite *ConnTestSuite) TestWriteOk() { + suite.connMock.On("Write", mock.Anything).Once().Return(len(suite.buf), nil) + + n, err := suite.c.Write(suite.buf) + suite.NoError(err) + suite.Equal(len(suite.buf), n) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestWriteErr() { + suite.connMock.On("Write", mock.Anything).Once().Return(0, io.EOF) + + _, err := suite.c.Write(suite.buf) + suite.True(errors.Is(err, io.EOF)) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestWriteContextDone() { + suite.connMock.On("Write", mock.Anything).Once().Return(len(suite.buf), nil) + suite.ctxCancel() + + suite.tickChannel <- struct{}{} + + suite.c.Write(suite.buf) // nolint: errcheck +} + +func (suite *ConnTestSuite) TearDownTest() { + select { + case <-suite.tickChannel: + default: + } + + close(suite.tickChannel) + + suite.connMock.AssertExpectations(suite.T()) +} + +func TestConn(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTestSuite{}) +} diff --git a/mtglib/internal/relay/init.go b/mtglib/internal/relay/init.go new file mode 100644 index 0000000..c381a4a --- /dev/null +++ b/mtglib/internal/relay/init.go @@ -0,0 +1,5 @@ +package relay + +type Logger interface { + Printf(msg string, args ...interface{}) +} diff --git a/mtglib/internal/relay/init_test.go b/mtglib/internal/relay/init_test.go new file mode 100644 index 0000000..e19ecf4 --- /dev/null +++ b/mtglib/internal/relay/init_test.go @@ -0,0 +1,49 @@ +package relay_test + +import ( + "bytes" + "io" + "sync" +) + +type loggerMock struct{} + +func (l loggerMock) Printf(format string, args ...interface{}) {} + +type rwcMock struct { + bytes.Buffer + + closed bool + mutex sync.Mutex +} + +func (r *rwcMock) Read(p []byte) (int, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + + if r.closed { + return 0, io.EOF + } + + return r.Buffer.Read(p) +} + +func (r *rwcMock) Write(p []byte) (int, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + + if r.closed { + return 0, io.EOF + } + + return r.Buffer.Write(p) +} + +func (r *rwcMock) Close() error { + r.mutex.Lock() + defer r.mutex.Unlock() + + r.closed = true + + return nil +} diff --git a/mtglib/internal/relay/pools.go b/mtglib/internal/relay/pools.go new file mode 100644 index 0000000..1fae858 --- /dev/null +++ b/mtglib/internal/relay/pools.go @@ -0,0 +1,41 @@ +package relay + +import ( + "context" + "sync" + "time" +) + +var relayPool = sync.Pool{ + New: func() interface{} { + return &Relay{ + tickChannel: make(chan struct{}), + errorChannel: make(chan error, 1), + } + }, +} + +func AcquireRelay(ctx context.Context, logger Logger, bufferSize int, idleTimeout time.Duration) *Relay { + ctx, cancel := context.WithCancel(ctx) + + r := relayPool.Get().(*Relay) + r.ctx = ctx + r.ctxCancel = cancel + r.logger = logger + r.tickTimeout = idleTimeout + + if len(r.eastBuffer) != bufferSize { + r.eastBuffer = make([]byte, bufferSize) + } + + if len(r.westBuffer) != bufferSize { + r.westBuffer = make([]byte, bufferSize) + } + + return r +} + +func ReleaseRelay(r *Relay) { + r.Reset() + relayPool.Put(r) +} diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go new file mode 100644 index 0000000..a9e105a --- /dev/null +++ b/mtglib/internal/relay/relay.go @@ -0,0 +1,128 @@ +package relay + +import ( + "context" + "io" + "sync" + "time" +) + +type Relay struct { + ctx context.Context + ctxCancel context.CancelFunc + logger Logger + processMutex sync.Mutex + eastBuffer []byte + westBuffer []byte + tickChannel chan struct{} + errorChannel chan error + tickTimeout time.Duration +} + +func (r *Relay) Reset() { + r.processMutex.Lock() + defer r.processMutex.Unlock() + + if r.ctxCancel != nil { + r.ctxCancel() + } + + r.ctx = nil + r.ctxCancel = nil + r.logger = nil +} + +func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { + r.processMutex.Lock() + defer r.processMutex.Unlock() + + eastConn = conn{ + ReadWriteCloser: eastConn, + ctx: r.ctx, + tickChannel: r.tickChannel, + } + westConn = conn{ + ReadWriteCloser: westConn, + ctx: r.ctx, + tickChannel: r.tickChannel, + } + + wg := &sync.WaitGroup{} + wg.Add(3) // nolint: gomnd + + go r.runObserver(eastConn, westConn, wg) + + go r.transmit(eastConn, westConn, r.westBuffer, "west", wg) + + r.transmit(westConn, eastConn, r.eastBuffer, "east", wg) + + wg.Wait() + + select { + case err := <-r.errorChannel: + return err + default: + return nil + } +} + +func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, + buffer []byte, direction string, wg *sync.WaitGroup) { + defer func() { + src.Close() + dst.Close() + wg.Done() + r.ctxCancel() + }() + + if _, err := io.CopyBuffer(dst, src, buffer); err != nil { + r.logger.Printf("error '%v' happened on direction %s", err, direction) + + select { + case <-r.ctx.Done(): + err = r.ctx.Err() + default: + } + + select { + case r.errorChannel <- err: + default: + } + } +} + +func (r *Relay) runObserver(one, another io.Closer, wg *sync.WaitGroup) { + ticker := time.NewTicker(time.Second) + + defer func() { + one.Close() + another.Close() + + ticker.Stop() + + select { + case <-ticker.C: + default: + } + + wg.Done() + }() + + lastTickAt := time.Now() + + for { + select { + case <-r.ctx.Done(): + return + case <-r.tickChannel: + lastTickAt = time.Now() + case <-ticker.C: + if time.Since(lastTickAt) > r.tickTimeout { + r.logger.Printf("exit due to a timeout") + r.ctxCancel() + + return + } + } + } +} diff --git a/mtglib/internal/relay/relay_test.go b/mtglib/internal/relay/relay_test.go new file mode 100644 index 0000000..9e3f19c --- /dev/null +++ b/mtglib/internal/relay/relay_test.go @@ -0,0 +1,77 @@ +package relay_test + +import ( + "context" + "io" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/9seconds/mtg/v2/mtglib/internal/relay" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type RelayTestSuite struct { + suite.Suite + + ctx context.Context + ctxCancel context.CancelFunc + r *relay.Relay +} + +func (suite *RelayTestSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) + suite.r = relay.AcquireRelay(suite.ctx, loggerMock{}, 4096, time.Second) +} + +func (suite *RelayTestSuite) TearDownTest() { + suite.ctxCancel() + relay.ReleaseRelay(suite.r) + suite.r = nil +} + +func (suite *RelayTestSuite) TestCancelled() { + suite.ctxCancel() + + eastConn := &rwcMock{} + eastConn.Write([]byte{1, 2, 3, 4, 5}) // nolint: errcheck + + westConn := &rwcMock{} + westConn.Write([]byte{100, 101, 102}) // nolint: errcheck + + suite.Nil(suite.r.Process(eastConn, westConn)) +} + +func (suite *RelayTestSuite) TestCopyFine() { + eastConn := &rwcMock{} + eastConn.Write([]byte{1, 2, 3, 4, 5}) // nolint: errcheck + + westConn := &rwcMock{} + westConn.Write([]byte{100, 101, 102}) // nolint: errcheck + + // yes, this test is not good enough. but apparently, if it hangs, + // we can debug most of possible issues. + _ = suite.r.Process(eastConn, westConn) +} + +func (suite *RelayTestSuite) TestTimeout() { + eastConn := &rwcMock{} + eastConn.Write([]byte{1, 2, 3, 4, 5}) // nolint: errcheck + + westConn := &testlib.NetConnMock{} + westConn.On("Close").Return(nil) + westConn.On("Read", mock.Anything).Return(0, io.EOF).Run(func(_ mock.Arguments) { + time.Sleep(2 * time.Second) + }) + westConn.On("Write", mock.Anything).Return(0, io.EOF).Run(func(_ mock.Arguments) { + time.Sleep(2 * time.Second) + }) + + suite.Error(suite.r.Process(eastConn, westConn)) +} + +func TestRelay(t *testing.T) { + t.Parallel() + suite.Run(t, &RelayTestSuite{}) +} diff --git a/mtglib/internal/telegram/init.go b/mtglib/internal/telegram/init.go new file mode 100644 index 0000000..af707d3 --- /dev/null +++ b/mtglib/internal/telegram/init.go @@ -0,0 +1,53 @@ +package telegram + +import ( + "context" + "net" +) + +type preferIP uint8 + +const ( + preferIPOnlyIPv4 preferIP = iota + preferIPOnlyIPv6 + preferIPPreferIPv4 + preferIPPreferIPv6 +) + +type tgAddr struct { + network string + address string +} + +// https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/mtproto_dc_options.cpp#L30 +var ( + v4Addresses = [5][]tgAddr{ + { + {network: "tcp4", address: "149.154.175.50:443"}, + }, + { + {network: "tcp4", address: "149.154.167.51:443"}, + {network: "tcp4", address: "95.161.76.100:443"}, + }, + { + {network: "tcp4", address: "149.154.175.100:443"}, + }, + { + {network: "tcp4", address: "149.154.167.91:443"}, + }, + { + {network: "tcp4", address: "149.154.171.5:443"}, + }, + } + v6Addresses = [5]tgAddr{ + {network: "tcp6", address: "[2001:b28:f23d:f001::a]:443"}, + {network: "tcp6", address: "[2001:67c:04e8:f002::a]:443"}, + {network: "tcp6", address: "[2001:b28:f23d:f003::a]:443"}, + {network: "tcp6", address: "[2001:67c:04e8:f004::a]:443"}, + {network: "tcp6", address: "[2001:b28:f23f:f005::a]:443"}, + } +) + +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} diff --git a/mtglib/internal/telegram/telegram.go b/mtglib/internal/telegram/telegram.go new file mode 100644 index 0000000..2ee86e2 --- /dev/null +++ b/mtglib/internal/telegram/telegram.go @@ -0,0 +1,75 @@ +package telegram + +import ( + "context" + "fmt" + "math/rand" + "net" + "strings" +) + +type Telegram struct { + dialer Dialer + preferIP preferIP +} + +func (t Telegram) Dial(ctx context.Context, dc int) (net.Conn, error) { + if dc <= 0 || dc > 5 { + return nil, fmt.Errorf("do not know how to dial to %d", dc) + } + + var addresses []tgAddr + + if t.preferIP == preferIPOnlyIPv6 { + addresses = []tgAddr{v6Addresses[dc-1]} + } else { + addresses = append(addresses, v4Addresses[dc-1]...) + rand.Shuffle(len(addresses), func(i, j int) { + addresses[i], addresses[j] = addresses[j], addresses[i] + }) + } + + switch t.preferIP { + case preferIPPreferIPv4: + addresses = append(addresses, v6Addresses[dc-1]) + case preferIPPreferIPv6: + addresses = append([]tgAddr{v6Addresses[dc-1]}, addresses...) + case preferIPOnlyIPv4, preferIPOnlyIPv6: + } + + var ( + conn net.Conn + err error + ) + + for _, v := range addresses { + conn, err = t.dialer.DialContext(ctx, v.network, v.address) + if err == nil { + return conn, nil + } + } + + return nil, fmt.Errorf("cannot dial to %d dc: %w", dc, err) +} + +func New(dialer Dialer, ipPreference string) (*Telegram, error) { + var pref preferIP + + switch strings.ToLower(ipPreference) { + case "prefer-ipv4": + pref = preferIPPreferIPv4 + case "prefer-ipv6": + pref = preferIPPreferIPv6 + case "only-ipv4": + pref = preferIPOnlyIPv4 + case "only-ipv6": + pref = preferIPOnlyIPv6 + default: + return nil, fmt.Errorf("unknown ip preference %s", ipPreference) + } + + return &Telegram{ + dialer: dialer, + preferIP: pref, + }, nil +} diff --git a/mtglib/internal/telegram/telegram_internal_test.go b/mtglib/internal/telegram/telegram_internal_test.go new file mode 100644 index 0000000..fe21ce2 --- /dev/null +++ b/mtglib/internal/telegram/telegram_internal_test.go @@ -0,0 +1,141 @@ +package telegram + +import ( + "context" + "errors" + "io" + "net" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type TelegramTestSuite struct { + suite.Suite + + dialerMock *testlib.MtglibNetworkMock + t *Telegram +} + +func (suite *TelegramTestSuite) SetupTest() { + suite.dialerMock = &testlib.MtglibNetworkMock{} + suite.t, _ = New(suite.dialerMock, "prefer-ipv4") +} + +func (suite *TelegramTestSuite) TearDownTest() { + suite.dialerMock.AssertExpectations(suite.T()) +} + +func (suite *TelegramTestSuite) TestUnknownDC() { + testData := []int{ + -1, + 0, + 6, + 100, + } + + for _, v := range testData { + value := v + + suite.T().Run(strconv.Itoa(value), func(t *testing.T) { + _, err := suite.t.Dial(context.Background(), value) + assert.Error(t, err) + }) + } +} + +func (suite *TelegramTestSuite) TestDialToCorrectIPs() { + testData := map[int][]tgAddr{} + + for i := 1; i <= 5; i++ { + testData[i] = []tgAddr{} + testData[i] = append(testData[i], v4Addresses[i-1]...) + testData[i] = append(testData[i], v6Addresses[i-1]) + } + + for i, v := range testData { + idx := i + addresses := v + + suite.T().Run(strconv.Itoa(idx), func(t *testing.T) { + for _, addr := range addresses { + suite.dialerMock. + On("DialContext", mock.Anything, addr.network, addr.address). + Once(). + Return((*net.TCPConn)(nil), io.EOF) + } + + _, err := suite.t.Dial(context.Background(), idx) + assert.True(t, errors.Is(err, io.EOF)) + }) + } +} + +func (suite *TelegramTestSuite) TestDialPreferIPRange() { + testData := map[string][]tgAddr{ + "prefer-ipv4": {v4Addresses[0][0], v6Addresses[0]}, + "prefer-ipv6": {v6Addresses[0], v4Addresses[0][0]}, + "only-ipv4": {v4Addresses[0][0]}, + "only-ipv6": {v6Addresses[0]}, + } + + for k, v := range testData { + name := k + addresses := v + + suite.T().Run(name, func(t *testing.T) { + for _, addr := range addresses { + suite.dialerMock. + On("DialContext", mock.Anything, addr.network, addr.address). + Once(). + Return((*net.TCPConn)(nil), io.EOF) + } + + tg, _ := New(suite.dialerMock, name) + _, err := tg.Dial(context.Background(), 1) + + assert.True(t, errors.Is(err, io.EOF)) + }) + } +} + +func (suite *TelegramTestSuite) TestDialPreferIPPriority() { + testData := map[string]tgAddr{ + "prefer-ipv4": v4Addresses[0][0], + "prefer-ipv6": v6Addresses[0], + } + + for k, v := range testData { + name := k + addr := v + + suite.T().Run(name, func(t *testing.T) { + conn := &net.TCPConn{} + + suite.dialerMock. + On("DialContext", mock.Anything, addr.network, addr.address). + Once(). + Return(conn, nil) + + tg, _ := New(suite.dialerMock, name) + + res, err := tg.Dial(context.Background(), 1) + assert.NoError(t, err) + assert.Equal(t, conn, res) + }) + } +} + +func (suite *TelegramTestSuite) TestUnknownPreferIP() { + _, err := New(suite.dialerMock, "xxx") + suite.Error(err) +} + +func TestTelegram(t *testing.T) { + t.Parallel() + suite.Run(t, &TelegramTestSuite{}) +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go new file mode 100644 index 0000000..e07a9fe --- /dev/null +++ b/mtglib/proxy.go @@ -0,0 +1,308 @@ +package mtglib + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "sync" + "time" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/9seconds/mtg/v2/mtglib/internal/relay" + "github.com/9seconds/mtg/v2/mtglib/internal/telegram" + "github.com/panjf2000/ants/v2" +) + +// Proxy is an MTPROTO proxy structure. +type Proxy struct { + ctx context.Context + ctxCancel context.CancelFunc + streamWaitGroup sync.WaitGroup + + idleTimeout time.Duration + tolerateTimeSkewness time.Duration + bufferSize int + domainFrontingPort int + workerPool *ants.PoolWithFunc + telegram *telegram.Telegram + + secret Secret + network Network + antiReplayCache AntiReplayCache + ipBlocklist IPBlocklist + eventStream EventStream + logger Logger +} + +// DomainFrontingAddress returns a host:port pair for a fronting domain. +func (p *Proxy) DomainFrontingAddress() string { + return net.JoinHostPort(p.secret.Host, strconv.Itoa(p.domainFrontingPort)) +} + +// ServeConn serves a connection. We do not check IP blocklist and +// concurrency limit here. +func (p *Proxy) ServeConn(conn net.Conn) { + p.streamWaitGroup.Add(1) + defer p.streamWaitGroup.Done() + + ctx := newStreamContext(p.ctx, p.logger, conn) + defer ctx.Close() + + go func() { + <-ctx.Done() + ctx.Close() + }() + + p.eventStream.Send(ctx, NewEventStart(ctx.streamID, ctx.ClientIP())) + ctx.logger.Info("Stream has been started") + + defer func() { + p.eventStream.Send(ctx, NewEventFinish(ctx.streamID)) + ctx.logger.Info("Stream has been finished") + }() + + if !p.doFakeTLSHandshake(ctx) { + return + } + + if err := p.doObfuscated2Handshake(ctx); err != nil { + p.logger.InfoError("obfuscated2 handshake is failed", err) + + return + } + + if err := p.doTelegramCall(ctx); err != nil { + p.logger.WarningError("cannot dial to telegram", err) + + return + } + + rel := relay.AcquireRelay(ctx, + p.logger.Named("relay"), p.bufferSize, p.idleTimeout) + defer relay.ReleaseRelay(rel) + + if err := rel.Process(ctx.clientConn, ctx.telegramConn); err != nil { + p.logger.DebugError("relay has been finished", err) + } +} + +// Serve starts a proxy on a given listener. +func (p *Proxy) Serve(listener net.Listener) error { + p.streamWaitGroup.Add(1) + defer p.streamWaitGroup.Done() + + for { + conn, err := listener.Accept() + if err != nil { + select { + case <-p.ctx.Done(): + return nil + default: + return fmt.Errorf("cannot accept a new connection: %w", err) + } + } + + ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP + logger := p.logger.BindStr("ip", ipAddr.String()) + + if p.ipBlocklist.Contains(ipAddr) { + conn.Close() + logger.Info("ip was blacklisted") + p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr)) + + continue + } + + err = p.workerPool.Invoke(conn) + + switch { + case err == nil: + case errors.Is(err, ants.ErrPoolClosed): + return nil + case errors.Is(err, ants.ErrPoolOverload): + logger.Info("connection was concurrency limited") + p.eventStream.Send(p.ctx, NewEventConcurrencyLimited()) + } + } +} + +// Shutdown 'gracefully' shutdowns all connections. Please remember that +// it does not close an underlying listener. +func (p *Proxy) Shutdown() { + p.ctxCancel() + p.streamWaitGroup.Wait() + p.workerPool.Release() +} + +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rewind := newConnRewind(ctx.clientConn) + + if err := rec.Read(rewind); err != nil { + p.logger.InfoError("cannot read client hello", err) + p.doDomainFronting(ctx, rewind) + + return false + } + + hello, err := faketls.ParseClientHello(p.secret.Key[:], rec.Payload.Bytes()) + if err != nil { + p.logger.InfoError("cannot parse client hello", err) + p.doDomainFronting(ctx, rewind) + + return false + } + + if err := hello.Valid(p.secret.Host, p.tolerateTimeSkewness); err != nil { + p.logger. + BindStr("hostname", hello.Host). + BindStr("hello-time", hello.Time.String()). + InfoError("invalid faketls client hello", err) + p.doDomainFronting(ctx, rewind) + + return false + } + + if p.antiReplayCache.SeenBefore(hello.SessionID) { + p.logger.Warning("replay attack has been detected!") + p.eventStream.Send(p.ctx, NewEventReplayAttack(ctx.streamID)) + p.doDomainFronting(ctx, rewind) + + return false + } + + if err := faketls.SendWelcomePacket(rewind, p.secret.Key[:], hello); err != nil { + p.logger.InfoError("cannot send welcome packet", err) + + return false + } + + ctx.clientConn = &faketls.Conn{ + Conn: ctx.clientConn, + } + + return true +} + +func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { + dc, encryptor, decryptor, err := obfuscated2.ClientHandshake(p.secret.Key[:], ctx.clientConn) + if err != nil { + return fmt.Errorf("cannot process client handshake: %w", err) + } + + ctx.dc = dc + ctx.logger = ctx.logger.BindInt("dc", dc) + ctx.clientConn = obfuscated2.Conn{ + Conn: ctx.clientConn, + Encryptor: encryptor, + Decryptor: decryptor, + } + + return nil +} + +func (p *Proxy) doTelegramCall(ctx *streamContext) error { + conn, err := p.telegram.Dial(ctx, ctx.dc) + if err != nil { + return fmt.Errorf("cannot dial to Telegram: %w", err) + } + + encryptor, decryptor, err := obfuscated2.ServerHandshake(conn) + if err != nil { + conn.Close() + + return fmt.Errorf("cannot perform obfuscated2 handshake: %w", err) + } + + ctx.telegramConn = obfuscated2.Conn{ + Conn: connTraffic{ + Conn: conn, + streamID: ctx.streamID, + stream: p.eventStream, + ctx: ctx, + }, + Encryptor: encryptor, + Decryptor: decryptor, + } + + p.eventStream.Send(ctx, + NewEventConnectedToDC(ctx.streamID, conn.RemoteAddr().(*net.TCPAddr).IP, ctx.dc)) + + return nil +} + +func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { + p.eventStream.Send(p.ctx, NewEventDomainFronting(ctx.streamID)) + conn.Rewind() + + frontConn, err := p.network.DialContext(ctx, "tcp", p.DomainFrontingAddress()) + if err != nil { + p.logger.WarningError("cannot dial to the fronting domain", err) + + return + } + + frontConn = connTraffic{ + Conn: frontConn, + ctx: ctx, + streamID: ctx.streamID, + stream: p.eventStream, + } + + rel := relay.AcquireRelay(ctx, + p.logger.Named("domain-fronting"), p.bufferSize, p.idleTimeout) + defer relay.ReleaseRelay(rel) + + if err := rel.Process(conn, frontConn); err != nil { + p.logger.DebugError("domain fronting relay has been finished", err) + } +} + +// NewProxy makes a new proxy instance. +func NewProxy(opts ProxyOpts) (*Proxy, error) { + if err := opts.valid(); err != nil { + return nil, fmt.Errorf("invalid settings: %w", err) + } + + tg, err := telegram.New(opts.Network, opts.getPreferIP()) + if err != nil { + return nil, fmt.Errorf("cannot build telegram dialer: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + proxy := &Proxy{ + ctx: ctx, + ctxCancel: cancel, + secret: opts.Secret, + network: opts.Network, + antiReplayCache: opts.AntiReplayCache, + ipBlocklist: opts.IPBlocklist, + eventStream: opts.EventStream, + logger: opts.getLogger("proxy"), + domainFrontingPort: opts.getDomainFrontingPort(), + tolerateTimeSkewness: opts.getTolerateTimeSkewness(), + idleTimeout: opts.getIdleTimeout(), + bufferSize: opts.getBufferSize(), + telegram: tg, + } + + pool, err := ants.NewPoolWithFunc(opts.getConcurrency(), + func(arg interface{}) { + proxy.ServeConn(arg.(net.Conn)) + }, + ants.WithLogger(opts.getLogger("ants")), + ants.WithNonblocking(true)) + if err != nil { + panic(err) + } + + proxy.workerPool = pool + + return proxy, nil +} diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go new file mode 100644 index 0000000..c2fdc69 --- /dev/null +++ b/mtglib/proxy_opts.go @@ -0,0 +1,163 @@ +package mtglib + +import "time" + +// ProxyOpts is a structure with settings to mtg proxy. +// +// This is not required per se, but this is to shorten function +// signature and give an ability to conveniently provide default values. +type ProxyOpts struct { + // Secret defines a secret which should be used by a proxy. + // + // This is a mandatory setting. + Secret Secret + + // Network defines a network instance which should be used for all + // network communications made by proxies. + // + // This is a mandatory setting. + Network Network + + // AntiReplayCache defines an instance of antireplay cache. + // + // This is a mandatory setting. + AntiReplayCache AntiReplayCache + + // IPBlocklist defines an instance of IP blocklist. + // + // This is a mandatory setting. + IPBlocklist IPBlocklist + + // EventStream defines an instance of event stream. + // + // This ia a mandatory setting. + EventStream EventStream + + // Logger defines an instance of the logger. + // + // This is a mandatory setting. + Logger Logger + + // BufferSize is a size of the copy buffer in bytes. + // + // Please remember that we multiply this number in 2, because when + // we relay between proxies, we have to create 2 intermediate + // buffers: to and from. + // + // This is an optional setting. + BufferSize uint + + // Concurrency is a size of the worker pool for connection management. + // + // If we have more connections than this number, they are going to be + // rejected. + // + // This is an optional setting. + Concurrency uint + + // DomainFrontingPort is a port we use to connect to a fronting + // domain. + // + // This is required because secret does not specify a port. It + // specifies a hostname only. + // + // This is an optional setting. + DomainFrontingPort uint + + // IdleTimeout is a timeout for relay when we have to break a + // stream. + // + // This is a timeout for any activity. So, if we have any message + // which will pass to either direction, a timer is reset. If we have + // no any reads or writes for this timeout, a connection will be + // aborted. + // + // This is an optional setting. + IdleTimeout time.Duration + + // TolerateTimeSkewness is a time boundary that defines a time + // range where faketls timestamp is acceptable. + // + // This means that if if you got a timestamp X, now is Y, then + // if |X-Y| < TolerateTimeSkewness, then you accept a packet. + // + // This is an optional setting. + TolerateTimeSkewness time.Duration + + // PreferIP defines an IP connectivity preference. Valid values are: + // 'prefer-ipv4', 'prefer-ipv6', 'only-ipv4', 'only-ipv6'. + // + // This is an optional setting. + PreferIP string +} + +func (p ProxyOpts) valid() error { + switch { + case p.Network == nil: + return ErrNetworkIsNotDefined + case p.AntiReplayCache == nil: + return ErrAntiReplayCacheIsNotDefined + case p.IPBlocklist == nil: + return ErrIPBlocklistIsNotDefined + case p.EventStream == nil: + return ErrEventStreamIsNotDefined + case p.Logger == nil: + return ErrLoggerIsNotDefined + case !p.Secret.Valid(): + return ErrSecretInvalid + } + + return nil +} + +func (p ProxyOpts) getBufferSize() int { + if p.BufferSize < 1 { + return DefaultBufferSize + } + + return int(p.BufferSize) +} + +func (p ProxyOpts) getConcurrency() int { + if p.Concurrency == 0 { + return DefaultConcurrency + } + + return int(p.Concurrency) +} + +func (p ProxyOpts) getDomainFrontingPort() int { + if p.DomainFrontingPort == 0 { + return DefaultDomainFrontingPort + } + + return int(p.DomainFrontingPort) +} + +func (p ProxyOpts) getIdleTimeout() time.Duration { + if p.IdleTimeout == 0 { + return DefaultIdleTimeout + } + + return p.IdleTimeout +} + +func (p ProxyOpts) getTolerateTimeSkewness() time.Duration { + if p.TolerateTimeSkewness == 0 { + return DefaultTolerateTimeSkewness + } + + return p.TolerateTimeSkewness +} + +func (p ProxyOpts) getPreferIP() string { + if p.PreferIP == "" { + return DefaultPreferIP + } + + return p.PreferIP +} + +func (p ProxyOpts) getLogger(name string) Logger { + return p.Logger.Named(name) +} diff --git a/mtglib/proxy_test.go b/mtglib/proxy_test.go new file mode 100644 index 0000000..3d3171b --- /dev/null +++ b/mtglib/proxy_test.go @@ -0,0 +1,174 @@ +package mtglib_test + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "testing" + "time" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/network" + "github.com/stretchr/testify/suite" +) + +type ProxyTestSuite struct { + suite.Suite + + opts *mtglib.ProxyOpts + p *mtglib.Proxy + listener net.Listener +} + +func (suite *ProxyTestSuite) ProxyAddress() string { + _, port, _ := net.SplitHostPort(suite.listener.Addr().String()) + + return net.JoinHostPort("127.0.0.1", port) +} + +func (suite *ProxyTestSuite) ProxySecret() string { + return suite.opts.Secret.Hex() +} + +func (suite *ProxyTestSuite) SetupSuite() { + dialer, err := network.NewDefaultDialer(0, 0) + suite.NoError(err) + + ntw, err := network.NewNetwork(dialer, "mtgtest", "1.1.1.1", 0) + suite.NoError(err) + + suite.opts = &mtglib.ProxyOpts{ + Secret: mtglib.GenerateSecret("httpbin.org"), + Network: ntw, + AntiReplayCache: antireplay.NewNoop(), + IPBlocklist: ipblocklist.NewNoop(), + EventStream: events.NewNoopStream(), + Logger: logger.NewNoopLogger(), + } + + proxy, err := mtglib.NewProxy(*suite.opts) + suite.NoError(err) + + suite.p = proxy + + listener, err := net.Listen("tcp", ":0") + suite.NoError(err) + + suite.listener = listener + + go suite.p.Serve(suite.listener) // nolint: errcheck +} + +func (suite *ProxyTestSuite) TearDownSuite() { + if suite.listener != nil { + suite.listener.Close() + } + + if suite.p != nil { + suite.p.Shutdown() + } +} + +func (suite *ProxyTestSuite) TestCannotInitNoSecret() { + opts := *suite.opts + opts.Secret = mtglib.Secret{} + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoNetwork() { + opts := *suite.opts + opts.Network = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoAntiReplayCache() { + opts := *suite.opts + opts.AntiReplayCache = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoIPBlocklist() { + opts := *suite.opts + opts.IPBlocklist = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoEventStream() { + opts := *suite.opts + opts.EventStream = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoLogger() { + opts := *suite.opts + opts.Logger = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitIncorrectPreferIP() { + opts := *suite.opts + opts.PreferIP = "xxx" + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestDomainFrontingAddress() { + suite.Equal("httpbin.org:443", suite.p.DomainFrontingAddress()) +} + +func (suite *ProxyTestSuite) TestHTTPSRequest() { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: 5 * time.Second, + } + + addr := fmt.Sprintf("https://%s/headers", suite.ProxyAddress()) + + resp, err := client.Get(addr) // nolint: noctx + suite.NoError(err) + + defer resp.Body.Close() + + suite.Equal(http.StatusOK, resp.StatusCode) + + data, err := io.ReadAll(resp.Body) + suite.NoError(err) + + jsonStruct := struct { + Headers struct { + TraceID string `json:"X-Amzn-Trace-Id"` + } `json:"headers"` + }{} + + suite.NoError(json.Unmarshal(data, &jsonStruct)) + suite.NotEmpty(jsonStruct.Headers.TraceID) +} + +func TestProxy(t *testing.T) { + t.Parallel() + suite.Run(t, &ProxyTestSuite{}) +} diff --git a/mtglib/secret.go b/mtglib/secret.go new file mode 100644 index 0000000..d9c1ff3 --- /dev/null +++ b/mtglib/secret.go @@ -0,0 +1,143 @@ +package mtglib + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "fmt" +) + +const secretFakeTLSFirstByte byte = 0xee + +var secretEmptyKey [SecretKeyLength]byte + +// Secret is a data structure that presents a secret. +// +// Telegram secret is not a simple string like +// "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d". +// Actually, this is a serialized datastructure of 2 parts: key and host. +// +// ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d +// |-|-------------------------------|------------------------------------------- +// p key hostname +// +// Serialized secret starts with 'ee'. Actually, in the past we also had +// 'dd' secrets and prefixless ones. But this is history. Currently, +// we do have only 'ee' secrets which mean faketls + protection from +// statistical attacks on a length. 'ee' is a byte 238 (0xee). +// +// After that, we have 16 bytes of the key. This is a random generated +// secret data of the proxy and this data is used to derive +// authentication schemas. These secrets are mixed into hmacs and sha256 +// checksums which are used to build AEAD ciphers for obfuscated2 +// protocol and ensure faketls handshake. +// +// Host is a domain fronting hostname in latin1 (ASCII) encoding. This +// hostname should be used for SNI in faketls and MTG verifies it. Also, +// this is when mtg gets about a domain fronting hostname. +// +// Secrets can be serialized into 2 forms: hex and base64. If +// you decode both forms into bytes, you'll get the same byte array. +// Telegram clients nowadays accept all forms. +type Secret struct { + // Key is a set of bytes used for traffic authentication. + Key [SecretKeyLength]byte + + // Host is a domain fronting hostname. + Host string +} + +// MarshalText is to support text.Marshaller interface. +func (s Secret) MarshalText() ([]byte, error) { + if s.Valid() { + return []byte(s.String()), nil + } + + return nil, nil +} + +// MarshalText is to support text.Unmarshaller interface. +func (s *Secret) UnmarshalText(data []byte) error { + text := string(data) + if text == "" { + return ErrSecretEmpty + } + + decoded, err := hex.DecodeString(text) + if err != nil { + decoded, err = base64.RawURLEncoding.DecodeString(text) + } + + if err != nil { + return fmt.Errorf("incorrect secret format: %w", err) + } + + if len(decoded) < 2 { // nolint: gomnd // we need at least 1 byte here + return fmt.Errorf("secret is truncated, length=%d", len(decoded)) + } + + if decoded[0] != secretFakeTLSFirstByte { + return fmt.Errorf("incorrect first byte of secret: %#x", decoded[0]) + } + + decoded = decoded[1:] + if len(decoded) < SecretKeyLength { + return fmt.Errorf("secret has incorrect length %d", len(decoded)) + } + + copy(s.Key[:], decoded[:SecretKeyLength]) + s.Host = string(decoded[SecretKeyLength:]) + + if s.Host == "" { + return fmt.Errorf("hostname cannot be empty: %s", text) + } + + return nil +} + +// Valid checks if this secret is valid and can be used in proxy. +func (s Secret) Valid() bool { + return s.Key != secretEmptyKey && s.Host != "" +} + +// String is to support fmt.Stringer interface. +func (s Secret) String() string { + return s.Base64() +} + +// Base64 returns a base64-encoded form of this secret. +func (s Secret) Base64() string { + return base64.RawURLEncoding.EncodeToString(s.makeBytes()) +} + +// Hex returns a hex-encoded form of this secret (ee-secret). +func (s Secret) Hex() string { + return hex.EncodeToString(s.makeBytes()) +} + +func (s *Secret) makeBytes() []byte { + data := append([]byte{secretFakeTLSFirstByte}, s.Key[:]...) + data = append(data, s.Host...) + + return data +} + +// GenerateSecret makes a new secret with a given hostname. +func GenerateSecret(hostname string) Secret { + s := Secret{ + Host: hostname, + } + + if _, err := rand.Read(s.Key[:]); err != nil { + panic(err) + } + + return s +} + +// ParseSecret parses a secret (both hex and base64 forms). +func ParseSecret(secret string) (Secret, error) { + s := Secret{} + + return s, s.UnmarshalText([]byte(secret)) +} diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go new file mode 100644 index 0000000..c1f2923 --- /dev/null +++ b/mtglib/secret_test.go @@ -0,0 +1,127 @@ +package mtglib_test + +import ( + "encoding/hex" + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type SecretTestSuite struct { + suite.Suite +} + +func (suite *SecretTestSuite) TestParseSecret() { + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") + s := mtglib.Secret{ + Host: "google.com", + } + + copy(s.Key[:], secretData) + + testData := map[string]string{ + "hex": "eed11c6cbbd9efe7fed5bc0db220b09665676f6f676c652e636f6d", + "base64": "7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t", + } + + for name, value := range testData { + param := value + + suite.T().Run(name, func(t *testing.T) { + parsed, err := mtglib.ParseSecret(param) + assert.NoError(t, err) + assert.Equal(t, s.Key, parsed.Key) + assert.Equal(t, s.Host, parsed.Host) + + newSecret := mtglib.Secret{} + assert.NoError(t, newSecret.UnmarshalText([]byte(param))) + assert.Equal(t, s.Key, newSecret.Key) + assert.Equal(t, s.Host, newSecret.Host) + }) + } +} + +func (suite *SecretTestSuite) TestSerialize() { + s := mtglib.Secret{} + + data, err := s.MarshalText() + suite.NoError(err) + suite.Empty(data) + + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") + s = mtglib.Secret{ + Host: "google.com", + } + + copy(s.Key[:], secretData) + + suite.Equal("eed11c6cbbd9efe7fed5bc0db220b09665676f6f676c652e636f6d", s.Hex()) + suite.Equal("7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t", s.Base64()) +} + +func (suite *SecretTestSuite) TestMarshalData() { + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") + s := mtglib.Secret{ + Host: "google.com", + } + + copy(s.Key[:], secretData) + + data, err := json.Marshal(&s) + suite.NoError(err) + suite.Equal(string(data), `"7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t"`) +} + +func (suite *SecretTestSuite) TestIncorrectSecret() { + testData := []string{ + "aaa", + "d11c6cbbd9efe7fed5bc0db220b09665", + "ddd11c6cbbd9efe7fed5bc0db220b09665", + "+ueJ0q91t5XOnFYP8Xac3A", + "eed11c6cbbd9efe7fed5bc0db220b09665", + "eed11c6cbbd9efe7fed5bc0db220b096", + "ed11c6cbbd9efe7fed5bc0db220b09665", + "", + "+**", + "ee", + "efd11c6cbbd9efe7fed5bc0db220b09665", + } + + for _, v := range testData { + param := v + + suite.T().Run(param, func(t *testing.T) { + _, err := mtglib.ParseSecret(param) + assert.Error(t, err) + }) + } +} + +func (suite *SecretTestSuite) TestInvariant() { + generated := mtglib.GenerateSecret("google.com") + + parsed, err := mtglib.ParseSecret(generated.Hex()) + suite.NoError(err) + suite.Equal(generated.Key, parsed.Key) + suite.Equal(generated.Host, parsed.Host) + suite.Equal("google.com", parsed.Host) +} + +func (suite *SecretTestSuite) TestValid() { + s := mtglib.Secret{} + suite.False(s.Valid()) + + s.Key[0] = 1 + suite.False(s.Valid()) + + s.Host = "11" + suite.True(s.Valid()) +} + +func TestSecret(t *testing.T) { + t.Parallel() + suite.Run(t, &SecretTestSuite{}) +} diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go new file mode 100644 index 0000000..f704f37 --- /dev/null +++ b/mtglib/stream_context.go @@ -0,0 +1,72 @@ +package mtglib + +import ( + "context" + "crypto/rand" + "encoding/base64" + "net" + "time" +) + +type streamContext struct { + ctx context.Context + ctxCancel context.CancelFunc + clientConn net.Conn + telegramConn net.Conn + streamID string + dc int + logger Logger +} + +func (s *streamContext) Deadline() (time.Time, bool) { + return s.ctx.Deadline() +} + +func (s *streamContext) Done() <-chan struct{} { + return s.ctx.Done() +} + +func (s *streamContext) Err() error { + return s.ctx.Err() +} + +func (s *streamContext) Value(key interface{}) interface{} { + return s.ctx.Value(key) +} + +func (s *streamContext) Close() { + s.ctxCancel() + + if s.clientConn != nil { + s.clientConn.Close() + } + + if s.telegramConn != nil { + s.telegramConn.Close() + } +} + +func (s *streamContext) ClientIP() net.IP { + return s.clientConn.RemoteAddr().(*net.TCPAddr).IP +} + +func newStreamContext(ctx context.Context, logger Logger, clientConn net.Conn) *streamContext { + connIDBytes := make([]byte, ConnectionIDBytesLength) + + if _, err := rand.Read(connIDBytes); err != nil { + panic(err) + } + + ctx, cancel := context.WithCancel(ctx) + streamCtx := &streamContext{ + ctx: ctx, + ctxCancel: cancel, + clientConn: clientConn, + streamID: base64.RawURLEncoding.EncodeToString(connIDBytes), + } + streamCtx.logger = logger. + BindStr("stream-id", streamCtx.streamID). + BindStr("client-ip", streamCtx.ClientIP().String()) + + return streamCtx +} diff --git a/mtglib/stream_context_internal_test.go b/mtglib/stream_context_internal_test.go new file mode 100644 index 0000000..17f90ea --- /dev/null +++ b/mtglib/stream_context_internal_test.go @@ -0,0 +1,95 @@ +package mtglib + +import ( + "context" + "net" + "testing" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/suite" +) + +type StreamContextTestSuite struct { + suite.Suite + + connMock *testlib.NetConnMock + logger NoopLogger + ctx *streamContext + ctxCancel context.CancelFunc +} + +func (suite *StreamContextTestSuite) SetupSuite() { + suite.logger = NoopLogger{} +} + +func (suite *StreamContextTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, "key", "value") // nolint: golint, revive, staticcheck + + suite.ctxCancel = cancel + suite.connMock = &testlib.NetConnMock{} + + addr := &net.TCPAddr{ + IP: net.ParseIP("10.0.0.10"), + Port: 6676, + } + suite.connMock.On("RemoteAddr").Return(addr) + + suite.ctx = newStreamContext(ctx, suite.logger, suite.connMock) +} + +func (suite *StreamContextTestSuite) TearDownTest() { + suite.ctxCancel() + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *StreamContextTestSuite) TestContextInterface() { + _, ok := suite.ctx.Deadline() + suite.False(ok) + + select { + case <-suite.ctx.Done(): + suite.FailNow("unexpectedly done") + default: + } + + suite.NoError(suite.ctx.Err()) + suite.Equal("value", suite.ctx.Value("key")) + + suite.ctxCancel() + + select { + case <-suite.ctx.Done(): + suite.Error(suite.ctx.Err()) + default: + suite.FailNow("unexpectedly not done") + } +} + +func (suite *StreamContextTestSuite) TestClientIP() { + suite.Equal("10.0.0.10", suite.ctx.ClientIP().String()) +} + +func (suite *StreamContextTestSuite) TestClose() { + suite.connMock.On("Close").Once().Return(nil) + + tgConnMock := &testlib.NetConnMock{} + tgConnMock.On("Close").Once().Return(nil) + + suite.ctx.telegramConn = tgConnMock + suite.ctx.Close() + + select { + case <-suite.ctx.Done(): + suite.Error(suite.ctx.Err()) + default: + suite.FailNow("unexpectedly not done") + } + + tgConnMock.AssertExpectations(suite.T()) +} + +func TestStreamContext(t *testing.T) { + t.Parallel() + suite.Run(t, &StreamContextTestSuite{}) +} diff --git a/mtproto/protocol.go b/mtproto/protocol.go deleted file mode 100644 index a880d10..0000000 --- a/mtproto/protocol.go +++ /dev/null @@ -1,102 +0,0 @@ -package mtproto - -import ( - "fmt" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/wrappers/packet" - "github.com/9seconds/mtg/wrappers/stream" -) - -func TelegramProtocol(req *protocol.TelegramRequest) (conntypes.PacketReadWriteCloser, error) { - conn, err := telegram.Middle.Dial(req.ClientProtocol.DC(), - req.ClientProtocol.ConnectionProtocol()) - if err != nil { - return nil, fmt.Errorf("cannot connect to telegram: %w", err) - } - - rpcNonceConn := packet.NewMtprotoFrame(conn, rpc.SeqNoNonce) - - rpcNonceReq, err := doRPCNonceRequest(rpcNonceConn) - if err != nil { - return nil, fmt.Errorf("cannot do nonce request: %w", err) - } - - rpcNonceResp, err := getRPCNonceResponse(rpcNonceConn, rpcNonceReq) - if err != nil { - return nil, fmt.Errorf("cannot get nonce response: %w", err) - } - - secureConn := stream.NewMiddleProxyCipher(conn, rpcNonceReq, rpcNonceResp, telegram.Middle.Secret()) - frameConn := packet.NewMtprotoFrame(secureConn, rpc.SeqNoHandshake) - - if err := doRPCHandshakeRequest(frameConn); err != nil { - return nil, fmt.Errorf("cannot do handshake request: %w", err) - } - - if err := getRPCHandshakeResponse(frameConn); err != nil { - return nil, fmt.Errorf("cannot get handshake response: %w", err) - } - - return frameConn, nil -} - -func doRPCNonceRequest(conn conntypes.BasePacketWriter) (*rpc.NonceRequest, error) { - rpcNonceReq, err := rpc.NewNonceRequest(telegram.Middle.Secret()) - if err != nil { - panic(err) - } - - if err := conn.Write(rpcNonceReq.Bytes()); err != nil { - return nil, err // nolint: wrapcheck - } - - return rpcNonceReq, nil -} - -func getRPCNonceResponse(conn conntypes.BasePacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { - packet, err := conn.Read() - if err != nil { - return nil, fmt.Errorf("cannot read from connection: %w", err) - } - - resp, err := rpc.NewNonceResponse(packet) - if err != nil { - return nil, fmt.Errorf("cannot build rpc nonce response: %w", err) - } - - if err = resp.Valid(req); err != nil { - return nil, fmt.Errorf("invalid nonce response: %w", err) - } - - return resp, nil -} - -func doRPCHandshakeRequest(conn conntypes.BasePacketWriter) error { - if err := conn.Write(rpc.HandshakeRequest); err != nil { - return fmt.Errorf("cannot make a request: %w", err) - } - - return nil -} - -func getRPCHandshakeResponse(conn conntypes.BasePacketReader) error { - packet, err := conn.Read() - if err != nil { - return fmt.Errorf("cannot read a response: %w", err) - } - - resp, err := rpc.NewHandshakeResponse(packet) - if err != nil { - return fmt.Errorf("cannot build a handshake response: %w", err) - } - - if err := resp.Valid(); err != nil { - return fmt.Errorf("invalid handshake response: %w", err) - } - - return nil -} diff --git a/mtproto/rpc/consts.go b/mtproto/rpc/consts.go deleted file mode 100644 index 9cb7ed4..0000000 --- a/mtproto/rpc/consts.go +++ /dev/null @@ -1,28 +0,0 @@ -package rpc - -// SeqNo* is the number of the sequence which have special meaning for -// the Telegram. -const ( - SeqNoNonce = -2 - SeqNoHandshake = -1 -) - -// Different constants for RPC protocol. -var ( - TagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} - TagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} - TagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} - TagHandshake = []byte{0xf5, 0xee, 0x82, 0x76} - TagNonce = []byte{0xaa, 0x87, 0xcb, 0x7a} - TagProxyRequest = []byte{0xee, 0xf1, 0xce, 0x36} - - NonceCryptoAES = []byte{0x01, 0x00, 0x00, 0x00} - - HandshakeFlags = []byte{0x00, 0x00, 0x00, 0x00} - - ProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} - ProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} - - HandshakeSenderPID = []byte("IPIPPRPDTIME") - HandshakePeerPID = []byte("IPIPPRPDTIME") -) diff --git a/mtproto/rpc/handshake_request.go b/mtproto/rpc/handshake_request.go deleted file mode 100644 index e0971cc..0000000 --- a/mtproto/rpc/handshake_request.go +++ /dev/null @@ -1,5 +0,0 @@ -package rpc - -var HandshakeRequest = append(TagHandshake, - append(HandshakeFlags, - append(HandshakeSenderPID, HandshakePeerPID...)...)...) diff --git a/mtproto/rpc/handshake_response.go b/mtproto/rpc/handshake_response.go deleted file mode 100644 index dc25109..0000000 --- a/mtproto/rpc/handshake_response.go +++ /dev/null @@ -1,54 +0,0 @@ -package rpc - -import ( - "bytes" - "errors" - "fmt" -) - -type HandshakeResponse struct { - Type []byte - Flags []byte - SenderPID []byte - PeerPID []byte -} - -// Bytes returns a serialized handshake response. -func (r *HandshakeResponse) Bytes() []byte { - buf := &bytes.Buffer{} - - buf.Write(r.Type) - buf.Write(r.Flags) - buf.Write(r.SenderPID) - buf.Write(r.PeerPID) - - return buf.Bytes() -} - -// Valid checks that handshake response compliments request. -func (r *HandshakeResponse) Valid() error { - if !bytes.Equal(r.Type, TagHandshake) { - return errors.New("unexpected handshake tag") - } - - if !bytes.Equal(r.PeerPID, HandshakeSenderPID) { - return errors.New("incorrect sender PID") - } - - return nil -} - -// NewHandshakeResponse constructs new handshake response from the given -// data. -func NewHandshakeResponse(data []byte) (*HandshakeResponse, error) { - if len(data) != 32 { - return nil, fmt.Errorf("incorrect handshake response length %d", len(data)) - } - - return &HandshakeResponse{ - Type: data[:4], - Flags: data[4:8], - SenderPID: data[8:20], - PeerPID: data[20:], - }, nil -} diff --git a/mtproto/rpc/nonce_request.go b/mtproto/rpc/nonce_request.go deleted file mode 100644 index 8a06c26..0000000 --- a/mtproto/rpc/nonce_request.go +++ /dev/null @@ -1,50 +0,0 @@ -package rpc - -import ( - "bytes" - "crypto/rand" - "encoding/binary" - "fmt" - "time" -) - -type NonceRequest struct { - KeySelector []byte - CryptoTS []byte - Nonce []byte -} - -// Bytes returns serialized nonce request. -func (r *NonceRequest) Bytes() []byte { - buf := &bytes.Buffer{} - - buf.Write(TagNonce) - buf.Write(r.KeySelector) - buf.Write(NonceCryptoAES) - buf.Write(r.CryptoTS) - buf.Write(r.Nonce) - - return buf.Bytes() -} - -// NewNonceRequest builds new none request based on proxy secret. -func NewNonceRequest(proxySecret []byte) (*NonceRequest, error) { - nonce := make([]byte, 16) - keySelector := make([]byte, 4) - cryptoTS := make([]byte, 4) - - if _, err := rand.Read(nonce); err != nil { - return nil, fmt.Errorf("cannot generate nonce: %w", err) - } - - copy(keySelector, proxySecret) - - timestamp := time.Now().Truncate(time.Second).Unix() % 4294967296 // 256 ^ 4 - do not know how to name - binary.LittleEndian.PutUint32(cryptoTS, uint32(timestamp)) - - return &NonceRequest{ - KeySelector: keySelector, - CryptoTS: cryptoTS, - Nonce: nonce, - }, nil -} diff --git a/mtproto/rpc/nonce_response.go b/mtproto/rpc/nonce_response.go deleted file mode 100644 index a6d831e..0000000 --- a/mtproto/rpc/nonce_response.go +++ /dev/null @@ -1,60 +0,0 @@ -package rpc - -import ( - "bytes" - "errors" - "fmt" -) - -type NonceResponse struct { - NonceRequest - - Type []byte - Crypto []byte -} - -// Bytes returns serialized form of the nonce response. -func (r *NonceResponse) Bytes() []byte { - buf := bytes.Buffer{} - - buf.Write(r.Type) - buf.Write(r.KeySelector) - buf.Write(r.Crypto) - buf.Write(r.CryptoTS) - buf.Write(r.Nonce) - - return buf.Bytes() -} - -func (r *NonceResponse) Valid(req *NonceRequest) error { - if !bytes.Equal(r.Type, TagNonce) { - return errors.New("unexpected RPC type") - } - - if !bytes.Equal(r.Crypto, NonceCryptoAES) { - return errors.New("unexpected crypto type") - } - - if !bytes.Equal(r.KeySelector, req.KeySelector) { - return errors.New("unexpected key selector") - } - - return nil -} - -// NewNonceResponse build new nonce response based on the given data. -func NewNonceResponse(data []byte) (*NonceResponse, error) { - if len(data) != 32 { - return nil, fmt.Errorf("unexpected message length %d", len(data)) - } - - return &NonceResponse{ - NonceRequest: NonceRequest{ - KeySelector: data[4:8], - CryptoTS: data[12:16], - Nonce: data[16:], - }, - Type: data[:4], - Crypto: data[8:12], - }, nil -} diff --git a/mtproto/rpc/proxy_flags.go b/mtproto/rpc/proxy_flags.go deleted file mode 100644 index 2179c66..0000000 --- a/mtproto/rpc/proxy_flags.go +++ /dev/null @@ -1,66 +0,0 @@ -package rpc - -import ( - "encoding/binary" - "strings" -) - -type ProxyRequestFlags uint32 - -const ( - ProxyRequestFlagsHasAdTag ProxyRequestFlags = 0x8 - ProxyRequestFlagsEncrypted ProxyRequestFlags = 0x2 - ProxyRequestFlagsMagic ProxyRequestFlags = 0x1000 - ProxyRequestFlagsExtMode2 ProxyRequestFlags = 0x20000 - ProxyRequestFlagsIntermediate ProxyRequestFlags = 0x20000000 - ProxyRequestFlagsAbdridged ProxyRequestFlags = 0x40000000 - ProxyRequestFlagsQuickAck ProxyRequestFlags = 0x80000000 - ProxyRequestFlagsPad ProxyRequestFlags = 0x8000000 -) - -var ProxyRequestFlagsEncryptedPrefix [8]byte - -func (r ProxyRequestFlags) Bytes() []byte { - converted := make([]byte, 4) - binary.LittleEndian.PutUint32(converted, uint32(r)) - - return converted -} - -func (r ProxyRequestFlags) String() string { - flags := make([]string, 0, 7) - - if r&ProxyRequestFlagsHasAdTag != 0 { - flags = append(flags, "HAS_AD_TAG") - } - - if r&ProxyRequestFlagsEncrypted != 0 { - flags = append(flags, "ENCRYPTED") - } - - if r&ProxyRequestFlagsMagic != 0 { - flags = append(flags, "MAGIC") - } - - if r&ProxyRequestFlagsExtMode2 != 0 { - flags = append(flags, "EXT_MODE_2") - } - - if r&ProxyRequestFlagsIntermediate != 0 { - flags = append(flags, "INTERMEDIATE") - } - - if r&ProxyRequestFlagsAbdridged != 0 { - flags = append(flags, "ABRIDGED") - } - - if r&ProxyRequestFlagsQuickAck != 0 { - flags = append(flags, "QUICK_ACK") - } - - if r&ProxyRequestFlagsPad != 0 { - flags = append(flags, "PAD") - } - - return strings.Join(flags, " | ") -} diff --git a/mtproto/rpc/proxy_response.go b/mtproto/rpc/proxy_response.go deleted file mode 100644 index 5366f34..0000000 --- a/mtproto/rpc/proxy_response.go +++ /dev/null @@ -1,53 +0,0 @@ -package rpc - -import ( - "bytes" - "fmt" - - "github.com/9seconds/mtg/conntypes" -) - -type ProxyResponseType uint8 - -const ( - ProxyResponseTypeAns ProxyResponseType = iota - ProxyResponseTypeSimpleAck - ProxyResponseTypeCloseExt -) - -type ProxyResponse struct { - Type ProxyResponseType - ConnID conntypes.ConnID - Payload conntypes.Packet -} - -func ParseProxyResponse(packet conntypes.Packet) (*ProxyResponse, error) { - var response ProxyResponse - - if len(packet) < 4 { - return nil, fmt.Errorf("incorrect packet length: %d", len(packet)) - } - - tag := packet[:4] - - switch { - case bytes.Equal(tag, TagProxyAns): - response.Type = ProxyResponseTypeAns - copy(response.ConnID[:], packet[8:16]) - response.Payload = packet[16:] - - return &response, nil - case bytes.Equal(tag, TagSimpleAck): - response.Type = ProxyResponseTypeSimpleAck - copy(response.ConnID[:], packet[4:12]) - response.Payload = packet[12:] - - return &response, nil - case bytes.Equal(tag, TagCloseExt): - response.Type = ProxyResponseTypeCloseExt - - return &response, nil - } - - return nil, fmt.Errorf("unknown response type %x", tag) -} diff --git a/network/circuit_breaker.go b/network/circuit_breaker.go new file mode 100644 index 0000000..4a02bda --- /dev/null +++ b/network/circuit_breaker.go @@ -0,0 +1,196 @@ +package network + +import ( + "context" + "net" + "sync/atomic" + "time" +) + +const ( + circuitBreakerStateClosed uint32 = iota + circuitBreakerStateHalfOpened + circuitBreakerStateOpened +) + +type circuitBreakerDialer struct { + Dialer + + stateMutexChan chan bool + + halfOpenTimer *time.Timer + failuresCleanupTimer *time.Timer + + state uint32 + halfOpenAttempts uint32 + failuresCount uint32 + + openThreshold uint32 + halfOpenTimeout time.Duration + resetFailuresTimeout time.Duration +} + +func (c *circuitBreakerDialer) Dial(network, address string) (net.Conn, error) { + return c.DialContext(context.Background(), network, address) +} + +func (c *circuitBreakerDialer) DialContext(ctx context.Context, + network, address string) (net.Conn, error) { + switch atomic.LoadUint32(&c.state) { + case circuitBreakerStateClosed: + return c.doClosed(ctx, network, address) + case circuitBreakerStateHalfOpened: + return c.doHalfOpened(ctx, network, address) + default: + return nil, ErrCircuitBreakerOpened + } +} + +func (c *circuitBreakerDialer) doClosed(ctx context.Context, + network, address string) (net.Conn, error) { + conn, err := c.Dialer.DialContext(ctx, network, address) + + select { + case <-ctx.Done(): + if conn != nil { + conn.Close() + } + + return nil, ctx.Err() + case c.stateMutexChan <- true: + defer func() { + <-c.stateMutexChan + }() + } + + if err == nil { + c.switchState(circuitBreakerStateClosed) + + return conn, err // nolint: wrapcheck + } + + c.failuresCount++ + + if c.state == circuitBreakerStateClosed && c.failuresCount >= c.openThreshold { + c.switchState(circuitBreakerStateOpened) + } + + return conn, err // nolint: wrapcheck +} + +func (c *circuitBreakerDialer) doHalfOpened(ctx context.Context, network, address string) (net.Conn, error) { + if !atomic.CompareAndSwapUint32(&c.halfOpenAttempts, 0, 1) { + return nil, ErrCircuitBreakerOpened + } + + conn, err := c.Dialer.DialContext(ctx, network, address) + + select { + case <-ctx.Done(): + if conn != nil { + conn.Close() + } + + return nil, ctx.Err() + case c.stateMutexChan <- true: + defer func() { + <-c.stateMutexChan + }() + } + + if c.state != circuitBreakerStateHalfOpened { + return conn, err // nolint: wrapcheck + } + + if err == nil { + c.switchState(circuitBreakerStateClosed) + } else { + c.switchState(circuitBreakerStateOpened) + } + + return conn, err // nolint: wrapcheck +} + +func (c *circuitBreakerDialer) switchState(state uint32) { + switch state { + case circuitBreakerStateClosed: + c.stopTimer(&c.halfOpenTimer) + c.ensureTimer(&c.failuresCleanupTimer, c.resetFailuresTimeout, c.resetFailures) + case circuitBreakerStateHalfOpened: + c.stopTimer(&c.failuresCleanupTimer) + c.stopTimer(&c.halfOpenTimer) + case circuitBreakerStateOpened: + c.stopTimer(&c.failuresCleanupTimer) + c.ensureTimer(&c.halfOpenTimer, c.halfOpenTimeout, c.tryHalfOpen) + } + + c.failuresCount = 0 + atomic.StoreUint32(&c.halfOpenAttempts, 0) + atomic.StoreUint32(&c.state, state) +} + +func (c *circuitBreakerDialer) resetFailures() { + c.stateMutexChan <- true + + defer func() { + <-c.stateMutexChan + }() + + c.stopTimer(&c.failuresCleanupTimer) + + if c.state == circuitBreakerStateClosed { + c.switchState(circuitBreakerStateClosed) + } +} + +func (c *circuitBreakerDialer) tryHalfOpen() { + c.stateMutexChan <- true + + defer func() { + <-c.stateMutexChan + }() + + if c.state == circuitBreakerStateOpened { + c.switchState(circuitBreakerStateHalfOpened) + } +} + +func (c *circuitBreakerDialer) stopTimer(timerRef **time.Timer) { + timer := *timerRef + if timer == nil { + return + } + + timer.Stop() + + select { + case <-timer.C: + default: + } + + *timerRef = nil +} + +func (c *circuitBreakerDialer) ensureTimer(timerRef **time.Timer, + timeout time.Duration, callback func()) { + if *timerRef == nil { + *timerRef = time.AfterFunc(timeout, callback) + } +} + +func newCircuitBreakerDialer(baseDialer Dialer, + openThreshold uint32, halfOpenTimeout, resetFailuresTimeout time.Duration) Dialer { + cb := &circuitBreakerDialer{ + Dialer: baseDialer, + stateMutexChan: make(chan bool, 1), + openThreshold: openThreshold, + halfOpenTimeout: halfOpenTimeout, + resetFailuresTimeout: resetFailuresTimeout, + } + + cb.stateMutexChan <- true // to convince race detector we are good + cb.switchState(circuitBreakerStateClosed) + <-cb.stateMutexChan + + return cb +} diff --git a/network/circuit_breaker_internal_test.go b/network/circuit_breaker_internal_test.go new file mode 100644 index 0000000..a26f594 --- /dev/null +++ b/network/circuit_breaker_internal_test.go @@ -0,0 +1,139 @@ +package network + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/9seconds/mtg/v2/internal/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type CircuitBreakerTestSuite struct { + suite.Suite + + d Dialer + mutex sync.Mutex + ctx context.Context + ctxCancel context.CancelFunc + connMock *testlib.NetConnMock + baseDialerMock *DialerMock +} + +func (suite *CircuitBreakerTestSuite) SetupTest() { + suite.mutex = sync.Mutex{} + suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) + suite.baseDialerMock = &DialerMock{} + suite.connMock = &testlib.NetConnMock{} + suite.d = newCircuitBreakerDialer(suite.baseDialerMock, + 3, 100*time.Millisecond, 50*time.Millisecond) +} + +func (suite *CircuitBreakerTestSuite) TearDownTest() { + suite.ctxCancel() + suite.baseDialerMock.AssertExpectations(suite.T()) + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *CircuitBreakerTestSuite) TestMultipleRunsOk() { + suite.connMock.On("RemoteAddr"). + Times(5). + Return(&net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 3128, + }) + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.1"). + Times(5). + Return(suite.connMock, nil) + + wg := &sync.WaitGroup{} + wg.Add(5) + + go func() { + wg.Wait() + suite.ctxCancel() + }() + + for i := 0; i < 5; i++ { + go func() { + defer wg.Done() + + conn, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + + suite.mutex.Lock() + defer suite.mutex.Unlock() + + suite.NoError(err) + suite.Equal("127.0.0.1:3128", conn.RemoteAddr().String()) + }() + } + + suite.Eventually(func() bool { + _, ok := <-suite.ctx.Done() + + return !ok + }, time.Second, 10*time.Millisecond) +} + +func (suite *CircuitBreakerTestSuite) TestFromClosedToOpen() { + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.1"). + Times(3). + Return(&net.TCPConn{}, io.EOF) + + _, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, ErrCircuitBreakerOpened)) +} + +func (suite *CircuitBreakerTestSuite) TestHalfOpen() { + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.1"). + Times(4). + Return(&net.TCPConn{}, io.EOF) + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.2"). + Twice(). + Return(suite.connMock, nil) + suite.connMock.On("RemoteAddr").Return(&net.TCPAddr{ + IP: net.ParseIP("10.0.0.10"), + Port: 80, + }) + + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + + time.Sleep(500 * time.Millisecond) + + _, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, ErrCircuitBreakerOpened)) + + time.Sleep(500 * time.Millisecond) + + conn, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.2") + suite.NoError(err) + suite.Equal("10.0.0.10:80", conn.RemoteAddr().String()) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.2") + suite.NoError(err) +} + +func TestCircuitBreaker(t *testing.T) { + t.Parallel() + suite.Run(t, &CircuitBreakerTestSuite{}) +} diff --git a/network/default.go b/network/default.go new file mode 100644 index 0000000..cad8799 --- /dev/null +++ b/network/default.go @@ -0,0 +1,91 @@ +package network + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/libp2p/go-reuseport" +) + +type defaultDialer struct { + net.Dialer + + bufferSize int +} + +func (d *defaultDialer) Dial(network, address string) (net.Conn, error) { + return d.DialContext(context.Background(), network, address) +} + +func (d *defaultDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + switch network { + case "tcp", "tcp4", "tcp6": // nolint: goconst + default: + return nil, fmt.Errorf("unsupported network %s", network) + } + + conn, err := d.Dialer.DialContext(ctx, network, address) + if err != nil { + return nil, fmt.Errorf("cannot dial to %s: %w", address, err) + } + + tcpConn := conn.(*net.TCPConn) + + if err := tcpConn.SetNoDelay(true); err != nil { + conn.Close() + + return nil, fmt.Errorf("cannot set TCP_NO_DELAY: %w", err) + } + + if err := tcpConn.SetReadBuffer(d.bufferSize); err != nil { + tcpConn.Close() + + return nil, fmt.Errorf("cannot set read buffer size: %w", err) + } + + if err := tcpConn.SetWriteBuffer(d.bufferSize); err != nil { + tcpConn.Close() + + return nil, fmt.Errorf("cannot set write buffer size: %w", err) + } + + if err := tcpConn.SetKeepAlive(true); err != nil { + tcpConn.Close() + + return nil, fmt.Errorf("cannot enable keep-alive: %w", err) + } + + return tcpConn, nil +} + +// NewDefaultDialer build a new dialer which dials bypassing proxies +// etc. +// +// The most default one you can imagine. But it has tunes TCP +// connections and setups SO_REUSEPORT. +func NewDefaultDialer(timeout time.Duration, bufferSize int) (Dialer, error) { + switch { + case timeout < 0: + return nil, fmt.Errorf("timeout %v should be positive number", timeout) + case bufferSize < 0: + return nil, fmt.Errorf("buffer size %d should be positive number", bufferSize) + } + + if timeout == 0 { + timeout = DefaultTimeout + } + + if bufferSize == 0 { + bufferSize = DefaultBufferSize + } + + return &defaultDialer{ + Dialer: net.Dialer{ + Timeout: timeout, + Control: reuseport.Control, + }, + bufferSize: bufferSize, + }, nil +} diff --git a/network/default_test.go b/network/default_test.go new file mode 100644 index 0000000..a6b24c7 --- /dev/null +++ b/network/default_test.go @@ -0,0 +1,77 @@ +package network_test + +import ( + "context" + "net/http" + "testing" + + "github.com/9seconds/mtg/v2/network" + "github.com/stretchr/testify/suite" +) + +type DefaultDialerTestSuite struct { + suite.Suite + HTTPServerTestSuite + + d network.Dialer +} + +func (suite *DefaultDialerTestSuite) SetupSuite() { + suite.HTTPServerTestSuite.SetupSuite() + + d, err := network.NewDefaultDialer(0, 0) + suite.NoError(err) + + suite.d = d +} + +func (suite *DefaultDialerTestSuite) TestNegativeTimeout() { + _, err := network.NewDefaultDialer(-1, 0) + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestNegativeBufferSize() { + _, err := network.NewDefaultDialer(0, -1) + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestUnsupportedProtocol() { + _, err := suite.d.DialContext(context.Background(), + "udp", + suite.HTTPServerAddress()) + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestCannotDial() { + _, err := suite.d.DialContext(context.Background(), + "tcp", + suite.HTTPServerAddress()+suite.HTTPServerAddress()) + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestConnectOk() { + conn, err := suite.d.DialContext(context.Background(), + "tcp", + suite.HTTPServerAddress()) + suite.NoError(err) + suite.NotNil(conn) + + conn.Close() +} + +func (suite *DefaultDialerTestSuite) TestHTTPRequest() { + httpClient := suite.MakeHTTPClient(suite.d) + + resp, err := httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } + + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) +} + +func TestDefaultDialer(t *testing.T) { + t.Parallel() + suite.Run(t, &DefaultDialerTestSuite{}) +} diff --git a/network/dns_resolver.go b/network/dns_resolver.go new file mode 100644 index 0000000..23a4c37 --- /dev/null +++ b/network/dns_resolver.go @@ -0,0 +1,95 @@ +package network + +import ( + "net/http" + "sync" + "time" + + doh "github.com/babolivier/go-doh-client" +) + +const dnsResolverKeepTime = 10 * time.Minute + +type dnsResolverCacheEntry struct { + ips []string + createdAt time.Time +} + +func (c dnsResolverCacheEntry) Ok() bool { + return time.Since(c.createdAt) < dnsResolverKeepTime +} + +type dnsResolver struct { + resolver doh.Resolver + cache map[string]dnsResolverCacheEntry + cacheMutex sync.RWMutex +} + +func (d *dnsResolver) LookupA(hostname string) []string { + key := "\x00" + hostname + + d.cacheMutex.RLock() + entry, ok := d.cache[key] + d.cacheMutex.RUnlock() + + if ok && entry.Ok() { + return entry.ips + } + + var ips []string + + if recs, _, err := d.resolver.LookupA(hostname); err == nil { + for _, v := range recs { + ips = append(ips, v.IP4) + } + + d.cacheMutex.Lock() + d.cache[key] = dnsResolverCacheEntry{ + ips: ips, + createdAt: time.Now(), + } + d.cacheMutex.Unlock() + } + + return ips +} + +func (d *dnsResolver) LookupAAAA(hostname string) []string { + key := "\x01" + hostname + + d.cacheMutex.RLock() + entry, ok := d.cache[key] + d.cacheMutex.RUnlock() + + if ok && entry.Ok() { + return entry.ips + } + + var ips []string + + if recs, _, err := d.resolver.LookupAAAA(hostname); err == nil { + for _, v := range recs { + ips = append(ips, v.IP6) + } + + d.cacheMutex.Lock() + d.cache[key] = dnsResolverCacheEntry{ + ips: ips, + createdAt: time.Now(), + } + d.cacheMutex.Unlock() + } + + return ips +} + +func newDNSResolver(hostname string, httpClient *http.Client) *dnsResolver { + return &dnsResolver{ + resolver: doh.Resolver{ + Host: hostname, + Class: doh.IN, + HTTPClient: httpClient, + }, + cache: map[string]dnsResolverCacheEntry{}, + } +} diff --git a/network/dns_resolver_internal_test.go b/network/dns_resolver_internal_test.go new file mode 100644 index 0000000..ba54c0c --- /dev/null +++ b/network/dns_resolver_internal_test.go @@ -0,0 +1,50 @@ +package network + +import ( + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type DNSResolverTestSuite struct { + suite.Suite + + d *dnsResolver +} + +func (suite *DNSResolverTestSuite) TestLookupA() { + suite.d.LookupA("google.com") + time.Sleep(10 * time.Millisecond) + + addrs := suite.d.LookupA("google.com") + + for _, v := range addrs { + suite.NotEmpty(v) + suite.NotNil(net.ParseIP(v).To4()) + } +} + +func (suite *DNSResolverTestSuite) TestLookupAAAA() { + suite.d.LookupAAAA("google.com") + time.Sleep(10 * time.Millisecond) + + addrs := suite.d.LookupAAAA("google.com") + + for _, v := range addrs { + suite.NotEmpty(v) + suite.Nil(net.ParseIP(v).To4()) + suite.NotNil(net.ParseIP(v).To16()) + } +} + +func (suite *DNSResolverTestSuite) SetupTest() { + suite.d = newDNSResolver("1.1.1.1", &http.Client{}) +} + +func TestDNSResolver(t *testing.T) { + t.Parallel() + suite.Run(t, &DNSResolverTestSuite{}) +} diff --git a/network/init.go b/network/init.go new file mode 100644 index 0000000..0dc9660 --- /dev/null +++ b/network/init.go @@ -0,0 +1,84 @@ +// Network contains a default implementation of the network. +// +// Please see mtglib.Network interface to get some basic idea behind +// this abstraction. +// +// Some notable feature of this implementation: +// +// 1. It detaches dialer from a network. Dialer is something which +// implements a real dialer and network completes it with more higher +// level details. +// +// 2. It uses only TCP connections. Even for DNS it uses DNS-Over-HTTPS +// +// 3. It has some simple implementation of DNS cache which is good +// enough for our purpose. +// +// 4. It sets uses SO_REUSEPORT port if applicable. +package network + +import ( + "context" + "errors" + "net" + "time" +) + +const ( + // DefaultTimeout is a default timeout for establishing TCP + // connection. + DefaultTimeout = 10 * time.Second + + // DefaultHTTPTimeout defines a default timeout for making HTTP + // request. + DefaultHTTPTimeout = 10 * time.Second + + // DefaultBufferSize defines a TCP buffer size. Both read and write, so + // for real size, please multiply this number by 2. + DefaultBufferSize = 16 * 1024 // 16 kib + + // ProxyDialerOpenThreshold is used for load balancing SOCKS5 dialer + // only. + // + // This dialer uses circuit breaker with of 3 stages: OPEN, + // HALF_OPEN and CLOSED. If state is CLOSED, all requests go in + // a normal mode. If you get more that ProxyDialerOpenThreshold + // errors, circuit breaker goes into OPEN mode. + // + // When circuit breaker is in OPEN mode, it forbids all request to + // a given proxy. But after ProxyDialerHalfOpenTimeout it gives a + // second chance and opens an access for a SINGLE request. If this + // request success, then circuit breaker closes, otherwise opens + // again. + // + // When circuit breaker is closed, it clears an error states each + // ProxyDialerResetFailuresTimeout. + ProxyDialerOpenThreshold = 5 + ProxyDialerHalfOpenTimeout = time.Minute + ProxyDialerResetFailuresTimeout = 10 * time.Second + + // DefaultDOHHostname defines a default IP address for DOH host. + // Since mtg is simple, please pass IP address here. We do not + // have bootstrap servers here embedded. + DefaultDOHHostname = "9.9.9.9" + + // DNSTimeout defines a timeout for DNS queries. + DNSTimeout = 5 * time.Second +) + +var ( + // ErrCircuitBreakerOpened is returned when proxy is being accessed + // but circuit breaker is opened. + ErrCircuitBreakerOpened = errors.New("circuit breaker is opened") + + // ErrCannotDialWithAllProxies is returned when load balancing + // client is trying to access proxies but all of them are failed. + ErrCannotDialWithAllProxies = errors.New("cannot dial with all proxies") +) + +// Dialer defines an interface which is required to bootstrap a network +// instance from. +type Dialer interface { + Dial(network, address string) (net.Conn, error) + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} diff --git a/network/init_internal_test.go b/network/init_internal_test.go new file mode 100644 index 0000000..9335818 --- /dev/null +++ b/network/init_internal_test.go @@ -0,0 +1,24 @@ +package network + +import ( + "context" + "net" + + "github.com/stretchr/testify/mock" +) + +type DialerMock struct { + mock.Mock +} + +func (d *DialerMock) Dial(network, address string) (net.Conn, error) { + args := d.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := d.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} diff --git a/network/init_test.go b/network/init_test.go new file mode 100644 index 0000000..f42d1c3 --- /dev/null +++ b/network/init_test.go @@ -0,0 +1,87 @@ +package network_test + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + + "github.com/9seconds/mtg/v2/network" + socks5 "github.com/armon/go-socks5" + "github.com/mccutchen/go-httpbin/httpbin" + "github.com/stretchr/testify/mock" +) + +type DialerMock struct { + mock.Mock +} + +func (d *DialerMock) Dial(network, address string) (net.Conn, error) { + args := d.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := d.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +type HTTPServerTestSuite struct { + httpServer *httptest.Server +} + +func (suite *HTTPServerTestSuite) SetupSuite() { + suite.httpServer = httptest.NewServer(httpbin.NewHTTPBin().Handler()) +} + +func (suite *HTTPServerTestSuite) TearDownSuite() { + suite.httpServer.Close() +} + +func (suite *HTTPServerTestSuite) HTTPServerAddress() string { + return strings.TrimPrefix(suite.httpServer.URL, "http://") +} + +func (suite *HTTPServerTestSuite) MakeURL(path string) string { + return suite.httpServer.URL + path +} + +func (suite *HTTPServerTestSuite) MakeHTTPClient(dialer network.Dialer) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DialContext: dialer.DialContext, + }, + } +} + +type Socks5ServerTestSuite struct { + socks5Listener net.Listener + socks5Server *socks5.Server +} + +func (suite *Socks5ServerTestSuite) SetupSuite() { + suite.socks5Listener, _ = net.Listen("tcp", "127.0.0.1:0") + suite.socks5Server, _ = socks5.New(&socks5.Config{ + Credentials: socks5.StaticCredentials{ + "user": "password", + }, + }) + + go suite.socks5Server.Serve(suite.socks5Listener) // nolint: errcheck +} + +func (suite *Socks5ServerTestSuite) TearDownSuite() { + suite.socks5Listener.Close() +} + +func (suite *Socks5ServerTestSuite) MakeSocks5URL(user, password string) *url.URL { + return &url.URL{ + Scheme: "socks5", + User: url.UserPassword(user, password), + Host: suite.socks5Listener.Addr().String(), + } +} diff --git a/network/load_balanced_socks5.go b/network/load_balanced_socks5.go new file mode 100644 index 0000000..a52004d --- /dev/null +++ b/network/load_balanced_socks5.go @@ -0,0 +1,60 @@ +package network + +import ( + "context" + "fmt" + "math/rand" + "net" + "net/url" +) + +type loadBalancedSocks5Dialer struct { + dialers []Dialer +} + +func (l loadBalancedSocks5Dialer) Dial(network, address string) (net.Conn, error) { + return l.DialContext(context.Background(), network, address) +} + +func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + length := len(l.dialers) + start := rand.Intn(length) + moved := false + + for i := start; i != start || !moved; i = (i + 1) % length { + moved = true + + if conn, err := l.dialers[i].DialContext(ctx, network, address); err == nil { + return conn, nil + } + } + + return nil, ErrCannotDialWithAllProxies +} + +// NewLoadBalancedSocks5Dialer builds a new load balancing SOCKS5 +// dialer. +// +// The main difference from one which is made by NewSocks5Dialer is that +// we actually have a list of these proxies. When dial is requested, +// any proxy is picked and used. If proxy fails for some reason, we try +// another one. +// +// So, it is mostly useful if you have some routes with proxies which +// are not always online or having buggy network. +func NewLoadBalancedSocks5Dialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { + dialers := make([]Dialer, 0, len(proxyURLs)) + + for _, u := range proxyURLs { + dialer, err := NewSocks5Dialer(newProxyDialer(baseDialer, u), u) + if err != nil { + return nil, fmt.Errorf("cannot build dialer for %s: %w", u.String(), err) + } + + dialers = append(dialers, dialer) + } + + return loadBalancedSocks5Dialer{ + dialers: dialers, + }, nil +} diff --git a/network/load_balanced_socks5_test.go b/network/load_balanced_socks5_test.go new file mode 100644 index 0000000..b6983c8 --- /dev/null +++ b/network/load_balanced_socks5_test.go @@ -0,0 +1,88 @@ +package network_test + +import ( + "errors" + "io" + "net" + "net/http" + "net/url" + "testing" + + "github.com/9seconds/mtg/v2/network" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type LoadBalancedSocks5TestSuite struct { + suite.Suite + HTTPServerTestSuite + Socks5ServerTestSuite + + httpClient *http.Client +} + +func (suite *LoadBalancedSocks5TestSuite) SetupSuite() { + suite.HTTPServerTestSuite.SetupSuite() + suite.Socks5ServerTestSuite.SetupSuite() +} + +func (suite *LoadBalancedSocks5TestSuite) SetupTest() { + baseDialer, _ := network.NewDefaultDialer(0, 0) + lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ + suite.MakeSocks5URL("user", "password"), + suite.MakeSocks5URL("user2", "password"), + }) + suite.NoError(err) + + suite.httpClient = suite.MakeHTTPClient(lbDialer) +} + +func (suite *LoadBalancedSocks5TestSuite) TearDownSuite() { + suite.Socks5ServerTestSuite.SetupSuite() + suite.HTTPServerTestSuite.SetupSuite() +} + +func (suite *LoadBalancedSocks5TestSuite) TestIncorrectURL() { + _, err := network.NewLoadBalancedSocks5Dialer(&DialerMock{}, []*url.URL{ + {Scheme: "http"}, + }) + suite.Error(err) +} + +func (suite *LoadBalancedSocks5TestSuite) TestCannotDial() { + baseDialer := &DialerMock{} + baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.1:1080"). + Times(network.ProxyDialerOpenThreshold). + Return(&net.TCPConn{}, io.EOF) + baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.2:1080"). + Times(network.ProxyDialerOpenThreshold). + Return(&net.TCPConn{}, io.EOF) + + lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ + {Scheme: "socks5", User: url.UserPassword("user", "password"), Host: "127.0.0.1:1080"}, + {Scheme: "socks5", User: url.UserPassword("user", "password"), Host: "127.0.0.2:1080"}, + }) + suite.NoError(err) + + for i := 0; i < network.ProxyDialerOpenThreshold*2; i++ { + _, err = lbDialer.Dial("tcp", "127.1.1.1:80") + suite.True(errors.Is(err, network.ErrCannotDialWithAllProxies)) + } + + baseDialer.AssertExpectations(suite.T()) +} + +func (suite *LoadBalancedSocks5TestSuite) TestDialOk() { + resp, err := suite.httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } + + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) +} + +func TestLoadBalancedSocks5(t *testing.T) { + t.Parallel() + suite.Run(t, &LoadBalancedSocks5TestSuite{}) +} diff --git a/network/network.go b/network/network.go new file mode 100644 index 0000000..3ff9af6 --- /dev/null +++ b/network/network.go @@ -0,0 +1,157 @@ +package network + +import ( + "context" + "fmt" + "math/rand" + "net" + "net/http" + "sync" + "time" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type networkHTTPTransport struct { + userAgent string + next http.RoundTripper +} + +func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", n.userAgent) + + return n.next.RoundTrip(req) +} + +type network struct { + dialer Dialer + httpTimeout time.Duration + userAgent string + dns *dnsResolver +} + +func (n *network) Dial(protocol, address string) (net.Conn, error) { + return n.DialContext(context.Background(), protocol, address) +} + +func (n *network) DialContext(ctx context.Context, protocol, address string) (net.Conn, error) { + host, port, _ := net.SplitHostPort(address) + + ips, err := n.dnsResolve(protocol, host) + if err != nil { + return nil, fmt.Errorf("cannot resolve dns names: %w", err) + } + + rand.Shuffle(len(ips), func(i, j int) { + ips[i], ips[j] = ips[j], ips[i] + }) + + var conn net.Conn + for _, v := range ips { + conn, err = n.dialer.DialContext(ctx, protocol, net.JoinHostPort(v, port)) + + if err == nil { + return conn, nil + } + } + + return nil, fmt.Errorf("cannot dial to %s:%s: %w", protocol, address, err) +} + +func (n *network) MakeHTTPClient(dialFunc func(ctx context.Context, + network, address string) (net.Conn, error)) *http.Client { + if dialFunc == nil { + dialFunc = n.DialContext + } + + return makeHTTPClient(n.userAgent, n.httpTimeout, dialFunc) +} + +func (n *network) dnsResolve(protocol, address string) ([]string, error) { + if net.ParseIP(address) != nil { + return []string{address}, nil + } + + ips := []string{} + wg := &sync.WaitGroup{} + mutex := &sync.Mutex{} + + switch protocol { + case "tcp", "tcp4": + wg.Add(1) + + go func() { + defer wg.Done() + + resolved := n.dns.LookupA(address) + + mutex.Lock() + ips = append(ips, resolved...) + mutex.Unlock() + }() + } + + switch protocol { + case "tcp", "tcp6": + wg.Add(1) + + go func() { + defer wg.Done() + + resolved := n.dns.LookupAAAA(address) + + mutex.Lock() + ips = append(ips, resolved...) + mutex.Unlock() + }() + } + + wg.Wait() + + if len(ips) == 0 { + return nil, fmt.Errorf("cannot find any ips for %s:%s", protocol, address) + } + + return ips, nil +} + +// NewNetwork assembles an mtglib.Network compatible structure +// based on a dialer and given params. +// +// It brings simple DNS cache and DNS-Over-HTTPS when necessary. +func NewNetwork(dialer Dialer, + userAgent, dohHostname string, + httpTimeout time.Duration) (mtglib.Network, error) { + switch { + case httpTimeout < 0: + return nil, fmt.Errorf("timeout should be positive number %s", httpTimeout) + case httpTimeout == 0: + httpTimeout = DefaultHTTPTimeout + } + + if net.ParseIP(dohHostname) == nil { + return nil, fmt.Errorf("hostname %s should be IP address", dohHostname) + } + + return &network{ + dialer: dialer, + httpTimeout: httpTimeout, + userAgent: userAgent, + dns: newDNSResolver(dohHostname, + makeHTTPClient(userAgent, DNSTimeout, dialer.DialContext)), + }, nil +} + +func makeHTTPClient(userAgent string, + timeout time.Duration, + dialFunc func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: networkHTTPTransport{ + userAgent: userAgent, + next: &http.Transport{ + DialContext: dialFunc, + }, + }, + } +} diff --git a/network/network_test.go b/network/network_test.go new file mode 100644 index 0000000..86b5f0c --- /dev/null +++ b/network/network_test.go @@ -0,0 +1,91 @@ +package network_test + +import ( + "encoding/json" + "io" + "net/http" + "testing" + "time" + + "github.com/9seconds/mtg/v2/network" + "github.com/stretchr/testify/suite" +) + +type NetworkTestSuite struct { + suite.Suite + HTTPServerTestSuite + + dialer network.Dialer +} + +func (suite *NetworkTestSuite) SetupTest() { + dialer, err := network.NewDefaultDialer(0, 0) + suite.NoError(err) + + suite.dialer = dialer +} + +func (suite *NetworkTestSuite) TestLocalHTTPRequest() { + ntw, err := network.NewNetwork(suite.dialer, "itsme", "1.1.1.1", 0) + suite.NoError(err) + + client := ntw.MakeHTTPClient(nil) + + resp, err := client.Get(suite.httpServer.URL + "/headers") // nolint: noctx + suite.NoError(err) + + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) + + jsonStruct := struct { + Headers struct { + UserAgent []string `json:"User-Agent"` + } `json:"headers"` + }{} + + suite.NoError(json.Unmarshal(data, &jsonStruct)) + suite.Equal([]string{"itsme"}, jsonStruct.Headers.UserAgent) +} + +func (suite *NetworkTestSuite) TestRealHTTPRequest() { + ntw, err := network.NewNetwork(suite.dialer, "itsme", "1.1.1.1", 0) + suite.NoError(err) + + client := ntw.MakeHTTPClient(nil) + + resp, err := client.Get("https://httpbin.org/headers") // nolint: noctx + suite.NoError(err) + + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) + + jsonStruct := struct { + Headers struct { + UserAgent string `json:"User-Agent"` + } `json:"headers"` + }{} + + suite.NoError(json.Unmarshal(data, &jsonStruct)) + suite.Equal("itsme", jsonStruct.Headers.UserAgent) +} + +func (suite *NetworkTestSuite) TestIncorrectTimeout() { + _, err := network.NewNetwork(suite.dialer, "itsme", "1.1.1.1", -time.Second) + suite.Error(err) +} + +func (suite *NetworkTestSuite) TestIncorrectDOHHostname() { + _, err := network.NewNetwork(suite.dialer, "itsme", "doh.com", 0) + suite.Error(err) +} + +func TestNetwork(t *testing.T) { + t.Parallel() + suite.Run(t, &NetworkTestSuite{}) +} diff --git a/network/proxy_dialer.go b/network/proxy_dialer.go new file mode 100644 index 0000000..9c3f0f7 --- /dev/null +++ b/network/proxy_dialer.go @@ -0,0 +1,37 @@ +package network + +import ( + "net/url" + "strconv" + "time" +) + +func newProxyDialer(baseDialer Dialer, proxyURL *url.URL) Dialer { + params := proxyURL.Query() + + var ( + openThreshold uint32 = ProxyDialerOpenThreshold + halfOpenTimeout = ProxyDialerHalfOpenTimeout + resetFailuresTimeout = ProxyDialerResetFailuresTimeout + ) + + if param := params.Get("open_threshold"); param != "" { + if intNum, err := strconv.ParseUint(param, 10, 32); err == nil { + openThreshold = uint32(intNum) + } + } + + if param := params.Get("half_open_timeout"); param != "" { + if dur, err := time.ParseDuration(param); err == nil && dur > 0 { + halfOpenTimeout = dur + } + } + + if param := params.Get("reset_failures_timeout"); param != "" { + if dur, err := time.ParseDuration(param); err == nil && dur > 0 { + resetFailuresTimeout = dur + } + } + + return newCircuitBreakerDialer(baseDialer, openThreshold, halfOpenTimeout, resetFailuresTimeout) +} diff --git a/network/proxy_dialer_internal_test.go b/network/proxy_dialer_internal_test.go new file mode 100644 index 0000000..33aa2b7 --- /dev/null +++ b/network/proxy_dialer_internal_test.go @@ -0,0 +1,94 @@ +package network + +import ( + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type ProxyDialerTestSuite struct { + suite.Suite + + u *url.URL +} + +func (suite *ProxyDialerTestSuite) SetupSuite() { + u, _ := url.Parse("socks5://hello:world@10.0.0.10:3128") + suite.u = u +} + +func (suite *ProxyDialerTestSuite) TestSetupDefaults() { + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + suite.EqualValues(ProxyDialerOpenThreshold, d.openThreshold) + suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + suite.EqualValues(ProxyDialerResetFailuresTimeout, d.resetFailuresTimeout) +} + +func (suite *ProxyDialerTestSuite) TestSetupValuesAllOk() { + query := url.Values{} + query.Set("open_threshold", "30") + query.Set("reset_failures_timeout", "1s") + query.Set("half_open_timeout", "2s") + suite.u.RawQuery = query.Encode() + + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + suite.EqualValues(30, d.openThreshold) + suite.EqualValues(2*time.Second, d.halfOpenTimeout) + suite.EqualValues(time.Second, d.resetFailuresTimeout) +} + +func (suite *ProxyDialerTestSuite) TestOpenThreshold() { + query := url.Values{} + params := []string{"-30", "aaa", "1.0", "-1.0"} + + for _, v := range params { + param := v + suite.T().Run(v, func(t *testing.T) { + query.Set("open_threshold", param) + suite.u.RawQuery = query.Encode() + + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + assert.EqualValues(t, ProxyDialerOpenThreshold, d.openThreshold) + }) + } +} + +func (suite *ProxyDialerTestSuite) TestHalfOpenTimeout() { + query := url.Values{} + params := []string{"-30", "30", "aaa", "-3.0", "3.0"} + + for _, v := range params { + param := v + suite.T().Run(v, func(t *testing.T) { + query.Set("half_open_timeout", param) + suite.u.RawQuery = query.Encode() + + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + assert.EqualValues(t, ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + }) + } +} + +func (suite *ProxyDialerTestSuite) TestResetFailuresTimeout() { + query := url.Values{} + params := []string{"-30", "30", "aaa", "-3.0", "3.0"} + + for _, v := range params { + param := v + suite.T().Run(v, func(t *testing.T) { + query.Set("reset_failures_timeout", param) + suite.u.RawQuery = query.Encode() + + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + assert.EqualValues(t, ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + }) + } +} + +func TestProxyDialer(t *testing.T) { + t.Parallel() + suite.Run(t, &ProxyDialerTestSuite{}) +} diff --git a/network/socks5.go b/network/socks5.go new file mode 100644 index 0000000..43aac4a --- /dev/null +++ b/network/socks5.go @@ -0,0 +1,21 @@ +package network + +import ( + "fmt" + "net/url" + + "golang.org/x/net/proxy" +) + +// NewSocks5Dialer build a new dialer from a given one (so, in theory +// you can chain here). Proxy parameters are passed with URI in a form of: +// +// socks5://[user:[password]]@host:port +func NewSocks5Dialer(baseDialer Dialer, proxyURL *url.URL) (Dialer, error) { + rv, err := proxy.FromURL(proxyURL, baseDialer) + if err != nil { + return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) + } + + return rv.(Dialer), nil +} diff --git a/network/socks5_test.go b/network/socks5_test.go new file mode 100644 index 0000000..3460f4d --- /dev/null +++ b/network/socks5_test.go @@ -0,0 +1,61 @@ +package network_test + +import ( + "net/http" + "testing" + + "github.com/9seconds/mtg/v2/network" + "github.com/stretchr/testify/suite" +) + +type Socks5TestSuite struct { + suite.Suite + HTTPServerTestSuite + Socks5ServerTestSuite + + d network.Dialer +} + +func (suite *Socks5TestSuite) SetupSuite() { + suite.HTTPServerTestSuite.SetupSuite() + suite.Socks5ServerTestSuite.SetupSuite() + + suite.d, _ = network.NewDefaultDialer(0, 0) +} + +func (suite *Socks5TestSuite) TearDownSuite() { + suite.Socks5ServerTestSuite.TearDownSuite() + suite.HTTPServerTestSuite.TearDownSuite() +} + +func (suite *Socks5TestSuite) TestRequestFailed() { + proxyURL := suite.MakeSocks5URL("user2", "password") + dialer, _ := network.NewSocks5Dialer(suite.d, proxyURL) + httpClient := suite.MakeHTTPClient(dialer) + + resp, err := httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } + + suite.Error(err) +} + +func (suite *Socks5TestSuite) TestRequestOk() { + proxyURL := suite.MakeSocks5URL("user", "password") + dialer, _ := network.NewSocks5Dialer(suite.d, proxyURL) + httpClient := suite.MakeHTTPClient(dialer) + + resp, err := httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } + + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) +} + +func TestSocks5TestSuite(t *testing.T) { + t.Parallel() + suite.Run(t, &Socks5TestSuite{}) +} diff --git a/ntp/ntp.go b/ntp/ntp.go deleted file mode 100644 index 97d5d63..0000000 --- a/ntp/ntp.go +++ /dev/null @@ -1,57 +0,0 @@ -package ntp - -import ( - "fmt" - "math/rand" - "time" - - "github.com/9seconds/mtg/config" - "github.com/beevik/ntp" - "go.uber.org/zap" -) - -const autoUpdatePeriod = time.Minute - -// Fetch fetches the data on time drift. -func Fetch() (time.Duration, error) { - url := config.C.NTPServers[rand.Intn(len(config.C.NTPServers))] // nolint: gosec - - resp, err := ntp.Query(url) - if err != nil { - return 0, fmt.Errorf("cannot fetch NTP server %s: %w", url, err) - } - - offsetInt := int64(resp.ClockOffset) - if offsetInt < 0 { - offsetInt = -offsetInt - } - - offset := time.Duration(offsetInt) - - return offset, nil -} - -// AutoUpdate runs periodic check of current time .drift state. -func AutoUpdate() { - logger := zap.S().Named("ntp") - - for range time.Tick(autoUpdatePeriod) { - diff, err := Fetch() - if err != nil { - logger.Debugw("Cannot fetch time from NTP", "error", err) - - continue - } - - switch { - case diff < 400*time.Millisecond: - logger.Debugw("NTP time drift", "value", diff.String()) - case diff < 600*time.Millisecond: - logger.Infow("NTP time drift", "value", diff.String()) - case diff < 800*time.Millisecond: - logger.Warnw("NTP time drift", "value", diff.String()) - default: - logger.Errorw("NTP time drift", "value", diff.String()) - } - } -} diff --git a/obfuscated2/client_protocol.go b/obfuscated2/client_protocol.go deleted file mode 100644 index 955ae9e..0000000 --- a/obfuscated2/client_protocol.go +++ /dev/null @@ -1,114 +0,0 @@ -package obfuscated2 - -import ( - "bytes" - "crypto/sha256" - "encoding/binary" - "errors" - "fmt" - "io" - "time" - - "github.com/9seconds/mtg/antireplay" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" -) - -const clientProtocolHandshakeTimeout = 10 * time.Second - -type ClientProtocol struct { - connectionType conntypes.ConnectionType - connectionProtocol conntypes.ConnectionProtocol - dc conntypes.DC -} - -func (c *ClientProtocol) ConnectionType() conntypes.ConnectionType { - return c.connectionType -} - -func (c *ClientProtocol) ConnectionProtocol() conntypes.ConnectionProtocol { - return c.connectionProtocol -} - -func (c *ClientProtocol) DC() conntypes.DC { - return c.dc -} - -func (c *ClientProtocol) Handshake(socket conntypes.StreamReadWriteCloser) (conntypes.StreamReadWriteCloser, error) { - fm, err := c.ReadFrame(socket) - if err != nil { - return nil, fmt.Errorf("cannot make a client handshake: %w", err) - } - - decHasher := sha256.New() - decHasher.Write(fm.Key()) // nolint: errcheck - decHasher.Write(config.C.Secret) // nolint: errcheck - decryptor := utils.MakeStreamCipher(decHasher.Sum(nil), fm.IV()) - - invertedFrame := fm.Invert() - encHasher := sha256.New() - encHasher.Write(invertedFrame.Key()) // nolint: errcheck - encHasher.Write(config.C.Secret) // nolint: errcheck - encryptor := utils.MakeStreamCipher(encHasher.Sum(nil), invertedFrame.IV()) - - decryptedFrame := Frame{} - decryptor.XORKeyStream(decryptedFrame.Bytes(), fm.Bytes()) - - magic := decryptedFrame.Magic() - - switch { - case bytes.Equal(magic, conntypes.ConnectionTagAbridged): - c.connectionType = conntypes.ConnectionTypeAbridged - case bytes.Equal(magic, conntypes.ConnectionTagIntermediate): - c.connectionType = conntypes.ConnectionTypeIntermediate - case bytes.Equal(magic, conntypes.ConnectionTagSecure): - c.connectionType = conntypes.ConnectionTypeSecure - default: - return nil, errors.New("unknown connection type") - } - - c.connectionProtocol = conntypes.ConnectionProtocolIPv4 - if socket.LocalAddr().IP.To4() == nil { - c.connectionProtocol = conntypes.ConnectionProtocolIPv6 - } - - buf := bytes.NewReader(decryptedFrame.DC()) - if err := binary.Read(buf, binary.LittleEndian, &c.dc); err != nil { - c.dc = conntypes.DCDefaultIdx - } - - replayKey := decryptedFrame.Unique() - if antireplay.Cache.HasObfuscated2(replayKey) { - stats.Stats.ReplayDetected() - - return nil, errors.New("replay attack is detected") - } - - antireplay.Cache.AddObfuscated2(replayKey) - - return stream.NewObfuscated2(socket, encryptor, decryptor), nil -} - -func (c *ClientProtocol) ReadFrame(socket conntypes.StreamReader) (fm Frame, err error) { - if _, err = io.ReadFull(handshakeReader{socket}, fm.Bytes()); err != nil { - err = fmt.Errorf("cannot extract obfuscated2 frame: %w", err) - } - - return -} - -type handshakeReader struct { - parent conntypes.StreamReader -} - -func (h handshakeReader) Read(p []byte) (int, error) { - return h.parent.ReadTimeout(p, clientProtocolHandshakeTimeout) -} - -func MakeClientProtocol() protocol.ClientProtocol { - return &ClientProtocol{} -} diff --git a/obfuscated2/frame.go b/obfuscated2/frame.go deleted file mode 100644 index e05cbd0..0000000 --- a/obfuscated2/frame.go +++ /dev/null @@ -1,54 +0,0 @@ -package obfuscated2 - -const ( - frameLenKey = 32 - frameLenIV = 16 - frameLenMagic = 4 - frameLenDC = 2 - - frameOffsetFirst = 8 - frameOffsetKey = frameOffsetFirst + frameLenKey - frameOffsetIV = frameOffsetKey + frameLenIV - frameOffsetMagic = frameOffsetIV + frameLenMagic - frameOffsetDC = frameOffsetMagic + frameLenDC - - frameLen = 64 -) - -// [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd]. -type Frame struct { - data [frameLen]byte -} - -func (f *Frame) Bytes() []byte { - return f.data[:] -} - -func (f *Frame) Key() []byte { - return f.data[frameOffsetFirst:frameOffsetKey] -} - -func (f *Frame) IV() []byte { - return f.data[frameOffsetKey:frameOffsetIV] -} - -func (f *Frame) Magic() []byte { - return f.data[frameOffsetIV:frameOffsetMagic] -} - -func (f *Frame) DC() []byte { - return f.data[frameOffsetMagic:frameOffsetDC] -} - -func (f *Frame) Unique() []byte { - return f.data[frameOffsetFirst:frameOffsetDC] -} - -func (f *Frame) Invert() (nf Frame) { - nf = *f - for i := 0; i < frameLenKey+frameLenIV; i++ { - nf.data[frameOffsetFirst+i] = f.data[frameOffsetIV-1-i] - } - - return -} diff --git a/obfuscated2/telegram_protocol.go b/obfuscated2/telegram_protocol.go deleted file mode 100644 index cf17099..0000000 --- a/obfuscated2/telegram_protocol.go +++ /dev/null @@ -1,68 +0,0 @@ -package obfuscated2 - -import ( - "crypto/rand" - "fmt" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" -) - -func TelegramProtocol(req *protocol.TelegramRequest) (conntypes.StreamReadWriteCloser, error) { - conn, err := telegram.Direct.Dial(req.ClientProtocol.DC(), - req.ClientProtocol.ConnectionProtocol()) - if err != nil { - return nil, fmt.Errorf("cannot dial to telegram: %w", err) - } - - conn = stream.NewTimeout(conn) - conn = stream.NewCtx(req.Ctx, req.Cancel, conn) - fm := generateFrame(req.ClientProtocol) - data := fm.Bytes() - - encryptor := utils.MakeStreamCipher(fm.Key(), fm.IV()) - decryptedFrame := fm.Invert() - decryptor := utils.MakeStreamCipher(decryptedFrame.Key(), decryptedFrame.IV()) - - copyFrame := make([]byte, frameLen) - copy(copyFrame[:frameOffsetIV], data[:frameOffsetIV]) - encryptor.XORKeyStream(data, data) - copy(data[:frameOffsetIV], copyFrame[:frameOffsetIV]) - - if _, err := conn.Write(data); err != nil { - return nil, fmt.Errorf("cannot write handshake frame to telegram: %w", err) - } - - return stream.NewObfuscated2(conn, encryptor, decryptor), nil -} - -func generateFrame(cp protocol.ClientProtocol) (fm Frame) { - data := fm.Bytes() - - for { - if _, err := rand.Read(data); err != nil { - continue - } - - if data[0] == 0xef { - continue - } - - val := (uint32(data[3]) << 24) | (uint32(data[2]) << 16) | (uint32(data[1]) << 8) | uint32(data[0]) - if val == 0x44414548 || val == 0x54534f50 || val == 0x20544547 || val == 0x4954504f || val == 0xeeeeeeee { - continue - } - - val = (uint32(data[7]) << 24) | (uint32(data[6]) << 16) | (uint32(data[5]) << 8) | uint32(data[4]) - if val == 0x00000000 { - continue - } - - copy(fm.Magic(), cp.ConnectionType().Tag()) - - return - } -} diff --git a/protocol/interfaces.go b/protocol/interfaces.go deleted file mode 100644 index 2e27fc3..0000000 --- a/protocol/interfaces.go +++ /dev/null @@ -1,12 +0,0 @@ -package protocol - -import "github.com/9seconds/mtg/conntypes" - -type ClientProtocol interface { - Handshake(conntypes.StreamReadWriteCloser) (conntypes.StreamReadWriteCloser, error) - ConnectionType() conntypes.ConnectionType - ConnectionProtocol() conntypes.ConnectionProtocol - DC() conntypes.DC -} - -type ClientProtocolMaker func() ClientProtocol diff --git a/protocol/request.go b/protocol/request.go deleted file mode 100644 index 5f5d98d..0000000 --- a/protocol/request.go +++ /dev/null @@ -1,17 +0,0 @@ -package protocol - -import ( - "context" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type TelegramRequest struct { - Logger *zap.SugaredLogger - ClientConn conntypes.StreamReadWriteCloser - ConnID conntypes.ConnID - Ctx context.Context - Cancel context.CancelFunc - ClientProtocol ClientProtocol -} diff --git a/proxy/direct.go b/proxy/direct.go deleted file mode 100644 index 03fd9af..0000000 --- a/proxy/direct.go +++ /dev/null @@ -1,49 +0,0 @@ -package proxy - -import ( - "io" - "sync" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/obfuscated2" - "github.com/9seconds/mtg/protocol" - "go.uber.org/zap" -) - -const directPipeBufferSize = 1024 - -func directConnection(request *protocol.TelegramRequest) error { - telegramConnRaw, err := obfuscated2.TelegramProtocol(request) - if err != nil { - return err // nolint: wrapcheck - } - - telegramConn := telegramConnRaw.(conntypes.StreamReadWriteCloser) - - defer telegramConn.Close() - - wg := &sync.WaitGroup{} - wg.Add(2) - - go directPipe(telegramConn, request.ClientConn, wg, request.Logger) - - go directPipe(request.ClientConn, telegramConn, wg, request.Logger) - - wg.Wait() - - return nil -} - -func directPipe(dst io.WriteCloser, src io.ReadCloser, wg *sync.WaitGroup, logger *zap.SugaredLogger) { - defer func() { - dst.Close() - src.Close() - wg.Done() - }() - - buf := [directPipeBufferSize]byte{} - - if _, err := io.CopyBuffer(dst, src, buf[:]); err != nil { - logger.Debugw("Cannot pump sockets", "error", err) - } -} diff --git a/proxy/middle.go b/proxy/middle.go deleted file mode 100644 index 7f56991..0000000 --- a/proxy/middle.go +++ /dev/null @@ -1,70 +0,0 @@ -package proxy - -import ( - "sync" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/wrappers/packetack" - "go.uber.org/zap" -) - -func middleConnection(request *protocol.TelegramRequest) { - telegramConn, err := packetack.NewProxy(request) - if err != nil { - request.Logger.Debugw("Cannot dial to Telegram", "error", err) - - return - } - defer telegramConn.Close() - - var clientConn conntypes.PacketAckFullReadWriteCloser - - switch request.ClientProtocol.ConnectionType() { - case conntypes.ConnectionTypeAbridged: - clientConn = packetack.NewClientAbridged(request.ClientConn) - case conntypes.ConnectionTypeIntermediate: - clientConn = packetack.NewClientIntermediate(request.ClientConn) - case conntypes.ConnectionTypeSecure: - clientConn = packetack.NewClientIntermediateSecure(request.ClientConn) - case conntypes.ConnectionTypeUnknown: - panic("unknown connection type") - } - - wg := &sync.WaitGroup{} - wg.Add(2) - - go middlePipe(telegramConn, clientConn, wg, request.Logger) - - go middlePipe(clientConn, telegramConn, wg, request.Logger) - - wg.Wait() -} - -func middlePipe(dst conntypes.PacketAckWriteCloser, - src conntypes.PacketAckReadCloser, - wg *sync.WaitGroup, - logger *zap.SugaredLogger) { - defer func() { - dst.Close() - src.Close() - wg.Done() - }() - - for { - acks := conntypes.ConnectionAcks{} - - packet, err := src.Read(&acks) - if err != nil { - logger.Debugw("Cannot read packet", "error", err) - - return - } - - if err = dst.Write(packet, &acks); err != nil { - logger.Debugw("Cannot send packet", "error", err) - - return - } - } -} diff --git a/proxy/proxy.go b/proxy/proxy.go deleted file mode 100644 index 7be2a19..0000000 --- a/proxy/proxy.go +++ /dev/null @@ -1,100 +0,0 @@ -package proxy - -import ( - "context" - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" - "go.uber.org/zap" -) - -type Proxy struct { - Logger *zap.SugaredLogger - Context context.Context - ClientProtocolMaker protocol.ClientProtocolMaker -} - -func (p *Proxy) Serve(listener net.Listener) { - doneChan := p.Context.Done() - - for { - conn, err := listener.Accept() - if err != nil { - select { - case <-doneChan: - return - default: - p.Logger.Fatalw("Cannot allocate incoming connection", "error", err) - } - } - - go p.accept(conn) - } -} - -func (p *Proxy) accept(conn net.Conn) { - defer func() { - conn.Close() - - if err := recover(); err != nil { - stats.Stats.Crash() - p.Logger.Errorw("Crash of accept handler", "error", err) - } - }() - - connID := conntypes.NewConnID() - logger := p.Logger.With("connection_id", connID) - - if err := utils.InitTCP(conn, config.C.ClientReadBuffer(), config.C.ClientWriteBuffer()); err != nil { - logger.Errorw("Cannot initialize client TCP connection", "error", err) - - return - } - - ctx, cancel := context.WithCancel(p.Context) - defer cancel() - - clientConn := stream.NewClientConn(conn, connID) - clientConn = stream.NewCtx(ctx, cancel, clientConn) - clientConn = stream.NewTimeout(clientConn) - - defer clientConn.Close() - - clientProtocol := p.ClientProtocolMaker() - - clientConn, err := clientProtocol.Handshake(clientConn) - if err != nil { - stats.Stats.AuthenticationFailed() - logger.Warnw("Cannot perform client handshake", "error", err) - - return - } - - stats.Stats.ClientConnected(clientProtocol.ConnectionType(), clientConn.RemoteAddr()) - defer stats.Stats.ClientDisconnected(clientProtocol.ConnectionType(), clientConn.RemoteAddr()) - logger.Infow("Client connected", "addr", conn.RemoteAddr()) - - req := &protocol.TelegramRequest{ - Logger: logger, - ClientConn: clientConn, - ConnID: connID, - Ctx: ctx, - Cancel: cancel, - ClientProtocol: clientProtocol, - } - - err = nil - - if config.C.MiddleProxyMode() { - middleConnection(req) - } else { - err = directConnection(req) - } - - logger.Infow("Client disconnected", "error", err, "addr", conn.RemoteAddr()) -} diff --git a/run.sh b/run.sh deleted file mode 100755 index 338a4c8..0000000 --- a/run.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# -# Configuration options (set by environment variables during script execution) -# - MTG_CONFIG - directory where mtg stores its configuration -# - MTG_IMAGENAME - a name of the docker image to use -# - MTG_PORT - which port of the host system should be used -# - MTG_CONTAINER - a name of the container to use -# -# Example: -# export MTG_CONFIG="$HOME/mtg_config" -# export MTG_IMAGENAME="nineseconds/mtg:latest" -# curl -sfL --compressed https://raw.githubusercontent.com/9seconds/mtg/master/run.sh | bash - -set -eu - -export XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-$HOME/.config}" -export MTG_CONFIG="${MTG_CONFIG:-$XDG_CONFIG_HOME/mtg}" - -if ! [ -x "$(command -v docker)" ]; then - echo 'Error: docker is not installed.' >&2 - exit 1 -fi - -id -Gn "$USER" | grep -qw 'docker' > /dev/null -if [ $? -eq 0 ] || [ "$(id -u)" -eq '0' ]; then - DOCKER_CMD="$(command -v docker)" -else - DOCKER_CMD="sudo $(command -v docker)" -fi - -mkdir -p "$MTG_CONFIG" || true - -MTG_SECRET="$MTG_CONFIG/secret" -MTG_ENV="$MTG_CONFIG/env" - -if [ ! -f "$MTG_ENV" ]; then - MTG_IMAGENAME="${MTG_IMAGENAME:-nineseconds/mtg:stable}" - MTG_PORT="${MTG_PORT:-3128}" - MTG_CONTAINER="${MTG_CONTAINER:-mtg}" - - echo "MTG_IMAGENAME=$MTG_IMAGENAME" > "$MTG_ENV" - echo "MTG_PORT=$MTG_PORT" >> "$MTG_ENV" - echo "MTG_CONTAINER=$MTG_CONTAINER" >> "$MTG_ENV" -fi - -set -a -source "$MTG_ENV" -set +a - -$DOCKER_CMD pull "$MTG_IMAGENAME" > /dev/null -if [ ! -f "$MTG_SECRET" ]; then - $DOCKER_CMD run \ - --rm \ - "$MTG_IMAGENAME" \ - generate-secret tls -c "$(openssl rand -hex 16).com" \ - > "$MTG_SECRET" -fi - -echo "Proxy secret is $(cat "$MTG_SECRET"). Port is $MTG_PORT." - -$DOCKER_CMD ps --filter "Name=$MTG_CONTAINER" -aq | xargs -r $DOCKER_CMD rm -fv > /dev/null -$DOCKER_CMD run \ - -d \ - --restart=unless-stopped \ - --name "$MTG_CONTAINER" \ - --ulimit nofile=51200:51200 \ - -p "$MTG_PORT:3128" \ - "$MTG_IMAGENAME" run "$(cat "$MTG_SECRET")" > /dev/null diff --git a/stats/init.go b/stats/init.go new file mode 100644 index 0000000..3a2bae7 --- /dev/null +++ b/stats/init.go @@ -0,0 +1,107 @@ +// Stats package has implementations of events.Observers for different +// monitoring systems. +// +// Observer is a consumer of events produced by mtg. Consumers, defined +// in this package, process these events and provide information used by +// different monitoring system or time series databases. +package stats + +const ( + // DefaultMetricPrefix defines a base prefix for all metrics. + DefaultMetricPrefix = "mtg" + + // DefaultStatsdMetricPrefix defines a base prefix for metrics + // which are passed to statsd. + DefaultStatsdMetricPrefix = DefaultMetricPrefix + "." + + // DefaultStatsdTagFormat defines a format of tags for statsd + // observer. + DefaultStatsdTagFormat = "datadog" + + // MetricClientConnections defines a metric which is responsible for a + // number of currently active connections established by client. + // + // Type: gauge + // Tags: + // ip_family | A type of ip (ipv4 or ipv6) of the client. + MetricClientConnections = "client_connections" + + // MetricTelegramConnections defines a metric which is responsible for + // a count of active connections to Telegram servers. + // + // Type: gauge + // Tags: + // telegram_ip | IP address of the telegram server. + // dc | Index of the datacenter to connect to. + MetricTelegramConnections = "telegram_connections" + + // MetricDomainFrontingConnections defines a metric which is + // responsible for a count of active connections to a fronting domain. + // Fronting domain is that one that is encoded in a secret. + // + // Type: gauge + // Tags: + // ip_family | A type of IP (ipv4 or ipv6) that was used. + MetricDomainFrontingConnections = "domain_fronting_connections" + + // MetricTelegramTraffic defines a metric for traffic (in bytes) that + // is sent to and from Telegram servers. + // + // Type: counter + // Tags: + // telegram_ip | IP address of the telegram server. + // dc | Index of the datacenter + // direction | Direction of the traffc flow. Values are + // | 'to_client' and 'from_client' + MetricTelegramTraffic = "telegram_traffic" + + // MetricDomainFrontingTraffic defines a metric for traffic (in bytes) + // that is sent to and from fronting domain. + // + // Type: counter + // Tags: + // direction | Direction of the traffc flow. Values are + // | 'to_client' and 'from_client' + MetricDomainFrontingTraffic = "domain_fronting_traffic" + + // MetricDomainFronting defines a metric for a number of domain + // fronting routing events. + // + // Type: counter + MetricDomainFronting = "domain_fronting" + + // MetricConcurrencyLimited defines a metric for a count of events, + // when the client was blocked due to the concurrency limit. + // + // Type: counter + MetricConcurrencyLimited = "concurrency_limited" + + // MetricIPBlocklisted defines a metric for a count of events, when + // client was blocked because her IP address was found in blocklists. + // + // Type: counter + MetricIPBlocklisted = "ip_blocklisted" + + // MetricReplayAttacks defines a metric for a count of events, when + // mtg has detected a replay attack. Just a reminder: mtg immediately + // routes a connection to a fronting domain if such event is detected. + // + // Type: counter + MetricReplayAttacks = "replay_attacks" + + // TagIPFamily defines a name of the 'ip_family' tag and all values. + TagIPFamily = "ip_family" + TagIPFamilyIPv4 = "ipv4" + TagIPFamilyIPv6 = "ipv6" + + // TagTelegramIP defines a name of the 'telegram_ip' tag. + TagTelegramIP = "telegram_ip" + + // TagDC defines a name of the 'dc' tag. + TagDC = "dc" + + // TagDirection defines a name of the 'direction' tag. + TagDirection = "direction" + TagDirectionToClient = "to_client" + TagDirectionFromClient = "from_client" +) diff --git a/stats/interfaces.go b/stats/interfaces.go deleted file mode 100644 index 083a8ea..0000000 --- a/stats/interfaces.go +++ /dev/null @@ -1,60 +0,0 @@ -package stats - -import ( - "net" - - "github.com/9seconds/mtg/conntypes" -) - -type IngressTrafficInterface interface { - IngressTraffic(int) -} - -type EgressTrafficInterface interface { - EgressTraffic(int) -} - -type ClientConnectedInterface interface { - ClientConnected(conntypes.ConnectionType, *net.TCPAddr) -} - -type ClientDisconnectedInterface interface { - ClientDisconnected(conntypes.ConnectionType, *net.TCPAddr) -} - -type TelegramConnectedInterface interface { - TelegramConnected(conntypes.DC, *net.TCPAddr) -} - -type TelegramDisconnectedInterface interface { - TelegramDisconnected(conntypes.DC, *net.TCPAddr) -} - -type CrashInterface interface { - Crash() -} - -type ReplayDetectedInterface interface { - ReplayDetected() -} - -type AuthenticationFailedInterface interface { - AuthenticationFailed() -} - -type CloakedRequestInterface interface { - CloakedRequest() -} - -type Interface interface { - IngressTrafficInterface - EgressTrafficInterface - ClientConnectedInterface - ClientDisconnectedInterface - TelegramConnectedInterface - TelegramDisconnectedInterface - CrashInterface - ReplayDetectedInterface - AuthenticationFailedInterface - CloakedRequestInterface -} diff --git a/stats/multi_stats.go b/stats/multi_stats.go deleted file mode 100644 index 569c93c..0000000 --- a/stats/multi_stats.go +++ /dev/null @@ -1,69 +0,0 @@ -package stats - -import ( - "net" - - "github.com/9seconds/mtg/conntypes" -) - -type multiStats []Interface - -func (m multiStats) IngressTraffic(traffic int) { - for i := range m { - go m[i].IngressTraffic(traffic) - } -} - -func (m multiStats) EgressTraffic(traffic int) { - for i := range m { - go m[i].EgressTraffic(traffic) - } -} - -func (m multiStats) ClientConnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - for i := range m { - go m[i].ClientConnected(connectionType, addr) - } -} - -func (m multiStats) ClientDisconnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - for i := range m { - go m[i].ClientDisconnected(connectionType, addr) - } -} - -func (m multiStats) TelegramConnected(dc conntypes.DC, addr *net.TCPAddr) { - for i := range m { - go m[i].TelegramConnected(dc, addr) - } -} - -func (m multiStats) TelegramDisconnected(dc conntypes.DC, addr *net.TCPAddr) { - for i := range m { - go m[i].TelegramDisconnected(dc, addr) - } -} - -func (m multiStats) Crash() { - for i := range m { - go m[i].Crash() - } -} - -func (m multiStats) ReplayDetected() { - for i := range m { - go m[i].ReplayDetected() - } -} - -func (m multiStats) AuthenticationFailed() { - for i := range m { - go m[i].AuthenticationFailed() - } -} - -func (m multiStats) CloakedRequest() { - for i := range m { - go m[i].CloakedRequest() - } -} diff --git a/stats/pools.go b/stats/pools.go new file mode 100644 index 0000000..b9b6255 --- /dev/null +++ b/stats/pools.go @@ -0,0 +1,20 @@ +package stats + +import "sync" + +var streamInfoPool = sync.Pool{ + New: func() interface{} { + return &streamInfo{ + tags: make(map[string]string), + } + }, +} + +func acquireStreamInfo() *streamInfo { + return streamInfoPool.Get().(*streamInfo) +} + +func releaseStreamInfo(info *streamInfo) { + info.Reset() + streamInfoPool.Put(info) +} diff --git a/stats/prometheus.go b/stats/prometheus.go new file mode 100644 index 0000000..634be95 --- /dev/null +++ b/stats/prometheus.go @@ -0,0 +1,248 @@ +package stats + +import ( + "context" + "net" + "net/http" + "strconv" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type prometheusProcessor struct { + streams map[string]*streamInfo + factory *PrometheusFactory +} + +func (p prometheusProcessor) EventStart(evt mtglib.EventStart) { + info := acquireStreamInfo() + + if evt.RemoteIP.To4() != nil { + info.tags[TagIPFamily] = TagIPFamilyIPv4 + } else { + info.tags[TagIPFamily] = TagIPFamilyIPv6 + } + + p.streams[evt.StreamID()] = info + + p.factory.metricClientConnections. + WithLabelValues(info.tags[TagIPFamily]). + Inc() +} + +func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + info, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + info.tags[TagTelegramIP] = evt.RemoteIP.String() + info.tags[TagDC] = strconv.Itoa(evt.DC) + + p.factory.metricTelegramConnections. + WithLabelValues(info.tags[TagTelegramIP], info.tags[TagDC]). + Inc() +} + +func (p prometheusProcessor) EventDomainFronting(evt mtglib.EventDomainFronting) { + info, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + info.isDomainFronted = true + + p.factory.metricDomainFronting.Inc() + p.factory.metricDomainFrontingConnections. + WithLabelValues(info.tags[TagIPFamily]). + Inc() +} + +func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) { + info, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + direction := getDirection(evt.IsRead) + + if info.isDomainFronted { + p.factory.metricDomainFrontingTraffic. + WithLabelValues(direction). + Add(float64(evt.Traffic)) + } else { + p.factory.metricTelegramTraffic. + WithLabelValues(info.tags[TagTelegramIP], info.tags[TagDC], direction). + Add(float64(evt.Traffic)) + } +} + +func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { + info, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + defer func() { + delete(p.streams, evt.StreamID()) + releaseStreamInfo(info) + }() + + p.factory.metricClientConnections. + WithLabelValues(info.tags[TagIPFamily]). + Dec() + + if info.isDomainFronted { + p.factory.metricDomainFrontingConnections. + WithLabelValues(info.tags[TagIPFamily]). + Dec() + } else if telegramIP, ok := info.tags[TagTelegramIP]; ok { + p.factory.metricTelegramConnections. + WithLabelValues(telegramIP, info.tags[TagDC]). + Dec() + } +} + +func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { + p.factory.metricConcurrencyLimited.Inc() +} + +func (p prometheusProcessor) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) { + p.factory.metricIPBlocklisted.Inc() +} + +func (p prometheusProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { + p.factory.metricReplayAttacks.Inc() +} + +func (p prometheusProcessor) Shutdown() { + for _, v := range p.streams { + releaseStreamInfo(v) + } + + p.streams = make(map[string]*streamInfo) +} + +// PrometheusFactory is a factory of events.Observers which collect +// information in a format suitable for Prometheus. +// +// This factory can also serve on a given listener. In that case it +// starts HTTP server with a single endpoint - a Prometheus-compatible +// scrape output. +type PrometheusFactory struct { + httpServer *http.Server + + metricClientConnections *prometheus.GaugeVec + metricTelegramConnections *prometheus.GaugeVec + metricDomainFrontingConnections *prometheus.GaugeVec + + metricTelegramTraffic *prometheus.CounterVec + metricDomainFrontingTraffic *prometheus.CounterVec + + metricDomainFronting prometheus.Counter + metricConcurrencyLimited prometheus.Counter + metricIPBlocklisted prometheus.Counter + metricReplayAttacks prometheus.Counter +} + +// Make builds a new observer. +func (p *PrometheusFactory) Make() events.Observer { + return prometheusProcessor{ + streams: make(map[string]*streamInfo), + factory: p, + } +} + +// Serve starts an HTTP server on a given listener. +func (p *PrometheusFactory) Serve(listener net.Listener) error { + return p.httpServer.Serve(listener) +} + +// Close stops a factory. Please pay attention that underlying listener +// is not closed. +func (p *PrometheusFactory) Close() error { + return p.httpServer.Shutdown(context.Background()) +} + +// NewPrometheus builds an events.ObserverFactory which can serve HTTP +// endpoint with Prometheus scrape data. +func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint: funlen + registry := prometheus.NewPedanticRegistry() + httpHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ + EnableOpenMetrics: true, + }) + mux := http.NewServeMux() + + mux.Handle(httpPath, httpHandler) + + factory := &PrometheusFactory{ + httpServer: &http.Server{ + Handler: mux, + }, + + metricClientConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricPrefix, + Name: MetricClientConnections, + Help: "A number of actively processing client connections.", + }, []string{TagIPFamily}), + metricTelegramConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricPrefix, + Name: MetricTelegramConnections, + Help: "A number of connections to Telegram servers.", + }, []string{TagTelegramIP, TagDC}), + metricDomainFrontingConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricPrefix, + Name: MetricDomainFrontingConnections, + Help: "A number of connections which talk to front domain.", + }, []string{TagIPFamily}), + + metricTelegramTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricTelegramTraffic, + Help: "Traffic which is generated talking with Telegram servers.", + }, []string{TagTelegramIP, TagDC, TagDirection}), + metricDomainFrontingTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricDomainFrontingTraffic, + Help: "Traffic which is generated talking with front domain.", + }, []string{TagDirection}), + + metricDomainFronting: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricDomainFronting, + Help: "A number of routings to front domain.", + }), + metricConcurrencyLimited: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricConcurrencyLimited, + Help: "A number of sessions that were rejected by concurrency limiter.", + }), + metricIPBlocklisted: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricIPBlocklisted, + Help: "A number of rejected sessions due to ip blocklisting.", + }), + metricReplayAttacks: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricReplayAttacks, + Help: "A number of detected replay attacks.", + }), + } + + registry.MustRegister(factory.metricClientConnections) + registry.MustRegister(factory.metricTelegramConnections) + registry.MustRegister(factory.metricDomainFrontingConnections) + + registry.MustRegister(factory.metricTelegramTraffic) + registry.MustRegister(factory.metricDomainFrontingTraffic) + + registry.MustRegister(factory.metricDomainFronting) + registry.MustRegister(factory.metricConcurrencyLimited) + registry.MustRegister(factory.metricIPBlocklisted) + registry.MustRegister(factory.metricReplayAttacks) + + return factory +} diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go new file mode 100644 index 0000000..6e44c0b --- /dev/null +++ b/stats/prometheus_test.go @@ -0,0 +1,175 @@ +package stats_test + +import ( + "fmt" + "io" + "net" + "net/http" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/stats" + "github.com/stretchr/testify/suite" +) + +type PrometheusTestSuite struct { + suite.Suite + + httpListener net.Listener + factory *stats.PrometheusFactory + prometheus events.Observer +} + +func (suite *PrometheusTestSuite) Get() (string, error) { + addr := fmt.Sprintf("http://%s/", suite.httpListener.Addr().String()) + + resp, err := http.Get(addr) // nolint: noctx + if err != nil { + return "", err // nolint: wrapcheck + } + + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return "", err // nolint: wrapcheck + } + + return string(data), nil +} + +func (suite *PrometheusTestSuite) SetupTest() { + suite.httpListener, _ = net.Listen("tcp", "127.0.0.1:0") + suite.factory = stats.NewPrometheus("mtg", "/") + suite.prometheus = suite.factory.Make() + + go suite.factory.Serve(suite.httpListener) // nolint: errcheck +} + +func (suite *PrometheusTestSuite) TearDownTest() { + suite.prometheus.Shutdown() + suite.NoError(suite.factory.Close()) + suite.httpListener.Close() +} + +func (suite *PrometheusTestSuite) TestTelegramPath() { + suite.prometheus.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 1`) + + suite.prometheus.EventConnectedToDC( + mtglib.NewEventConnectedToDC("connID", net.ParseIP("10.0.0.1"), 4)) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 1`) + + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 200, true)) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="to_client",telegram_ip="10.0.0.1"} 200`) + + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 100, false)) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="from_client",telegram_ip="10.0.0.1"} 100`) + + suite.prometheus.EventFinish(mtglib.NewEventFinish("connID")) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 0`) + suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 0`) +} + +func (suite *PrometheusTestSuite) TestDomainFrontingPath() { + suite.prometheus.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 1`) + + suite.prometheus.EventDomainFronting(mtglib.NewEventDomainFronting("connID")) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_domain_fronting 1`) + suite.Contains(data, `mtg_domain_fronting_connections{ip_family="ipv4"} 1`) + + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 200, true)) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_domain_fronting_traffic{direction="to_client"} 200`) + + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 100, false)) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_domain_fronting_traffic{direction="from_client"} 100`) + + suite.prometheus.EventFinish(mtglib.NewEventFinish("connID")) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 0`) + suite.Contains(data, `mtg_domain_fronting_connections{ip_family="ipv4"} 0`) +} + +func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { + suite.prometheus.EventConcurrencyLimited(mtglib.NewEventConcurrencyLimited()) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_concurrency_limited 1`) +} + +func (suite *PrometheusTestSuite) TestEventIPBlocklisted() { + suite.prometheus.EventIPBlocklisted( + mtglib.NewEventIPBlocklisted(net.ParseIP("2001:db8::68"))) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_ip_blocklisted 1`) +} + +func (suite *PrometheusTestSuite) TestEventReplayAttack() { + suite.prometheus.EventReplayAttack(mtglib.NewEventReplayAttack("connID")) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_replay_attacks 1`) +} + +func TestPrometheus(t *testing.T) { + t.Parallel() + suite.Run(t, &PrometheusTestSuite{}) +} diff --git a/stats/stats.go b/stats/stats.go deleted file mode 100644 index 80a3250..0000000 --- a/stats/stats.go +++ /dev/null @@ -1,41 +0,0 @@ -package stats - -import ( - "context" - "fmt" - "net" - "net/http" - - "github.com/9seconds/mtg/config" -) - -var Stats Interface - -func Init(ctx context.Context) error { - mux := http.NewServeMux() - - stats := []Interface{newStatsPrometheus(mux)} - if config.C.StatsdAddr != nil { - stats = append(stats, newStatsStatsd()) - } - - listener, err := net.Listen("tcp", config.C.StatsBind.String()) - if err != nil { - return fmt.Errorf("cannot initialize stats server: %w", err) - } - - srv := http.Server{ - Handler: mux, - } - - go srv.Serve(listener) // nolint: errcheck - - go func() { - <-ctx.Done() - srv.Shutdown(context.Background()) // nolint: errcheck - }() - - Stats = multiStats(stats) - - return nil -} diff --git a/stats/stats_prometheus.go b/stats/stats_prometheus.go deleted file mode 100644 index 687c635..0000000 --- a/stats/stats_prometheus.go +++ /dev/null @@ -1,156 +0,0 @@ -package stats - -import ( - "net" - "net/http" - "strconv" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -type statsPrometheus struct { - connections *prometheus.GaugeVec - telegramConnections *prometheus.GaugeVec - traffic *prometheus.GaugeVec - crashes prometheus.Counter - replayAttacks prometheus.Counter - authenticationFailed prometheus.Counter - cloakedRequests prometheus.Counter -} - -func (s *statsPrometheus) IngressTraffic(traffic int) { - s.traffic.WithLabelValues("ingress").Add(float64(traffic)) -} - -func (s *statsPrometheus) EgressTraffic(traffic int) { - s.traffic.WithLabelValues("egress").Add(float64(traffic)) -} - -func (s *statsPrometheus) ClientConnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, 1.0) -} - -func (s *statsPrometheus) ClientDisconnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, -1.0) -} - -func (s *statsPrometheus) changeConnections(connectionType conntypes.ConnectionType, - addr *net.TCPAddr, - increment float64) { - labels := [...]string{ - "intermediate", - "ipv4", - } - - switch connectionType { - case conntypes.ConnectionTypeAbridged: - labels[0] = "abridged" - case conntypes.ConnectionTypeSecure: - labels[0] = "secured" - case conntypes.ConnectionTypeIntermediate: - labels[0] = "intermediate" - case conntypes.ConnectionTypeUnknown: - panic("unknown connection type") - } - - if addr.IP.To4() == nil { - labels[1] = "ipv6" - } - - s.connections.WithLabelValues(labels[:]...).Add(increment) -} - -func (s *statsPrometheus) TelegramConnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, 1.0) -} - -func (s *statsPrometheus) TelegramDisconnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, -1.0) -} - -func (s *statsPrometheus) changeTelegramConnections(dc conntypes.DC, addr *net.TCPAddr, increment float64) { - labels := [...]string{ - strconv.Itoa(int(dc)), - "ipv4", - } - - if addr.IP.To4() == nil { - labels[1] = "ipv6" - } - - s.telegramConnections.WithLabelValues(labels[:]...).Add(increment) -} - -func (s *statsPrometheus) Crash() { - s.crashes.Inc() -} - -func (s *statsPrometheus) ReplayDetected() { - s.replayAttacks.Inc() -} - -func (s *statsPrometheus) AuthenticationFailed() { - s.authenticationFailed.Inc() -} - -func (s *statsPrometheus) CloakedRequest() { - s.cloakedRequests.Inc() -} - -func newStatsPrometheus(mux *http.ServeMux) Interface { - registry := prometheus.NewPedanticRegistry() - - instance := &statsPrometheus{ - connections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: config.C.StatsNamespace, - Name: "connections", - Help: "Current number of client connections to the proxy.", - }, []string{"type", "protocol"}), - telegramConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: config.C.StatsNamespace, - Name: "telegram_connections", - Help: "Current number of telegram connections established by this proxy.", - }, []string{"dc", "protocol"}), - traffic: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: config.C.StatsNamespace, - Name: "traffic", - Help: "Traffic passed through the proxy in bytes.", - }, []string{"direction"}), - crashes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "crashes", - Help: "How many crashes happened.", - }), - replayAttacks: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "replay_attacks", - Help: "How many replay attacks were prevented.", - }), - authenticationFailed: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "authentication_failed", - Help: "How many authentication failed events we've seen.", - }), - cloakedRequests: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "cloaked_requests", - Help: "How many requests were proxified during cloaking.", - }), - } - - registry.MustRegister(instance.connections) - registry.MustRegister(instance.telegramConnections) - registry.MustRegister(instance.traffic) - registry.MustRegister(instance.crashes) - registry.MustRegister(instance.replayAttacks) - registry.MustRegister(instance.authenticationFailed) - registry.MustRegister(instance.cloakedRequests) - - handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) - mux.Handle("/", handler) - - return instance -} diff --git a/stats/stats_statsd.go b/stats/stats_statsd.go deleted file mode 100644 index 26fce1c..0000000 --- a/stats/stats_statsd.go +++ /dev/null @@ -1,204 +0,0 @@ -package stats - -import ( - "fmt" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - statsd "github.com/smira/go-statsd" - "go.uber.org/zap" -) - -var ( - tagTrafficIngress = &statsStatsdTag{ - name: "ingress", - tag: statsd.StringTag("type", "ingress"), - } - tagTrafficEgress = &statsStatsdTag{ - name: "egress", - tag: statsd.StringTag("type", "egress"), - } - - tagConnectionTypeAbridged = &statsStatsdTag{ - name: "abridged", - tag: statsd.StringTag("type", "abridged"), - } - tagConnectionTypeIntermediate = &statsStatsdTag{ - name: "intermediate", - tag: statsd.StringTag("type", "intermediate"), - } - tagConnectionTypeSecured = &statsStatsdTag{ - name: "secured", - tag: statsd.StringTag("type", "secured"), - } - - tagConnectionProtocol4 = &statsStatsdTag{ - name: "ipv4", - tag: statsd.StringTag("protocol", "ipv4"), - } - tagConnectionProtocol6 = &statsStatsdTag{ - name: "ipv6", - tag: statsd.StringTag("protocol", "ipv6"), - } -) - -type statsStatsdTag struct { - tag statsd.Tag - name string -} - -type statsStatsdLogger struct { - log *zap.SugaredLogger -} - -func (s statsStatsdLogger) Printf(msg string, args ...interface{}) { - s.log.Debugw(fmt.Sprintf(msg, args...)) -} - -type statsStatsd struct { - seen map[string]struct{} - seenMutex sync.RWMutex - client *statsd.Client -} - -func (s *statsStatsd) IngressTraffic(traffic int) { - s.gauge("traffic", int64(traffic), tagTrafficIngress) -} - -func (s *statsStatsd) EgressTraffic(traffic int) { - s.gauge("traffic", int64(traffic), tagTrafficEgress) -} - -func (s *statsStatsd) ClientConnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, 1) -} - -func (s *statsStatsd) ClientDisconnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, -1) -} - -func (s *statsStatsd) changeConnections(connectionType conntypes.ConnectionType, addr *net.TCPAddr, increment int64) { - tags := make([]*statsStatsdTag, 0, 2) - - switch connectionType { - case conntypes.ConnectionTypeAbridged: - tags = append(tags, tagConnectionTypeAbridged) - case conntypes.ConnectionTypeIntermediate: - tags = append(tags, tagConnectionTypeIntermediate) - case conntypes.ConnectionTypeSecure: - tags = append(tags, tagConnectionTypeSecured) - case conntypes.ConnectionTypeUnknown: - panic("Unknown connection type") - } - - if addr.IP.To4() == nil { - tags = append(tags, tagConnectionProtocol6) - } else { - tags = append(tags, tagConnectionProtocol4) - } - - s.gauge("connections", increment, tags...) -} - -func (s *statsStatsd) TelegramConnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, 1) -} - -func (s *statsStatsd) TelegramDisconnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, -1) -} - -func (s *statsStatsd) changeTelegramConnections(dc conntypes.DC, addr *net.TCPAddr, increment int64) { - tags := []*statsStatsdTag{ - { - name: "dc" + strconv.Itoa(int(dc)), - tag: statsd.IntTag("dc", int(dc)), - }, - } - - if addr.IP.To4() == nil { - tags = append(tags, tagConnectionProtocol6) - } else { - tags = append(tags, tagConnectionProtocol4) - } - - s.gauge("telegram_connections", increment, tags...) -} - -func (s *statsStatsd) Crash() { - s.gauge("crashes", 1) -} - -func (s *statsStatsd) ReplayDetected() { - s.gauge("replay_attacks", 1) -} - -func (s *statsStatsd) AuthenticationFailed() { - s.gauge("authentication_failed", 1) -} - -func (s *statsStatsd) CloakedRequest() { - s.gauge("cloaked_requests", 1) -} - -func (s *statsStatsd) gauge(metric string, value int64, tags ...*statsStatsdTag) { - key, tagList := s.prepareVals(metric, tags) - s.initGauge(metric, key, tagList) - s.client.GaugeDelta(metric, value, tagList...) -} - -func (s *statsStatsd) prepareVals(metric string, tags []*statsStatsdTag) (string, []statsd.Tag) { - tagList := make([]statsd.Tag, len(tags)) - builder := strings.Builder{} - builder.WriteString(metric) - - for i, v := range tags { - builder.WriteRune('.') - builder.WriteString(v.name) - tagList[i] = v.tag - } - - return builder.String(), tagList -} - -func (s *statsStatsd) initGauge(metric, key string, tags []statsd.Tag) { - s.seenMutex.RLock() - if _, ok := s.seen[key]; ok { - s.seenMutex.RUnlock() - - return - } else { // nolint: golint,revive - s.seenMutex.RUnlock() - } - - s.seenMutex.Lock() - defer s.seenMutex.Unlock() - - if _, ok := s.seen[key]; !ok { - s.seen[key] = struct{}{} - s.client.Gauge(metric, 0, tags...) - } -} - -func newStatsStatsd() Interface { - prefix := strings.TrimSuffix(config.C.StatsNamespace, ".") + "." - logger := statsStatsdLogger{ - log: zap.S().Named("stats").Named("statsd"), - } - - return &statsStatsd{ - seen: make(map[string]struct{}), - client: statsd.NewClient(config.C.StatsdAddr.String(), - statsd.SendLoopCount(2), - statsd.ReconnectInterval(10*time.Second), - statsd.Logger(logger), - statsd.MetricPrefix(prefix), - statsd.TagStyle(config.C.StatsdTagsFormat), - ), - } -} diff --git a/stats/statsd.go b/stats/statsd.go new file mode 100644 index 0000000..f394ec6 --- /dev/null +++ b/stats/statsd.go @@ -0,0 +1,185 @@ +package stats + +import ( + "fmt" + "strconv" + "strings" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + statsd "github.com/smira/go-statsd" +) + +type statsdProcessor struct { + streams map[string]*streamInfo + client *statsd.Client +} + +func (s statsdProcessor) EventStart(evt mtglib.EventStart) { + info := acquireStreamInfo() + + if evt.RemoteIP.To4() != nil { + info.tags[TagIPFamily] = TagIPFamilyIPv4 + } else { + info.tags[TagIPFamily] = TagIPFamilyIPv6 + } + + s.streams[evt.StreamID()] = info + + s.client.GaugeDelta(MetricClientConnections, + 1, + info.T(TagIPFamily)) +} + +func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + info, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + info.tags[TagTelegramIP] = evt.RemoteIP.String() + info.tags[TagDC] = strconv.Itoa(evt.DC) + + s.client.GaugeDelta(MetricTelegramConnections, + 1, + info.T(TagTelegramIP), + info.T(TagDC)) +} + +func (s statsdProcessor) EventDomainFronting(evt mtglib.EventDomainFronting) { + info, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + info.isDomainFronted = true + + s.client.Incr(MetricDomainFronting, 1) + s.client.GaugeDelta(MetricDomainFrontingConnections, + 1, + info.T(TagIPFamily)) +} + +func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { + info, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + directionTag := statsd.StringTag(TagDirection, getDirection(evt.IsRead)) + + if info.isDomainFronted { + s.client.Incr(MetricDomainFrontingTraffic, + int64(evt.Traffic), + directionTag) + } else { + s.client.Incr(MetricTelegramTraffic, + int64(evt.Traffic), + info.T(TagTelegramIP), + info.T(TagDC), + directionTag) + } +} + +func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { + info, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + defer func() { + delete(s.streams, evt.StreamID()) + releaseStreamInfo(info) + }() + + s.client.GaugeDelta(MetricClientConnections, + -1, + info.T(TagIPFamily)) + + if info.isDomainFronted { + s.client.GaugeDelta(MetricDomainFrontingConnections, + -1, + info.T(TagIPFamily)) + } else if _, ok := info.tags[TagTelegramIP]; ok { + s.client.GaugeDelta(MetricTelegramConnections, + -1, + info.T(TagTelegramIP), + info.T(TagDC)) + } +} + +func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { + s.client.Incr(MetricConcurrencyLimited, 1) +} + +func (s statsdProcessor) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) { + s.client.Incr(MetricIPBlocklisted, 1) +} + +func (s statsdProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { + s.client.Incr(MetricReplayAttacks, 1) +} + +func (s statsdProcessor) Shutdown() { + events := make([]mtglib.EventFinish, 0, len(s.streams)) + + for k := range s.streams { + events = append(events, mtglib.NewEventFinish(k)) + } + + for i := range events { + s.EventFinish(events[i]) + } +} + +// StatsdFactory is a factory of events.Observers which dumps +// information to statsd. +// +// Please beware that we support ONLY UDP endpoints there. And this +// factory won't use mtglib.Network so it won't use a proxy if you +// provide any. If you need it, I would recommend starting a local +// statsd and route metrics further by features of the chosen server. +type StatsdFactory struct { + client *statsd.Client +} + +// Close stops sending requests to statsd. +func (s StatsdFactory) Close() error { + return s.client.Close() +} + +// Make build a new observer. +func (s StatsdFactory) Make() events.Observer { + return statsdProcessor{ + client: s.client, + streams: make(map[string]*streamInfo), + } +} + +// NewStatsd builds an events.ObserverFactory that sends events +// to statsd. +// +// Valid tagFormats are 'datadog', 'influxdb' and 'graphite'. +func NewStatsd(address string, log logger.StdLikeLogger, + metricPrefix, tagFormat string) (StatsdFactory, error) { + options := []statsd.Option{ + statsd.MetricPrefix(metricPrefix), + statsd.Logger(log), + } + + switch strings.ToLower(tagFormat) { + case "datadog": + options = append(options, statsd.TagStyle(statsd.TagFormatDatadog)) + case "influxdb": + options = append(options, statsd.TagStyle(statsd.TagFormatInfluxDB)) + case "graphite": + options = append(options, statsd.TagStyle(statsd.TagFormatGraphite)) + default: + return StatsdFactory{}, fmt.Errorf("unknown tag format %s", tagFormat) + } + + return StatsdFactory{ + client: statsd.NewClient(address, options...), + }, nil +} diff --git a/stats/statsd_test.go b/stats/statsd_test.go new file mode 100644 index 0000000..4383509 --- /dev/null +++ b/stats/statsd_test.go @@ -0,0 +1,202 @@ +package stats_test + +import ( + "bytes" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/stats" + statsd "github.com/smira/go-statsd" + "github.com/stretchr/testify/suite" +) + +const statsdSleepTime = 4 * statsd.DefaultFlushInterval + +type statsdFakeServer struct { + conn *net.UDPConn + buf *bytes.Buffer + mutex sync.Mutex +} + +func (s *statsdFakeServer) Addr() string { + return s.conn.LocalAddr().String() +} + +func (s *statsdFakeServer) Close() error { + if s.conn != nil { + return s.conn.Close() + } + + return nil +} + +func (s *statsdFakeServer) String() string { + s.mutex.Lock() + defer s.mutex.Unlock() + + return strings.TrimSpace(s.buf.String()) +} + +func statsdNewFakeServer() *statsdFakeServer { + conn, err := net.ListenUDP("udp", &net.UDPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }) + if err != nil { + panic(err) + } + + rv := &statsdFakeServer{ + conn: conn, + buf: &bytes.Buffer{}, + } + + go func() { + currentBuffer := make([]byte, 4096) + + for { + n, _, err := conn.ReadFromUDP(currentBuffer) + if n > 0 { + rv.mutex.Lock() + rv.buf.Write(currentBuffer[:n]) + rv.mutex.Unlock() + } + + if err != nil { + return + } + } + }() + + return rv +} + +type StatsdTestSuite struct { + suite.Suite + + statsdServer *statsdFakeServer + factory stats.StatsdFactory + statsd events.Observer +} + +func (suite *StatsdTestSuite) SetupTest() { + suite.statsdServer = statsdNewFakeServer() + + factory, err := stats.NewStatsd(suite.statsdServer.Addr(), + logger.NewNoopLogger(), "mtg.", "datadog") + if err != nil { + panic(err) + } + + suite.factory = factory + suite.statsd = suite.factory.Make() +} + +func (suite *StatsdTestSuite) TearDownTest() { + suite.statsd.Shutdown() + suite.factory.Close() + suite.statsdServer.Close() +} + +func (suite *StatsdTestSuite) TestTelegramPath() { + suite.statsd.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) + time.Sleep(statsdSleepTime) + suite.Equal("mtg.client_connections:+1|g|#ip_family:ipv4", suite.statsdServer.String()) + + suite.statsd.EventConnectedToDC( + mtglib.NewEventConnectedToDC("connID", net.ParseIP("10.1.0.10"), 2)) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.telegram_connections:+1|g|#telegram_ip:10.1.0.10,dc:2") + + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 30, true)) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.telegram_traffic:30|c|#telegram_ip:10.1.0.10,dc:2,direction:to_client") + + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 90, false)) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.telegram_traffic:90|c|#telegram_ip:10.1.0.10,dc:2,direction:from_client") + + suite.statsd.EventFinish(mtglib.NewEventFinish("connID")) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.telegram_connections:-1|g|#telegram_ip:10.1.0.10,dc:2") + suite.Contains(suite.statsdServer.String(), + "mtg.client_connections:-1|g|#ip_family:ipv4") + + suite.NotContains(suite.statsdServer.String(), "domain_fronting_traffic") + suite.NotContains(suite.statsdServer.String(), "domain_fronting_connections") +} + +func (suite *StatsdTestSuite) TestDomainFrontingPath() { + suite.statsd.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) + time.Sleep(statsdSleepTime) + suite.Equal("mtg.client_connections:+1|g|#ip_family:ipv4", suite.statsdServer.String()) + + suite.statsd.EventDomainFronting(mtglib.NewEventDomainFronting("connID")) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), "mtg.domain_fronting:1|c") + suite.Contains(suite.statsdServer.String(), + `mtg.domain_fronting_connections:+1|g|#ip_family:ipv4`) + + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 30, true)) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + `mtg.domain_fronting_traffic:30|c|#direction:to_client`) + + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 90, false)) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + `mtg.domain_fronting_traffic:90|c|#direction:from_client`) + + suite.statsd.EventFinish(mtglib.NewEventFinish("connID")) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.domain_fronting_connections:-1|g|#ip_family:ipv4") + suite.Contains(suite.statsdServer.String(), + "mtg.client_connections:-1|g|#ip_family:ipv4") + + suite.NotContains(suite.statsdServer.String(), "telegram_traffic") + suite.NotContains(suite.statsdServer.String(), "telegram_connections") +} + +func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { + suite.statsd.EventConcurrencyLimited(mtglib.NewEventConcurrencyLimited()) + + time.Sleep(statsdSleepTime) + suite.Equal("mtg.concurrency_limited:1|c", suite.statsdServer.String()) +} + +func (suite *StatsdTestSuite) TestEventIPBlocklisted() { + suite.statsd.EventIPBlocklisted( + mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10"))) + + time.Sleep(statsdSleepTime) + suite.Equal("mtg.ip_blocklisted:1|c", suite.statsdServer.String()) +} + +func (suite *StatsdTestSuite) TestEventReplayAttack() { + suite.statsd.EventReplayAttack(mtglib.NewEventReplayAttack("connID")) + + time.Sleep(statsdSleepTime) + suite.Equal("mtg.replay_attacks:1|c", suite.statsdServer.String()) +} + +func TestStatsd(t *testing.T) { + t.Parallel() + suite.Run(t, &StatsdTestSuite{}) +} diff --git a/stats/stream_info.go b/stats/stream_info.go new file mode 100644 index 0000000..b28bccb --- /dev/null +++ b/stats/stream_info.go @@ -0,0 +1,28 @@ +package stats + +import statsd "github.com/smira/go-statsd" + +type streamInfo struct { + isDomainFronted bool + tags map[string]string +} + +func (s streamInfo) T(key string) statsd.Tag { + return statsd.StringTag(key, s.tags[key]) +} + +func (s *streamInfo) Reset() { + s.isDomainFronted = false + + for k := range s.tags { + delete(s.tags, k) + } +} + +func getDirection(isRead bool) string { + if isRead { // for telegram + return TagDirectionToClient + } + + return TagDirectionFromClient +} diff --git a/telegram/api/addresses.go b/telegram/api/addresses.go deleted file mode 100644 index c7573e4..0000000 --- a/telegram/api/addresses.go +++ /dev/null @@ -1,109 +0,0 @@ -package api - -import ( - "bufio" - "fmt" - "net" - "regexp" - "strconv" - "strings" - - "github.com/9seconds/mtg/conntypes" -) - -const ( - addressesURLV4 = "https://core.telegram.org/getProxyConfig" // nolint: gas - addressesURLV6 = "https://core.telegram.org/getProxyConfigV6" // nolint: gas -) - -var addressesProxyForSplitter = regexp.MustCompile(`\s+`) - -func AddressesV4() (map[conntypes.DC][]string, conntypes.DC, error) { - return getAddresses(addressesURLV4) -} - -func AddressesV6() (map[conntypes.DC][]string, conntypes.DC, error) { - return getAddresses(addressesURLV6) -} - -func getAddresses(url string) (map[conntypes.DC][]string, conntypes.DC, error) { - resp, err := request(url) - if err != nil { - return nil, 0, fmt.Errorf("cannot get http response: %w", err) - } - - defer resp.Close() - - scanner := bufio.NewScanner(resp) - data := map[conntypes.DC][]string{} - defaultDC := conntypes.DCDefaultIdx - - for scanner.Scan() { - text := strings.TrimSpace(scanner.Text()) - - switch { - case strings.HasPrefix(text, "#"): - continue - case strings.HasPrefix(text, "proxy_for"): - addr, idx, err := addressesParseProxyFor(text) - if err != nil { - return nil, 0, fmt.Errorf("cannot parse 'proxy_for' section: %w", err) - } - - if addresses, ok := data[idx]; ok { - data[idx] = append(addresses, addr) - } else { - data[idx] = []string{addr} - } - case strings.HasPrefix(text, "default"): - idx, err := addressesParseDefault(text) - if err != nil { - return nil, 0, fmt.Errorf("cannot parse 'default' section: %w", err) - } - - defaultDC = idx - } - } - - err = scanner.Err() - if err != nil { - return nil, 0, fmt.Errorf("cannot parse http response: %w", err) - } - - return data, defaultDC, nil -} - -func addressesParseProxyFor(text string) (string, conntypes.DC, error) { - chunks := addressesProxyForSplitter.Split(text, 3) - if len(chunks) != 3 || chunks[0] != "proxy_for" { - return "", 0, fmt.Errorf("incorrect config %s", text) - } - - dc, err := strconv.ParseInt(chunks[1], 10, 16) - if err != nil { - return "", 0, fmt.Errorf("incorrect config '%s': %w", text, err) - } - - addr := strings.TrimRight(chunks[2], ";") - if _, _, err = net.SplitHostPort(addr); err != nil { - return "", 0, fmt.Errorf("incorrect config '%s': %w", text, err) - } - - return addr, conntypes.DC(dc), nil -} - -func addressesParseDefault(text string) (conntypes.DC, error) { - chunks := addressesProxyForSplitter.Split(text, 2) - if len(chunks) != 2 || chunks[0] != "default" { - return 0, fmt.Errorf("incorrect config '%s'", text) - } - - dcString := strings.TrimRight(chunks[1], ";") - - dc, err := strconv.ParseInt(dcString, 10, 16) - if err != nil { - return 0, fmt.Errorf("incorrect config '%s': %w", text, err) - } - - return conntypes.DC(dc), nil -} diff --git a/telegram/api/api.go b/telegram/api/api.go deleted file mode 100644 index 7cdef71..0000000 --- a/telegram/api/api.go +++ /dev/null @@ -1,44 +0,0 @@ -package api - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/http" - "time" -) - -const ( - apiUserAgent = "github.com/9seconds/mtg" - apiHTTPTimeout = 30 * time.Second -) - -var httpClient = http.Client{ - Timeout: apiHTTPTimeout, -} - -func request(url string) (io.ReadCloser, error) { - ctx, cancel := context.WithTimeout(context.Background(), apiHTTPTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - panic(err) - } - - req.Header.Set("Accept", "text/plan") - req.Header.Set("User-Agent", apiUserAgent) - - resp, err := httpClient.Do(req) - if err != nil { - if resp != nil { - io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck - resp.Body.Close() - } - - return nil, fmt.Errorf("cannot perform a request: %w", err) - } - - return resp.Body, err // nolint: wrapcheck -} diff --git a/telegram/api/secret.go b/telegram/api/secret.go deleted file mode 100644 index c5a1137..0000000 --- a/telegram/api/secret.go +++ /dev/null @@ -1,24 +0,0 @@ -package api - -import ( - "fmt" - "io/ioutil" -) - -const secretURL = "https://core.telegram.org/getProxySecret" // nolint: gas - -func Secret() ([]byte, error) { - resp, err := request(secretURL) - if err != nil { - return nil, fmt.Errorf("cannot access telegram server: %w", err) - } - - defer resp.Close() - - secret, err := ioutil.ReadAll(resp) - if err != nil { - return nil, fmt.Errorf("cannot read response: %w", err) - } - - return secret, nil -} diff --git a/telegram/base.go b/telegram/base.go deleted file mode 100644 index 4a6c383..0000000 --- a/telegram/base.go +++ /dev/null @@ -1,91 +0,0 @@ -package telegram - -import ( - "errors" - "math/rand" - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" - "go.uber.org/zap" -) - -type baseTelegram struct { - dialer net.Dialer - logger *zap.SugaredLogger - - secret []byte - v4DefaultDC conntypes.DC - v6DefaultDC conntypes.DC - v4Addresses map[conntypes.DC][]string - v6Addresses map[conntypes.DC][]string -} - -func (b *baseTelegram) Secret() []byte { - return b.secret -} - -func (b *baseTelegram) dial(dc conntypes.DC, - protocol conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) { - for _, addr := range b.getAddresses(dc, protocol) { - conn, err := b.dialer.Dial("tcp", addr) - if err != nil { - b.logger.Infow("Cannot dial to Telegram", "address", addr, "error", err) - - continue - } - - if err := utils.InitTCP(conn, config.C.ProxyReadBuffer(), config.C.ProxyWriteBuffer()); err != nil { - b.logger.Infow("Cannot initialize TCP socket", "address", addr, "error", err) - - continue - } - - return stream.NewTelegramConn(dc, conn), nil - } - - return nil, errors.New("cannot dial to the chosen DC") -} - -func (b *baseTelegram) getAddresses(dc conntypes.DC, protocol conntypes.ConnectionProtocol) []string { - addresses := make([]string, 0, 2) - protos := []conntypes.ConnectionProtocol{ - conntypes.ConnectionProtocolIPv6, - conntypes.ConnectionProtocolIPv4, - } - - if config.C.PreferIP == config.PreferIPv4 { - protos[0], protos[1] = protos[1], protos[0] - } - - for _, proto := range protos { - switch { - case proto&protocol == 0: - case proto&conntypes.ConnectionProtocolIPv6 != 0: - addresses = append(addresses, b.chooseAddress(b.v6Addresses, dc, b.v6DefaultDC)) - case proto&conntypes.ConnectionProtocolIPv4 != 0: - addresses = append(addresses, b.chooseAddress(b.v4Addresses, dc, b.v4DefaultDC)) - } - } - - return addresses -} - -func (b *baseTelegram) chooseAddress(addresses map[conntypes.DC][]string, - dc, defaultDC conntypes.DC) string { - addrs, ok := addresses[dc] - if !ok { - addrs = addresses[defaultDC] - } - - switch { - case len(addrs) == 1: - return addrs[0] - case len(addrs) > 1: - return addrs[rand.Intn(len(addrs))] // nolint: gosec - } - - return "" -} diff --git a/telegram/direct.go b/telegram/direct.go deleted file mode 100644 index 5a7183c..0000000 --- a/telegram/direct.go +++ /dev/null @@ -1,41 +0,0 @@ -package telegram - -import "github.com/9seconds/mtg/conntypes" - -const ( - directV4DefaultIdx conntypes.DC = 1 - directV6DefaultIdx conntypes.DC = 1 -) - -var ( - directV4Addresses = map[conntypes.DC][]string{ - 0: {"149.154.175.50:443"}, - 1: {"149.154.167.51:443"}, - 2: {"149.154.175.100:443"}, - 3: {"149.154.167.91:443"}, - 4: {"149.154.171.5:443"}, - } - directV6Addresses = map[conntypes.DC][]string{ - 0: {"[2001:b28:f23d:f001::a]:443"}, - 1: {"[2001:67c:04e8:f002::a]:443"}, - 2: {"[2001:b28:f23d:f003::a]:443"}, - 3: {"[2001:67c:04e8:f004::a]:443"}, - 4: {"[2001:b28:f23f:f005::a]:443"}, - } -) - -type directTelegram struct { - baseTelegram -} - -func (d *directTelegram) Dial(dc conntypes.DC, - protocol conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) { - switch { - case dc < 0: - dc = -dc - case dc == 0: - dc = conntypes.DCDefaultIdx - } - - return d.baseTelegram.dial(dc-1, conntypes.ConnectionProtocolAny) -} diff --git a/telegram/init.go b/telegram/init.go deleted file mode 100644 index 72da84e..0000000 --- a/telegram/init.go +++ /dev/null @@ -1,48 +0,0 @@ -package telegram - -import ( - "net" - "sync" - "time" - - "go.uber.org/zap" -) - -const telegramDialTimeout = 10 * time.Second - -var ( - Direct Telegram - Middle Telegram - - initOnce sync.Once -) - -func Init() { - initOnce.Do(func() { - logger := zap.S().Named("telegram") - - Direct = &directTelegram{ - baseTelegram: baseTelegram{ - dialer: net.Dialer{Timeout: telegramDialTimeout}, - logger: logger.Named("direct"), - v4DefaultDC: directV4DefaultIdx, - v6DefaultDC: directV6DefaultIdx, - v4Addresses: directV4Addresses, - v6Addresses: directV6Addresses, - }, - } - - tg := &middleTelegram{ - baseTelegram: baseTelegram{ - dialer: net.Dialer{Timeout: telegramDialTimeout}, - logger: logger.Named("middle"), - }, - } - if err := tg.update(); err != nil { - panic(err) - } - go tg.backgroundUpdate() - - Middle = tg - }) -} diff --git a/telegram/interfaces.go b/telegram/interfaces.go deleted file mode 100644 index e419b8b..0000000 --- a/telegram/interfaces.go +++ /dev/null @@ -1,8 +0,0 @@ -package telegram - -import "github.com/9seconds/mtg/conntypes" - -type Telegram interface { - Dial(conntypes.DC, conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) - Secret() []byte -} diff --git a/telegram/middle.go b/telegram/middle.go deleted file mode 100644 index 3683f4d..0000000 --- a/telegram/middle.go +++ /dev/null @@ -1,75 +0,0 @@ -package telegram - -import ( - "fmt" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/telegram/api" - "go.uber.org/zap" -) - -const middleTelegramBackgroundUpdateEvery = time.Hour - -type middleTelegram struct { - baseTelegram - - mutex sync.RWMutex -} - -func (m *middleTelegram) Secret() []byte { - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.baseTelegram.Secret() -} - -func (m *middleTelegram) update() error { - secret, err := api.Secret() - if err != nil { - return fmt.Errorf("cannot fetch secret: %w", err) - } - - v4Addresses, v4DefaultDC, err := api.AddressesV4() - if err != nil { - return fmt.Errorf("cannot fetch addresses for ipv4: %w", err) - } - - v6Addresses, v6DefaultDC, err := api.AddressesV6() - if err != nil { - return fmt.Errorf("cannot fetch addresses for ipv6: %w", err) - } - - m.mutex.Lock() - m.secret = secret - m.v4DefaultDC = v4DefaultDC - m.v6DefaultDC = v6DefaultDC - m.v4Addresses = v4Addresses - m.v6Addresses = v6Addresses - m.mutex.Unlock() - - return nil -} - -func (m *middleTelegram) backgroundUpdate() { - logger := zap.S().Named("telegram") - - for range time.Tick(middleTelegramBackgroundUpdateEvery) { - if err := m.update(); err != nil { - logger.Warnw("Cannot update Telegram proxies", "error", err) - } - } -} - -func (m *middleTelegram) Dial(dc conntypes.DC, - protocol conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) { - if dc == 0 { - dc = conntypes.DCDefaultIdx - } - - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.baseTelegram.dial(dc, protocol) -} diff --git a/tlstypes/client_hello.go b/tlstypes/client_hello.go deleted file mode 100644 index ffb6e87..0000000 --- a/tlstypes/client_hello.go +++ /dev/null @@ -1,86 +0,0 @@ -package tlstypes - -import ( - "bytes" - "crypto/hmac" - "crypto/sha256" - "fmt" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/utils" -) - -type ClientHello struct { - Handshake -} - -func (c ClientHello) Digest() []byte { - dirtyDigest := c.Random - c.Random = [32]byte{} - - rec := Record{ - Type: RecordTypeHandshake, - Version: Version10, - Data: &c, - } - - mac := hmac.New(sha256.New, config.C.Secret) - rec.WriteBytes(mac) - computedDigest := mac.Sum(nil) - - for i := range computedDigest { - computedDigest[i] ^= dirtyDigest[i] - } - - return computedDigest -} - -func ParseClientHello(raw []byte) (*ClientHello, error) { - rv := &ClientHello{} - - rv.Type = HandshakeType(raw[0]) - if rv.Type != HandshakeTypeClient { - return nil, fmt.Errorf("incorrect handshake type %v", rv.Type) - } - - raw = raw[1:] - sizeUint24 := utils.Uint24{} - copy(sizeUint24[:], utils.ReverseBytes(raw[:3])) - size := int(utils.FromUint24(sizeUint24)) - - raw = raw[3:] - if len(raw) != size { - return nil, fmt.Errorf("payload size mismatch (%d != %d)", len(raw), size) - } - - versionRaw := raw[:2] - - switch { - case bytes.Equal(versionRaw, Version13Bytes): - rv.Version = Version13 - case bytes.Equal(versionRaw, Version12Bytes): - rv.Version = Version12 - case bytes.Equal(versionRaw, Version11Bytes): - rv.Version = Version11 - case bytes.Equal(versionRaw, Version10Bytes): - rv.Version = Version10 - default: - return nil, fmt.Errorf("unknown protocol version %v", versionRaw) - } - - raw = raw[2:] - copy(rv.Random[:], raw[:32]) - raw = raw[32:] - - sessionIDLength := int(raw[0]) - raw = raw[1:] - rv.SessionID = make([]byte, sessionIDLength) - copy(rv.SessionID, raw) - raw = raw[sessionIDLength:] - - tail := make([]byte, len(raw)) - copy(tail, raw) - rv.Tail = RawBytes(tail) - - return rv, nil -} diff --git a/tlstypes/consts.go b/tlstypes/consts.go deleted file mode 100644 index 79d5a10..0000000 --- a/tlstypes/consts.go +++ /dev/null @@ -1,90 +0,0 @@ -package tlstypes - -import "io" - -type RecordType uint8 - -const ( - RecordTypeHandshake RecordType = 0x16 - RecordTypeApplicationData RecordType = 0x17 - RecordTypeChangeCipherSpec RecordType = 0x14 -) - -type HandshakeType uint8 - -const ( - HandshakeTypeClient HandshakeType = 0x01 - HandshakeTypeServer HandshakeType = 0x02 -) - -type CipherSuiteType uint8 - -const ( - CipherSuiteType_TLS_AES_128_GCM_SHA256 CipherSuiteType = iota // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_AES_256_GCM_SHA384 // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256 // nolint: stylecheck,golint,revive -) - -func (c CipherSuiteType) Bytes() []byte { - switch c { - case CipherSuiteType_TLS_AES_128_GCM_SHA256: - return CipherSuiteType_TLS_AES_128_GCM_SHA256_Bytes - case CipherSuiteType_TLS_AES_256_GCM_SHA384: - return CipherSuiteType_TLS_AES_256_GCM_SHA384_Bytes - case CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256: - return CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256_Bytes - } - - return CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256_Bytes -} - -type Version uint8 - -func (v Version) Bytes() []byte { - switch v { - case Version13: - return Version13Bytes - case Version12: - return Version12Bytes - case Version11: - return Version11Bytes - case Version10, VersionUnknown: - return Version10Bytes - } - - return Version10Bytes -} - -const ( - VersionUnknown Version = iota - Version10 - Version11 - Version12 - Version13 -) - -var ( - Version10Bytes = []byte{0x03, 0x01} - Version11Bytes = []byte{0x03, 0x02} - Version12Bytes = []byte{0x03, 0x03} - Version13Bytes = []byte{0x03, 0x04} - - CipherSuiteType_TLS_AES_128_GCM_SHA256_Bytes = []byte{0x13, 0x01} // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_AES_256_GCM_SHA384_Bytes = []byte{0x13, 0x02} // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256_Bytes = []byte{0x13, 0x03} // nolint: stylecheck,golint,revive -) - -type Byter interface { - WriteBytes(io.Writer) - Len() int -} - -type RawBytes []byte - -func (r RawBytes) WriteBytes(writer io.Writer) { - writer.Write(r) // nolint: errcheck -} - -func (r RawBytes) Len() int { - return len(r) -} diff --git a/tlstypes/handshake.go b/tlstypes/handshake.go deleted file mode 100644 index 3775b45..0000000 --- a/tlstypes/handshake.go +++ /dev/null @@ -1,43 +0,0 @@ -package tlstypes - -import ( - "bytes" - "io" - - "github.com/9seconds/mtg/utils" -) - -type Handshake struct { - Type HandshakeType - Version Version - Random [32]byte - SessionID []byte - Tail Byter -} - -func (h *Handshake) WriteBytes(writer io.Writer) { - packetBuf := bytes.Buffer{} - - writer.Write([]byte{byte(h.Type)}) // nolint: errcheck - - packetBuf.Write(h.Version.Bytes()) - packetBuf.Write(h.Random[:]) - packetBuf.WriteByte(byte(len(h.SessionID))) - packetBuf.Write(h.SessionID) - h.Tail.WriteBytes(&packetBuf) - - sizeUint24 := utils.ToUint24(uint32(packetBuf.Len())) - sizeUint24Bytes := sizeUint24[:] - sizeUint24Bytes[0], sizeUint24Bytes[2] = sizeUint24Bytes[2], sizeUint24Bytes[0] - - writer.Write(sizeUint24Bytes) // nolint: errcheck - packetBuf.WriteTo(writer) // nolint: errcheck -} - -func (h *Handshake) Len() int { - buf := bytes.Buffer{} - - h.WriteBytes(&buf) - - return buf.Len() -} diff --git a/tlstypes/record.go b/tlstypes/record.go deleted file mode 100644 index 5dae77e..0000000 --- a/tlstypes/record.go +++ /dev/null @@ -1,84 +0,0 @@ -package tlstypes - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" -) - -const recordMaxChunkSize = 16384 + 24 - -type Record struct { - Type RecordType - Version Version - Data Byter -} - -func (r Record) WriteBytes(writer io.Writer) { - writer.Write([]byte{byte(r.Type)}) // nolint: errcheck - writer.Write(r.Version.Bytes()) // nolint: errcheck - binary.Write(writer, binary.BigEndian, uint16(r.Data.Len())) // nolint: errcheck - r.Data.WriteBytes(writer) -} - -func (r Record) Len() int { - return 1 + 2 + 2 + r.Data.Len() -} - -func ReadRecord(reader io.Reader) (Record, error) { - buf := [2]byte{} - rec := Record{} - - if _, err := io.ReadFull(reader, buf[:1]); err != nil { - return rec, fmt.Errorf("cannot read record type: %w", err) - } - - rec.Type = RecordType(buf[0]) - - if _, err := io.ReadFull(reader, buf[:]); err != nil { - return rec, fmt.Errorf("cannot read version: %w", err) - } - - switch { - case bytes.Equal(buf[:], Version13Bytes): - rec.Version = Version13 - case bytes.Equal(buf[:], Version12Bytes): - rec.Version = Version12 - case bytes.Equal(buf[:], Version11Bytes): - rec.Version = Version11 - case bytes.Equal(buf[:], Version10Bytes): - rec.Version = Version10 - } - - if _, err := io.ReadFull(reader, buf[:]); err != nil { - return rec, fmt.Errorf("cannot read data length: %w", err) - } - - data := make([]byte, binary.BigEndian.Uint16(buf[:])) - if _, err := io.ReadFull(reader, data); err != nil { - return rec, fmt.Errorf("cannot read data: %w", err) - } - - rec.Data = RawBytes(data) - - return rec, nil -} - -func MakeRecords(raw []byte) (arr []Record) { - for len(raw) > 0 { - chunkSize := recordMaxChunkSize - if chunkSize > len(raw) { - chunkSize = len(raw) - } - - arr = append(arr, Record{ - Type: RecordTypeApplicationData, - Version: Version12, - Data: RawBytes(raw[:chunkSize]), - }) - raw = raw[chunkSize:] - } - - return -} diff --git a/tlstypes/server_hello.go b/tlstypes/server_hello.go deleted file mode 100644 index efd624c..0000000 --- a/tlstypes/server_hello.go +++ /dev/null @@ -1,97 +0,0 @@ -package tlstypes - -import ( - "bytes" - "crypto/hmac" - "crypto/rand" - "crypto/sha256" - "io" - mrand "math/rand" - - "github.com/9seconds/mtg/config" - "golang.org/x/crypto/curve25519" -) - -type ServerHello struct { - Handshake - - clientHello *ClientHello -} - -func (s ServerHello) WelcomePacket() []byte { - buf := &bytes.Buffer{} - - s.Random = [32]byte{} - rec := Record{ - Type: RecordTypeHandshake, - Version: Version12, - Data: &s, - } - rec.WriteBytes(buf) - - recChangeCipher := Record{ - Type: RecordTypeChangeCipherSpec, - Version: Version12, - Data: RawBytes([]byte{0x01}), - } - recChangeCipher.WriteBytes(buf) - - hostCert := make([]byte, 1024+mrand.Intn(3092)) // nolint: gosec - rand.Read(hostCert) // nolint: errcheck - - recData := Record{ - Type: RecordTypeApplicationData, - Version: Version12, - Data: RawBytes(hostCert), - } - recData.WriteBytes(buf) - - packet := buf.Bytes() - - mac := hmac.New(sha256.New, config.C.Secret) - mac.Write(s.clientHello.Random[:]) // nolint: errcheck - mac.Write(packet) // nolint: errcheck - copy(packet[11:], mac.Sum(nil)) - - return packet -} - -func NewServerHello(clientHello *ClientHello) *ServerHello { - rv := &ServerHello{ - clientHello: clientHello, - } - - rv.Type = HandshakeTypeServer - rv.Version = Version12 - rv.SessionID = make([]byte, len(clientHello.SessionID)) - copy(rv.SessionID, clientHello.SessionID) - - tail := bytes.NewBuffer(CipherSuiteType_TLS_AES_128_GCM_SHA256_Bytes) - tail.WriteByte(0x00) // no compression - makeTLSExtensions(tail) - rv.Tail = RawBytes(tail.Bytes()) - - return rv -} - -func makeTLSExtensions(buf io.Writer) { - buf.Write([]byte{ // nolint: errcheck - 0x00, 0x2e, // 46 bytes of data - 0x00, 0x33, // Extension - Key Share - 0x00, 0x24, // 36 bytes - 0x00, 0x1d, // x25519 curve - 0x00, 0x20, // 32 bytes of key - }) - - var scalar [32]byte - - rand.Read(scalar[:]) // nolint: errcheck - curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) - buf.Write(curve) // nolint: errcheck - - buf.Write([]byte{ // nolint: errcheck - 0x00, 0x2b, // Extension - Supported Versions - 0x00, 0x02, // 2 bytes are following - 0x03, 0x04, // TLS 1.3 - }) -} diff --git a/utils/init_tcp.go b/utils/init_tcp.go deleted file mode 100644 index d491472..0000000 --- a/utils/init_tcp.go +++ /dev/null @@ -1,28 +0,0 @@ -package utils - -import ( - "fmt" - "net" -) - -func InitTCP(conn net.Conn, readBufferSize int, writeBufferSize int) error { - tcpConn := conn.(*net.TCPConn) - - if err := tcpConn.SetNoDelay(true); err != nil { - return fmt.Errorf("cannot set TCP_NO_DELAY: %w", err) - } - - if err := tcpConn.SetReadBuffer(readBufferSize); err != nil { - return fmt.Errorf("cannot set read buffer size: %w", err) - } - - if err := tcpConn.SetWriteBuffer(writeBufferSize); err != nil { - return fmt.Errorf("cannot set write buffer size: %w", err) - } - - if err := tcpConn.SetKeepAlive(true); err != nil { - return fmt.Errorf("cannot enable keep-alive: %w", err) - } - - return nil -} diff --git a/utils/read_full.go b/utils/read_full.go deleted file mode 100644 index 1a86ab1..0000000 --- a/utils/read_full.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import "io" - -const readFullBufferSize = 1024 + 1 // +1 because telegram opreates with blocks mod 4 - -func ReadFull(src io.Reader) (rv []byte, err error) { - buf := make([]byte, readFullBufferSize) - n := readFullBufferSize - - for n == len(buf) { - n, err = src.Read(buf) - if err != nil { - return nil, err // nolint: wrapcheck - } - - rv = append(rv, buf[:n]...) - } - - return rv, nil -} diff --git a/utils/reverse_bytes.go b/utils/reverse_bytes.go deleted file mode 100644 index 87ad8f6..0000000 --- a/utils/reverse_bytes.go +++ /dev/null @@ -1,15 +0,0 @@ -package utils - -// ReverseBytes is a common slice reverser. -func ReverseBytes(data []byte) []byte { - dataLen := len(data) - rv := make([]byte, dataLen) - rv[dataLen/2] = data[dataLen/2] - - for i := dataLen/2 - 1; i >= 0; i-- { - opp := dataLen - i - 1 - rv[i], rv[opp] = data[opp], data[i] - } - - return rv -} diff --git a/utils/stream_cipher.go b/utils/stream_cipher.go deleted file mode 100644 index 44f0b27..0000000 --- a/utils/stream_cipher.go +++ /dev/null @@ -1,12 +0,0 @@ -package utils - -import ( - "crypto/aes" - "crypto/cipher" -) - -func MakeStreamCipher(key, iv []byte) cipher.Stream { - block, _ := aes.NewCipher(key) - - return cipher.NewCTR(block, iv) -} diff --git a/utils/uint24.go b/utils/uint24.go deleted file mode 100644 index 350f3d5..0000000 --- a/utils/uint24.go +++ /dev/null @@ -1,11 +0,0 @@ -package utils - -type Uint24 [3]byte - -func ToUint24(number uint32) Uint24 { - return Uint24{byte(number), byte(number >> 8), byte(number >> 16)} -} - -func FromUint24(number Uint24) uint32 { - return uint32(number[0]) + (uint32(number[1]) << 8) + (uint32(number[2]) << 16) -} diff --git a/wrappers/packet/mtproto_frame.go b/wrappers/packet/mtproto_frame.go deleted file mode 100644 index 805f361..0000000 --- a/wrappers/packet/mtproto_frame.go +++ /dev/null @@ -1,165 +0,0 @@ -package packet - -import ( - "bytes" - "crypto/aes" - "encoding/binary" - "fmt" - "hash/crc32" - "io" - "io/ioutil" - "net" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -const ( - mtprotoFrameMinMessageLength = 12 - mtprotoFrameMaxMessageLength = 16777216 -) - -var mtprotoFramePadding = []byte{0x04, 0x00, 0x00, 0x00} - -// MTProtoFrame is a wrapper which converts written data to the MTProtoFrame. -// The format of the frame: -// -// [ MSGLEN(4) | SEQNO(4) | MSG(...) | CRC32(4) | PADDING(4*x) ] -// -// MSGLEN is the length of the message + len of seqno and msglen. -// SEQNO is the number of frame in the receive/send sequence. If client -// sends a message with SeqNo 18, it has to receive message with SeqNo 18. -// MSG is the data which has to be written -// CRC32 is the CRC32 checksum of MSGLEN + SEQNO + MSG -// PADDING is custom padding schema to complete frame length to such that -// len(frame) % 16 == 0 -type wrapperMtprotoFrame struct { - parent conntypes.StreamReadWriteCloser - logger *zap.SugaredLogger - readSeqNo int32 - writeSeqNo int32 -} - -func (w *wrapperMtprotoFrame) Read() (conntypes.Packet, error) { // nolint: funlen, cyclop - buf := &bytes.Buffer{} - - sum := crc32.NewIEEE() - writer := io.MultiWriter(buf, sum) - - for { - buf.Reset() - sum.Reset() - - if _, err := io.CopyN(writer, w.parent, 4); err != nil { - return nil, fmt.Errorf("cannot read frame padding: %w", err) - } - - if !bytes.Equal(buf.Bytes(), mtprotoFramePadding) { - break - } - } - - messageLength := binary.LittleEndian.Uint32(buf.Bytes()) - w.logger.Debugw("Read MTProto frame", - "messageLength", messageLength, - "sequence_number", w.readSeqNo, - ) - - if messageLength%4 != 0 || messageLength < mtprotoFrameMinMessageLength || - messageLength > mtprotoFrameMaxMessageLength { - return nil, fmt.Errorf("incorrect frame message length %d", messageLength) - } - - buf.Reset() - - if _, err := io.CopyN(writer, w.parent, int64(messageLength)-4-4); err != nil { - return nil, fmt.Errorf("cannot read the message frame: %w", err) - } - - var seqNo int32 - - binary.Read(buf, binary.LittleEndian, &seqNo) // nolint: errcheck - - if seqNo != w.readSeqNo { - return nil, fmt.Errorf("unexpected sequence number %d (wait for %d)", seqNo, w.readSeqNo) - } - - data, _ := ioutil.ReadAll(buf) - buf.Reset() - // write to buf, not to writer. This is because we are going to fetch - // crc32 checksum. - if _, err := io.CopyN(buf, w.parent, 4); err != nil { - return nil, fmt.Errorf("cannot read checksum: %w", err) - } - - checksum := binary.LittleEndian.Uint32(buf.Bytes()) - if checksum != sum.Sum32() { - return nil, fmt.Errorf("CRC32 checksum mismatch. wait for %d, got %d", sum.Sum32(), checksum) - } - - w.logger.Debugw("Read MTProto frame", - "messageLength", messageLength, - "sequence_number", w.readSeqNo, - "dataLength", len(data), - "checksum", checksum, - ) - w.readSeqNo++ - - return data, nil -} - -func (w *wrapperMtprotoFrame) Write(p conntypes.Packet) error { - messageLength := 4 + 4 + len(p) + 4 - paddingLength := (aes.BlockSize - messageLength%aes.BlockSize) % aes.BlockSize - - buf := &bytes.Buffer{} - - binary.Write(buf, binary.LittleEndian, uint32(messageLength)) // nolint: errcheck - binary.Write(buf, binary.LittleEndian, w.writeSeqNo) // nolint: errcheck - buf.Write(p) - - checksum := crc32.ChecksumIEEE(buf.Bytes()) - binary.Write(buf, binary.LittleEndian, checksum) // nolint: errcheck - buf.Write(bytes.Repeat(mtprotoFramePadding, paddingLength/4)) - - w.logger.Debugw("Write MTProto frame", - "length", len(p), - "sequence_number", w.writeSeqNo, - "crc32", checksum, - "frame_length", buf.Len(), - ) - w.writeSeqNo++ - - _, err := w.parent.Write(buf.Bytes()) - - return err // nolint: wrapcheck -} - -func (w *wrapperMtprotoFrame) Close() error { - return w.parent.Close() -} - -func (w *wrapperMtprotoFrame) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperMtprotoFrame) Logger() *zap.SugaredLogger { - return w.logger -} - -func (w *wrapperMtprotoFrame) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperMtprotoFrame) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func NewMtprotoFrame(parent conntypes.StreamReadWriteCloser, seqNo int32) conntypes.PacketReadWriteCloser { - return &wrapperMtprotoFrame{ - parent: parent, - logger: parent.Logger().Named("mtproto-frame"), - readSeqNo: seqNo, - writeSeqNo: seqNo, - } -} diff --git a/wrappers/packetack/client_abridged.go b/wrappers/packetack/client_abridged.go deleted file mode 100644 index 2952de0..0000000 --- a/wrappers/packetack/client_abridged.go +++ /dev/null @@ -1,130 +0,0 @@ -package packetack - -import ( - "bytes" - "fmt" - "io" - "net" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/utils" - "go.uber.org/zap" -) - -const ( - clientAbridgedSmallPacketLength = 0x7f - clientAbridgedQuickAckLength = 0x80 - clientAbridgedLargePacketLength = 16777216 // 256 ^ 3 -) - -type wrapperClientAbridged struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperClientAbridged) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - buf := bytes.Buffer{} - - buf.Grow(1) - - if _, err := io.CopyN(&buf, w.parent, 1); err != nil { - return nil, fmt.Errorf("cannot read message length: %w", err) - } - - msgLength := uint32(buf.Bytes()[0]) - buf.Reset() - - if msgLength >= clientAbridgedQuickAckLength { - acks.Quick = true - msgLength -= clientAbridgedQuickAckLength - } - - if msgLength == clientAbridgedSmallPacketLength { - buf.Grow(3) - - if _, err := io.CopyN(&buf, w.parent, 3); err != nil { - return nil, fmt.Errorf("cannot read correct message length: %w", err) - } - - number := utils.Uint24{} - copy(number[:], buf.Bytes()) - msgLength = utils.FromUint24(number) - } - - msgLength *= 4 - - buf.Reset() - buf.Grow(int(msgLength)) - - if _, err := io.CopyN(&buf, w.parent, int64(msgLength)); err != nil { - return nil, fmt.Errorf("cannot read message: %w", err) - } - - return conntypes.Packet(buf.Bytes()), nil -} - -func (w *wrapperClientAbridged) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - if len(packet)%4 != 0 { - return fmt.Errorf("incorrect packet length %d", len(packet)) - } - - if acks.Simple { - if _, err := w.parent.Write(utils.ReverseBytes(packet)); err != nil { - return fmt.Errorf("cannot send a simpleacked packet: %w", err) - } - - return nil - } - - packetLength := len(packet) / 4 - - switch { - case packetLength < clientAbridgedSmallPacketLength: - data := append([]byte{byte(packetLength)}, packet...) - if _, err := w.parent.Write(data); err != nil { - return fmt.Errorf("cannot send small packet: %w", err) - } - - return nil - case packetLength < clientAbridgedLargePacketLength: - length24 := utils.ToUint24(uint32(packetLength)) - buf := bytes.Buffer{} - - buf.WriteByte(byte(clientAbridgedSmallPacketLength)) - buf.Write(length24[:]) - buf.Write(packet) - - if _, err := w.parent.Write(buf.Bytes()); err != nil { - return fmt.Errorf("cannot send large packet: %w", err) - } - - return nil - } - - return fmt.Errorf("packet is too big: %d", len(packet)) -} - -func (w *wrapperClientAbridged) Close() error { - return w.parent.Close() -} - -func (w *wrapperClientAbridged) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperClientAbridged) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperClientAbridged) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperClientAbridged) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("client-abridged") -} - -func NewClientAbridged(parent conntypes.StreamReadWriteCloser) conntypes.PacketAckFullReadWriteCloser { - return &wrapperClientAbridged{ - parent: parent, - } -} diff --git a/wrappers/packetack/client_intermediate.go b/wrappers/packetack/client_intermediate.go deleted file mode 100644 index ac60fb4..0000000 --- a/wrappers/packetack/client_intermediate.go +++ /dev/null @@ -1,89 +0,0 @@ -package packetack - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "net" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -const clientIntermediateQuickAckLength = 0x80000000 - -type wrapperClientIntermediate struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperClientIntermediate) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - buf := bytes.Buffer{} - - buf.Grow(4) - - if _, err := io.CopyN(&buf, w.parent, 4); err != nil { - return nil, fmt.Errorf("cannot read message length: %w", err) - } - - length := binary.LittleEndian.Uint32(buf.Bytes()) - - if length > clientIntermediateQuickAckLength { - acks.Quick = true - length -= clientIntermediateQuickAckLength - } - - buf.Reset() - buf.Grow(int(length)) - - if _, err := io.CopyN(&buf, w.parent, int64(length)); err != nil { - return nil, fmt.Errorf("cannot read the message: %w", err) - } - - return buf.Bytes(), nil -} - -func (w *wrapperClientIntermediate) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - if acks.Simple { - if _, err := w.parent.Write(packet); err != nil { - return fmt.Errorf("cannot send simpleacked packet: %w", err) - } - - return nil - } - - length := [4]byte{} - binary.LittleEndian.PutUint32(length[:], uint32(len(packet))) - - if _, err := w.parent.Write(append(length[:], packet...)); err != nil { - return fmt.Errorf("cannot send packet: %w", err) - } - - return nil -} - -func (w *wrapperClientIntermediate) Close() error { - return w.parent.Close() -} - -func (w *wrapperClientIntermediate) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperClientIntermediate) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperClientIntermediate) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperClientIntermediate) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("client-intermediate") -} - -func NewClientIntermediate(parent conntypes.StreamReadWriteCloser) conntypes.PacketAckFullReadWriteCloser { - return &wrapperClientIntermediate{ - parent: parent, - } -} diff --git a/wrappers/packetack/client_intermediate_secure.go b/wrappers/packetack/client_intermediate_secure.go deleted file mode 100644 index 1151a37..0000000 --- a/wrappers/packetack/client_intermediate_secure.go +++ /dev/null @@ -1,63 +0,0 @@ -package packetack - -import ( - "bytes" - "encoding/binary" - "fmt" - "math/rand" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type wrapperClientIntermediateSecure struct { - wrapperClientIntermediate -} - -func (w *wrapperClientIntermediateSecure) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - data, err := w.wrapperClientIntermediate.Read(acks) - if err != nil { - return nil, err - } - - length := len(data) - (len(data) % 4) - - return data[:length], nil -} - -func (w *wrapperClientIntermediateSecure) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - if acks.Simple { - if _, err := w.parent.Write(packet); err != nil { - return fmt.Errorf("cannot send simpleacked packet: %w", err) - } - - return nil - } - - buf := &bytes.Buffer{} - paddingLength := rand.Intn(4) // nolint: gosec - - buf.Grow(4 + len(packet) + paddingLength) - - binary.Write(buf, binary.LittleEndian, uint32(len(packet)+paddingLength)) // nolint: errcheck - buf.Write(packet) - buf.Write(make([]byte, paddingLength)) - - if _, err := w.parent.Write(buf.Bytes()); err != nil { - return fmt.Errorf("cannot send packet: %w", err) - } - - return nil -} - -func (w *wrapperClientIntermediateSecure) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("client-intermediate-secure") -} - -func NewClientIntermediateSecure(parent conntypes.StreamReadWriteCloser) conntypes.PacketAckFullReadWriteCloser { - return &wrapperClientIntermediateSecure{ - wrapperClientIntermediate: wrapperClientIntermediate{ - parent: parent, - }, - } -} diff --git a/wrappers/packetack/proxy.go b/wrappers/packetack/proxy.go deleted file mode 100644 index facbaa4..0000000 --- a/wrappers/packetack/proxy.go +++ /dev/null @@ -1,108 +0,0 @@ -package packetack - -import ( - "bytes" - "encoding/binary" - "fmt" - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/hub" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" -) - -type wrapperProxy struct { - request *protocol.TelegramRequest - proxy *hub.ProxyConn - clientIPPort []byte - ourIPPort []byte - flags rpc.ProxyRequestFlags -} - -func (w *wrapperProxy) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - buf := bytes.Buffer{} - flags := w.flags - - if acks.Quick { - flags |= rpc.ProxyRequestFlagsQuickAck - } - - if bytes.HasPrefix(packet, rpc.ProxyRequestFlagsEncryptedPrefix[:]) { - flags |= rpc.ProxyRequestFlagsEncrypted - } - - buf.Write(rpc.TagProxyRequest) - buf.Write(flags.Bytes()) - buf.Write(w.request.ConnID[:]) - buf.Write(w.clientIPPort) - buf.Write(w.ourIPPort) - buf.Write(rpc.ProxyRequestExtraSize) - buf.Write(rpc.ProxyRequestProxyTag) - buf.WriteByte(byte(len(config.C.AdTag))) - buf.Write(config.C.AdTag) - buf.Write(make([]byte, (4-buf.Len()%4)%4)) - buf.Grow(len(packet)) - buf.Write(packet) - - return w.proxy.Write(buf.Bytes()) -} - -func (w *wrapperProxy) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - resp, err := w.proxy.Read() - if err != nil { - return nil, fmt.Errorf("cannot read a response: %w", err) - } - - if resp.Type == rpc.ProxyResponseTypeSimpleAck { - acks.Simple = true - } - - return resp.Payload, nil -} - -func (w *wrapperProxy) Close() error { - w.proxy.Close() - - return nil -} - -func NewProxy(request *protocol.TelegramRequest) (conntypes.PacketAckReadWriteCloser, error) { - flags := rpc.ProxyRequestFlagsHasAdTag | rpc.ProxyRequestFlagsMagic | rpc.ProxyRequestFlagsExtMode2 - - switch request.ClientProtocol.ConnectionType() { - case conntypes.ConnectionTypeAbridged: - flags |= rpc.ProxyRequestFlagsAbdridged - case conntypes.ConnectionTypeIntermediate: - flags |= rpc.ProxyRequestFlagsIntermediate - case conntypes.ConnectionTypeSecure: - flags |= rpc.ProxyRequestFlagsIntermediate | rpc.ProxyRequestFlagsPad - case conntypes.ConnectionTypeUnknown: - panic("unknown connection type") - } - - proxy, err := hub.Hub.Register(request) - if err != nil { - return nil, fmt.Errorf("cannot make a new proxy wrapper: %w", err) - } - - return &wrapperProxy{ - flags: flags, - request: request, - proxy: proxy, - clientIPPort: proxyGetIPPort(request.ClientConn.RemoteAddr()), - ourIPPort: proxyGetIPPort(request.ClientConn.LocalAddr()), - }, nil -} - -func proxyGetIPPort(addr *net.TCPAddr) []byte { - rv := [16 + 4]byte{} - port := [4]byte{} - - copy(rv[:16], addr.IP.To16()) - binary.LittleEndian.PutUint32(port[:], uint32(addr.Port)) - copy(rv[16:], port[:]) - - return rv[:] -} diff --git a/wrappers/rwc/ping.go b/wrappers/rwc/ping.go deleted file mode 100644 index 548bef8..0000000 --- a/wrappers/rwc/ping.go +++ /dev/null @@ -1,48 +0,0 @@ -package rwc - -import ( - "context" - "io" -) - -type wrapperPing struct { - parent io.ReadWriteCloser - ctx context.Context - channelPing chan<- struct{} -} - -func (w *wrapperPing) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - if err == nil { - select { - case <-w.ctx.Done(): - case w.channelPing <- struct{}{}: - } - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperPing) Write(p []byte) (int, error) { - n, err := w.parent.Write(p) - if err == nil { - select { - case <-w.ctx.Done(): - case w.channelPing <- struct{}{}: - } - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperPing) Close() error { - return w.parent.Close() -} - -func NewPing(ctx context.Context, parent io.ReadWriteCloser, channelPing chan<- struct{}) io.ReadWriteCloser { - return &wrapperPing{ - parent: parent, - ctx: ctx, - channelPing: channelPing, - } -} diff --git a/wrappers/stream/base.go b/wrappers/stream/base.go deleted file mode 100644 index 0fd50ca..0000000 --- a/wrappers/stream/base.go +++ /dev/null @@ -1,21 +0,0 @@ -package stream - -import ( - "net" - - "github.com/9seconds/mtg/conntypes" -) - -func NewClientConn(parent net.Conn, connID conntypes.ConnID) conntypes.StreamReadWriteCloser { - conn := newConn(parent, connID, connPurposeClient) - conn = NewTrafficStats(conn) - - return conn -} - -func NewTelegramConn(dc conntypes.DC, parent net.Conn) conntypes.StreamReadWriteCloser { - conn := newConn(parent, conntypes.ConnID{}, connPurposeTelegram) - conn = NewTelegramStats(dc, conn) - - return conn -} diff --git a/wrappers/stream/blockcipher.go b/wrappers/stream/blockcipher.go deleted file mode 100644 index 6503e4f..0000000 --- a/wrappers/stream/blockcipher.go +++ /dev/null @@ -1,96 +0,0 @@ -package stream - -import ( - "crypto/aes" - "crypto/cipher" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/utils" - "go.uber.org/zap" -) - -type wrapperBlockCipher struct { - bufferedReader - - parent conntypes.StreamReadWriteCloser - encryptor cipher.BlockMode - decryptor cipher.BlockMode -} - -func (w *wrapperBlockCipher) Write(p []byte) (int, error) { - encrypted, err := w.encrypt(p) - if err != nil { - return 0, err - } - - return w.parent.Write(encrypted) -} - -func (w *wrapperBlockCipher) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - encrypted, err := w.encrypt(p) - if err != nil { - return 0, err - } - - return w.parent.WriteTimeout(encrypted, timeout) -} - -func (w *wrapperBlockCipher) encrypt(p []byte) ([]byte, error) { - if len(p)%aes.BlockSize > 0 { - return nil, fmt.Errorf("incorrect block size %d", len(p)) - } - - encrypted := make([]byte, len(p)) - w.encryptor.CryptBlocks(encrypted, p) - - return encrypted, nil -} - -func (w *wrapperBlockCipher) Close() error { - return w.parent.Close() -} - -func (w *wrapperBlockCipher) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperBlockCipher) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("block-cipher") -} - -func (w *wrapperBlockCipher) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperBlockCipher) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func newBlockCipher(parent conntypes.StreamReadWriteCloser, - encryptor, decryptor cipher.BlockMode) conntypes.StreamReadWriteCloser { - cipher := &wrapperBlockCipher{ - parent: parent, - encryptor: encryptor, - decryptor: decryptor, - } - - cipher.readFunc = func() ([]byte, error) { - var currentBuffer []byte - for len(currentBuffer) == 0 || len(currentBuffer)%aes.BlockSize != 0 { - rv, err := utils.ReadFull(cipher.parent) - if err != nil { - return nil, fmt.Errorf("cannot read data: %w", err) - } - - currentBuffer = append(currentBuffer, rv...) - } - cipher.decryptor.CryptBlocks(currentBuffer, currentBuffer) - - return currentBuffer, nil - } - - return cipher -} diff --git a/wrappers/stream/buffered_reader.go b/wrappers/stream/buffered_reader.go deleted file mode 100644 index 6a551f7..0000000 --- a/wrappers/stream/buffered_reader.go +++ /dev/null @@ -1,44 +0,0 @@ -package stream - -import ( - "bytes" - "time" -) - -type bufferedReaderReadFunc func() ([]byte, error) - -type bufferedReader struct { - buf bytes.Buffer - readFunc bufferedReaderReadFunc -} - -func (b *bufferedReader) Read(p []byte) (int, error) { - if b.buf.Len() > 0 { - return b.flush(p) - } - - res, err := b.readFunc() - if err != nil { - return 0, err - } - - b.buf.Write(res) - - return b.flush(p) -} - -func (b *bufferedReader) ReadTimeout(p []byte, _ time.Duration) (int, error) { - return b.Read(p) -} - -func (b *bufferedReader) flush(p []byte) (int, error) { - if b.buf.Len() > len(p) { - return b.buf.Read(p) - } - - sizeToReturn := b.buf.Len() - copy(p, b.buf.Bytes()) - b.buf.Reset() - - return sizeToReturn, nil -} diff --git a/wrappers/stream/conn.go b/wrappers/stream/conn.go deleted file mode 100644 index d55c55c..0000000 --- a/wrappers/stream/conn.go +++ /dev/null @@ -1,123 +0,0 @@ -package stream - -import ( - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type connPurpose uint8 - -const ( - connPurposeClient connPurpose = 1 << iota - connPurposeTelegram -) - -type wrapperConn struct { - parent net.Conn - connID conntypes.ConnID - logger *zap.SugaredLogger - localAddr *net.TCPAddr - remoteAddr *net.TCPAddr -} - -func (w *wrapperConn) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - if err := w.parent.SetWriteDeadline(time.Now().Add(timeout)); err != nil { - w.Close() - - return 0, fmt.Errorf("cannot set write deadline to the socket: %w", err) - } - - return w.Write(p) -} - -func (w *wrapperConn) Write(p []byte) (int, error) { - n, err := w.parent.Write(p) - w.logger.Debugw("write to stream", "bytes", n, "error", err) - - if err != nil { - w.Close() - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperConn) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - if err := w.parent.SetReadDeadline(time.Now().Add(timeout)); err != nil { - w.Close() - - return 0, fmt.Errorf("cannot set read deadline to the socket: %w", err) - } - - return w.Read(p) -} - -func (w *wrapperConn) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - w.logger.Debugw("Read from stream", "bytes", n, "error", err) - - if err != nil { - w.Close() - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperConn) Close() error { - w.logger.Debugw("Close connection") - - return w.parent.Close() -} - -func (w *wrapperConn) Conn() net.Conn { - return w.parent -} - -func (w *wrapperConn) Logger() *zap.SugaredLogger { - return w.logger -} - -func (w *wrapperConn) LocalAddr() *net.TCPAddr { - return w.localAddr -} - -func (w *wrapperConn) RemoteAddr() *net.TCPAddr { - return w.remoteAddr -} - -func newConn(parent net.Conn, - connID conntypes.ConnID, - purpose connPurpose) conntypes.StreamReadWriteCloser { - localAddr := *parent.LocalAddr().(*net.TCPAddr) - - if parent.RemoteAddr().(*net.TCPAddr).IP.To4() != nil { - if config.C.PublicIPv4.IP != nil { - localAddr.IP = config.C.PublicIPv4.IP - } - } else if config.C.PublicIPv6.IP != nil { - localAddr.IP = config.C.PublicIPv6.IP - } - - logger := zap.S().With( - "local_address", localAddr, - "remote_address", parent.RemoteAddr(), - ).Named("conn") - - if purpose == connPurposeClient { - logger = logger.Named("client").With("connection_id", connID.String()) - } else { - logger = logger.Named("telegram") - } - - return &wrapperConn{ - parent: parent, - connID: connID, - logger: logger, - remoteAddr: parent.RemoteAddr().(*net.TCPAddr), - localAddr: &localAddr, - } -} diff --git a/wrappers/stream/ctx.go b/wrappers/stream/ctx.go deleted file mode 100644 index 86e7359..0000000 --- a/wrappers/stream/ctx.go +++ /dev/null @@ -1,93 +0,0 @@ -package stream - -import ( - "context" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type wrapperCtx struct { - parent conntypes.StreamReadWriteCloser - ctx context.Context - cancel context.CancelFunc -} - -func (w *wrapperCtx) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.WriteTimeout(p, timeout) - } -} - -func (w *wrapperCtx) Write(p []byte) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.Write(p) - } -} - -func (w *wrapperCtx) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.ReadTimeout(p, timeout) - } -} - -func (w *wrapperCtx) Read(p []byte) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.Read(p) - } -} - -func (w *wrapperCtx) Close() error { - w.cancel() - - return w.parent.Close() -} - -func (w *wrapperCtx) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperCtx) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("ctx") -} - -func (w *wrapperCtx) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperCtx) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func NewCtx(ctx context.Context, - cancel context.CancelFunc, - parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - return &wrapperCtx{ - parent: parent, - ctx: ctx, - cancel: cancel, - } -} diff --git a/wrappers/stream/faketls.go b/wrappers/stream/faketls.go deleted file mode 100644 index 17a1fe6..0000000 --- a/wrappers/stream/faketls.go +++ /dev/null @@ -1,107 +0,0 @@ -package stream - -import ( - "bytes" - "errors" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/tlstypes" - "go.uber.org/zap" -) - -type wrapperFakeTLS struct { - bufferedReader - - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperFakeTLS) Write(p []byte) (int, error) { - return w.write(p, func(b []byte) (int, error) { - return w.parent.Write(b) - }) -} - -func (w *wrapperFakeTLS) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - startTime := time.Now() - - return w.write(p, func(b []byte) (int, error) { - elapsed := time.Since(startTime) - if elapsed > timeout { - return w.parent.WriteTimeout(b, timeout-elapsed) - } - - return 0, errors.New("timeout") - }) -} - -func (w *wrapperFakeTLS) write(p []byte, writeFunc func([]byte) (int, error)) (int, error) { - sum := 0 - buf := bytes.Buffer{} - - for _, v := range tlstypes.MakeRecords(p) { - buf.Reset() - v.WriteBytes(&buf) - - _, err := writeFunc(buf.Bytes()) - if err != nil { - return sum, err - } - - sum += v.Data.Len() - } - - return sum, nil -} - -func (w *wrapperFakeTLS) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperFakeTLS) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("faketls") -} - -func (w *wrapperFakeTLS) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperFakeTLS) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperFakeTLS) Close() error { - return w.parent.Close() -} - -func NewFakeTLS(socket conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - faketls := &wrapperFakeTLS{ - parent: socket, - } - - faketls.readFunc = func() ([]byte, error) { - for { - rec, err := tlstypes.ReadRecord(faketls.parent) - if err != nil { - return nil, err // nolint: wrapcheck - } - - switch rec.Type { - case tlstypes.RecordTypeChangeCipherSpec: - case tlstypes.RecordTypeApplicationData: - buf := &bytes.Buffer{} - rec.Data.WriteBytes(buf) - - return buf.Bytes(), nil - case tlstypes.RecordTypeHandshake: - return nil, errors.New("unsupported record type handshake") - default: - return nil, fmt.Errorf("unsupported record type %v", rec.Type) - } - } - } - - return faketls -} diff --git a/wrappers/stream/mtproto_cipher.go b/wrappers/stream/mtproto_cipher.go deleted file mode 100644 index 5a59f7c..0000000 --- a/wrappers/stream/mtproto_cipher.go +++ /dev/null @@ -1,117 +0,0 @@ -package stream - -import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "crypto/md5" - "crypto/sha1" - "encoding/binary" - "net" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/utils" -) - -type mtprotoCipherPurpose uint8 - -const ( - mtprotoCipherPurposeClient mtprotoCipherPurpose = iota - mtprotoCipherPurposeServer -) - -var mtprotoEmptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} - -func NewMiddleProxyCipher(parent conntypes.StreamReadWriteCloser, - req *rpc.NonceRequest, - resp *rpc.NonceResponse, - secret []byte) conntypes.StreamReadWriteCloser { - localAddr := parent.LocalAddr() - remoteAddr := parent.RemoteAddr() - - encKey, encIV := mtprotoDeriveKeys(mtprotoCipherPurposeClient, - req, - resp, - localAddr, - remoteAddr, - secret) - decKey, decIV := mtprotoDeriveKeys(mtprotoCipherPurposeServer, - req, - resp, - localAddr, - remoteAddr, - secret) - - enc, _ := mtprotoMakeEncrypterDecrypter(encKey, encIV) - _, dec := mtprotoMakeEncrypterDecrypter(decKey, decIV) - - return newBlockCipher(parent, enc, dec) -} - -func mtprotoDeriveKeys(purpose mtprotoCipherPurpose, - req *rpc.NonceRequest, - resp *rpc.NonceResponse, - client, remote *net.TCPAddr, - secret []byte) ([]byte, []byte) { - message := bytes.Buffer{} - - message.Write(resp.Nonce) - message.Write(req.Nonce) - message.Write(req.CryptoTS) - - clientIPv4 := mtprotoEmptyIP[:] - serverIPv4 := mtprotoEmptyIP[:] - - if client.IP.To4() != nil { - clientIPv4 = utils.ReverseBytes(client.IP.To4()) - serverIPv4 = utils.ReverseBytes(remote.IP.To4()) - } - - message.Write(serverIPv4) - - var port [2]byte - - binary.LittleEndian.PutUint16(port[:], uint16(client.Port)) - message.Write(port[:]) - - switch purpose { - case mtprotoCipherPurposeClient: - message.WriteString("CLIENT") - case mtprotoCipherPurposeServer: - message.WriteString("SERVER") - default: - panic("Unexpected cipher purpose") - } - - message.Write(clientIPv4) - binary.LittleEndian.PutUint16(port[:], uint16(remote.Port)) - message.Write(port[:]) - message.Write(secret) - message.Write(resp.Nonce) - - if client.IP.To4() == nil { - message.Write(client.IP.To16()) - message.Write(remote.IP.To16()) - } - - message.Write(req.Nonce) - - data := message.Bytes() - md5sum := md5.Sum(data[1:]) // nolint: gas - sha1sum := sha1.Sum(data) // nolint: gosec - - key := append(md5sum[:12], sha1sum[:]...) - iv := md5.Sum(data[2:]) // nolint: gas - - return key, iv[:] -} - -func mtprotoMakeEncrypterDecrypter(key, iv []byte) (cipher.BlockMode, cipher.BlockMode) { - block, err := aes.NewCipher(key) - if err != nil { - panic(err) - } - - return cipher.NewCBCEncrypter(block, iv), cipher.NewCBCDecrypter(block, iv) -} diff --git a/wrappers/stream/obfuscated2.go b/wrappers/stream/obfuscated2.go deleted file mode 100644 index da2199a..0000000 --- a/wrappers/stream/obfuscated2.go +++ /dev/null @@ -1,93 +0,0 @@ -package stream - -import ( - "bytes" - "crypto/cipher" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type wrapperObfuscated2 struct { - encryptor cipher.Stream - decryptor cipher.Stream - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperObfuscated2) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - n, err := w.parent.ReadTimeout(p, timeout) - if err != nil { - return 0, fmt.Errorf("cannot read stream ciphered data: %w", err) - } - - w.decryptor.XORKeyStream(p, p[:n]) - - return n, nil -} - -func (w *wrapperObfuscated2) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - if err != nil { - return n, err // nolint: wrapcheck - } - - w.decryptor.XORKeyStream(p, p[:n]) - - return n, nil -} - -func (w *wrapperObfuscated2) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - buffer := bytes.Buffer{} - - buffer.Write(p) - - buf := buffer.Bytes() - - w.encryptor.XORKeyStream(buf, buf) - - return w.parent.WriteTimeout(buf, timeout) -} - -func (w *wrapperObfuscated2) Write(p []byte) (int, error) { - buffer := bytes.Buffer{} - - buffer.Write(p) - - buf := buffer.Bytes() - - w.encryptor.XORKeyStream(buf, buf) - - return w.parent.Write(buf) -} - -func (w *wrapperObfuscated2) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperObfuscated2) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("obfuscated2") -} - -func (w *wrapperObfuscated2) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperObfuscated2) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperObfuscated2) Close() error { - return w.parent.Close() -} - -func NewObfuscated2(socket conntypes.StreamReadWriteCloser, - encryptor, decryptor cipher.Stream) conntypes.StreamReadWriteCloser { - return &wrapperObfuscated2{ - parent: socket, - encryptor: encryptor, - decryptor: decryptor, - } -} diff --git a/wrappers/stream/rewind.go b/wrappers/stream/rewind.go deleted file mode 100644 index d8d7425..0000000 --- a/wrappers/stream/rewind.go +++ /dev/null @@ -1,83 +0,0 @@ -package stream - -import ( - "bytes" - "io" - "net" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type ReadWriteCloseRewinder interface { - conntypes.StreamReadWriteCloser - Rewind() -} - -type wrapperRewind struct { - parent conntypes.StreamReadWriteCloser - activeReader io.Reader - buf bytes.Buffer - mutex sync.Mutex -} - -func (w *wrapperRewind) Write(p []byte) (int, error) { - return w.parent.Write(p) -} - -func (w *wrapperRewind) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.WriteTimeout(p, timeout) -} - -func (w *wrapperRewind) Read(p []byte) (int, error) { - w.mutex.Lock() - defer w.mutex.Unlock() - - return w.activeReader.Read(p) -} - -func (w *wrapperRewind) ReadTimeout(p []byte, _ time.Duration) (int, error) { - w.mutex.Lock() - defer w.mutex.Unlock() - - return w.activeReader.Read(p) -} - -func (w *wrapperRewind) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperRewind) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("rewinded") -} - -func (w *wrapperRewind) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperRewind) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperRewind) Close() error { - w.buf.Reset() - - return w.parent.Close() -} - -func (w *wrapperRewind) Rewind() { - w.mutex.Lock() - w.activeReader = io.MultiReader(&w.buf, w.parent) - w.mutex.Unlock() -} - -func NewRewind(parent conntypes.StreamReadWriteCloser) ReadWriteCloseRewinder { - rv := &wrapperRewind{ - parent: parent, - } - rv.activeReader = io.TeeReader(parent, &rv.buf) - - return rv -} diff --git a/wrappers/stream/stats_telegram.go b/wrappers/stream/stats_telegram.go deleted file mode 100644 index 9fe6d9e..0000000 --- a/wrappers/stream/stats_telegram.go +++ /dev/null @@ -1,71 +0,0 @@ -package stream - -import ( - "net" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/stats" - "go.uber.org/zap" -) - -type wrapperTelegramStats struct { - parent conntypes.StreamReadWriteCloser - dc conntypes.DC - once sync.Once -} - -func (w *wrapperTelegramStats) Write(p []byte) (int, error) { - return w.parent.Write(p) -} - -func (w *wrapperTelegramStats) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.WriteTimeout(p, timeout) -} - -func (w *wrapperTelegramStats) Read(p []byte) (int, error) { - return w.parent.Read(p) -} - -func (w *wrapperTelegramStats) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.ReadTimeout(p, timeout) -} - -func (w *wrapperTelegramStats) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperTelegramStats) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("stats-telegram") -} - -func (w *wrapperTelegramStats) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperTelegramStats) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperTelegramStats) Close() error { - var err error - - w.once.Do(func() { - err = w.parent.Close() - stats.Stats.TelegramDisconnected(w.dc, w.RemoteAddr()) - }) - - return err // nolint: wrapcheck -} - -func NewTelegramStats(dc conntypes.DC, parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - conn := &wrapperTelegramStats{ - parent: parent, - dc: dc, - } - - stats.Stats.TelegramConnected(dc, parent.RemoteAddr()) - - return conn -} diff --git a/wrappers/stream/stats_traffic.go b/wrappers/stream/stats_traffic.go deleted file mode 100644 index cc7d224..0000000 --- a/wrappers/stream/stats_traffic.go +++ /dev/null @@ -1,66 +0,0 @@ -package stream - -import ( - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/stats" - "go.uber.org/zap" -) - -type wrapperTrafficStats struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperTrafficStats) Write(p []byte) (int, error) { - n, err := w.parent.Write(p) - stats.Stats.EgressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - n, err := w.parent.WriteTimeout(p, timeout) - stats.Stats.EgressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - stats.Stats.IngressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - n, err := w.parent.ReadTimeout(p, timeout) - stats.Stats.IngressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperTrafficStats) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("stats-traffic") -} - -func (w *wrapperTrafficStats) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperTrafficStats) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperTrafficStats) Close() error { - return w.parent.Close() -} - -func NewTrafficStats(parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - return &wrapperTrafficStats{parent} -} diff --git a/wrappers/stream/timeout.go b/wrappers/stream/timeout.go deleted file mode 100644 index 2447fe2..0000000 --- a/wrappers/stream/timeout.go +++ /dev/null @@ -1,60 +0,0 @@ -package stream - -import ( - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -const ( - timeoutRead = 2 * time.Minute - timeoutWrite = 2 * time.Minute -) - -type wrapperTimeout struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperTimeout) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.WriteTimeout(p, timeout) -} - -func (w *wrapperTimeout) Write(p []byte) (int, error) { - return w.parent.WriteTimeout(p, timeoutWrite) -} - -func (w *wrapperTimeout) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.ReadTimeout(p, timeout) -} - -func (w *wrapperTimeout) Read(p []byte) (int, error) { - return w.parent.ReadTimeout(p, timeoutRead) -} - -func (w *wrapperTimeout) Close() error { - return w.parent.Close() -} - -func (w *wrapperTimeout) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperTimeout) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("timeout") -} - -func (w *wrapperTimeout) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperTimeout) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func NewTimeout(parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - return &wrapperTimeout{ - parent: parent, - } -}