From baee322cd746f460f2f6626cbce4c34a9476250c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 4 Mar 2021 10:15:34 +0300 Subject: [PATCH 001/206] Reset a project --- antireplay/cache.go | 36 -- antireplay/init.go | 30 -- antireplay/nilcache.go | 8 - cli/generate.go | 26 - cli/proxy.go | 101 ---- cli/utils.go | 43 -- config/config.go | 312 ------------ config/global_ips.go | 78 --- config/urls.go | 99 ---- conntypes/acks.go | 6 - conntypes/dc.go | 5 - conntypes/id.go | 24 - conntypes/packet.go | 3 - conntypes/protocol.go | 22 - conntypes/type.go | 29 -- conntypes/wrap_interfaces.go | 14 - conntypes/wrap_packet_ack_interfaces.go | 41 -- conntypes/wrap_packet_interfaces.go | 51 -- conntypes/wrap_stream_interfaces.go | 56 --- faketls/client_protocol.go | 122 ----- faketls/cloak.go | 73 --- faketls/consts.go | 30 -- go.mod | 26 +- go.sum | 472 ------------------ hub/connection.go | 172 ------- hub/connection_list.go | 70 --- hub/hub.go | 40 -- hub/init.go | 24 - hub/interface.go | 7 - hub/mux.go | 90 ---- hub/proxy_conn.go | 79 --- main.go | 178 ------- mtproto/protocol.go | 102 ---- mtproto/rpc/consts.go | 28 -- mtproto/rpc/handshake_request.go | 5 - mtproto/rpc/handshake_response.go | 54 -- mtproto/rpc/nonce_request.go | 50 -- mtproto/rpc/nonce_response.go | 60 --- mtproto/rpc/proxy_flags.go | 66 --- mtproto/rpc/proxy_response.go | 53 -- ntp/ntp.go | 57 --- obfuscated2/client_protocol.go | 114 ----- obfuscated2/frame.go | 54 -- obfuscated2/telegram_protocol.go | 68 --- protocol/interfaces.go | 12 - protocol/request.go | 17 - proxy/direct.go | 49 -- proxy/middle.go | 70 --- proxy/proxy.go | 100 ---- run.sh | 68 --- stats/interfaces.go | 60 --- stats/multi_stats.go | 69 --- stats/stats.go | 41 -- stats/stats_prometheus.go | 156 ------ stats/stats_statsd.go | 204 -------- telegram/api/addresses.go | 109 ---- telegram/api/api.go | 44 -- telegram/api/secret.go | 24 - telegram/base.go | 91 ---- telegram/direct.go | 41 -- telegram/init.go | 48 -- telegram/interfaces.go | 8 - telegram/middle.go | 75 --- tlstypes/client_hello.go | 86 ---- tlstypes/consts.go | 90 ---- tlstypes/handshake.go | 43 -- tlstypes/record.go | 84 ---- tlstypes/server_hello.go | 97 ---- utils/init_tcp.go | 28 -- utils/read_full.go | 21 - utils/reverse_bytes.go | 15 - utils/rlimit.go | 24 - utils/rlimit_windows.go | 7 - utils/signal_context.go | 25 - utils/signal_context_windows.go | 23 - utils/stream_cipher.go | 12 - utils/uint24.go | 11 - wrappers/packet/mtproto_frame.go | 165 ------ wrappers/packetack/client_abridged.go | 130 ----- wrappers/packetack/client_intermediate.go | 89 ---- .../packetack/client_intermediate_secure.go | 63 --- wrappers/packetack/proxy.go | 108 ---- wrappers/rwc/ping.go | 48 -- wrappers/stream/base.go | 21 - wrappers/stream/blockcipher.go | 96 ---- wrappers/stream/buffered_reader.go | 44 -- wrappers/stream/conn.go | 123 ----- wrappers/stream/ctx.go | 93 ---- wrappers/stream/faketls.go | 107 ---- wrappers/stream/mtproto_cipher.go | 117 ----- wrappers/stream/obfuscated2.go | 93 ---- wrappers/stream/rewind.go | 83 --- wrappers/stream/stats_telegram.go | 71 --- wrappers/stream/stats_traffic.go | 66 --- wrappers/stream/timeout.go | 60 --- 95 files changed, 2 insertions(+), 6505 deletions(-) delete mode 100644 antireplay/cache.go delete mode 100644 antireplay/init.go delete mode 100644 antireplay/nilcache.go delete mode 100644 cli/generate.go delete mode 100644 cli/proxy.go delete mode 100644 cli/utils.go delete mode 100644 config/config.go delete mode 100644 config/global_ips.go delete mode 100644 config/urls.go delete mode 100644 conntypes/acks.go delete mode 100644 conntypes/dc.go delete mode 100644 conntypes/id.go delete mode 100644 conntypes/packet.go delete mode 100644 conntypes/protocol.go delete mode 100644 conntypes/type.go delete mode 100644 conntypes/wrap_interfaces.go delete mode 100644 conntypes/wrap_packet_ack_interfaces.go delete mode 100644 conntypes/wrap_packet_interfaces.go delete mode 100644 conntypes/wrap_stream_interfaces.go delete mode 100644 faketls/client_protocol.go delete mode 100644 faketls/cloak.go delete mode 100644 faketls/consts.go delete mode 100644 hub/connection.go delete mode 100644 hub/connection_list.go delete mode 100644 hub/hub.go delete mode 100644 hub/init.go delete mode 100644 hub/interface.go delete mode 100644 hub/mux.go delete mode 100644 hub/proxy_conn.go delete mode 100644 mtproto/protocol.go delete mode 100644 mtproto/rpc/consts.go delete mode 100644 mtproto/rpc/handshake_request.go delete mode 100644 mtproto/rpc/handshake_response.go delete mode 100644 mtproto/rpc/nonce_request.go delete mode 100644 mtproto/rpc/nonce_response.go delete mode 100644 mtproto/rpc/proxy_flags.go delete mode 100644 mtproto/rpc/proxy_response.go delete mode 100644 ntp/ntp.go delete mode 100644 obfuscated2/client_protocol.go delete mode 100644 obfuscated2/frame.go delete mode 100644 obfuscated2/telegram_protocol.go delete mode 100644 protocol/interfaces.go delete mode 100644 protocol/request.go delete mode 100644 proxy/direct.go delete mode 100644 proxy/middle.go delete mode 100644 proxy/proxy.go delete mode 100755 run.sh delete mode 100644 stats/interfaces.go delete mode 100644 stats/multi_stats.go delete mode 100644 stats/stats.go delete mode 100644 stats/stats_prometheus.go delete mode 100644 stats/stats_statsd.go delete mode 100644 telegram/api/addresses.go delete mode 100644 telegram/api/api.go delete mode 100644 telegram/api/secret.go delete mode 100644 telegram/base.go delete mode 100644 telegram/direct.go delete mode 100644 telegram/init.go delete mode 100644 telegram/interfaces.go delete mode 100644 telegram/middle.go delete mode 100644 tlstypes/client_hello.go delete mode 100644 tlstypes/consts.go delete mode 100644 tlstypes/handshake.go delete mode 100644 tlstypes/record.go delete mode 100644 tlstypes/server_hello.go delete mode 100644 utils/init_tcp.go delete mode 100644 utils/read_full.go delete mode 100644 utils/reverse_bytes.go delete mode 100644 utils/rlimit.go delete mode 100644 utils/rlimit_windows.go delete mode 100644 utils/signal_context.go delete mode 100644 utils/signal_context_windows.go delete mode 100644 utils/stream_cipher.go delete mode 100644 utils/uint24.go delete mode 100644 wrappers/packet/mtproto_frame.go delete mode 100644 wrappers/packetack/client_abridged.go delete mode 100644 wrappers/packetack/client_intermediate.go delete mode 100644 wrappers/packetack/client_intermediate_secure.go delete mode 100644 wrappers/packetack/proxy.go delete mode 100644 wrappers/rwc/ping.go delete mode 100644 wrappers/stream/base.go delete mode 100644 wrappers/stream/blockcipher.go delete mode 100644 wrappers/stream/buffered_reader.go delete mode 100644 wrappers/stream/conn.go delete mode 100644 wrappers/stream/ctx.go delete mode 100644 wrappers/stream/faketls.go delete mode 100644 wrappers/stream/mtproto_cipher.go delete mode 100644 wrappers/stream/obfuscated2.go delete mode 100644 wrappers/stream/rewind.go delete mode 100644 wrappers/stream/stats_telegram.go delete mode 100644 wrappers/stream/stats_traffic.go delete mode 100644 wrappers/stream/timeout.go diff --git a/antireplay/cache.go b/antireplay/cache.go deleted file mode 100644 index dab3af4..0000000 --- a/antireplay/cache.go +++ /dev/null @@ -1,36 +0,0 @@ -package antireplay - -import "github.com/VictoriaMetrics/fastcache" - -var ( - prefixObfuscated2 = []byte{0x00} - prefixTLS = []byte{0x01} -) - -type cache struct { - data *fastcache.Cache -} - -func (c cache) AddObfuscated2(data []byte) { - c.data.Set(keyObfuscated2(data), nil) -} - -func (c cache) AddTLS(data []byte) { - c.data.Set(keyTLS(data), nil) -} - -func (c cache) HasObfuscated2(data []byte) bool { - return c.data.Has(keyObfuscated2(data)) -} - -func (c cache) HasTLS(data []byte) bool { - return c.data.Has(keyTLS(data)) -} - -func keyObfuscated2(data []byte) []byte { - return append(prefixObfuscated2, data...) -} - -func keyTLS(data []byte) []byte { - return append(prefixTLS, data...) -} diff --git a/antireplay/init.go b/antireplay/init.go deleted file mode 100644 index 4ad9950..0000000 --- a/antireplay/init.go +++ /dev/null @@ -1,30 +0,0 @@ -package antireplay - -import ( - "sync" - - "github.com/9seconds/mtg/config" - "github.com/VictoriaMetrics/fastcache" -) - -type CacheInterface interface { - AddObfuscated2([]byte) - AddTLS([]byte) - HasObfuscated2([]byte) bool - HasTLS([]byte) bool -} - -var ( - Cache CacheInterface - initOnce sync.Once -) - -func Init() { - initOnce.Do(func() { - if config.C.AntiReplayMaxSize == 0 { - Cache = nilCache{} - } else { - Cache = cache{fastcache.New(config.C.AntiReplayMaxSize)} - } - }) -} diff --git a/antireplay/nilcache.go b/antireplay/nilcache.go deleted file mode 100644 index baa0947..0000000 --- a/antireplay/nilcache.go +++ /dev/null @@ -1,8 +0,0 @@ -package antireplay - -type nilCache struct{} - -func (n nilCache) AddObfuscated2(_ []byte) {} -func (n nilCache) AddTLS(_ []byte) {} -func (n nilCache) HasObfuscated2(_ []byte) bool { return false } -func (n nilCache) HasTLS(_ []byte) bool { return false } diff --git a/cli/generate.go b/cli/generate.go deleted file mode 100644 index 5811798..0000000 --- a/cli/generate.go +++ /dev/null @@ -1,26 +0,0 @@ -package cli - -import ( - "crypto/rand" - "encoding/hex" - - "github.com/9seconds/mtg/config" -) - -func Generate(secretType, hostname string) { - data := make([]byte, config.SimpleSecretLength) - if _, err := rand.Read(data); err != nil { - panic(err) - } - - secret := hex.EncodeToString(data) - - switch secretType { - case "simple": - PrintStdout(secret) - case "secured": - PrintStdout("dd" + secret) - default: - PrintStdout("ee" + secret + hex.EncodeToString([]byte(hostname))) - } -} diff --git a/cli/proxy.go b/cli/proxy.go deleted file mode 100644 index 038239e..0000000 --- a/cli/proxy.go +++ /dev/null @@ -1,101 +0,0 @@ -package cli - -import ( - "net" - "os" - "time" - - "github.com/9seconds/mtg/antireplay" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/faketls" - "github.com/9seconds/mtg/hub" - "github.com/9seconds/mtg/ntp" - "github.com/9seconds/mtg/obfuscated2" - "github.com/9seconds/mtg/proxy" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/utils" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func Proxy() error { // nolint: funlen,cyclop - ctx := utils.GetSignalContext() - - atom := zap.NewAtomicLevel() - - switch { - case config.C.Debug: - atom.SetLevel(zapcore.DebugLevel) - case config.C.Verbose: - atom.SetLevel(zapcore.InfoLevel) - default: - atom.SetLevel(zapcore.ErrorLevel) - } - - encoderCfg := zap.NewProductionEncoderConfig() - logger := zap.New(zapcore.NewCore( - zapcore.NewJSONEncoder(encoderCfg), - zapcore.Lock(os.Stderr), - atom, - )) - - zap.ReplaceGlobals(logger) - defer logger.Sync() // nolint: errcheck - - if err := config.InitPublicAddress(ctx); err != nil { - Fatal(err) - } - - zap.S().Debugw("Configuration", "config", config.Printable()) - - if config.C.MiddleProxyMode() { - zap.S().Infow("Use middle proxy connection to Telegram") - - diff, err := ntp.Fetch() - if err != nil { - Fatal("Cannot fetch time data from NTP") - } - - if diff > time.Second { - Fatal("Your local time is skewed and drift is bigger than a second. Please sync your time.") - } - - go ntp.AutoUpdate() - } else { - zap.S().Infow("Use direct connection to Telegram") - } - - PrintJSONStdout(config.GetURLs()) - - if err := stats.Init(ctx); err != nil { - Fatal(err) - } - - antireplay.Init() - telegram.Init() - hub.Init(ctx) - - proxyListener, err := net.Listen("tcp", config.C.Bind.String()) - if err != nil { - Fatal(err) - } - - go func() { - <-ctx.Done() - proxyListener.Close() - }() - - app := &proxy.Proxy{ - Logger: zap.S().Named("proxy"), - Context: ctx, - ClientProtocolMaker: obfuscated2.MakeClientProtocol, - } - if config.C.SecretMode == config.SecretModeTLS { - app.ClientProtocolMaker = faketls.MakeClientProtocol - } - - app.Serve(proxyListener) - - return nil -} diff --git a/cli/utils.go b/cli/utils.go deleted file mode 100644 index 948d875..0000000 --- a/cli/utils.go +++ /dev/null @@ -1,43 +0,0 @@ -package cli - -import ( - "encoding/json" - "fmt" - "io" - "os" -) - -func Fatal(arg interface{}) { - if value, ok := arg.(error); ok { - arg = fmt.Errorf("fatal error: %+v", value) // nolint: errorlint - } - - PrintStderr(arg) - os.Exit(1) -} - -func PrintStderr(args ...interface{}) { - fmt.Fprintln(os.Stderr, args...) -} - -func PrintStdout(args ...interface{}) { - fmt.Println(args...) // nolint: forbidigo -} - -func PrintJSONStderr(data interface{}) { - printJSON(os.Stderr, data) -} - -func PrintJSONStdout(data interface{}) { - printJSON(os.Stdout, data) -} - -func printJSON(writer io.Writer, data interface{}) { - encoder := json.NewEncoder(writer) - encoder.SetEscapeHTML(false) - encoder.SetIndent("", " ") - - if err := encoder.Encode(data); err != nil { - panic(err) - } -} diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 5e97f86..0000000 --- a/config/config.go +++ /dev/null @@ -1,312 +0,0 @@ -package config - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "math" - "net" - - "github.com/alecthomas/units" - statsd "github.com/smira/go-statsd" - "go.uber.org/zap" -) - -type SecretMode uint8 - -func (s SecretMode) String() string { - switch s { - case SecretModeSimple: - return "simple" - case SecretModeSecured: - return "secured" - case SecretModeTLS: - return "tls" - } - - return "tls" -} - -const ( - SecretModeSimple SecretMode = iota - SecretModeSecured - SecretModeTLS -) - -type PreferIP uint8 - -const ( - PreferIPv4 PreferIP = iota - PreferIPv6 -) - -const SimpleSecretLength = 16 - -type OptionType uint8 - -const ( - OptionTypeDebug OptionType = iota - OptionTypeVerbose - - OptionTypePreferIP - - OptionTypeBind - OptionTypePublicIPv4 - OptionTypePublicIPv6 - - OptionTypeStatsBind - OptionTypeStatsNamespace - OptionTypeStatsdAddress - OptionTypeStatsdTagsFormat - OptionTypeStatsdTags - - OptionTypeWriteBufferSize - OptionTypeReadBufferSize - - OptionTypeCloakPort - - OptionTypeAntiReplayMaxSize - - OptionTypeMultiplexPerConnection - - OptionTypeNTPServers - - OptionTypeSecret - OptionTypeAdtag -) - -type Config struct { - Bind *net.TCPAddr `json:"bind"` - PublicIPv4 *net.TCPAddr `json:"public_ipv4"` - PublicIPv6 *net.TCPAddr `json:"public_ipv6"` - StatsBind *net.TCPAddr `json:"stats_bind"` - StatsdAddr *net.TCPAddr `json:"stats_addr"` - StatsdTagsFormat *statsd.TagFormat `json:"statsd_tags_format"` - - StatsNamespace string `json:"stats_namespace"` - CloakHost string `json:"cloak_host"` - StatsdTags map[string]string `json:"statsd_tags"` - - WriteBuffer int `json:"write_buffer"` - ReadBuffer int `json:"read_buffer"` - CloakPort int `json:"cloak_port"` - - AntiReplayMaxSize int `json:"anti_replay_max_size"` - - MultiplexPerConnection int `json:"multiplex_per_connection"` - - Debug bool `json:"debug"` - Verbose bool `json:"verbose"` - SecretMode SecretMode `json:"secret_mode"` - PreferIP PreferIP `json:"prefer_ip"` - NTPServers []string `json:"ntp_servers"` - - Secret []byte `json:"secret"` - AdTag []byte `json:"adtag"` -} - -func (c *Config) ClientReadBuffer() int { - return c.ReadBuffer -} - -func (c *Config) ClientWriteBuffer() int { - return c.WriteBuffer -} - -func (c *Config) MiddleProxyMode() bool { - return len(c.AdTag) > 0 -} - -func (c *Config) ProxyReadBuffer() int { - value := c.ReadBuffer - - if c.MiddleProxyMode() { - value = c.adjustProxyValue(value) - } - - return value -} - -func (c *Config) ProxyWriteBuffer() int { - value := c.WriteBuffer - - if c.MiddleProxyMode() { - value = c.adjustProxyValue(value) - } - - return value -} - -func (c *Config) adjustProxyValue(value int) int { - if c.MultiplexPerConnection == 0 { - return value - } - - fvalue := float64(value) - - newValue := fvalue * 2 * math.Log(float64(c.MultiplexPerConnection)) - newValue = math.Ceil(newValue) - newValue = math.Max(fvalue, newValue) - - return int(newValue) -} - -type Opt struct { - Option OptionType - Value interface{} -} - -var C = Config{} - -func Init(options ...Opt) error { // nolint: gocyclo, funlen, cyclop - for _, opt := range options { - switch opt.Option { - case OptionTypeDebug: - C.Debug = opt.Value.(bool) - case OptionTypeVerbose: - C.Verbose = opt.Value.(bool) - case OptionTypePreferIP: - value := opt.Value.(string) - switch value { - case "ipv4": - C.PreferIP = PreferIPv4 - case "ipv6": - C.PreferIP = PreferIPv6 - default: - return fmt.Errorf("incorrect direct IP mode %s", value) - } - case OptionTypeBind: - C.Bind = opt.Value.(*net.TCPAddr) - case OptionTypePublicIPv4: - C.PublicIPv4 = opt.Value.(*net.TCPAddr) - if C.PublicIPv4 == nil { - C.PublicIPv4 = &net.TCPAddr{} - } - case OptionTypePublicIPv6: - C.PublicIPv6 = opt.Value.(*net.TCPAddr) - if C.PublicIPv6 == nil { - C.PublicIPv6 = &net.TCPAddr{} - } - case OptionTypeStatsBind: - C.StatsBind = opt.Value.(*net.TCPAddr) - case OptionTypeStatsNamespace: - C.StatsNamespace = opt.Value.(string) - case OptionTypeStatsdAddress: - C.StatsdAddr = opt.Value.(*net.TCPAddr) - case OptionTypeStatsdTagsFormat: - value := opt.Value.(string) - switch value { - case "datadog": - C.StatsdTagsFormat = statsd.TagFormatDatadog - case "influxdb": - C.StatsdTagsFormat = statsd.TagFormatInfluxDB - default: - return fmt.Errorf("incorrect statsd tag %s", value) - } - case OptionTypeStatsdTags: - C.StatsdTags = opt.Value.(map[string]string) - case OptionTypeWriteBufferSize: - C.WriteBuffer = int(opt.Value.(units.Base2Bytes)) - case OptionTypeReadBufferSize: - C.ReadBuffer = int(opt.Value.(units.Base2Bytes)) - case OptionTypeCloakPort: - C.CloakPort = int(opt.Value.(uint16)) - case OptionTypeAntiReplayMaxSize: - C.AntiReplayMaxSize = int(opt.Value.(units.Base2Bytes)) - case OptionTypeMultiplexPerConnection: - C.MultiplexPerConnection = int(opt.Value.(uint)) - case OptionTypeNTPServers: - C.NTPServers = opt.Value.([]string) - if len(C.NTPServers) == 0 { - return errors.New("ntp server list is empty") - } - case OptionTypeSecret: - C.Secret = opt.Value.([]byte) - case OptionTypeAdtag: - C.AdTag = opt.Value.([]byte) - default: - return fmt.Errorf("unknown tag %v", opt.Option) - } - } - - switch { - case len(C.Secret) == 1+SimpleSecretLength && bytes.HasPrefix(C.Secret, []byte{0xdd}): - C.SecretMode = SecretModeSecured - C.Secret = bytes.TrimPrefix(C.Secret, []byte{0xdd}) - case len(C.Secret) > SimpleSecretLength && bytes.HasPrefix(C.Secret, []byte{0xee}): - C.SecretMode = SecretModeTLS - secret := bytes.TrimPrefix(C.Secret, []byte{0xee}) - C.Secret = secret[:SimpleSecretLength] - C.CloakHost = string(secret[SimpleSecretLength:]) - case len(C.Secret) == SimpleSecretLength: - C.SecretMode = SecretModeSimple - default: - return errors.New("incorrect secret") - } - - if C.MultiplexPerConnection == 0 { - return errors.New("cannot use 0 clients per connection for multiplexing") - } - - if C.CloakHost != "" { - if _, err := net.LookupHost(C.CloakHost); err != nil { - zap.S().Warnw("Cannot resolve address of host", "hostname", C.CloakHost, "error", err) - } - } - - return nil -} - -func InitPublicAddress(ctx context.Context) error { - if C.PublicIPv4.Port == 0 { - C.PublicIPv4.Port = C.Bind.Port - } - - if C.PublicIPv6.Port == 0 { - C.PublicIPv6.Port = C.Bind.Port - } - - foundAddress := C.PublicIPv4.IP != nil || C.PublicIPv6.IP != nil - - if C.PublicIPv4.IP == nil { - ip, err := getGlobalIPv4(ctx) - if err != nil { - zap.S().Warnw("Cannot resolve public address", "error", err) - } else { - C.PublicIPv4.IP = ip - foundAddress = true - } - } - - if C.PublicIPv6.IP == nil { - ip, err := getGlobalIPv6(ctx) - if err != nil { - zap.S().Warnw("Cannot resolve public address", "error", err) - } else { - C.PublicIPv6.IP = ip - foundAddress = true - } - } - - if !foundAddress { - return errors.New("cannot resolve any public address") - } - - return nil -} - -func Printable() interface{} { - data, err := json.Marshal(C) - if err != nil { - panic(err) - } - - rv := map[string]interface{}{} - if err := json.Unmarshal(data, &rv); err != nil { - panic(err) - } - - return rv -} diff --git a/config/global_ips.go b/config/global_ips.go deleted file mode 100644 index 0dd78da..0000000 --- a/config/global_ips.go +++ /dev/null @@ -1,78 +0,0 @@ -package config - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "strings" - "time" -) - -const ( - ifconfigAddress = "https://ifconfig.co/ip" - ifconfigTimeout = 10 * time.Second -) - -func getGlobalIPv4(ctx context.Context) (net.IP, error) { - ip, err := fetchIP(ctx, "tcp4") - if err != nil || ip.To4() == nil { - return nil, fmt.Errorf("cannot find public ipv4 address: %w", err) - } - - return ip, nil -} - -func getGlobalIPv6(ctx context.Context) (net.IP, error) { - ip, err := fetchIP(ctx, "tcp6") - if err != nil || ip.To4() != nil { - return nil, fmt.Errorf("cannot find public ipv6 address: %w", err) - } - - return ip, nil -} - -func fetchIP(ctx context.Context, network string) (net.IP, error) { - dialer := &net.Dialer{FallbackDelay: -1} - client := &http.Client{ - Jar: nil, - Timeout: ifconfigTimeout, - Transport: &http.Transport{ - DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { - return dialer.DialContext(ctx, network, addr) - }, - }, - } - - req, err := http.NewRequest("GET", ifconfigAddress, nil) - if err != nil { - return nil, fmt.Errorf("cannot create a request: %w", err) - } - - resp, err := client.Do(req.WithContext(ctx)) - if err != nil { - if resp != nil { - io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck - } - - return nil, fmt.Errorf("cannot perform a request: %w", err) - } - - defer resp.Body.Close() - - respDataBytes, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("cannot read response body: %w", err) - } - - respData := strings.TrimSpace(string(respDataBytes)) - - ip := net.ParseIP(respData) - if ip == nil { - return nil, fmt.Errorf("ifconfig.co returns incorrect IP %s", respData) - } - - return ip, nil -} diff --git a/config/urls.go b/config/urls.go deleted file mode 100644 index f657e82..0000000 --- a/config/urls.go +++ /dev/null @@ -1,99 +0,0 @@ -package config - -import ( - "encoding/hex" - "net" - "net/url" - "strconv" -) - -type URLs struct { - TG string `json:"tg_url"` - TMe string `json:"tme_url"` - TGQRCode string `json:"tg_qrcode"` - TMeQRCode string `json:"tme_qrcode"` -} - -type IPURLs struct { - IPv4 *URLs `json:"ipv4,omitempty"` - IPv6 *URLs `json:"ipv6,omitempty"` - BotSecret string `json:"secret_for_mtproxybot"` -} - -func GetURLs() (urls IPURLs) { - secret := "" - - switch C.SecretMode { - case SecretModeSimple: - secret = hex.EncodeToString(C.Secret) - case SecretModeSecured: - secret = "dd" + hex.EncodeToString(C.Secret) - case SecretModeTLS: - secret = "ee" + hex.EncodeToString(C.Secret) + hex.EncodeToString([]byte(C.CloakHost)) - } - - if C.PublicIPv4.IP != nil { - urls.IPv4 = makeURLs(C.PublicIPv4, secret) - } - - if C.PublicIPv6.IP != nil { - urls.IPv6 = makeURLs(C.PublicIPv6, secret) - } - - urls.BotSecret = hex.EncodeToString(C.Secret) - - return urls -} - -func makeURLs(addr *net.TCPAddr, secret string) *URLs { - urls := &URLs{} - - values := url.Values{} - values.Set("server", addr.IP.String()) - values.Set("port", strconv.Itoa(addr.Port)) - values.Set("secret", secret) - - return &URLs{ - TG: makeTGURL(values), - TMe: makeTMeURL(values), - TGQRCode: makeQRCodeURL(urls.TG), - TMeQRCode: makeQRCodeURL(urls.TMe), - } -} - -func makeTGURL(values url.Values) string { - tgURL := url.URL{ - Scheme: "tg", - Host: "proxy", - RawQuery: values.Encode(), - } - - return tgURL.String() -} - -func makeTMeURL(values url.Values) string { - tMeURL := url.URL{ - Scheme: "https", - Host: "t.me", - Path: "proxy", - RawQuery: values.Encode(), - } - - return tMeURL.String() -} - -func makeQRCodeURL(data string) string { - qr := url.URL{ - Scheme: "https", - Host: "api.qrserver.com", - Path: "v1/create-qr-code", - } - - values := url.Values{} - values.Set("qzone", "4") - values.Set("format", "svg") - values.Set("data", data) - qr.RawQuery = values.Encode() - - return qr.String() -} diff --git a/conntypes/acks.go b/conntypes/acks.go deleted file mode 100644 index d3dbd8d..0000000 --- a/conntypes/acks.go +++ /dev/null @@ -1,6 +0,0 @@ -package conntypes - -type ConnectionAcks struct { - Simple bool - Quick bool -} diff --git a/conntypes/dc.go b/conntypes/dc.go deleted file mode 100644 index 3cf2850..0000000 --- a/conntypes/dc.go +++ /dev/null @@ -1,5 +0,0 @@ -package conntypes - -type DC int16 - -const DCDefaultIdx DC = 1 diff --git a/conntypes/id.go b/conntypes/id.go deleted file mode 100644 index 1aa482c..0000000 --- a/conntypes/id.go +++ /dev/null @@ -1,24 +0,0 @@ -package conntypes - -import ( - "crypto/rand" - "encoding/hex" -) - -const ConnIDLength = 8 - -type ConnID [ConnIDLength]byte - -func (c ConnID) String() string { - return hex.EncodeToString(c[:]) -} - -func NewConnID() ConnID { - var id ConnID - - if _, err := rand.Read(id[:]); err != nil { - panic(err) - } - - return id -} diff --git a/conntypes/packet.go b/conntypes/packet.go deleted file mode 100644 index fa72022..0000000 --- a/conntypes/packet.go +++ /dev/null @@ -1,3 +0,0 @@ -package conntypes - -type Packet []byte diff --git a/conntypes/protocol.go b/conntypes/protocol.go deleted file mode 100644 index b89ac65..0000000 --- a/conntypes/protocol.go +++ /dev/null @@ -1,22 +0,0 @@ -package conntypes - -type ConnectionProtocol uint8 - -func (c ConnectionProtocol) String() string { - switch c { - case ConnectionProtocolAny: - return "any" - case ConnectionProtocolIPv4: - return "ipv4" - case ConnectionProtocolIPv6: - return "ipv6" - } - - return "ipv6" -} - -const ( - ConnectionProtocolIPv4 ConnectionProtocol = 1 - ConnectionProtocolIPv6 = ConnectionProtocolIPv4 << 1 - ConnectionProtocolAny = ConnectionProtocolIPv4 | ConnectionProtocolIPv6 -) diff --git a/conntypes/type.go b/conntypes/type.go deleted file mode 100644 index c41d42f..0000000 --- a/conntypes/type.go +++ /dev/null @@ -1,29 +0,0 @@ -package conntypes - -type ConnectionType uint8 - -const ( - ConnectionTypeUnknown ConnectionType = iota - ConnectionTypeAbridged - ConnectionTypeIntermediate - ConnectionTypeSecure -) - -var ( - ConnectionTagAbridged = []byte{0xef, 0xef, 0xef, 0xef} - ConnectionTagIntermediate = []byte{0xee, 0xee, 0xee, 0xee} - ConnectionTagSecure = []byte{0xdd, 0xdd, 0xdd, 0xdd} -) - -func (t ConnectionType) Tag() []byte { - switch t { - case ConnectionTypeAbridged: - return ConnectionTagAbridged - case ConnectionTypeIntermediate: - return ConnectionTagIntermediate - case ConnectionTypeSecure, ConnectionTypeUnknown: - return ConnectionTagSecure - } - - return ConnectionTagSecure -} diff --git a/conntypes/wrap_interfaces.go b/conntypes/wrap_interfaces.go deleted file mode 100644 index f21eb91..0000000 --- a/conntypes/wrap_interfaces.go +++ /dev/null @@ -1,14 +0,0 @@ -package conntypes - -import ( - "net" - - "go.uber.org/zap" -) - -type Wrap interface { - Conn() net.Conn - Logger() *zap.SugaredLogger - LocalAddr() *net.TCPAddr - RemoteAddr() *net.TCPAddr -} diff --git a/conntypes/wrap_packet_ack_interfaces.go b/conntypes/wrap_packet_ack_interfaces.go deleted file mode 100644 index 6388f0c..0000000 --- a/conntypes/wrap_packet_ack_interfaces.go +++ /dev/null @@ -1,41 +0,0 @@ -package conntypes - -import "io" - -type PacketAckReader interface { - Read(*ConnectionAcks) (Packet, error) -} - -type PacketAckWriter interface { - Write(Packet, *ConnectionAcks) error -} - -type PacketAckCloser interface { - io.Closer -} - -type PacketAckReadCloser interface { - PacketAckReader - PacketAckCloser -} - -type PacketAckWriteCloser interface { - PacketAckWriter - PacketAckCloser -} - -type PacketAckReadWriter interface { - PacketAckReader - PacketAckWriter -} - -type PacketAckReadWriteCloser interface { - PacketAckReader - PacketAckWriter - PacketAckCloser -} - -type PacketAckFullReadWriteCloser interface { - Wrap - PacketAckReadWriteCloser -} diff --git a/conntypes/wrap_packet_interfaces.go b/conntypes/wrap_packet_interfaces.go deleted file mode 100644 index 6c573d4..0000000 --- a/conntypes/wrap_packet_interfaces.go +++ /dev/null @@ -1,51 +0,0 @@ -package conntypes - -import "io" - -type BasePacketReader interface { - Read() (Packet, error) -} - -type BasePacketWriter interface { - Write(Packet) error -} - -type PacketReader interface { - Wrap - BasePacketReader -} - -type PacketWriter interface { - Wrap - BasePacketWriter -} - -type PacketCloser interface { - Wrap - io.Closer -} - -type PacketReadCloser interface { - Wrap - BasePacketReader - io.Closer -} - -type PacketWriteCloser interface { - Wrap - BasePacketWriter - io.Closer -} - -type PacketReadWriter interface { - Wrap - BasePacketWriter - BasePacketReader -} - -type PacketReadWriteCloser interface { - Wrap - BasePacketWriter - BasePacketReader - io.Closer -} diff --git a/conntypes/wrap_stream_interfaces.go b/conntypes/wrap_stream_interfaces.go deleted file mode 100644 index 5924d66..0000000 --- a/conntypes/wrap_stream_interfaces.go +++ /dev/null @@ -1,56 +0,0 @@ -package conntypes - -import ( - "io" - "time" -) - -type BaseStreamReaderWithTimeout interface { - ReadTimeout([]byte, time.Duration) (int, error) -} - -type BaseStreamWriterWithTimeout interface { - WriteTimeout([]byte, time.Duration) (int, error) -} - -type StreamReader interface { - Wrap - io.Reader - BaseStreamReaderWithTimeout -} - -type StreamWriter interface { - Wrap - io.Writer - BaseStreamWriterWithTimeout -} - -type StreamCloser interface { - Wrap - io.Closer -} - -type StreamReadCloser interface { - Wrap - io.ReadCloser - BaseStreamReaderWithTimeout -} - -type StreamWriteCloser interface { - Wrap - io.WriteCloser - BaseStreamWriterWithTimeout -} - -type StreamReadWriter interface { - Wrap - io.ReadWriter - BaseStreamReaderWithTimeout -} - -type StreamReadWriteCloser interface { - Wrap - io.ReadWriteCloser - BaseStreamReaderWithTimeout - BaseStreamWriterWithTimeout -} diff --git a/faketls/client_protocol.go b/faketls/client_protocol.go deleted file mode 100644 index 8208ac0..0000000 --- a/faketls/client_protocol.go +++ /dev/null @@ -1,122 +0,0 @@ -package faketls - -import ( - "bufio" - "bytes" - "encoding/binary" - "errors" - "fmt" - "io" - "net" - "strconv" - "time" - - "github.com/9seconds/mtg/antireplay" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/obfuscated2" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/tlstypes" - "github.com/9seconds/mtg/wrappers/stream" -) - -type ClientProtocol struct { - obfuscated2.ClientProtocol -} - -func (c *ClientProtocol) Handshake(socket conntypes.StreamReadWriteCloser) (conntypes.StreamReadWriteCloser, error) { - rewinded := stream.NewRewind(socket) - bufferedReader := bufio.NewReader(rewinded) - - for _, expected := range faketlsStartBytes { - if actual, err := bufferedReader.ReadByte(); err != nil || actual != expected { - rewinded.Rewind() - c.cloakHost(rewinded) - - return nil, errors.New("failed first bytes of tls handshake") - } - } - - rewinded.Rewind() - rewinded = stream.NewRewind(rewinded) - - if err := c.tlsHandshake(rewinded); err != nil { - rewinded.Rewind() - c.cloakHost(rewinded) - - return nil, fmt.Errorf("failed tls handshake: %w", err) - } - - conn := stream.NewFakeTLS(socket) - - conn, err := c.ClientProtocol.Handshake(conn) - if err != nil { - return nil, err // nolint: wrapcheck - } - - return conn, err // nolint: wrapcheck -} - -func (c *ClientProtocol) tlsHandshake(conn io.ReadWriter) error { - helloRecord, err := tlstypes.ReadRecord(conn) - if err != nil { - return fmt.Errorf("cannot read initial record: %w", err) - } - - buf := &bytes.Buffer{} - helloRecord.Data.WriteBytes(buf) - - clientHello, err := tlstypes.ParseClientHello(buf.Bytes()) - if err != nil { - return fmt.Errorf("cannot parse client hello: %w", err) - } - - digest := clientHello.Digest() - for i := 0; i < len(digest)-4; i++ { - if digest[i] != 0 { - return errBadDigest - } - } - - timestamp := int64(binary.LittleEndian.Uint32(digest[len(digest)-4:])) - createdAt := time.Unix(timestamp, 0) - timeDiff := time.Since(createdAt) - - if (timeDiff > TimeSkew || timeDiff < -TimeSkew) && timestamp > TimeFromBoot { - return errBadTime - } - - if antireplay.Cache.HasTLS(clientHello.Random[:]) { - stats.Stats.ReplayDetected() - - return errors.New("replay attack is detected") - } - - antireplay.Cache.AddTLS(clientHello.Random[:]) - serverHello := tlstypes.NewServerHello(clientHello) - serverHelloPacket := serverHello.WelcomePacket() - - if _, err := conn.Write(serverHelloPacket); err != nil { - return fmt.Errorf("cannot send welcome packet: %w", err) - } - - return nil -} - -func (c *ClientProtocol) cloakHost(clientConn io.ReadWriteCloser) { - stats.Stats.CloakedRequest() - - addr := net.JoinHostPort(config.C.CloakHost, strconv.Itoa(config.C.CloakPort)) - - hostConn, err := net.Dial("tcp", addr) - if err != nil { - return - } - - cloak(clientConn, hostConn) -} - -func MakeClientProtocol() protocol.ClientProtocol { - return &ClientProtocol{} -} diff --git a/faketls/cloak.go b/faketls/cloak.go deleted file mode 100644 index b134de8..0000000 --- a/faketls/cloak.go +++ /dev/null @@ -1,73 +0,0 @@ -package faketls - -import ( - "context" - "io" - "sync" - "time" - - "github.com/9seconds/mtg/wrappers/rwc" -) - -const ( - cloakLastActivityTimeout = 5 * time.Second - cloakMaxTimeout = 30 * time.Second -) - -func cloak(one, another io.ReadWriteCloser) { - defer func() { - one.Close() - another.Close() - }() - - channelPing := make(chan struct{}, 1) - ctx, cancel := context.WithCancel(context.Background()) - one = rwc.NewPing(ctx, one, channelPing) - another = rwc.NewPing(ctx, another, channelPing) - wg := &sync.WaitGroup{} - - wg.Add(2) - - go cloakPipe(one, another, wg) - - go cloakPipe(another, one, wg) - - go func() { - wg.Wait() - cancel() - }() - - go func() { - lastActivityTimer := time.NewTimer(cloakLastActivityTimeout) - defer lastActivityTimer.Stop() - - maxTimer := time.NewTimer(cloakMaxTimeout) - defer maxTimer.Stop() - - for { - select { - case <-channelPing: - lastActivityTimer.Stop() - lastActivityTimer = time.NewTimer(cloakLastActivityTimeout) - case <-ctx.Done(): - return - case <-lastActivityTimer.C: - cancel() - - return - case <-maxTimer.C: - cancel() - - return - } - } - }() - - <-ctx.Done() -} - -func cloakPipe(one io.Writer, another io.Reader, wg *sync.WaitGroup) { - defer wg.Done() - - io.Copy(one, another) // nolint: errcheck -} diff --git a/faketls/consts.go b/faketls/consts.go deleted file mode 100644 index 9bd2353..0000000 --- a/faketls/consts.go +++ /dev/null @@ -1,30 +0,0 @@ -package faketls - -import ( - "errors" - "time" -) - -const ( - TimeSkew = 5 * time.Second - TimeFromBoot = 24 * 60 * 60 -) - -var ( - errBadDigest = errors.New("bad digest") - errBadTime = errors.New("bad time") - - faketlsStartBytes = [...]byte{ - 0x16, - 0x03, - 0x01, - 0x02, - 0x00, - 0x01, - 0x00, - 0x01, - 0xfc, - 0x03, - 0x03, - } -) diff --git a/go.mod b/go.mod index a2c7190..2b87317 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,3 @@ -module github.com/9seconds/mtg +module github.com/9seconds/mtg/v2 -go 1.13 - -require ( - github.com/VictoriaMetrics/fastcache v1.5.7 - github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 - github.com/beevik/ntp v0.3.0 - github.com/golang/snappy v0.0.3 // indirect - github.com/prometheus/client_golang v1.9.0 - github.com/prometheus/common v0.18.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect - github.com/smira/go-statsd v1.3.2 - go.uber.org/multierr v1.6.0 // indirect - go.uber.org/zap v1.16.0 - golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 - golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect - golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect - golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 - golang.org/x/tools v0.1.0 // indirect - google.golang.org/protobuf v1.25.0 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 - honnef.co/go/tools v0.0.1-2020.1.3 // indirect -) +go 1.16 diff --git a/go.sum b/go.sum index 2433d93..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,472 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VictoriaMetrics/fastcache v1.5.7 h1:4y6y0G8PRzszQUYIQHHssv/jgPHAb5qQuuDNdCbyAgw= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= -github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/beevik/ntp v0.3.0 h1:xzVrPrE4ziasFXgBVBZJDP0Wg/KpMwk2KHJ4Ba8GrDw= -github.com/beevik/ntp v0.3.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= -github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smira/go-statsd v1.3.2 h1:1EeuzxNZ/TD9apbTOFSM9nulqfcsQFmT4u1A2DREabI= -github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04 h1:cEhElsAv9LUt9ZUUocxzWe05oFLVd+AA2nstydTeI8g= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/hub/connection.go b/hub/connection.go deleted file mode 100644 index e96cae9..0000000 --- a/hub/connection.go +++ /dev/null @@ -1,172 +0,0 @@ -package hub - -import ( - "fmt" - "math/rand" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" - "go.uber.org/zap" -) - -const connectionTTL = time.Hour - -type connection struct { - conn conntypes.PacketReadWriteCloser - proxyConns map[string]*ProxyConn - closeOnce sync.Once - proxyConnsMutex sync.RWMutex - id int - logger *zap.SugaredLogger - - channelDone chan struct{} - channelWrite chan conntypes.Packet - channelRead chan *rpc.ProxyResponse - channelConnAttach chan *ProxyConn - channelConnDetach chan conntypes.ConnID -} - -func (c *connection) run() { // nolint: cyclop - defer c.Close() - - ttl := time.NewTimer(connectionTTL) - defer ttl.Stop() - - for { - select { - case <-c.channelDone: - for _, v := range c.proxyConns { - v.Close() - } - - return - case <-ttl.C: - c.logger.Debugw("Closing connection by TTL") - c.Close() - case resp := <-c.channelRead: - if channel, ok := c.proxyConns[string(resp.ConnID[:])]; ok { - if resp.Type == rpc.ProxyResponseTypeCloseExt { - channel.Close() - } else { - channel.put(resp) - } - } - case packet := <-c.channelWrite: - if err := c.conn.Write(packet); err != nil { - c.logger.Debugw("Cannot write packet", "error", err) - c.Close() - } - case conn := <-c.channelConnAttach: - c.proxyConnsMutex.Lock() - c.proxyConns[string(conn.req.ConnID[:])] = conn - c.proxyConnsMutex.Unlock() - conn.channelWrite = c.channelWrite - case connID := <-c.channelConnDetach: - if conn, ok := c.proxyConns[string(connID[:])]; ok { - c.proxyConnsMutex.Lock() - delete(c.proxyConns, string(connID[:])) - c.proxyConnsMutex.Unlock() - conn.Close() - } - } - } -} - -func (c *connection) readLoop() { - for { - packet, err := c.conn.Read() - if err != nil { - c.logger.Debugw("Cannot read packet", "error", err) - c.Close() - - return - } - - response, err := rpc.ParseProxyResponse(packet) - if err != nil { - c.logger.Debugw("Failed response", "error", err) - - continue - } - - select { - case <-c.channelDone: - return - case c.channelRead <- response: - } - } -} - -func (c *connection) Close() { - c.closeOnce.Do(func() { - c.logger.Debugw("Closing connection") - - close(c.channelDone) - c.conn.Close() - }) -} - -func (c *connection) Done() bool { - select { - case <-c.channelDone: - return true - default: - return c.Len() == 0 - } -} - -func (c *connection) Len() int { - c.proxyConnsMutex.RLock() - defer c.proxyConnsMutex.RUnlock() - - return len(c.proxyConns) -} - -func (c *connection) Attach(conn *ProxyConn) error { - select { - case <-c.channelDone: - return ErrClosed - case c.channelConnAttach <- conn: - return nil - } -} - -func (c *connection) Detach(connID conntypes.ConnID) { - select { - case <-c.channelDone: - case c.channelConnDetach <- connID: - } -} - -func newConnection(req *protocol.TelegramRequest) (*connection, error) { - conn, err := mtproto.TelegramProtocol(req) - if err != nil { - return nil, fmt.Errorf("cannot create a new connection: %w", err) - } - - id := rand.Int() // nolint: gosec - rv := &connection{ - conn: conn, - id: id, - logger: zap.S().Named("hub-connection").With("id", id, - "dc", req.ClientProtocol.DC(), - "protocol", req.ClientProtocol.ConnectionProtocol()), - proxyConns: make(map[string]*ProxyConn), - - channelRead: make(chan *rpc.ProxyResponse, 1), - channelDone: make(chan struct{}), - channelWrite: make(chan conntypes.Packet), - channelConnAttach: make(chan *ProxyConn), - channelConnDetach: make(chan conntypes.ConnID), - } - - go rv.readLoop() - - go rv.run() - - return rv, nil -} diff --git a/hub/connection_list.go b/hub/connection_list.go deleted file mode 100644 index d1d6a61..0000000 --- a/hub/connection_list.go +++ /dev/null @@ -1,70 +0,0 @@ -package hub - -import ( - "fmt" - "sort" - - "github.com/9seconds/mtg/config" -) - -type connectionList struct { - connections []*connection -} - -func (c *connectionList) get(conn *ProxyConn) (*connection, error) { - if len(c.connections) > 0 && c.connections[0].Len() < config.C.MultiplexPerConnection { - if err := c.connections[0].Attach(conn); err == nil { - return c.connections[0], nil - } - } - - newConn, err := newConnection(conn.req) - if err != nil { - return nil, fmt.Errorf("cannot allocate a new connection: %w", err) - } - - if err = newConn.Attach(conn); err != nil { - newConn.Close() - - return nil, fmt.Errorf("cannot attach to the newly created connection: %w", err) - } - - c.connections = append(c.connections, newConn) - lastIndex := len(c.connections) - 1 - c.connections[0], c.connections[lastIndex] = c.connections[lastIndex], c.connections[0] - - return newConn, nil -} - -func (c *connectionList) gc() { - prevLen := len(c.connections) - if prevLen == 0 { - return - } - - for i := len(c.connections) - 1; i >= 0; i-- { - lastIndex := len(c.connections) - 1 - - if c.connections[i].Done() { - c.connections[i].Close() - - if len(c.connections)-1 == i { - c.connections = c.connections[:lastIndex] - } else { - c.connections[i], c.connections[lastIndex] = c.connections[lastIndex], c.connections[i] - } - } - } - - if prevLen != len(c.connections) { - c.sort() - } -} - -func (c *connectionList) sort() { - if len(c.connections) > 1 { - sort.Slice(c.connections, func(i, j int) bool { - return c.connections[i].Len() < c.connections[j].Len() - }) - } -} diff --git a/hub/hub.go b/hub/hub.go deleted file mode 100644 index edfc442..0000000 --- a/hub/hub.go +++ /dev/null @@ -1,40 +0,0 @@ -package hub - -import ( - "context" - "sync" - - "github.com/9seconds/mtg/protocol" -) - -type hub struct { - muxes map[int32]*mux - mutex sync.RWMutex - ctx context.Context -} - -func (h *hub) Register(req *protocol.TelegramRequest) (*ProxyConn, error) { - return h.getMux(req).Get(req) -} - -func (h *hub) getMux(req *protocol.TelegramRequest) *mux { - var key int32 = 32767 + int32(req.ClientProtocol.DC()) + 100000*int32(req.ClientProtocol.ConnectionProtocol()) - - h.mutex.RLock() - m, ok := h.muxes[key] - h.mutex.RUnlock() - - if !ok { - h.mutex.Lock() - m, ok = h.muxes[key] - - if !ok { - m = newMux(h.ctx) - h.muxes[key] = m - } - - h.mutex.Unlock() - } - - return m -} diff --git a/hub/init.go b/hub/init.go deleted file mode 100644 index 19f5289..0000000 --- a/hub/init.go +++ /dev/null @@ -1,24 +0,0 @@ -package hub - -import ( - "context" - "errors" - "sync" -) - -var ( - ErrTimeout = errors.New("timeout") - ErrClosed = errors.New("context is closed") - - Hub Interface - initOnce sync.Once -) - -func Init(ctx context.Context) { - initOnce.Do(func() { - Hub = &hub{ - muxes: make(map[int32]*mux), - ctx: ctx, - } - }) -} diff --git a/hub/interface.go b/hub/interface.go deleted file mode 100644 index a1efcdc..0000000 --- a/hub/interface.go +++ /dev/null @@ -1,7 +0,0 @@ -package hub - -import "github.com/9seconds/mtg/protocol" - -type Interface interface { - Register(*protocol.TelegramRequest) (*ProxyConn, error) -} diff --git a/hub/mux.go b/hub/mux.go deleted file mode 100644 index f812b07..0000000 --- a/hub/mux.go +++ /dev/null @@ -1,90 +0,0 @@ -package hub - -import ( - "context" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" -) - -const muxGCEvery = time.Minute - -type muxNewRequest struct { - req *protocol.TelegramRequest - resp chan<- muxNewResponse -} - -type muxNewResponse struct { - conn *ProxyConn - err error -} - -type mux struct { - connections connectionList - clients map[string]*connection - ctx context.Context - channelClosed chan conntypes.ConnID - channelNew chan muxNewRequest -} - -func (m *mux) run() { - gcTicker := time.NewTicker(muxGCEvery) - defer gcTicker.Stop() - - for { - select { - case <-m.ctx.Done(): - for _, v := range m.clients { - v.Close() - } - - return - case <-gcTicker.C: - m.connections.gc() - case req := <-m.channelNew: - m.connections.gc() - proxyConn := newProxyConn(req.req, m.channelClosed) - conn, err := m.connections.get(proxyConn) - - if err == nil { - m.clients[string(req.req.ConnID[:])] = conn - } - - req.resp <- muxNewResponse{ - conn: proxyConn, - err: err, - } - close(req.resp) - case connID := <-m.channelClosed: - if conn, ok := m.clients[string(connID[:])]; ok { - conn.Detach(connID) - delete(m.clients, string(connID[:])) - } - } - } -} - -func (m *mux) Get(req *protocol.TelegramRequest) (*ProxyConn, error) { - resp := make(chan muxNewResponse) - m.channelNew <- muxNewRequest{ - req: req, - resp: resp, - } - - rv := <-resp - - return rv.conn, rv.err -} - -func newMux(ctx context.Context) *mux { - m := &mux{ - ctx: ctx, - clients: make(map[string]*connection), - channelClosed: make(chan conntypes.ConnID, 1), - channelNew: make(chan muxNewRequest), - } - go m.run() - - return m -} diff --git a/hub/proxy_conn.go b/hub/proxy_conn.go deleted file mode 100644 index 331d4c8..0000000 --- a/hub/proxy_conn.go +++ /dev/null @@ -1,79 +0,0 @@ -package hub - -import ( - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" -) - -const ( - proxyConnWriteTimeout = 2 * time.Minute - proxyConnReadTimeout = 2 * time.Minute - - proxyConnBackpressureAfter = 10 -) - -type ProxyConn struct { - closeOnce sync.Once - req *protocol.TelegramRequest - channelResponse chan *rpc.ProxyResponse - channelClosed chan<- conntypes.ConnID - channelWrite chan<- conntypes.Packet - channelDone chan struct{} -} - -func (p *ProxyConn) Read() (*rpc.ProxyResponse, error) { - timer := time.NewTimer(proxyConnReadTimeout) - defer timer.Stop() - - select { - case <-timer.C: - return nil, ErrTimeout - case <-p.channelDone: - return nil, ErrClosed - case packet := <-p.channelResponse: - return packet, nil - } -} - -func (p *ProxyConn) Write(packet conntypes.Packet) error { - timer := time.NewTimer(proxyConnWriteTimeout) - defer timer.Stop() - - select { - case <-timer.C: - return ErrTimeout - case <-p.channelDone: - return ErrClosed - case p.channelWrite <- packet: - return nil - } -} - -func (p *ProxyConn) put(response *rpc.ProxyResponse) { - select { - case <-p.channelDone: - case p.channelResponse <- response: - } -} - -func (p *ProxyConn) Close() { - p.closeOnce.Do(func() { - close(p.channelDone) - go func() { - p.channelClosed <- p.req.ConnID - }() - }) -} - -func newProxyConn(req *protocol.TelegramRequest, channelClosed chan<- conntypes.ConnID) *ProxyConn { - return &ProxyConn{ - channelResponse: make(chan *rpc.ProxyResponse, proxyConnBackpressureAfter), - channelDone: make(chan struct{}), - channelClosed: channelClosed, - req: req, - } -} diff --git a/main.go b/main.go index b16f040..8c4cef3 100644 --- a/main.go +++ b/main.go @@ -2,189 +2,11 @@ package main import ( "math/rand" - "os" - "runtime/debug" - "strings" "time" - - "github.com/9seconds/mtg/cli" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/utils" - kingpin "gopkg.in/alecthomas/kingpin.v2" ) var version = "dev" // has to be set by ldflags -var ( - app = kingpin.New("mtg", "Simple MTPROTO proxy.") - - generateSecretCommand = app.Command("generate-secret", - "Generate new secret") - generateCloakHost = generateSecretCommand.Flag("cloak-host", - "A host to use for TLS cloaking."). - Short('c'). - Default("storage.googleapis.com"). - String() - generateSecretType = generateSecretCommand.Arg("type", - "A type of secret to generate. Valid options are 'simple', 'secured' and 'tls'"). - Required(). - Enum("simple", "secured", "tls") - - runCommand = app.Command("run", - "Run new proxy instance") - runDebug = runCommand.Flag("debug", - "Run in debug mode."). - Short('d'). - Envar("MTG_DEBUG"). - Bool() - runVerbose = runCommand.Flag("verbose", - "Run in verbose mode."). - Short('v'). - Envar("MTG_VERBOSE"). - Bool() - runPreferIP = runCommand.Flag("prefer-ip", - "Prefer this IP protocol if possible. Valid options are 'ipv4' and 'ipv6'"). - Envar("MTG_PREFER_DIRECT_IP"). - Default("ipv6"). - Enum("ipv4", "ipv6") - runBind = runCommand.Flag("bind", - "Host:Port to bind proxy to."). - Short('b'). - Envar("MTG_BIND"). - Default("0.0.0.0:3128"). - TCP() - runPublicIPv4 = runCommand.Flag("public-ipv4", - "Which IPv4 host:port to use."). - Short('4'). - Envar("MTG_IPV4"). - TCP() - runPublicIPv6 = runCommand.Flag("public-ipv6", - "Which IPv6 host:port to use."). - Short('6'). - Envar("MTG_IPV6"). - TCP() - runStatsBind = runCommand.Flag("stats-bind", - "Which Host:Port to bind stats server to."). - Short('t'). - Envar("MTG_STATS_BIND"). - Default("127.0.0.1:3129"). - TCP() - runStatsNamespace = runCommand.Flag("stats-namespace", - "Which namespace to use for Prometheus."). - Envar("MTG_STATS_NAMESPACE"). - Default("mtg"). - String() - runStatsdAddress = runCommand.Flag("statsd-addr", - "Host:port of statsd server"). - Envar("MTG_STATSD_ADDR"). - TCP() - runStatsdTagsFormat = runCommand.Flag("statsd-tags-format", - "Which tag format should we use to send stats metrics. Valid options are 'datadog' and 'influxdb'."). - Envar("MTG_STATSD_TAGS_FORMAT"). - Default("influxdb"). - Enum("datadog", "influxdb") - runStatsdTags = runCommand.Flag("statsd-tags", - "Tags to use for working with statsd (specified as 'key=value')."). - Envar("MTG_STATSD_TAGS"). - StringMap() - runWriteBufferSize = runCommand.Flag("write-buffer", - "Write buffer size. You can think about it as a buffer from client to Telegram."). - Short('w'). - Envar("MTG_BUFFER_WRITE"). - Default("32KB"). - Bytes() - runReadBufferSize = runCommand.Flag("read-buffer", - "Read buffer size. You can think about it as a buffer from Telegram to client."). - Short('r'). - Envar("MTG_BUFFER_READ"). - Default("32KB"). - Bytes() - runTLSCloakPort = runCommand.Flag("cloak-port", - "Port which should be used for host cloaking."). - Envar("MTG_CLOAK_PORT"). - Default("443"). - Uint16() - runAntiReplayMaxSize = runCommand.Flag("anti-replay-max-size", - "Max size of antireplay cache."). - Envar("MTG_ANTIREPLAY_MAXSIZE"). - Default("128MB"). - Bytes() - runMultiplexPerConnection = runCommand.Flag("multiplex-per-connection", - "How many clients can share a single connection to Telegram."). - Envar("MTG_MULTIPLEX_PERCONNECTION"). - Default("50"). - Uint() - runNTPServers = runCommand.Flag("ntp-server", - "A list of NTP servers to use."). - Envar("MTG_NTP_SERVERS"). - Default("0.pool.ntp.org", "1.pool.ntp.org", "2.pool.ntp.org", "3.pool.ntp.org"). - Strings() - runSecret = runCommand.Arg("secret", "Secret of this proxy.").Required().HexBytes() - runAdtag = runCommand.Arg("adtag", "ADTag of the proxy.").HexBytes() -) - func main() { rand.Seed(time.Now().UTC().UnixNano()) - app.Version(getVersion()) - app.HelpFlag.Short('h') - - if err := utils.SetLimits(); err != nil { - cli.Fatal(err) - } - - switch kingpin.MustParse(app.Parse(os.Args[1:])) { - case generateSecretCommand.FullCommand(): - cli.Generate(*generateSecretType, *generateCloakHost) - case runCommand.FullCommand(): - err := config.Init( - config.Opt{Option: config.OptionTypeDebug, Value: *runDebug}, - config.Opt{Option: config.OptionTypeVerbose, Value: *runVerbose}, - config.Opt{Option: config.OptionTypePreferIP, Value: *runPreferIP}, - config.Opt{Option: config.OptionTypeBind, Value: *runBind}, - config.Opt{Option: config.OptionTypePublicIPv4, Value: *runPublicIPv4}, - config.Opt{Option: config.OptionTypePublicIPv6, Value: *runPublicIPv6}, - config.Opt{Option: config.OptionTypeStatsBind, Value: *runStatsBind}, - config.Opt{Option: config.OptionTypeStatsNamespace, Value: *runStatsNamespace}, - config.Opt{Option: config.OptionTypeStatsdAddress, Value: *runStatsdAddress}, - config.Opt{Option: config.OptionTypeStatsdTagsFormat, Value: *runStatsdTagsFormat}, - config.Opt{Option: config.OptionTypeStatsdTags, Value: *runStatsdTags}, - config.Opt{Option: config.OptionTypeWriteBufferSize, Value: *runWriteBufferSize}, - config.Opt{Option: config.OptionTypeReadBufferSize, Value: *runReadBufferSize}, - config.Opt{Option: config.OptionTypeCloakPort, Value: *runTLSCloakPort}, - config.Opt{Option: config.OptionTypeAntiReplayMaxSize, Value: *runAntiReplayMaxSize}, - config.Opt{Option: config.OptionTypeMultiplexPerConnection, Value: *runMultiplexPerConnection}, - config.Opt{Option: config.OptionTypeNTPServers, Value: *runNTPServers}, - config.Opt{Option: config.OptionTypeSecret, Value: *runSecret}, - config.Opt{Option: config.OptionTypeAdtag, Value: *runAdtag}, - ) - if err != nil { - cli.Fatal(err) - } - - if err := cli.Proxy(); err != nil { - cli.Fatal(err) - } - } -} - -func getVersion() string { - if version != "dev" { - return version - } - - info, ok := debug.ReadBuildInfo() - if !ok { - return version - } - - builder := strings.Builder{} - builder.WriteString(info.Main.Version) - - if info.Main.Sum != "" { - builder.WriteString(" (checksum: ") - builder.WriteString(info.Main.Sum) - builder.WriteRune(')') - } - - return builder.String() } diff --git a/mtproto/protocol.go b/mtproto/protocol.go deleted file mode 100644 index a880d10..0000000 --- a/mtproto/protocol.go +++ /dev/null @@ -1,102 +0,0 @@ -package mtproto - -import ( - "fmt" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/wrappers/packet" - "github.com/9seconds/mtg/wrappers/stream" -) - -func TelegramProtocol(req *protocol.TelegramRequest) (conntypes.PacketReadWriteCloser, error) { - conn, err := telegram.Middle.Dial(req.ClientProtocol.DC(), - req.ClientProtocol.ConnectionProtocol()) - if err != nil { - return nil, fmt.Errorf("cannot connect to telegram: %w", err) - } - - rpcNonceConn := packet.NewMtprotoFrame(conn, rpc.SeqNoNonce) - - rpcNonceReq, err := doRPCNonceRequest(rpcNonceConn) - if err != nil { - return nil, fmt.Errorf("cannot do nonce request: %w", err) - } - - rpcNonceResp, err := getRPCNonceResponse(rpcNonceConn, rpcNonceReq) - if err != nil { - return nil, fmt.Errorf("cannot get nonce response: %w", err) - } - - secureConn := stream.NewMiddleProxyCipher(conn, rpcNonceReq, rpcNonceResp, telegram.Middle.Secret()) - frameConn := packet.NewMtprotoFrame(secureConn, rpc.SeqNoHandshake) - - if err := doRPCHandshakeRequest(frameConn); err != nil { - return nil, fmt.Errorf("cannot do handshake request: %w", err) - } - - if err := getRPCHandshakeResponse(frameConn); err != nil { - return nil, fmt.Errorf("cannot get handshake response: %w", err) - } - - return frameConn, nil -} - -func doRPCNonceRequest(conn conntypes.BasePacketWriter) (*rpc.NonceRequest, error) { - rpcNonceReq, err := rpc.NewNonceRequest(telegram.Middle.Secret()) - if err != nil { - panic(err) - } - - if err := conn.Write(rpcNonceReq.Bytes()); err != nil { - return nil, err // nolint: wrapcheck - } - - return rpcNonceReq, nil -} - -func getRPCNonceResponse(conn conntypes.BasePacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { - packet, err := conn.Read() - if err != nil { - return nil, fmt.Errorf("cannot read from connection: %w", err) - } - - resp, err := rpc.NewNonceResponse(packet) - if err != nil { - return nil, fmt.Errorf("cannot build rpc nonce response: %w", err) - } - - if err = resp.Valid(req); err != nil { - return nil, fmt.Errorf("invalid nonce response: %w", err) - } - - return resp, nil -} - -func doRPCHandshakeRequest(conn conntypes.BasePacketWriter) error { - if err := conn.Write(rpc.HandshakeRequest); err != nil { - return fmt.Errorf("cannot make a request: %w", err) - } - - return nil -} - -func getRPCHandshakeResponse(conn conntypes.BasePacketReader) error { - packet, err := conn.Read() - if err != nil { - return fmt.Errorf("cannot read a response: %w", err) - } - - resp, err := rpc.NewHandshakeResponse(packet) - if err != nil { - return fmt.Errorf("cannot build a handshake response: %w", err) - } - - if err := resp.Valid(); err != nil { - return fmt.Errorf("invalid handshake response: %w", err) - } - - return nil -} diff --git a/mtproto/rpc/consts.go b/mtproto/rpc/consts.go deleted file mode 100644 index 9cb7ed4..0000000 --- a/mtproto/rpc/consts.go +++ /dev/null @@ -1,28 +0,0 @@ -package rpc - -// SeqNo* is the number of the sequence which have special meaning for -// the Telegram. -const ( - SeqNoNonce = -2 - SeqNoHandshake = -1 -) - -// Different constants for RPC protocol. -var ( - TagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} - TagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} - TagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} - TagHandshake = []byte{0xf5, 0xee, 0x82, 0x76} - TagNonce = []byte{0xaa, 0x87, 0xcb, 0x7a} - TagProxyRequest = []byte{0xee, 0xf1, 0xce, 0x36} - - NonceCryptoAES = []byte{0x01, 0x00, 0x00, 0x00} - - HandshakeFlags = []byte{0x00, 0x00, 0x00, 0x00} - - ProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} - ProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} - - HandshakeSenderPID = []byte("IPIPPRPDTIME") - HandshakePeerPID = []byte("IPIPPRPDTIME") -) diff --git a/mtproto/rpc/handshake_request.go b/mtproto/rpc/handshake_request.go deleted file mode 100644 index e0971cc..0000000 --- a/mtproto/rpc/handshake_request.go +++ /dev/null @@ -1,5 +0,0 @@ -package rpc - -var HandshakeRequest = append(TagHandshake, - append(HandshakeFlags, - append(HandshakeSenderPID, HandshakePeerPID...)...)...) diff --git a/mtproto/rpc/handshake_response.go b/mtproto/rpc/handshake_response.go deleted file mode 100644 index dc25109..0000000 --- a/mtproto/rpc/handshake_response.go +++ /dev/null @@ -1,54 +0,0 @@ -package rpc - -import ( - "bytes" - "errors" - "fmt" -) - -type HandshakeResponse struct { - Type []byte - Flags []byte - SenderPID []byte - PeerPID []byte -} - -// Bytes returns a serialized handshake response. -func (r *HandshakeResponse) Bytes() []byte { - buf := &bytes.Buffer{} - - buf.Write(r.Type) - buf.Write(r.Flags) - buf.Write(r.SenderPID) - buf.Write(r.PeerPID) - - return buf.Bytes() -} - -// Valid checks that handshake response compliments request. -func (r *HandshakeResponse) Valid() error { - if !bytes.Equal(r.Type, TagHandshake) { - return errors.New("unexpected handshake tag") - } - - if !bytes.Equal(r.PeerPID, HandshakeSenderPID) { - return errors.New("incorrect sender PID") - } - - return nil -} - -// NewHandshakeResponse constructs new handshake response from the given -// data. -func NewHandshakeResponse(data []byte) (*HandshakeResponse, error) { - if len(data) != 32 { - return nil, fmt.Errorf("incorrect handshake response length %d", len(data)) - } - - return &HandshakeResponse{ - Type: data[:4], - Flags: data[4:8], - SenderPID: data[8:20], - PeerPID: data[20:], - }, nil -} diff --git a/mtproto/rpc/nonce_request.go b/mtproto/rpc/nonce_request.go deleted file mode 100644 index 8a06c26..0000000 --- a/mtproto/rpc/nonce_request.go +++ /dev/null @@ -1,50 +0,0 @@ -package rpc - -import ( - "bytes" - "crypto/rand" - "encoding/binary" - "fmt" - "time" -) - -type NonceRequest struct { - KeySelector []byte - CryptoTS []byte - Nonce []byte -} - -// Bytes returns serialized nonce request. -func (r *NonceRequest) Bytes() []byte { - buf := &bytes.Buffer{} - - buf.Write(TagNonce) - buf.Write(r.KeySelector) - buf.Write(NonceCryptoAES) - buf.Write(r.CryptoTS) - buf.Write(r.Nonce) - - return buf.Bytes() -} - -// NewNonceRequest builds new none request based on proxy secret. -func NewNonceRequest(proxySecret []byte) (*NonceRequest, error) { - nonce := make([]byte, 16) - keySelector := make([]byte, 4) - cryptoTS := make([]byte, 4) - - if _, err := rand.Read(nonce); err != nil { - return nil, fmt.Errorf("cannot generate nonce: %w", err) - } - - copy(keySelector, proxySecret) - - timestamp := time.Now().Truncate(time.Second).Unix() % 4294967296 // 256 ^ 4 - do not know how to name - binary.LittleEndian.PutUint32(cryptoTS, uint32(timestamp)) - - return &NonceRequest{ - KeySelector: keySelector, - CryptoTS: cryptoTS, - Nonce: nonce, - }, nil -} diff --git a/mtproto/rpc/nonce_response.go b/mtproto/rpc/nonce_response.go deleted file mode 100644 index a6d831e..0000000 --- a/mtproto/rpc/nonce_response.go +++ /dev/null @@ -1,60 +0,0 @@ -package rpc - -import ( - "bytes" - "errors" - "fmt" -) - -type NonceResponse struct { - NonceRequest - - Type []byte - Crypto []byte -} - -// Bytes returns serialized form of the nonce response. -func (r *NonceResponse) Bytes() []byte { - buf := bytes.Buffer{} - - buf.Write(r.Type) - buf.Write(r.KeySelector) - buf.Write(r.Crypto) - buf.Write(r.CryptoTS) - buf.Write(r.Nonce) - - return buf.Bytes() -} - -func (r *NonceResponse) Valid(req *NonceRequest) error { - if !bytes.Equal(r.Type, TagNonce) { - return errors.New("unexpected RPC type") - } - - if !bytes.Equal(r.Crypto, NonceCryptoAES) { - return errors.New("unexpected crypto type") - } - - if !bytes.Equal(r.KeySelector, req.KeySelector) { - return errors.New("unexpected key selector") - } - - return nil -} - -// NewNonceResponse build new nonce response based on the given data. -func NewNonceResponse(data []byte) (*NonceResponse, error) { - if len(data) != 32 { - return nil, fmt.Errorf("unexpected message length %d", len(data)) - } - - return &NonceResponse{ - NonceRequest: NonceRequest{ - KeySelector: data[4:8], - CryptoTS: data[12:16], - Nonce: data[16:], - }, - Type: data[:4], - Crypto: data[8:12], - }, nil -} diff --git a/mtproto/rpc/proxy_flags.go b/mtproto/rpc/proxy_flags.go deleted file mode 100644 index 2179c66..0000000 --- a/mtproto/rpc/proxy_flags.go +++ /dev/null @@ -1,66 +0,0 @@ -package rpc - -import ( - "encoding/binary" - "strings" -) - -type ProxyRequestFlags uint32 - -const ( - ProxyRequestFlagsHasAdTag ProxyRequestFlags = 0x8 - ProxyRequestFlagsEncrypted ProxyRequestFlags = 0x2 - ProxyRequestFlagsMagic ProxyRequestFlags = 0x1000 - ProxyRequestFlagsExtMode2 ProxyRequestFlags = 0x20000 - ProxyRequestFlagsIntermediate ProxyRequestFlags = 0x20000000 - ProxyRequestFlagsAbdridged ProxyRequestFlags = 0x40000000 - ProxyRequestFlagsQuickAck ProxyRequestFlags = 0x80000000 - ProxyRequestFlagsPad ProxyRequestFlags = 0x8000000 -) - -var ProxyRequestFlagsEncryptedPrefix [8]byte - -func (r ProxyRequestFlags) Bytes() []byte { - converted := make([]byte, 4) - binary.LittleEndian.PutUint32(converted, uint32(r)) - - return converted -} - -func (r ProxyRequestFlags) String() string { - flags := make([]string, 0, 7) - - if r&ProxyRequestFlagsHasAdTag != 0 { - flags = append(flags, "HAS_AD_TAG") - } - - if r&ProxyRequestFlagsEncrypted != 0 { - flags = append(flags, "ENCRYPTED") - } - - if r&ProxyRequestFlagsMagic != 0 { - flags = append(flags, "MAGIC") - } - - if r&ProxyRequestFlagsExtMode2 != 0 { - flags = append(flags, "EXT_MODE_2") - } - - if r&ProxyRequestFlagsIntermediate != 0 { - flags = append(flags, "INTERMEDIATE") - } - - if r&ProxyRequestFlagsAbdridged != 0 { - flags = append(flags, "ABRIDGED") - } - - if r&ProxyRequestFlagsQuickAck != 0 { - flags = append(flags, "QUICK_ACK") - } - - if r&ProxyRequestFlagsPad != 0 { - flags = append(flags, "PAD") - } - - return strings.Join(flags, " | ") -} diff --git a/mtproto/rpc/proxy_response.go b/mtproto/rpc/proxy_response.go deleted file mode 100644 index 5366f34..0000000 --- a/mtproto/rpc/proxy_response.go +++ /dev/null @@ -1,53 +0,0 @@ -package rpc - -import ( - "bytes" - "fmt" - - "github.com/9seconds/mtg/conntypes" -) - -type ProxyResponseType uint8 - -const ( - ProxyResponseTypeAns ProxyResponseType = iota - ProxyResponseTypeSimpleAck - ProxyResponseTypeCloseExt -) - -type ProxyResponse struct { - Type ProxyResponseType - ConnID conntypes.ConnID - Payload conntypes.Packet -} - -func ParseProxyResponse(packet conntypes.Packet) (*ProxyResponse, error) { - var response ProxyResponse - - if len(packet) < 4 { - return nil, fmt.Errorf("incorrect packet length: %d", len(packet)) - } - - tag := packet[:4] - - switch { - case bytes.Equal(tag, TagProxyAns): - response.Type = ProxyResponseTypeAns - copy(response.ConnID[:], packet[8:16]) - response.Payload = packet[16:] - - return &response, nil - case bytes.Equal(tag, TagSimpleAck): - response.Type = ProxyResponseTypeSimpleAck - copy(response.ConnID[:], packet[4:12]) - response.Payload = packet[12:] - - return &response, nil - case bytes.Equal(tag, TagCloseExt): - response.Type = ProxyResponseTypeCloseExt - - return &response, nil - } - - return nil, fmt.Errorf("unknown response type %x", tag) -} diff --git a/ntp/ntp.go b/ntp/ntp.go deleted file mode 100644 index 97d5d63..0000000 --- a/ntp/ntp.go +++ /dev/null @@ -1,57 +0,0 @@ -package ntp - -import ( - "fmt" - "math/rand" - "time" - - "github.com/9seconds/mtg/config" - "github.com/beevik/ntp" - "go.uber.org/zap" -) - -const autoUpdatePeriod = time.Minute - -// Fetch fetches the data on time drift. -func Fetch() (time.Duration, error) { - url := config.C.NTPServers[rand.Intn(len(config.C.NTPServers))] // nolint: gosec - - resp, err := ntp.Query(url) - if err != nil { - return 0, fmt.Errorf("cannot fetch NTP server %s: %w", url, err) - } - - offsetInt := int64(resp.ClockOffset) - if offsetInt < 0 { - offsetInt = -offsetInt - } - - offset := time.Duration(offsetInt) - - return offset, nil -} - -// AutoUpdate runs periodic check of current time .drift state. -func AutoUpdate() { - logger := zap.S().Named("ntp") - - for range time.Tick(autoUpdatePeriod) { - diff, err := Fetch() - if err != nil { - logger.Debugw("Cannot fetch time from NTP", "error", err) - - continue - } - - switch { - case diff < 400*time.Millisecond: - logger.Debugw("NTP time drift", "value", diff.String()) - case diff < 600*time.Millisecond: - logger.Infow("NTP time drift", "value", diff.String()) - case diff < 800*time.Millisecond: - logger.Warnw("NTP time drift", "value", diff.String()) - default: - logger.Errorw("NTP time drift", "value", diff.String()) - } - } -} diff --git a/obfuscated2/client_protocol.go b/obfuscated2/client_protocol.go deleted file mode 100644 index 955ae9e..0000000 --- a/obfuscated2/client_protocol.go +++ /dev/null @@ -1,114 +0,0 @@ -package obfuscated2 - -import ( - "bytes" - "crypto/sha256" - "encoding/binary" - "errors" - "fmt" - "io" - "time" - - "github.com/9seconds/mtg/antireplay" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" -) - -const clientProtocolHandshakeTimeout = 10 * time.Second - -type ClientProtocol struct { - connectionType conntypes.ConnectionType - connectionProtocol conntypes.ConnectionProtocol - dc conntypes.DC -} - -func (c *ClientProtocol) ConnectionType() conntypes.ConnectionType { - return c.connectionType -} - -func (c *ClientProtocol) ConnectionProtocol() conntypes.ConnectionProtocol { - return c.connectionProtocol -} - -func (c *ClientProtocol) DC() conntypes.DC { - return c.dc -} - -func (c *ClientProtocol) Handshake(socket conntypes.StreamReadWriteCloser) (conntypes.StreamReadWriteCloser, error) { - fm, err := c.ReadFrame(socket) - if err != nil { - return nil, fmt.Errorf("cannot make a client handshake: %w", err) - } - - decHasher := sha256.New() - decHasher.Write(fm.Key()) // nolint: errcheck - decHasher.Write(config.C.Secret) // nolint: errcheck - decryptor := utils.MakeStreamCipher(decHasher.Sum(nil), fm.IV()) - - invertedFrame := fm.Invert() - encHasher := sha256.New() - encHasher.Write(invertedFrame.Key()) // nolint: errcheck - encHasher.Write(config.C.Secret) // nolint: errcheck - encryptor := utils.MakeStreamCipher(encHasher.Sum(nil), invertedFrame.IV()) - - decryptedFrame := Frame{} - decryptor.XORKeyStream(decryptedFrame.Bytes(), fm.Bytes()) - - magic := decryptedFrame.Magic() - - switch { - case bytes.Equal(magic, conntypes.ConnectionTagAbridged): - c.connectionType = conntypes.ConnectionTypeAbridged - case bytes.Equal(magic, conntypes.ConnectionTagIntermediate): - c.connectionType = conntypes.ConnectionTypeIntermediate - case bytes.Equal(magic, conntypes.ConnectionTagSecure): - c.connectionType = conntypes.ConnectionTypeSecure - default: - return nil, errors.New("unknown connection type") - } - - c.connectionProtocol = conntypes.ConnectionProtocolIPv4 - if socket.LocalAddr().IP.To4() == nil { - c.connectionProtocol = conntypes.ConnectionProtocolIPv6 - } - - buf := bytes.NewReader(decryptedFrame.DC()) - if err := binary.Read(buf, binary.LittleEndian, &c.dc); err != nil { - c.dc = conntypes.DCDefaultIdx - } - - replayKey := decryptedFrame.Unique() - if antireplay.Cache.HasObfuscated2(replayKey) { - stats.Stats.ReplayDetected() - - return nil, errors.New("replay attack is detected") - } - - antireplay.Cache.AddObfuscated2(replayKey) - - return stream.NewObfuscated2(socket, encryptor, decryptor), nil -} - -func (c *ClientProtocol) ReadFrame(socket conntypes.StreamReader) (fm Frame, err error) { - if _, err = io.ReadFull(handshakeReader{socket}, fm.Bytes()); err != nil { - err = fmt.Errorf("cannot extract obfuscated2 frame: %w", err) - } - - return -} - -type handshakeReader struct { - parent conntypes.StreamReader -} - -func (h handshakeReader) Read(p []byte) (int, error) { - return h.parent.ReadTimeout(p, clientProtocolHandshakeTimeout) -} - -func MakeClientProtocol() protocol.ClientProtocol { - return &ClientProtocol{} -} diff --git a/obfuscated2/frame.go b/obfuscated2/frame.go deleted file mode 100644 index e05cbd0..0000000 --- a/obfuscated2/frame.go +++ /dev/null @@ -1,54 +0,0 @@ -package obfuscated2 - -const ( - frameLenKey = 32 - frameLenIV = 16 - frameLenMagic = 4 - frameLenDC = 2 - - frameOffsetFirst = 8 - frameOffsetKey = frameOffsetFirst + frameLenKey - frameOffsetIV = frameOffsetKey + frameLenIV - frameOffsetMagic = frameOffsetIV + frameLenMagic - frameOffsetDC = frameOffsetMagic + frameLenDC - - frameLen = 64 -) - -// [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd]. -type Frame struct { - data [frameLen]byte -} - -func (f *Frame) Bytes() []byte { - return f.data[:] -} - -func (f *Frame) Key() []byte { - return f.data[frameOffsetFirst:frameOffsetKey] -} - -func (f *Frame) IV() []byte { - return f.data[frameOffsetKey:frameOffsetIV] -} - -func (f *Frame) Magic() []byte { - return f.data[frameOffsetIV:frameOffsetMagic] -} - -func (f *Frame) DC() []byte { - return f.data[frameOffsetMagic:frameOffsetDC] -} - -func (f *Frame) Unique() []byte { - return f.data[frameOffsetFirst:frameOffsetDC] -} - -func (f *Frame) Invert() (nf Frame) { - nf = *f - for i := 0; i < frameLenKey+frameLenIV; i++ { - nf.data[frameOffsetFirst+i] = f.data[frameOffsetIV-1-i] - } - - return -} diff --git a/obfuscated2/telegram_protocol.go b/obfuscated2/telegram_protocol.go deleted file mode 100644 index cf17099..0000000 --- a/obfuscated2/telegram_protocol.go +++ /dev/null @@ -1,68 +0,0 @@ -package obfuscated2 - -import ( - "crypto/rand" - "fmt" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" -) - -func TelegramProtocol(req *protocol.TelegramRequest) (conntypes.StreamReadWriteCloser, error) { - conn, err := telegram.Direct.Dial(req.ClientProtocol.DC(), - req.ClientProtocol.ConnectionProtocol()) - if err != nil { - return nil, fmt.Errorf("cannot dial to telegram: %w", err) - } - - conn = stream.NewTimeout(conn) - conn = stream.NewCtx(req.Ctx, req.Cancel, conn) - fm := generateFrame(req.ClientProtocol) - data := fm.Bytes() - - encryptor := utils.MakeStreamCipher(fm.Key(), fm.IV()) - decryptedFrame := fm.Invert() - decryptor := utils.MakeStreamCipher(decryptedFrame.Key(), decryptedFrame.IV()) - - copyFrame := make([]byte, frameLen) - copy(copyFrame[:frameOffsetIV], data[:frameOffsetIV]) - encryptor.XORKeyStream(data, data) - copy(data[:frameOffsetIV], copyFrame[:frameOffsetIV]) - - if _, err := conn.Write(data); err != nil { - return nil, fmt.Errorf("cannot write handshake frame to telegram: %w", err) - } - - return stream.NewObfuscated2(conn, encryptor, decryptor), nil -} - -func generateFrame(cp protocol.ClientProtocol) (fm Frame) { - data := fm.Bytes() - - for { - if _, err := rand.Read(data); err != nil { - continue - } - - if data[0] == 0xef { - continue - } - - val := (uint32(data[3]) << 24) | (uint32(data[2]) << 16) | (uint32(data[1]) << 8) | uint32(data[0]) - if val == 0x44414548 || val == 0x54534f50 || val == 0x20544547 || val == 0x4954504f || val == 0xeeeeeeee { - continue - } - - val = (uint32(data[7]) << 24) | (uint32(data[6]) << 16) | (uint32(data[5]) << 8) | uint32(data[4]) - if val == 0x00000000 { - continue - } - - copy(fm.Magic(), cp.ConnectionType().Tag()) - - return - } -} diff --git a/protocol/interfaces.go b/protocol/interfaces.go deleted file mode 100644 index 2e27fc3..0000000 --- a/protocol/interfaces.go +++ /dev/null @@ -1,12 +0,0 @@ -package protocol - -import "github.com/9seconds/mtg/conntypes" - -type ClientProtocol interface { - Handshake(conntypes.StreamReadWriteCloser) (conntypes.StreamReadWriteCloser, error) - ConnectionType() conntypes.ConnectionType - ConnectionProtocol() conntypes.ConnectionProtocol - DC() conntypes.DC -} - -type ClientProtocolMaker func() ClientProtocol diff --git a/protocol/request.go b/protocol/request.go deleted file mode 100644 index 5f5d98d..0000000 --- a/protocol/request.go +++ /dev/null @@ -1,17 +0,0 @@ -package protocol - -import ( - "context" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type TelegramRequest struct { - Logger *zap.SugaredLogger - ClientConn conntypes.StreamReadWriteCloser - ConnID conntypes.ConnID - Ctx context.Context - Cancel context.CancelFunc - ClientProtocol ClientProtocol -} diff --git a/proxy/direct.go b/proxy/direct.go deleted file mode 100644 index 03fd9af..0000000 --- a/proxy/direct.go +++ /dev/null @@ -1,49 +0,0 @@ -package proxy - -import ( - "io" - "sync" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/obfuscated2" - "github.com/9seconds/mtg/protocol" - "go.uber.org/zap" -) - -const directPipeBufferSize = 1024 - -func directConnection(request *protocol.TelegramRequest) error { - telegramConnRaw, err := obfuscated2.TelegramProtocol(request) - if err != nil { - return err // nolint: wrapcheck - } - - telegramConn := telegramConnRaw.(conntypes.StreamReadWriteCloser) - - defer telegramConn.Close() - - wg := &sync.WaitGroup{} - wg.Add(2) - - go directPipe(telegramConn, request.ClientConn, wg, request.Logger) - - go directPipe(request.ClientConn, telegramConn, wg, request.Logger) - - wg.Wait() - - return nil -} - -func directPipe(dst io.WriteCloser, src io.ReadCloser, wg *sync.WaitGroup, logger *zap.SugaredLogger) { - defer func() { - dst.Close() - src.Close() - wg.Done() - }() - - buf := [directPipeBufferSize]byte{} - - if _, err := io.CopyBuffer(dst, src, buf[:]); err != nil { - logger.Debugw("Cannot pump sockets", "error", err) - } -} diff --git a/proxy/middle.go b/proxy/middle.go deleted file mode 100644 index 7f56991..0000000 --- a/proxy/middle.go +++ /dev/null @@ -1,70 +0,0 @@ -package proxy - -import ( - "sync" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/wrappers/packetack" - "go.uber.org/zap" -) - -func middleConnection(request *protocol.TelegramRequest) { - telegramConn, err := packetack.NewProxy(request) - if err != nil { - request.Logger.Debugw("Cannot dial to Telegram", "error", err) - - return - } - defer telegramConn.Close() - - var clientConn conntypes.PacketAckFullReadWriteCloser - - switch request.ClientProtocol.ConnectionType() { - case conntypes.ConnectionTypeAbridged: - clientConn = packetack.NewClientAbridged(request.ClientConn) - case conntypes.ConnectionTypeIntermediate: - clientConn = packetack.NewClientIntermediate(request.ClientConn) - case conntypes.ConnectionTypeSecure: - clientConn = packetack.NewClientIntermediateSecure(request.ClientConn) - case conntypes.ConnectionTypeUnknown: - panic("unknown connection type") - } - - wg := &sync.WaitGroup{} - wg.Add(2) - - go middlePipe(telegramConn, clientConn, wg, request.Logger) - - go middlePipe(clientConn, telegramConn, wg, request.Logger) - - wg.Wait() -} - -func middlePipe(dst conntypes.PacketAckWriteCloser, - src conntypes.PacketAckReadCloser, - wg *sync.WaitGroup, - logger *zap.SugaredLogger) { - defer func() { - dst.Close() - src.Close() - wg.Done() - }() - - for { - acks := conntypes.ConnectionAcks{} - - packet, err := src.Read(&acks) - if err != nil { - logger.Debugw("Cannot read packet", "error", err) - - return - } - - if err = dst.Write(packet, &acks); err != nil { - logger.Debugw("Cannot send packet", "error", err) - - return - } - } -} diff --git a/proxy/proxy.go b/proxy/proxy.go deleted file mode 100644 index 7be2a19..0000000 --- a/proxy/proxy.go +++ /dev/null @@ -1,100 +0,0 @@ -package proxy - -import ( - "context" - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/protocol" - "github.com/9seconds/mtg/stats" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" - "go.uber.org/zap" -) - -type Proxy struct { - Logger *zap.SugaredLogger - Context context.Context - ClientProtocolMaker protocol.ClientProtocolMaker -} - -func (p *Proxy) Serve(listener net.Listener) { - doneChan := p.Context.Done() - - for { - conn, err := listener.Accept() - if err != nil { - select { - case <-doneChan: - return - default: - p.Logger.Fatalw("Cannot allocate incoming connection", "error", err) - } - } - - go p.accept(conn) - } -} - -func (p *Proxy) accept(conn net.Conn) { - defer func() { - conn.Close() - - if err := recover(); err != nil { - stats.Stats.Crash() - p.Logger.Errorw("Crash of accept handler", "error", err) - } - }() - - connID := conntypes.NewConnID() - logger := p.Logger.With("connection_id", connID) - - if err := utils.InitTCP(conn, config.C.ClientReadBuffer(), config.C.ClientWriteBuffer()); err != nil { - logger.Errorw("Cannot initialize client TCP connection", "error", err) - - return - } - - ctx, cancel := context.WithCancel(p.Context) - defer cancel() - - clientConn := stream.NewClientConn(conn, connID) - clientConn = stream.NewCtx(ctx, cancel, clientConn) - clientConn = stream.NewTimeout(clientConn) - - defer clientConn.Close() - - clientProtocol := p.ClientProtocolMaker() - - clientConn, err := clientProtocol.Handshake(clientConn) - if err != nil { - stats.Stats.AuthenticationFailed() - logger.Warnw("Cannot perform client handshake", "error", err) - - return - } - - stats.Stats.ClientConnected(clientProtocol.ConnectionType(), clientConn.RemoteAddr()) - defer stats.Stats.ClientDisconnected(clientProtocol.ConnectionType(), clientConn.RemoteAddr()) - logger.Infow("Client connected", "addr", conn.RemoteAddr()) - - req := &protocol.TelegramRequest{ - Logger: logger, - ClientConn: clientConn, - ConnID: connID, - Ctx: ctx, - Cancel: cancel, - ClientProtocol: clientProtocol, - } - - err = nil - - if config.C.MiddleProxyMode() { - middleConnection(req) - } else { - err = directConnection(req) - } - - logger.Infow("Client disconnected", "error", err, "addr", conn.RemoteAddr()) -} diff --git a/run.sh b/run.sh deleted file mode 100755 index 338a4c8..0000000 --- a/run.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# -# Configuration options (set by environment variables during script execution) -# - MTG_CONFIG - directory where mtg stores its configuration -# - MTG_IMAGENAME - a name of the docker image to use -# - MTG_PORT - which port of the host system should be used -# - MTG_CONTAINER - a name of the container to use -# -# Example: -# export MTG_CONFIG="$HOME/mtg_config" -# export MTG_IMAGENAME="nineseconds/mtg:latest" -# curl -sfL --compressed https://raw.githubusercontent.com/9seconds/mtg/master/run.sh | bash - -set -eu - -export XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-$HOME/.config}" -export MTG_CONFIG="${MTG_CONFIG:-$XDG_CONFIG_HOME/mtg}" - -if ! [ -x "$(command -v docker)" ]; then - echo 'Error: docker is not installed.' >&2 - exit 1 -fi - -id -Gn "$USER" | grep -qw 'docker' > /dev/null -if [ $? -eq 0 ] || [ "$(id -u)" -eq '0' ]; then - DOCKER_CMD="$(command -v docker)" -else - DOCKER_CMD="sudo $(command -v docker)" -fi - -mkdir -p "$MTG_CONFIG" || true - -MTG_SECRET="$MTG_CONFIG/secret" -MTG_ENV="$MTG_CONFIG/env" - -if [ ! -f "$MTG_ENV" ]; then - MTG_IMAGENAME="${MTG_IMAGENAME:-nineseconds/mtg:stable}" - MTG_PORT="${MTG_PORT:-3128}" - MTG_CONTAINER="${MTG_CONTAINER:-mtg}" - - echo "MTG_IMAGENAME=$MTG_IMAGENAME" > "$MTG_ENV" - echo "MTG_PORT=$MTG_PORT" >> "$MTG_ENV" - echo "MTG_CONTAINER=$MTG_CONTAINER" >> "$MTG_ENV" -fi - -set -a -source "$MTG_ENV" -set +a - -$DOCKER_CMD pull "$MTG_IMAGENAME" > /dev/null -if [ ! -f "$MTG_SECRET" ]; then - $DOCKER_CMD run \ - --rm \ - "$MTG_IMAGENAME" \ - generate-secret tls -c "$(openssl rand -hex 16).com" \ - > "$MTG_SECRET" -fi - -echo "Proxy secret is $(cat "$MTG_SECRET"). Port is $MTG_PORT." - -$DOCKER_CMD ps --filter "Name=$MTG_CONTAINER" -aq | xargs -r $DOCKER_CMD rm -fv > /dev/null -$DOCKER_CMD run \ - -d \ - --restart=unless-stopped \ - --name "$MTG_CONTAINER" \ - --ulimit nofile=51200:51200 \ - -p "$MTG_PORT:3128" \ - "$MTG_IMAGENAME" run "$(cat "$MTG_SECRET")" > /dev/null diff --git a/stats/interfaces.go b/stats/interfaces.go deleted file mode 100644 index 083a8ea..0000000 --- a/stats/interfaces.go +++ /dev/null @@ -1,60 +0,0 @@ -package stats - -import ( - "net" - - "github.com/9seconds/mtg/conntypes" -) - -type IngressTrafficInterface interface { - IngressTraffic(int) -} - -type EgressTrafficInterface interface { - EgressTraffic(int) -} - -type ClientConnectedInterface interface { - ClientConnected(conntypes.ConnectionType, *net.TCPAddr) -} - -type ClientDisconnectedInterface interface { - ClientDisconnected(conntypes.ConnectionType, *net.TCPAddr) -} - -type TelegramConnectedInterface interface { - TelegramConnected(conntypes.DC, *net.TCPAddr) -} - -type TelegramDisconnectedInterface interface { - TelegramDisconnected(conntypes.DC, *net.TCPAddr) -} - -type CrashInterface interface { - Crash() -} - -type ReplayDetectedInterface interface { - ReplayDetected() -} - -type AuthenticationFailedInterface interface { - AuthenticationFailed() -} - -type CloakedRequestInterface interface { - CloakedRequest() -} - -type Interface interface { - IngressTrafficInterface - EgressTrafficInterface - ClientConnectedInterface - ClientDisconnectedInterface - TelegramConnectedInterface - TelegramDisconnectedInterface - CrashInterface - ReplayDetectedInterface - AuthenticationFailedInterface - CloakedRequestInterface -} diff --git a/stats/multi_stats.go b/stats/multi_stats.go deleted file mode 100644 index 569c93c..0000000 --- a/stats/multi_stats.go +++ /dev/null @@ -1,69 +0,0 @@ -package stats - -import ( - "net" - - "github.com/9seconds/mtg/conntypes" -) - -type multiStats []Interface - -func (m multiStats) IngressTraffic(traffic int) { - for i := range m { - go m[i].IngressTraffic(traffic) - } -} - -func (m multiStats) EgressTraffic(traffic int) { - for i := range m { - go m[i].EgressTraffic(traffic) - } -} - -func (m multiStats) ClientConnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - for i := range m { - go m[i].ClientConnected(connectionType, addr) - } -} - -func (m multiStats) ClientDisconnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - for i := range m { - go m[i].ClientDisconnected(connectionType, addr) - } -} - -func (m multiStats) TelegramConnected(dc conntypes.DC, addr *net.TCPAddr) { - for i := range m { - go m[i].TelegramConnected(dc, addr) - } -} - -func (m multiStats) TelegramDisconnected(dc conntypes.DC, addr *net.TCPAddr) { - for i := range m { - go m[i].TelegramDisconnected(dc, addr) - } -} - -func (m multiStats) Crash() { - for i := range m { - go m[i].Crash() - } -} - -func (m multiStats) ReplayDetected() { - for i := range m { - go m[i].ReplayDetected() - } -} - -func (m multiStats) AuthenticationFailed() { - for i := range m { - go m[i].AuthenticationFailed() - } -} - -func (m multiStats) CloakedRequest() { - for i := range m { - go m[i].CloakedRequest() - } -} diff --git a/stats/stats.go b/stats/stats.go deleted file mode 100644 index 80a3250..0000000 --- a/stats/stats.go +++ /dev/null @@ -1,41 +0,0 @@ -package stats - -import ( - "context" - "fmt" - "net" - "net/http" - - "github.com/9seconds/mtg/config" -) - -var Stats Interface - -func Init(ctx context.Context) error { - mux := http.NewServeMux() - - stats := []Interface{newStatsPrometheus(mux)} - if config.C.StatsdAddr != nil { - stats = append(stats, newStatsStatsd()) - } - - listener, err := net.Listen("tcp", config.C.StatsBind.String()) - if err != nil { - return fmt.Errorf("cannot initialize stats server: %w", err) - } - - srv := http.Server{ - Handler: mux, - } - - go srv.Serve(listener) // nolint: errcheck - - go func() { - <-ctx.Done() - srv.Shutdown(context.Background()) // nolint: errcheck - }() - - Stats = multiStats(stats) - - return nil -} diff --git a/stats/stats_prometheus.go b/stats/stats_prometheus.go deleted file mode 100644 index 687c635..0000000 --- a/stats/stats_prometheus.go +++ /dev/null @@ -1,156 +0,0 @@ -package stats - -import ( - "net" - "net/http" - "strconv" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -type statsPrometheus struct { - connections *prometheus.GaugeVec - telegramConnections *prometheus.GaugeVec - traffic *prometheus.GaugeVec - crashes prometheus.Counter - replayAttacks prometheus.Counter - authenticationFailed prometheus.Counter - cloakedRequests prometheus.Counter -} - -func (s *statsPrometheus) IngressTraffic(traffic int) { - s.traffic.WithLabelValues("ingress").Add(float64(traffic)) -} - -func (s *statsPrometheus) EgressTraffic(traffic int) { - s.traffic.WithLabelValues("egress").Add(float64(traffic)) -} - -func (s *statsPrometheus) ClientConnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, 1.0) -} - -func (s *statsPrometheus) ClientDisconnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, -1.0) -} - -func (s *statsPrometheus) changeConnections(connectionType conntypes.ConnectionType, - addr *net.TCPAddr, - increment float64) { - labels := [...]string{ - "intermediate", - "ipv4", - } - - switch connectionType { - case conntypes.ConnectionTypeAbridged: - labels[0] = "abridged" - case conntypes.ConnectionTypeSecure: - labels[0] = "secured" - case conntypes.ConnectionTypeIntermediate: - labels[0] = "intermediate" - case conntypes.ConnectionTypeUnknown: - panic("unknown connection type") - } - - if addr.IP.To4() == nil { - labels[1] = "ipv6" - } - - s.connections.WithLabelValues(labels[:]...).Add(increment) -} - -func (s *statsPrometheus) TelegramConnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, 1.0) -} - -func (s *statsPrometheus) TelegramDisconnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, -1.0) -} - -func (s *statsPrometheus) changeTelegramConnections(dc conntypes.DC, addr *net.TCPAddr, increment float64) { - labels := [...]string{ - strconv.Itoa(int(dc)), - "ipv4", - } - - if addr.IP.To4() == nil { - labels[1] = "ipv6" - } - - s.telegramConnections.WithLabelValues(labels[:]...).Add(increment) -} - -func (s *statsPrometheus) Crash() { - s.crashes.Inc() -} - -func (s *statsPrometheus) ReplayDetected() { - s.replayAttacks.Inc() -} - -func (s *statsPrometheus) AuthenticationFailed() { - s.authenticationFailed.Inc() -} - -func (s *statsPrometheus) CloakedRequest() { - s.cloakedRequests.Inc() -} - -func newStatsPrometheus(mux *http.ServeMux) Interface { - registry := prometheus.NewPedanticRegistry() - - instance := &statsPrometheus{ - connections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: config.C.StatsNamespace, - Name: "connections", - Help: "Current number of client connections to the proxy.", - }, []string{"type", "protocol"}), - telegramConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: config.C.StatsNamespace, - Name: "telegram_connections", - Help: "Current number of telegram connections established by this proxy.", - }, []string{"dc", "protocol"}), - traffic: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: config.C.StatsNamespace, - Name: "traffic", - Help: "Traffic passed through the proxy in bytes.", - }, []string{"direction"}), - crashes: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "crashes", - Help: "How many crashes happened.", - }), - replayAttacks: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "replay_attacks", - Help: "How many replay attacks were prevented.", - }), - authenticationFailed: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "authentication_failed", - Help: "How many authentication failed events we've seen.", - }), - cloakedRequests: prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: config.C.StatsNamespace, - Name: "cloaked_requests", - Help: "How many requests were proxified during cloaking.", - }), - } - - registry.MustRegister(instance.connections) - registry.MustRegister(instance.telegramConnections) - registry.MustRegister(instance.traffic) - registry.MustRegister(instance.crashes) - registry.MustRegister(instance.replayAttacks) - registry.MustRegister(instance.authenticationFailed) - registry.MustRegister(instance.cloakedRequests) - - handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) - mux.Handle("/", handler) - - return instance -} diff --git a/stats/stats_statsd.go b/stats/stats_statsd.go deleted file mode 100644 index 26fce1c..0000000 --- a/stats/stats_statsd.go +++ /dev/null @@ -1,204 +0,0 @@ -package stats - -import ( - "fmt" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - statsd "github.com/smira/go-statsd" - "go.uber.org/zap" -) - -var ( - tagTrafficIngress = &statsStatsdTag{ - name: "ingress", - tag: statsd.StringTag("type", "ingress"), - } - tagTrafficEgress = &statsStatsdTag{ - name: "egress", - tag: statsd.StringTag("type", "egress"), - } - - tagConnectionTypeAbridged = &statsStatsdTag{ - name: "abridged", - tag: statsd.StringTag("type", "abridged"), - } - tagConnectionTypeIntermediate = &statsStatsdTag{ - name: "intermediate", - tag: statsd.StringTag("type", "intermediate"), - } - tagConnectionTypeSecured = &statsStatsdTag{ - name: "secured", - tag: statsd.StringTag("type", "secured"), - } - - tagConnectionProtocol4 = &statsStatsdTag{ - name: "ipv4", - tag: statsd.StringTag("protocol", "ipv4"), - } - tagConnectionProtocol6 = &statsStatsdTag{ - name: "ipv6", - tag: statsd.StringTag("protocol", "ipv6"), - } -) - -type statsStatsdTag struct { - tag statsd.Tag - name string -} - -type statsStatsdLogger struct { - log *zap.SugaredLogger -} - -func (s statsStatsdLogger) Printf(msg string, args ...interface{}) { - s.log.Debugw(fmt.Sprintf(msg, args...)) -} - -type statsStatsd struct { - seen map[string]struct{} - seenMutex sync.RWMutex - client *statsd.Client -} - -func (s *statsStatsd) IngressTraffic(traffic int) { - s.gauge("traffic", int64(traffic), tagTrafficIngress) -} - -func (s *statsStatsd) EgressTraffic(traffic int) { - s.gauge("traffic", int64(traffic), tagTrafficEgress) -} - -func (s *statsStatsd) ClientConnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, 1) -} - -func (s *statsStatsd) ClientDisconnected(connectionType conntypes.ConnectionType, addr *net.TCPAddr) { - s.changeConnections(connectionType, addr, -1) -} - -func (s *statsStatsd) changeConnections(connectionType conntypes.ConnectionType, addr *net.TCPAddr, increment int64) { - tags := make([]*statsStatsdTag, 0, 2) - - switch connectionType { - case conntypes.ConnectionTypeAbridged: - tags = append(tags, tagConnectionTypeAbridged) - case conntypes.ConnectionTypeIntermediate: - tags = append(tags, tagConnectionTypeIntermediate) - case conntypes.ConnectionTypeSecure: - tags = append(tags, tagConnectionTypeSecured) - case conntypes.ConnectionTypeUnknown: - panic("Unknown connection type") - } - - if addr.IP.To4() == nil { - tags = append(tags, tagConnectionProtocol6) - } else { - tags = append(tags, tagConnectionProtocol4) - } - - s.gauge("connections", increment, tags...) -} - -func (s *statsStatsd) TelegramConnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, 1) -} - -func (s *statsStatsd) TelegramDisconnected(dc conntypes.DC, addr *net.TCPAddr) { - s.changeTelegramConnections(dc, addr, -1) -} - -func (s *statsStatsd) changeTelegramConnections(dc conntypes.DC, addr *net.TCPAddr, increment int64) { - tags := []*statsStatsdTag{ - { - name: "dc" + strconv.Itoa(int(dc)), - tag: statsd.IntTag("dc", int(dc)), - }, - } - - if addr.IP.To4() == nil { - tags = append(tags, tagConnectionProtocol6) - } else { - tags = append(tags, tagConnectionProtocol4) - } - - s.gauge("telegram_connections", increment, tags...) -} - -func (s *statsStatsd) Crash() { - s.gauge("crashes", 1) -} - -func (s *statsStatsd) ReplayDetected() { - s.gauge("replay_attacks", 1) -} - -func (s *statsStatsd) AuthenticationFailed() { - s.gauge("authentication_failed", 1) -} - -func (s *statsStatsd) CloakedRequest() { - s.gauge("cloaked_requests", 1) -} - -func (s *statsStatsd) gauge(metric string, value int64, tags ...*statsStatsdTag) { - key, tagList := s.prepareVals(metric, tags) - s.initGauge(metric, key, tagList) - s.client.GaugeDelta(metric, value, tagList...) -} - -func (s *statsStatsd) prepareVals(metric string, tags []*statsStatsdTag) (string, []statsd.Tag) { - tagList := make([]statsd.Tag, len(tags)) - builder := strings.Builder{} - builder.WriteString(metric) - - for i, v := range tags { - builder.WriteRune('.') - builder.WriteString(v.name) - tagList[i] = v.tag - } - - return builder.String(), tagList -} - -func (s *statsStatsd) initGauge(metric, key string, tags []statsd.Tag) { - s.seenMutex.RLock() - if _, ok := s.seen[key]; ok { - s.seenMutex.RUnlock() - - return - } else { // nolint: golint,revive - s.seenMutex.RUnlock() - } - - s.seenMutex.Lock() - defer s.seenMutex.Unlock() - - if _, ok := s.seen[key]; !ok { - s.seen[key] = struct{}{} - s.client.Gauge(metric, 0, tags...) - } -} - -func newStatsStatsd() Interface { - prefix := strings.TrimSuffix(config.C.StatsNamespace, ".") + "." - logger := statsStatsdLogger{ - log: zap.S().Named("stats").Named("statsd"), - } - - return &statsStatsd{ - seen: make(map[string]struct{}), - client: statsd.NewClient(config.C.StatsdAddr.String(), - statsd.SendLoopCount(2), - statsd.ReconnectInterval(10*time.Second), - statsd.Logger(logger), - statsd.MetricPrefix(prefix), - statsd.TagStyle(config.C.StatsdTagsFormat), - ), - } -} diff --git a/telegram/api/addresses.go b/telegram/api/addresses.go deleted file mode 100644 index c7573e4..0000000 --- a/telegram/api/addresses.go +++ /dev/null @@ -1,109 +0,0 @@ -package api - -import ( - "bufio" - "fmt" - "net" - "regexp" - "strconv" - "strings" - - "github.com/9seconds/mtg/conntypes" -) - -const ( - addressesURLV4 = "https://core.telegram.org/getProxyConfig" // nolint: gas - addressesURLV6 = "https://core.telegram.org/getProxyConfigV6" // nolint: gas -) - -var addressesProxyForSplitter = regexp.MustCompile(`\s+`) - -func AddressesV4() (map[conntypes.DC][]string, conntypes.DC, error) { - return getAddresses(addressesURLV4) -} - -func AddressesV6() (map[conntypes.DC][]string, conntypes.DC, error) { - return getAddresses(addressesURLV6) -} - -func getAddresses(url string) (map[conntypes.DC][]string, conntypes.DC, error) { - resp, err := request(url) - if err != nil { - return nil, 0, fmt.Errorf("cannot get http response: %w", err) - } - - defer resp.Close() - - scanner := bufio.NewScanner(resp) - data := map[conntypes.DC][]string{} - defaultDC := conntypes.DCDefaultIdx - - for scanner.Scan() { - text := strings.TrimSpace(scanner.Text()) - - switch { - case strings.HasPrefix(text, "#"): - continue - case strings.HasPrefix(text, "proxy_for"): - addr, idx, err := addressesParseProxyFor(text) - if err != nil { - return nil, 0, fmt.Errorf("cannot parse 'proxy_for' section: %w", err) - } - - if addresses, ok := data[idx]; ok { - data[idx] = append(addresses, addr) - } else { - data[idx] = []string{addr} - } - case strings.HasPrefix(text, "default"): - idx, err := addressesParseDefault(text) - if err != nil { - return nil, 0, fmt.Errorf("cannot parse 'default' section: %w", err) - } - - defaultDC = idx - } - } - - err = scanner.Err() - if err != nil { - return nil, 0, fmt.Errorf("cannot parse http response: %w", err) - } - - return data, defaultDC, nil -} - -func addressesParseProxyFor(text string) (string, conntypes.DC, error) { - chunks := addressesProxyForSplitter.Split(text, 3) - if len(chunks) != 3 || chunks[0] != "proxy_for" { - return "", 0, fmt.Errorf("incorrect config %s", text) - } - - dc, err := strconv.ParseInt(chunks[1], 10, 16) - if err != nil { - return "", 0, fmt.Errorf("incorrect config '%s': %w", text, err) - } - - addr := strings.TrimRight(chunks[2], ";") - if _, _, err = net.SplitHostPort(addr); err != nil { - return "", 0, fmt.Errorf("incorrect config '%s': %w", text, err) - } - - return addr, conntypes.DC(dc), nil -} - -func addressesParseDefault(text string) (conntypes.DC, error) { - chunks := addressesProxyForSplitter.Split(text, 2) - if len(chunks) != 2 || chunks[0] != "default" { - return 0, fmt.Errorf("incorrect config '%s'", text) - } - - dcString := strings.TrimRight(chunks[1], ";") - - dc, err := strconv.ParseInt(dcString, 10, 16) - if err != nil { - return 0, fmt.Errorf("incorrect config '%s': %w", text, err) - } - - return conntypes.DC(dc), nil -} diff --git a/telegram/api/api.go b/telegram/api/api.go deleted file mode 100644 index 7cdef71..0000000 --- a/telegram/api/api.go +++ /dev/null @@ -1,44 +0,0 @@ -package api - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "net/http" - "time" -) - -const ( - apiUserAgent = "github.com/9seconds/mtg" - apiHTTPTimeout = 30 * time.Second -) - -var httpClient = http.Client{ - Timeout: apiHTTPTimeout, -} - -func request(url string) (io.ReadCloser, error) { - ctx, cancel := context.WithTimeout(context.Background(), apiHTTPTimeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - panic(err) - } - - req.Header.Set("Accept", "text/plan") - req.Header.Set("User-Agent", apiUserAgent) - - resp, err := httpClient.Do(req) - if err != nil { - if resp != nil { - io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck - resp.Body.Close() - } - - return nil, fmt.Errorf("cannot perform a request: %w", err) - } - - return resp.Body, err // nolint: wrapcheck -} diff --git a/telegram/api/secret.go b/telegram/api/secret.go deleted file mode 100644 index c5a1137..0000000 --- a/telegram/api/secret.go +++ /dev/null @@ -1,24 +0,0 @@ -package api - -import ( - "fmt" - "io/ioutil" -) - -const secretURL = "https://core.telegram.org/getProxySecret" // nolint: gas - -func Secret() ([]byte, error) { - resp, err := request(secretURL) - if err != nil { - return nil, fmt.Errorf("cannot access telegram server: %w", err) - } - - defer resp.Close() - - secret, err := ioutil.ReadAll(resp) - if err != nil { - return nil, fmt.Errorf("cannot read response: %w", err) - } - - return secret, nil -} diff --git a/telegram/base.go b/telegram/base.go deleted file mode 100644 index 4a6c383..0000000 --- a/telegram/base.go +++ /dev/null @@ -1,91 +0,0 @@ -package telegram - -import ( - "errors" - "math/rand" - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers/stream" - "go.uber.org/zap" -) - -type baseTelegram struct { - dialer net.Dialer - logger *zap.SugaredLogger - - secret []byte - v4DefaultDC conntypes.DC - v6DefaultDC conntypes.DC - v4Addresses map[conntypes.DC][]string - v6Addresses map[conntypes.DC][]string -} - -func (b *baseTelegram) Secret() []byte { - return b.secret -} - -func (b *baseTelegram) dial(dc conntypes.DC, - protocol conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) { - for _, addr := range b.getAddresses(dc, protocol) { - conn, err := b.dialer.Dial("tcp", addr) - if err != nil { - b.logger.Infow("Cannot dial to Telegram", "address", addr, "error", err) - - continue - } - - if err := utils.InitTCP(conn, config.C.ProxyReadBuffer(), config.C.ProxyWriteBuffer()); err != nil { - b.logger.Infow("Cannot initialize TCP socket", "address", addr, "error", err) - - continue - } - - return stream.NewTelegramConn(dc, conn), nil - } - - return nil, errors.New("cannot dial to the chosen DC") -} - -func (b *baseTelegram) getAddresses(dc conntypes.DC, protocol conntypes.ConnectionProtocol) []string { - addresses := make([]string, 0, 2) - protos := []conntypes.ConnectionProtocol{ - conntypes.ConnectionProtocolIPv6, - conntypes.ConnectionProtocolIPv4, - } - - if config.C.PreferIP == config.PreferIPv4 { - protos[0], protos[1] = protos[1], protos[0] - } - - for _, proto := range protos { - switch { - case proto&protocol == 0: - case proto&conntypes.ConnectionProtocolIPv6 != 0: - addresses = append(addresses, b.chooseAddress(b.v6Addresses, dc, b.v6DefaultDC)) - case proto&conntypes.ConnectionProtocolIPv4 != 0: - addresses = append(addresses, b.chooseAddress(b.v4Addresses, dc, b.v4DefaultDC)) - } - } - - return addresses -} - -func (b *baseTelegram) chooseAddress(addresses map[conntypes.DC][]string, - dc, defaultDC conntypes.DC) string { - addrs, ok := addresses[dc] - if !ok { - addrs = addresses[defaultDC] - } - - switch { - case len(addrs) == 1: - return addrs[0] - case len(addrs) > 1: - return addrs[rand.Intn(len(addrs))] // nolint: gosec - } - - return "" -} diff --git a/telegram/direct.go b/telegram/direct.go deleted file mode 100644 index 5a7183c..0000000 --- a/telegram/direct.go +++ /dev/null @@ -1,41 +0,0 @@ -package telegram - -import "github.com/9seconds/mtg/conntypes" - -const ( - directV4DefaultIdx conntypes.DC = 1 - directV6DefaultIdx conntypes.DC = 1 -) - -var ( - directV4Addresses = map[conntypes.DC][]string{ - 0: {"149.154.175.50:443"}, - 1: {"149.154.167.51:443"}, - 2: {"149.154.175.100:443"}, - 3: {"149.154.167.91:443"}, - 4: {"149.154.171.5:443"}, - } - directV6Addresses = map[conntypes.DC][]string{ - 0: {"[2001:b28:f23d:f001::a]:443"}, - 1: {"[2001:67c:04e8:f002::a]:443"}, - 2: {"[2001:b28:f23d:f003::a]:443"}, - 3: {"[2001:67c:04e8:f004::a]:443"}, - 4: {"[2001:b28:f23f:f005::a]:443"}, - } -) - -type directTelegram struct { - baseTelegram -} - -func (d *directTelegram) Dial(dc conntypes.DC, - protocol conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) { - switch { - case dc < 0: - dc = -dc - case dc == 0: - dc = conntypes.DCDefaultIdx - } - - return d.baseTelegram.dial(dc-1, conntypes.ConnectionProtocolAny) -} diff --git a/telegram/init.go b/telegram/init.go deleted file mode 100644 index 72da84e..0000000 --- a/telegram/init.go +++ /dev/null @@ -1,48 +0,0 @@ -package telegram - -import ( - "net" - "sync" - "time" - - "go.uber.org/zap" -) - -const telegramDialTimeout = 10 * time.Second - -var ( - Direct Telegram - Middle Telegram - - initOnce sync.Once -) - -func Init() { - initOnce.Do(func() { - logger := zap.S().Named("telegram") - - Direct = &directTelegram{ - baseTelegram: baseTelegram{ - dialer: net.Dialer{Timeout: telegramDialTimeout}, - logger: logger.Named("direct"), - v4DefaultDC: directV4DefaultIdx, - v6DefaultDC: directV6DefaultIdx, - v4Addresses: directV4Addresses, - v6Addresses: directV6Addresses, - }, - } - - tg := &middleTelegram{ - baseTelegram: baseTelegram{ - dialer: net.Dialer{Timeout: telegramDialTimeout}, - logger: logger.Named("middle"), - }, - } - if err := tg.update(); err != nil { - panic(err) - } - go tg.backgroundUpdate() - - Middle = tg - }) -} diff --git a/telegram/interfaces.go b/telegram/interfaces.go deleted file mode 100644 index e419b8b..0000000 --- a/telegram/interfaces.go +++ /dev/null @@ -1,8 +0,0 @@ -package telegram - -import "github.com/9seconds/mtg/conntypes" - -type Telegram interface { - Dial(conntypes.DC, conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) - Secret() []byte -} diff --git a/telegram/middle.go b/telegram/middle.go deleted file mode 100644 index 3683f4d..0000000 --- a/telegram/middle.go +++ /dev/null @@ -1,75 +0,0 @@ -package telegram - -import ( - "fmt" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/telegram/api" - "go.uber.org/zap" -) - -const middleTelegramBackgroundUpdateEvery = time.Hour - -type middleTelegram struct { - baseTelegram - - mutex sync.RWMutex -} - -func (m *middleTelegram) Secret() []byte { - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.baseTelegram.Secret() -} - -func (m *middleTelegram) update() error { - secret, err := api.Secret() - if err != nil { - return fmt.Errorf("cannot fetch secret: %w", err) - } - - v4Addresses, v4DefaultDC, err := api.AddressesV4() - if err != nil { - return fmt.Errorf("cannot fetch addresses for ipv4: %w", err) - } - - v6Addresses, v6DefaultDC, err := api.AddressesV6() - if err != nil { - return fmt.Errorf("cannot fetch addresses for ipv6: %w", err) - } - - m.mutex.Lock() - m.secret = secret - m.v4DefaultDC = v4DefaultDC - m.v6DefaultDC = v6DefaultDC - m.v4Addresses = v4Addresses - m.v6Addresses = v6Addresses - m.mutex.Unlock() - - return nil -} - -func (m *middleTelegram) backgroundUpdate() { - logger := zap.S().Named("telegram") - - for range time.Tick(middleTelegramBackgroundUpdateEvery) { - if err := m.update(); err != nil { - logger.Warnw("Cannot update Telegram proxies", "error", err) - } - } -} - -func (m *middleTelegram) Dial(dc conntypes.DC, - protocol conntypes.ConnectionProtocol) (conntypes.StreamReadWriteCloser, error) { - if dc == 0 { - dc = conntypes.DCDefaultIdx - } - - m.mutex.RLock() - defer m.mutex.RUnlock() - - return m.baseTelegram.dial(dc, protocol) -} diff --git a/tlstypes/client_hello.go b/tlstypes/client_hello.go deleted file mode 100644 index ffb6e87..0000000 --- a/tlstypes/client_hello.go +++ /dev/null @@ -1,86 +0,0 @@ -package tlstypes - -import ( - "bytes" - "crypto/hmac" - "crypto/sha256" - "fmt" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/utils" -) - -type ClientHello struct { - Handshake -} - -func (c ClientHello) Digest() []byte { - dirtyDigest := c.Random - c.Random = [32]byte{} - - rec := Record{ - Type: RecordTypeHandshake, - Version: Version10, - Data: &c, - } - - mac := hmac.New(sha256.New, config.C.Secret) - rec.WriteBytes(mac) - computedDigest := mac.Sum(nil) - - for i := range computedDigest { - computedDigest[i] ^= dirtyDigest[i] - } - - return computedDigest -} - -func ParseClientHello(raw []byte) (*ClientHello, error) { - rv := &ClientHello{} - - rv.Type = HandshakeType(raw[0]) - if rv.Type != HandshakeTypeClient { - return nil, fmt.Errorf("incorrect handshake type %v", rv.Type) - } - - raw = raw[1:] - sizeUint24 := utils.Uint24{} - copy(sizeUint24[:], utils.ReverseBytes(raw[:3])) - size := int(utils.FromUint24(sizeUint24)) - - raw = raw[3:] - if len(raw) != size { - return nil, fmt.Errorf("payload size mismatch (%d != %d)", len(raw), size) - } - - versionRaw := raw[:2] - - switch { - case bytes.Equal(versionRaw, Version13Bytes): - rv.Version = Version13 - case bytes.Equal(versionRaw, Version12Bytes): - rv.Version = Version12 - case bytes.Equal(versionRaw, Version11Bytes): - rv.Version = Version11 - case bytes.Equal(versionRaw, Version10Bytes): - rv.Version = Version10 - default: - return nil, fmt.Errorf("unknown protocol version %v", versionRaw) - } - - raw = raw[2:] - copy(rv.Random[:], raw[:32]) - raw = raw[32:] - - sessionIDLength := int(raw[0]) - raw = raw[1:] - rv.SessionID = make([]byte, sessionIDLength) - copy(rv.SessionID, raw) - raw = raw[sessionIDLength:] - - tail := make([]byte, len(raw)) - copy(tail, raw) - rv.Tail = RawBytes(tail) - - return rv, nil -} diff --git a/tlstypes/consts.go b/tlstypes/consts.go deleted file mode 100644 index 79d5a10..0000000 --- a/tlstypes/consts.go +++ /dev/null @@ -1,90 +0,0 @@ -package tlstypes - -import "io" - -type RecordType uint8 - -const ( - RecordTypeHandshake RecordType = 0x16 - RecordTypeApplicationData RecordType = 0x17 - RecordTypeChangeCipherSpec RecordType = 0x14 -) - -type HandshakeType uint8 - -const ( - HandshakeTypeClient HandshakeType = 0x01 - HandshakeTypeServer HandshakeType = 0x02 -) - -type CipherSuiteType uint8 - -const ( - CipherSuiteType_TLS_AES_128_GCM_SHA256 CipherSuiteType = iota // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_AES_256_GCM_SHA384 // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256 // nolint: stylecheck,golint,revive -) - -func (c CipherSuiteType) Bytes() []byte { - switch c { - case CipherSuiteType_TLS_AES_128_GCM_SHA256: - return CipherSuiteType_TLS_AES_128_GCM_SHA256_Bytes - case CipherSuiteType_TLS_AES_256_GCM_SHA384: - return CipherSuiteType_TLS_AES_256_GCM_SHA384_Bytes - case CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256: - return CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256_Bytes - } - - return CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256_Bytes -} - -type Version uint8 - -func (v Version) Bytes() []byte { - switch v { - case Version13: - return Version13Bytes - case Version12: - return Version12Bytes - case Version11: - return Version11Bytes - case Version10, VersionUnknown: - return Version10Bytes - } - - return Version10Bytes -} - -const ( - VersionUnknown Version = iota - Version10 - Version11 - Version12 - Version13 -) - -var ( - Version10Bytes = []byte{0x03, 0x01} - Version11Bytes = []byte{0x03, 0x02} - Version12Bytes = []byte{0x03, 0x03} - Version13Bytes = []byte{0x03, 0x04} - - CipherSuiteType_TLS_AES_128_GCM_SHA256_Bytes = []byte{0x13, 0x01} // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_AES_256_GCM_SHA384_Bytes = []byte{0x13, 0x02} // nolint: stylecheck,golint,revive - CipherSuiteType_TLS_CHACHA20_POLY1305_SHA256_Bytes = []byte{0x13, 0x03} // nolint: stylecheck,golint,revive -) - -type Byter interface { - WriteBytes(io.Writer) - Len() int -} - -type RawBytes []byte - -func (r RawBytes) WriteBytes(writer io.Writer) { - writer.Write(r) // nolint: errcheck -} - -func (r RawBytes) Len() int { - return len(r) -} diff --git a/tlstypes/handshake.go b/tlstypes/handshake.go deleted file mode 100644 index 3775b45..0000000 --- a/tlstypes/handshake.go +++ /dev/null @@ -1,43 +0,0 @@ -package tlstypes - -import ( - "bytes" - "io" - - "github.com/9seconds/mtg/utils" -) - -type Handshake struct { - Type HandshakeType - Version Version - Random [32]byte - SessionID []byte - Tail Byter -} - -func (h *Handshake) WriteBytes(writer io.Writer) { - packetBuf := bytes.Buffer{} - - writer.Write([]byte{byte(h.Type)}) // nolint: errcheck - - packetBuf.Write(h.Version.Bytes()) - packetBuf.Write(h.Random[:]) - packetBuf.WriteByte(byte(len(h.SessionID))) - packetBuf.Write(h.SessionID) - h.Tail.WriteBytes(&packetBuf) - - sizeUint24 := utils.ToUint24(uint32(packetBuf.Len())) - sizeUint24Bytes := sizeUint24[:] - sizeUint24Bytes[0], sizeUint24Bytes[2] = sizeUint24Bytes[2], sizeUint24Bytes[0] - - writer.Write(sizeUint24Bytes) // nolint: errcheck - packetBuf.WriteTo(writer) // nolint: errcheck -} - -func (h *Handshake) Len() int { - buf := bytes.Buffer{} - - h.WriteBytes(&buf) - - return buf.Len() -} diff --git a/tlstypes/record.go b/tlstypes/record.go deleted file mode 100644 index 5dae77e..0000000 --- a/tlstypes/record.go +++ /dev/null @@ -1,84 +0,0 @@ -package tlstypes - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" -) - -const recordMaxChunkSize = 16384 + 24 - -type Record struct { - Type RecordType - Version Version - Data Byter -} - -func (r Record) WriteBytes(writer io.Writer) { - writer.Write([]byte{byte(r.Type)}) // nolint: errcheck - writer.Write(r.Version.Bytes()) // nolint: errcheck - binary.Write(writer, binary.BigEndian, uint16(r.Data.Len())) // nolint: errcheck - r.Data.WriteBytes(writer) -} - -func (r Record) Len() int { - return 1 + 2 + 2 + r.Data.Len() -} - -func ReadRecord(reader io.Reader) (Record, error) { - buf := [2]byte{} - rec := Record{} - - if _, err := io.ReadFull(reader, buf[:1]); err != nil { - return rec, fmt.Errorf("cannot read record type: %w", err) - } - - rec.Type = RecordType(buf[0]) - - if _, err := io.ReadFull(reader, buf[:]); err != nil { - return rec, fmt.Errorf("cannot read version: %w", err) - } - - switch { - case bytes.Equal(buf[:], Version13Bytes): - rec.Version = Version13 - case bytes.Equal(buf[:], Version12Bytes): - rec.Version = Version12 - case bytes.Equal(buf[:], Version11Bytes): - rec.Version = Version11 - case bytes.Equal(buf[:], Version10Bytes): - rec.Version = Version10 - } - - if _, err := io.ReadFull(reader, buf[:]); err != nil { - return rec, fmt.Errorf("cannot read data length: %w", err) - } - - data := make([]byte, binary.BigEndian.Uint16(buf[:])) - if _, err := io.ReadFull(reader, data); err != nil { - return rec, fmt.Errorf("cannot read data: %w", err) - } - - rec.Data = RawBytes(data) - - return rec, nil -} - -func MakeRecords(raw []byte) (arr []Record) { - for len(raw) > 0 { - chunkSize := recordMaxChunkSize - if chunkSize > len(raw) { - chunkSize = len(raw) - } - - arr = append(arr, Record{ - Type: RecordTypeApplicationData, - Version: Version12, - Data: RawBytes(raw[:chunkSize]), - }) - raw = raw[chunkSize:] - } - - return -} diff --git a/tlstypes/server_hello.go b/tlstypes/server_hello.go deleted file mode 100644 index efd624c..0000000 --- a/tlstypes/server_hello.go +++ /dev/null @@ -1,97 +0,0 @@ -package tlstypes - -import ( - "bytes" - "crypto/hmac" - "crypto/rand" - "crypto/sha256" - "io" - mrand "math/rand" - - "github.com/9seconds/mtg/config" - "golang.org/x/crypto/curve25519" -) - -type ServerHello struct { - Handshake - - clientHello *ClientHello -} - -func (s ServerHello) WelcomePacket() []byte { - buf := &bytes.Buffer{} - - s.Random = [32]byte{} - rec := Record{ - Type: RecordTypeHandshake, - Version: Version12, - Data: &s, - } - rec.WriteBytes(buf) - - recChangeCipher := Record{ - Type: RecordTypeChangeCipherSpec, - Version: Version12, - Data: RawBytes([]byte{0x01}), - } - recChangeCipher.WriteBytes(buf) - - hostCert := make([]byte, 1024+mrand.Intn(3092)) // nolint: gosec - rand.Read(hostCert) // nolint: errcheck - - recData := Record{ - Type: RecordTypeApplicationData, - Version: Version12, - Data: RawBytes(hostCert), - } - recData.WriteBytes(buf) - - packet := buf.Bytes() - - mac := hmac.New(sha256.New, config.C.Secret) - mac.Write(s.clientHello.Random[:]) // nolint: errcheck - mac.Write(packet) // nolint: errcheck - copy(packet[11:], mac.Sum(nil)) - - return packet -} - -func NewServerHello(clientHello *ClientHello) *ServerHello { - rv := &ServerHello{ - clientHello: clientHello, - } - - rv.Type = HandshakeTypeServer - rv.Version = Version12 - rv.SessionID = make([]byte, len(clientHello.SessionID)) - copy(rv.SessionID, clientHello.SessionID) - - tail := bytes.NewBuffer(CipherSuiteType_TLS_AES_128_GCM_SHA256_Bytes) - tail.WriteByte(0x00) // no compression - makeTLSExtensions(tail) - rv.Tail = RawBytes(tail.Bytes()) - - return rv -} - -func makeTLSExtensions(buf io.Writer) { - buf.Write([]byte{ // nolint: errcheck - 0x00, 0x2e, // 46 bytes of data - 0x00, 0x33, // Extension - Key Share - 0x00, 0x24, // 36 bytes - 0x00, 0x1d, // x25519 curve - 0x00, 0x20, // 32 bytes of key - }) - - var scalar [32]byte - - rand.Read(scalar[:]) // nolint: errcheck - curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) - buf.Write(curve) // nolint: errcheck - - buf.Write([]byte{ // nolint: errcheck - 0x00, 0x2b, // Extension - Supported Versions - 0x00, 0x02, // 2 bytes are following - 0x03, 0x04, // TLS 1.3 - }) -} diff --git a/utils/init_tcp.go b/utils/init_tcp.go deleted file mode 100644 index d491472..0000000 --- a/utils/init_tcp.go +++ /dev/null @@ -1,28 +0,0 @@ -package utils - -import ( - "fmt" - "net" -) - -func InitTCP(conn net.Conn, readBufferSize int, writeBufferSize int) error { - tcpConn := conn.(*net.TCPConn) - - if err := tcpConn.SetNoDelay(true); err != nil { - return fmt.Errorf("cannot set TCP_NO_DELAY: %w", err) - } - - if err := tcpConn.SetReadBuffer(readBufferSize); err != nil { - return fmt.Errorf("cannot set read buffer size: %w", err) - } - - if err := tcpConn.SetWriteBuffer(writeBufferSize); err != nil { - return fmt.Errorf("cannot set write buffer size: %w", err) - } - - if err := tcpConn.SetKeepAlive(true); err != nil { - return fmt.Errorf("cannot enable keep-alive: %w", err) - } - - return nil -} diff --git a/utils/read_full.go b/utils/read_full.go deleted file mode 100644 index 1a86ab1..0000000 --- a/utils/read_full.go +++ /dev/null @@ -1,21 +0,0 @@ -package utils - -import "io" - -const readFullBufferSize = 1024 + 1 // +1 because telegram opreates with blocks mod 4 - -func ReadFull(src io.Reader) (rv []byte, err error) { - buf := make([]byte, readFullBufferSize) - n := readFullBufferSize - - for n == len(buf) { - n, err = src.Read(buf) - if err != nil { - return nil, err // nolint: wrapcheck - } - - rv = append(rv, buf[:n]...) - } - - return rv, nil -} diff --git a/utils/reverse_bytes.go b/utils/reverse_bytes.go deleted file mode 100644 index 87ad8f6..0000000 --- a/utils/reverse_bytes.go +++ /dev/null @@ -1,15 +0,0 @@ -package utils - -// ReverseBytes is a common slice reverser. -func ReverseBytes(data []byte) []byte { - dataLen := len(data) - rv := make([]byte, dataLen) - rv[dataLen/2] = data[dataLen/2] - - for i := dataLen/2 - 1; i >= 0; i-- { - opp := dataLen - i - 1 - rv[i], rv[opp] = data[opp], data[i] - } - - return rv -} diff --git a/utils/rlimit.go b/utils/rlimit.go deleted file mode 100644 index b474076..0000000 --- a/utils/rlimit.go +++ /dev/null @@ -1,24 +0,0 @@ -// +build !windows - -package utils - -import ( - "fmt" - - "golang.org/x/sys/unix" -) - -func SetLimits() error { - rLimit := unix.Rlimit{} - if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit); err != nil { - return fmt.Errorf("cannot get rlimit: %w", err) - } - - rLimit.Cur = rLimit.Max - - if err := unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit); err != nil { - return fmt.Errorf("cannot set rlimit: %w", err) - } - - return nil -} diff --git a/utils/rlimit_windows.go b/utils/rlimit_windows.go deleted file mode 100644 index 58bdf39..0000000 --- a/utils/rlimit_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build windows - -package utils - -func SetLimits() error { - return nil -} diff --git a/utils/signal_context.go b/utils/signal_context.go deleted file mode 100644 index 6d3951f..0000000 --- a/utils/signal_context.go +++ /dev/null @@ -1,25 +0,0 @@ -// +build !windows - -package utils - -import ( - "context" - "os" - "os/signal" - "syscall" -) - -func GetSignalContext() context.Context { - ctx, cancel := context.WithCancel(context.Background()) - sigChan := make(chan os.Signal, 1) - - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go func() { - for range sigChan { - cancel() - } - }() - - return ctx -} diff --git a/utils/signal_context_windows.go b/utils/signal_context_windows.go deleted file mode 100644 index d18ce71..0000000 --- a/utils/signal_context_windows.go +++ /dev/null @@ -1,23 +0,0 @@ -// +build windows - -package utils - -import ( - "context" - "os" - "os/signal" -) - -func GetSignalContext() context.Context { - ctx, cancel := context.WithCancel(context.Background()) - sigChan := make(chan os.Signal, 1) - - signal.Notify(sigChan, os.Interrupt) - go func() { - for range sigChan { - cancel() - } - }() - - return ctx -} diff --git a/utils/stream_cipher.go b/utils/stream_cipher.go deleted file mode 100644 index 44f0b27..0000000 --- a/utils/stream_cipher.go +++ /dev/null @@ -1,12 +0,0 @@ -package utils - -import ( - "crypto/aes" - "crypto/cipher" -) - -func MakeStreamCipher(key, iv []byte) cipher.Stream { - block, _ := aes.NewCipher(key) - - return cipher.NewCTR(block, iv) -} diff --git a/utils/uint24.go b/utils/uint24.go deleted file mode 100644 index 350f3d5..0000000 --- a/utils/uint24.go +++ /dev/null @@ -1,11 +0,0 @@ -package utils - -type Uint24 [3]byte - -func ToUint24(number uint32) Uint24 { - return Uint24{byte(number), byte(number >> 8), byte(number >> 16)} -} - -func FromUint24(number Uint24) uint32 { - return uint32(number[0]) + (uint32(number[1]) << 8) + (uint32(number[2]) << 16) -} diff --git a/wrappers/packet/mtproto_frame.go b/wrappers/packet/mtproto_frame.go deleted file mode 100644 index 805f361..0000000 --- a/wrappers/packet/mtproto_frame.go +++ /dev/null @@ -1,165 +0,0 @@ -package packet - -import ( - "bytes" - "crypto/aes" - "encoding/binary" - "fmt" - "hash/crc32" - "io" - "io/ioutil" - "net" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -const ( - mtprotoFrameMinMessageLength = 12 - mtprotoFrameMaxMessageLength = 16777216 -) - -var mtprotoFramePadding = []byte{0x04, 0x00, 0x00, 0x00} - -// MTProtoFrame is a wrapper which converts written data to the MTProtoFrame. -// The format of the frame: -// -// [ MSGLEN(4) | SEQNO(4) | MSG(...) | CRC32(4) | PADDING(4*x) ] -// -// MSGLEN is the length of the message + len of seqno and msglen. -// SEQNO is the number of frame in the receive/send sequence. If client -// sends a message with SeqNo 18, it has to receive message with SeqNo 18. -// MSG is the data which has to be written -// CRC32 is the CRC32 checksum of MSGLEN + SEQNO + MSG -// PADDING is custom padding schema to complete frame length to such that -// len(frame) % 16 == 0 -type wrapperMtprotoFrame struct { - parent conntypes.StreamReadWriteCloser - logger *zap.SugaredLogger - readSeqNo int32 - writeSeqNo int32 -} - -func (w *wrapperMtprotoFrame) Read() (conntypes.Packet, error) { // nolint: funlen, cyclop - buf := &bytes.Buffer{} - - sum := crc32.NewIEEE() - writer := io.MultiWriter(buf, sum) - - for { - buf.Reset() - sum.Reset() - - if _, err := io.CopyN(writer, w.parent, 4); err != nil { - return nil, fmt.Errorf("cannot read frame padding: %w", err) - } - - if !bytes.Equal(buf.Bytes(), mtprotoFramePadding) { - break - } - } - - messageLength := binary.LittleEndian.Uint32(buf.Bytes()) - w.logger.Debugw("Read MTProto frame", - "messageLength", messageLength, - "sequence_number", w.readSeqNo, - ) - - if messageLength%4 != 0 || messageLength < mtprotoFrameMinMessageLength || - messageLength > mtprotoFrameMaxMessageLength { - return nil, fmt.Errorf("incorrect frame message length %d", messageLength) - } - - buf.Reset() - - if _, err := io.CopyN(writer, w.parent, int64(messageLength)-4-4); err != nil { - return nil, fmt.Errorf("cannot read the message frame: %w", err) - } - - var seqNo int32 - - binary.Read(buf, binary.LittleEndian, &seqNo) // nolint: errcheck - - if seqNo != w.readSeqNo { - return nil, fmt.Errorf("unexpected sequence number %d (wait for %d)", seqNo, w.readSeqNo) - } - - data, _ := ioutil.ReadAll(buf) - buf.Reset() - // write to buf, not to writer. This is because we are going to fetch - // crc32 checksum. - if _, err := io.CopyN(buf, w.parent, 4); err != nil { - return nil, fmt.Errorf("cannot read checksum: %w", err) - } - - checksum := binary.LittleEndian.Uint32(buf.Bytes()) - if checksum != sum.Sum32() { - return nil, fmt.Errorf("CRC32 checksum mismatch. wait for %d, got %d", sum.Sum32(), checksum) - } - - w.logger.Debugw("Read MTProto frame", - "messageLength", messageLength, - "sequence_number", w.readSeqNo, - "dataLength", len(data), - "checksum", checksum, - ) - w.readSeqNo++ - - return data, nil -} - -func (w *wrapperMtprotoFrame) Write(p conntypes.Packet) error { - messageLength := 4 + 4 + len(p) + 4 - paddingLength := (aes.BlockSize - messageLength%aes.BlockSize) % aes.BlockSize - - buf := &bytes.Buffer{} - - binary.Write(buf, binary.LittleEndian, uint32(messageLength)) // nolint: errcheck - binary.Write(buf, binary.LittleEndian, w.writeSeqNo) // nolint: errcheck - buf.Write(p) - - checksum := crc32.ChecksumIEEE(buf.Bytes()) - binary.Write(buf, binary.LittleEndian, checksum) // nolint: errcheck - buf.Write(bytes.Repeat(mtprotoFramePadding, paddingLength/4)) - - w.logger.Debugw("Write MTProto frame", - "length", len(p), - "sequence_number", w.writeSeqNo, - "crc32", checksum, - "frame_length", buf.Len(), - ) - w.writeSeqNo++ - - _, err := w.parent.Write(buf.Bytes()) - - return err // nolint: wrapcheck -} - -func (w *wrapperMtprotoFrame) Close() error { - return w.parent.Close() -} - -func (w *wrapperMtprotoFrame) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperMtprotoFrame) Logger() *zap.SugaredLogger { - return w.logger -} - -func (w *wrapperMtprotoFrame) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperMtprotoFrame) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func NewMtprotoFrame(parent conntypes.StreamReadWriteCloser, seqNo int32) conntypes.PacketReadWriteCloser { - return &wrapperMtprotoFrame{ - parent: parent, - logger: parent.Logger().Named("mtproto-frame"), - readSeqNo: seqNo, - writeSeqNo: seqNo, - } -} diff --git a/wrappers/packetack/client_abridged.go b/wrappers/packetack/client_abridged.go deleted file mode 100644 index 2952de0..0000000 --- a/wrappers/packetack/client_abridged.go +++ /dev/null @@ -1,130 +0,0 @@ -package packetack - -import ( - "bytes" - "fmt" - "io" - "net" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/utils" - "go.uber.org/zap" -) - -const ( - clientAbridgedSmallPacketLength = 0x7f - clientAbridgedQuickAckLength = 0x80 - clientAbridgedLargePacketLength = 16777216 // 256 ^ 3 -) - -type wrapperClientAbridged struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperClientAbridged) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - buf := bytes.Buffer{} - - buf.Grow(1) - - if _, err := io.CopyN(&buf, w.parent, 1); err != nil { - return nil, fmt.Errorf("cannot read message length: %w", err) - } - - msgLength := uint32(buf.Bytes()[0]) - buf.Reset() - - if msgLength >= clientAbridgedQuickAckLength { - acks.Quick = true - msgLength -= clientAbridgedQuickAckLength - } - - if msgLength == clientAbridgedSmallPacketLength { - buf.Grow(3) - - if _, err := io.CopyN(&buf, w.parent, 3); err != nil { - return nil, fmt.Errorf("cannot read correct message length: %w", err) - } - - number := utils.Uint24{} - copy(number[:], buf.Bytes()) - msgLength = utils.FromUint24(number) - } - - msgLength *= 4 - - buf.Reset() - buf.Grow(int(msgLength)) - - if _, err := io.CopyN(&buf, w.parent, int64(msgLength)); err != nil { - return nil, fmt.Errorf("cannot read message: %w", err) - } - - return conntypes.Packet(buf.Bytes()), nil -} - -func (w *wrapperClientAbridged) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - if len(packet)%4 != 0 { - return fmt.Errorf("incorrect packet length %d", len(packet)) - } - - if acks.Simple { - if _, err := w.parent.Write(utils.ReverseBytes(packet)); err != nil { - return fmt.Errorf("cannot send a simpleacked packet: %w", err) - } - - return nil - } - - packetLength := len(packet) / 4 - - switch { - case packetLength < clientAbridgedSmallPacketLength: - data := append([]byte{byte(packetLength)}, packet...) - if _, err := w.parent.Write(data); err != nil { - return fmt.Errorf("cannot send small packet: %w", err) - } - - return nil - case packetLength < clientAbridgedLargePacketLength: - length24 := utils.ToUint24(uint32(packetLength)) - buf := bytes.Buffer{} - - buf.WriteByte(byte(clientAbridgedSmallPacketLength)) - buf.Write(length24[:]) - buf.Write(packet) - - if _, err := w.parent.Write(buf.Bytes()); err != nil { - return fmt.Errorf("cannot send large packet: %w", err) - } - - return nil - } - - return fmt.Errorf("packet is too big: %d", len(packet)) -} - -func (w *wrapperClientAbridged) Close() error { - return w.parent.Close() -} - -func (w *wrapperClientAbridged) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperClientAbridged) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperClientAbridged) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperClientAbridged) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("client-abridged") -} - -func NewClientAbridged(parent conntypes.StreamReadWriteCloser) conntypes.PacketAckFullReadWriteCloser { - return &wrapperClientAbridged{ - parent: parent, - } -} diff --git a/wrappers/packetack/client_intermediate.go b/wrappers/packetack/client_intermediate.go deleted file mode 100644 index ac60fb4..0000000 --- a/wrappers/packetack/client_intermediate.go +++ /dev/null @@ -1,89 +0,0 @@ -package packetack - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "net" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -const clientIntermediateQuickAckLength = 0x80000000 - -type wrapperClientIntermediate struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperClientIntermediate) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - buf := bytes.Buffer{} - - buf.Grow(4) - - if _, err := io.CopyN(&buf, w.parent, 4); err != nil { - return nil, fmt.Errorf("cannot read message length: %w", err) - } - - length := binary.LittleEndian.Uint32(buf.Bytes()) - - if length > clientIntermediateQuickAckLength { - acks.Quick = true - length -= clientIntermediateQuickAckLength - } - - buf.Reset() - buf.Grow(int(length)) - - if _, err := io.CopyN(&buf, w.parent, int64(length)); err != nil { - return nil, fmt.Errorf("cannot read the message: %w", err) - } - - return buf.Bytes(), nil -} - -func (w *wrapperClientIntermediate) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - if acks.Simple { - if _, err := w.parent.Write(packet); err != nil { - return fmt.Errorf("cannot send simpleacked packet: %w", err) - } - - return nil - } - - length := [4]byte{} - binary.LittleEndian.PutUint32(length[:], uint32(len(packet))) - - if _, err := w.parent.Write(append(length[:], packet...)); err != nil { - return fmt.Errorf("cannot send packet: %w", err) - } - - return nil -} - -func (w *wrapperClientIntermediate) Close() error { - return w.parent.Close() -} - -func (w *wrapperClientIntermediate) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperClientIntermediate) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperClientIntermediate) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperClientIntermediate) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("client-intermediate") -} - -func NewClientIntermediate(parent conntypes.StreamReadWriteCloser) conntypes.PacketAckFullReadWriteCloser { - return &wrapperClientIntermediate{ - parent: parent, - } -} diff --git a/wrappers/packetack/client_intermediate_secure.go b/wrappers/packetack/client_intermediate_secure.go deleted file mode 100644 index 1151a37..0000000 --- a/wrappers/packetack/client_intermediate_secure.go +++ /dev/null @@ -1,63 +0,0 @@ -package packetack - -import ( - "bytes" - "encoding/binary" - "fmt" - "math/rand" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type wrapperClientIntermediateSecure struct { - wrapperClientIntermediate -} - -func (w *wrapperClientIntermediateSecure) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - data, err := w.wrapperClientIntermediate.Read(acks) - if err != nil { - return nil, err - } - - length := len(data) - (len(data) % 4) - - return data[:length], nil -} - -func (w *wrapperClientIntermediateSecure) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - if acks.Simple { - if _, err := w.parent.Write(packet); err != nil { - return fmt.Errorf("cannot send simpleacked packet: %w", err) - } - - return nil - } - - buf := &bytes.Buffer{} - paddingLength := rand.Intn(4) // nolint: gosec - - buf.Grow(4 + len(packet) + paddingLength) - - binary.Write(buf, binary.LittleEndian, uint32(len(packet)+paddingLength)) // nolint: errcheck - buf.Write(packet) - buf.Write(make([]byte, paddingLength)) - - if _, err := w.parent.Write(buf.Bytes()); err != nil { - return fmt.Errorf("cannot send packet: %w", err) - } - - return nil -} - -func (w *wrapperClientIntermediateSecure) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("client-intermediate-secure") -} - -func NewClientIntermediateSecure(parent conntypes.StreamReadWriteCloser) conntypes.PacketAckFullReadWriteCloser { - return &wrapperClientIntermediateSecure{ - wrapperClientIntermediate: wrapperClientIntermediate{ - parent: parent, - }, - } -} diff --git a/wrappers/packetack/proxy.go b/wrappers/packetack/proxy.go deleted file mode 100644 index facbaa4..0000000 --- a/wrappers/packetack/proxy.go +++ /dev/null @@ -1,108 +0,0 @@ -package packetack - -import ( - "bytes" - "encoding/binary" - "fmt" - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/hub" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/protocol" -) - -type wrapperProxy struct { - request *protocol.TelegramRequest - proxy *hub.ProxyConn - clientIPPort []byte - ourIPPort []byte - flags rpc.ProxyRequestFlags -} - -func (w *wrapperProxy) Write(packet conntypes.Packet, acks *conntypes.ConnectionAcks) error { - buf := bytes.Buffer{} - flags := w.flags - - if acks.Quick { - flags |= rpc.ProxyRequestFlagsQuickAck - } - - if bytes.HasPrefix(packet, rpc.ProxyRequestFlagsEncryptedPrefix[:]) { - flags |= rpc.ProxyRequestFlagsEncrypted - } - - buf.Write(rpc.TagProxyRequest) - buf.Write(flags.Bytes()) - buf.Write(w.request.ConnID[:]) - buf.Write(w.clientIPPort) - buf.Write(w.ourIPPort) - buf.Write(rpc.ProxyRequestExtraSize) - buf.Write(rpc.ProxyRequestProxyTag) - buf.WriteByte(byte(len(config.C.AdTag))) - buf.Write(config.C.AdTag) - buf.Write(make([]byte, (4-buf.Len()%4)%4)) - buf.Grow(len(packet)) - buf.Write(packet) - - return w.proxy.Write(buf.Bytes()) -} - -func (w *wrapperProxy) Read(acks *conntypes.ConnectionAcks) (conntypes.Packet, error) { - resp, err := w.proxy.Read() - if err != nil { - return nil, fmt.Errorf("cannot read a response: %w", err) - } - - if resp.Type == rpc.ProxyResponseTypeSimpleAck { - acks.Simple = true - } - - return resp.Payload, nil -} - -func (w *wrapperProxy) Close() error { - w.proxy.Close() - - return nil -} - -func NewProxy(request *protocol.TelegramRequest) (conntypes.PacketAckReadWriteCloser, error) { - flags := rpc.ProxyRequestFlagsHasAdTag | rpc.ProxyRequestFlagsMagic | rpc.ProxyRequestFlagsExtMode2 - - switch request.ClientProtocol.ConnectionType() { - case conntypes.ConnectionTypeAbridged: - flags |= rpc.ProxyRequestFlagsAbdridged - case conntypes.ConnectionTypeIntermediate: - flags |= rpc.ProxyRequestFlagsIntermediate - case conntypes.ConnectionTypeSecure: - flags |= rpc.ProxyRequestFlagsIntermediate | rpc.ProxyRequestFlagsPad - case conntypes.ConnectionTypeUnknown: - panic("unknown connection type") - } - - proxy, err := hub.Hub.Register(request) - if err != nil { - return nil, fmt.Errorf("cannot make a new proxy wrapper: %w", err) - } - - return &wrapperProxy{ - flags: flags, - request: request, - proxy: proxy, - clientIPPort: proxyGetIPPort(request.ClientConn.RemoteAddr()), - ourIPPort: proxyGetIPPort(request.ClientConn.LocalAddr()), - }, nil -} - -func proxyGetIPPort(addr *net.TCPAddr) []byte { - rv := [16 + 4]byte{} - port := [4]byte{} - - copy(rv[:16], addr.IP.To16()) - binary.LittleEndian.PutUint32(port[:], uint32(addr.Port)) - copy(rv[16:], port[:]) - - return rv[:] -} diff --git a/wrappers/rwc/ping.go b/wrappers/rwc/ping.go deleted file mode 100644 index 548bef8..0000000 --- a/wrappers/rwc/ping.go +++ /dev/null @@ -1,48 +0,0 @@ -package rwc - -import ( - "context" - "io" -) - -type wrapperPing struct { - parent io.ReadWriteCloser - ctx context.Context - channelPing chan<- struct{} -} - -func (w *wrapperPing) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - if err == nil { - select { - case <-w.ctx.Done(): - case w.channelPing <- struct{}{}: - } - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperPing) Write(p []byte) (int, error) { - n, err := w.parent.Write(p) - if err == nil { - select { - case <-w.ctx.Done(): - case w.channelPing <- struct{}{}: - } - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperPing) Close() error { - return w.parent.Close() -} - -func NewPing(ctx context.Context, parent io.ReadWriteCloser, channelPing chan<- struct{}) io.ReadWriteCloser { - return &wrapperPing{ - parent: parent, - ctx: ctx, - channelPing: channelPing, - } -} diff --git a/wrappers/stream/base.go b/wrappers/stream/base.go deleted file mode 100644 index 0fd50ca..0000000 --- a/wrappers/stream/base.go +++ /dev/null @@ -1,21 +0,0 @@ -package stream - -import ( - "net" - - "github.com/9seconds/mtg/conntypes" -) - -func NewClientConn(parent net.Conn, connID conntypes.ConnID) conntypes.StreamReadWriteCloser { - conn := newConn(parent, connID, connPurposeClient) - conn = NewTrafficStats(conn) - - return conn -} - -func NewTelegramConn(dc conntypes.DC, parent net.Conn) conntypes.StreamReadWriteCloser { - conn := newConn(parent, conntypes.ConnID{}, connPurposeTelegram) - conn = NewTelegramStats(dc, conn) - - return conn -} diff --git a/wrappers/stream/blockcipher.go b/wrappers/stream/blockcipher.go deleted file mode 100644 index 6503e4f..0000000 --- a/wrappers/stream/blockcipher.go +++ /dev/null @@ -1,96 +0,0 @@ -package stream - -import ( - "crypto/aes" - "crypto/cipher" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/utils" - "go.uber.org/zap" -) - -type wrapperBlockCipher struct { - bufferedReader - - parent conntypes.StreamReadWriteCloser - encryptor cipher.BlockMode - decryptor cipher.BlockMode -} - -func (w *wrapperBlockCipher) Write(p []byte) (int, error) { - encrypted, err := w.encrypt(p) - if err != nil { - return 0, err - } - - return w.parent.Write(encrypted) -} - -func (w *wrapperBlockCipher) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - encrypted, err := w.encrypt(p) - if err != nil { - return 0, err - } - - return w.parent.WriteTimeout(encrypted, timeout) -} - -func (w *wrapperBlockCipher) encrypt(p []byte) ([]byte, error) { - if len(p)%aes.BlockSize > 0 { - return nil, fmt.Errorf("incorrect block size %d", len(p)) - } - - encrypted := make([]byte, len(p)) - w.encryptor.CryptBlocks(encrypted, p) - - return encrypted, nil -} - -func (w *wrapperBlockCipher) Close() error { - return w.parent.Close() -} - -func (w *wrapperBlockCipher) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperBlockCipher) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("block-cipher") -} - -func (w *wrapperBlockCipher) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperBlockCipher) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func newBlockCipher(parent conntypes.StreamReadWriteCloser, - encryptor, decryptor cipher.BlockMode) conntypes.StreamReadWriteCloser { - cipher := &wrapperBlockCipher{ - parent: parent, - encryptor: encryptor, - decryptor: decryptor, - } - - cipher.readFunc = func() ([]byte, error) { - var currentBuffer []byte - for len(currentBuffer) == 0 || len(currentBuffer)%aes.BlockSize != 0 { - rv, err := utils.ReadFull(cipher.parent) - if err != nil { - return nil, fmt.Errorf("cannot read data: %w", err) - } - - currentBuffer = append(currentBuffer, rv...) - } - cipher.decryptor.CryptBlocks(currentBuffer, currentBuffer) - - return currentBuffer, nil - } - - return cipher -} diff --git a/wrappers/stream/buffered_reader.go b/wrappers/stream/buffered_reader.go deleted file mode 100644 index 6a551f7..0000000 --- a/wrappers/stream/buffered_reader.go +++ /dev/null @@ -1,44 +0,0 @@ -package stream - -import ( - "bytes" - "time" -) - -type bufferedReaderReadFunc func() ([]byte, error) - -type bufferedReader struct { - buf bytes.Buffer - readFunc bufferedReaderReadFunc -} - -func (b *bufferedReader) Read(p []byte) (int, error) { - if b.buf.Len() > 0 { - return b.flush(p) - } - - res, err := b.readFunc() - if err != nil { - return 0, err - } - - b.buf.Write(res) - - return b.flush(p) -} - -func (b *bufferedReader) ReadTimeout(p []byte, _ time.Duration) (int, error) { - return b.Read(p) -} - -func (b *bufferedReader) flush(p []byte) (int, error) { - if b.buf.Len() > len(p) { - return b.buf.Read(p) - } - - sizeToReturn := b.buf.Len() - copy(p, b.buf.Bytes()) - b.buf.Reset() - - return sizeToReturn, nil -} diff --git a/wrappers/stream/conn.go b/wrappers/stream/conn.go deleted file mode 100644 index d55c55c..0000000 --- a/wrappers/stream/conn.go +++ /dev/null @@ -1,123 +0,0 @@ -package stream - -import ( - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type connPurpose uint8 - -const ( - connPurposeClient connPurpose = 1 << iota - connPurposeTelegram -) - -type wrapperConn struct { - parent net.Conn - connID conntypes.ConnID - logger *zap.SugaredLogger - localAddr *net.TCPAddr - remoteAddr *net.TCPAddr -} - -func (w *wrapperConn) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - if err := w.parent.SetWriteDeadline(time.Now().Add(timeout)); err != nil { - w.Close() - - return 0, fmt.Errorf("cannot set write deadline to the socket: %w", err) - } - - return w.Write(p) -} - -func (w *wrapperConn) Write(p []byte) (int, error) { - n, err := w.parent.Write(p) - w.logger.Debugw("write to stream", "bytes", n, "error", err) - - if err != nil { - w.Close() - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperConn) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - if err := w.parent.SetReadDeadline(time.Now().Add(timeout)); err != nil { - w.Close() - - return 0, fmt.Errorf("cannot set read deadline to the socket: %w", err) - } - - return w.Read(p) -} - -func (w *wrapperConn) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - w.logger.Debugw("Read from stream", "bytes", n, "error", err) - - if err != nil { - w.Close() - } - - return n, err // nolint: wrapcheck -} - -func (w *wrapperConn) Close() error { - w.logger.Debugw("Close connection") - - return w.parent.Close() -} - -func (w *wrapperConn) Conn() net.Conn { - return w.parent -} - -func (w *wrapperConn) Logger() *zap.SugaredLogger { - return w.logger -} - -func (w *wrapperConn) LocalAddr() *net.TCPAddr { - return w.localAddr -} - -func (w *wrapperConn) RemoteAddr() *net.TCPAddr { - return w.remoteAddr -} - -func newConn(parent net.Conn, - connID conntypes.ConnID, - purpose connPurpose) conntypes.StreamReadWriteCloser { - localAddr := *parent.LocalAddr().(*net.TCPAddr) - - if parent.RemoteAddr().(*net.TCPAddr).IP.To4() != nil { - if config.C.PublicIPv4.IP != nil { - localAddr.IP = config.C.PublicIPv4.IP - } - } else if config.C.PublicIPv6.IP != nil { - localAddr.IP = config.C.PublicIPv6.IP - } - - logger := zap.S().With( - "local_address", localAddr, - "remote_address", parent.RemoteAddr(), - ).Named("conn") - - if purpose == connPurposeClient { - logger = logger.Named("client").With("connection_id", connID.String()) - } else { - logger = logger.Named("telegram") - } - - return &wrapperConn{ - parent: parent, - connID: connID, - logger: logger, - remoteAddr: parent.RemoteAddr().(*net.TCPAddr), - localAddr: &localAddr, - } -} diff --git a/wrappers/stream/ctx.go b/wrappers/stream/ctx.go deleted file mode 100644 index 86e7359..0000000 --- a/wrappers/stream/ctx.go +++ /dev/null @@ -1,93 +0,0 @@ -package stream - -import ( - "context" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type wrapperCtx struct { - parent conntypes.StreamReadWriteCloser - ctx context.Context - cancel context.CancelFunc -} - -func (w *wrapperCtx) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.WriteTimeout(p, timeout) - } -} - -func (w *wrapperCtx) Write(p []byte) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.Write(p) - } -} - -func (w *wrapperCtx) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.ReadTimeout(p, timeout) - } -} - -func (w *wrapperCtx) Read(p []byte) (int, error) { - select { - case <-w.ctx.Done(): - w.Close() - - return 0, fmt.Errorf("cannot write because context was closed: %w", w.ctx.Err()) - default: - return w.parent.Read(p) - } -} - -func (w *wrapperCtx) Close() error { - w.cancel() - - return w.parent.Close() -} - -func (w *wrapperCtx) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperCtx) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("ctx") -} - -func (w *wrapperCtx) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperCtx) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func NewCtx(ctx context.Context, - cancel context.CancelFunc, - parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - return &wrapperCtx{ - parent: parent, - ctx: ctx, - cancel: cancel, - } -} diff --git a/wrappers/stream/faketls.go b/wrappers/stream/faketls.go deleted file mode 100644 index 17a1fe6..0000000 --- a/wrappers/stream/faketls.go +++ /dev/null @@ -1,107 +0,0 @@ -package stream - -import ( - "bytes" - "errors" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/tlstypes" - "go.uber.org/zap" -) - -type wrapperFakeTLS struct { - bufferedReader - - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperFakeTLS) Write(p []byte) (int, error) { - return w.write(p, func(b []byte) (int, error) { - return w.parent.Write(b) - }) -} - -func (w *wrapperFakeTLS) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - startTime := time.Now() - - return w.write(p, func(b []byte) (int, error) { - elapsed := time.Since(startTime) - if elapsed > timeout { - return w.parent.WriteTimeout(b, timeout-elapsed) - } - - return 0, errors.New("timeout") - }) -} - -func (w *wrapperFakeTLS) write(p []byte, writeFunc func([]byte) (int, error)) (int, error) { - sum := 0 - buf := bytes.Buffer{} - - for _, v := range tlstypes.MakeRecords(p) { - buf.Reset() - v.WriteBytes(&buf) - - _, err := writeFunc(buf.Bytes()) - if err != nil { - return sum, err - } - - sum += v.Data.Len() - } - - return sum, nil -} - -func (w *wrapperFakeTLS) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperFakeTLS) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("faketls") -} - -func (w *wrapperFakeTLS) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperFakeTLS) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperFakeTLS) Close() error { - return w.parent.Close() -} - -func NewFakeTLS(socket conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - faketls := &wrapperFakeTLS{ - parent: socket, - } - - faketls.readFunc = func() ([]byte, error) { - for { - rec, err := tlstypes.ReadRecord(faketls.parent) - if err != nil { - return nil, err // nolint: wrapcheck - } - - switch rec.Type { - case tlstypes.RecordTypeChangeCipherSpec: - case tlstypes.RecordTypeApplicationData: - buf := &bytes.Buffer{} - rec.Data.WriteBytes(buf) - - return buf.Bytes(), nil - case tlstypes.RecordTypeHandshake: - return nil, errors.New("unsupported record type handshake") - default: - return nil, fmt.Errorf("unsupported record type %v", rec.Type) - } - } - } - - return faketls -} diff --git a/wrappers/stream/mtproto_cipher.go b/wrappers/stream/mtproto_cipher.go deleted file mode 100644 index 5a59f7c..0000000 --- a/wrappers/stream/mtproto_cipher.go +++ /dev/null @@ -1,117 +0,0 @@ -package stream - -import ( - "bytes" - "crypto/aes" - "crypto/cipher" - "crypto/md5" - "crypto/sha1" - "encoding/binary" - "net" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/utils" -) - -type mtprotoCipherPurpose uint8 - -const ( - mtprotoCipherPurposeClient mtprotoCipherPurpose = iota - mtprotoCipherPurposeServer -) - -var mtprotoEmptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} - -func NewMiddleProxyCipher(parent conntypes.StreamReadWriteCloser, - req *rpc.NonceRequest, - resp *rpc.NonceResponse, - secret []byte) conntypes.StreamReadWriteCloser { - localAddr := parent.LocalAddr() - remoteAddr := parent.RemoteAddr() - - encKey, encIV := mtprotoDeriveKeys(mtprotoCipherPurposeClient, - req, - resp, - localAddr, - remoteAddr, - secret) - decKey, decIV := mtprotoDeriveKeys(mtprotoCipherPurposeServer, - req, - resp, - localAddr, - remoteAddr, - secret) - - enc, _ := mtprotoMakeEncrypterDecrypter(encKey, encIV) - _, dec := mtprotoMakeEncrypterDecrypter(decKey, decIV) - - return newBlockCipher(parent, enc, dec) -} - -func mtprotoDeriveKeys(purpose mtprotoCipherPurpose, - req *rpc.NonceRequest, - resp *rpc.NonceResponse, - client, remote *net.TCPAddr, - secret []byte) ([]byte, []byte) { - message := bytes.Buffer{} - - message.Write(resp.Nonce) - message.Write(req.Nonce) - message.Write(req.CryptoTS) - - clientIPv4 := mtprotoEmptyIP[:] - serverIPv4 := mtprotoEmptyIP[:] - - if client.IP.To4() != nil { - clientIPv4 = utils.ReverseBytes(client.IP.To4()) - serverIPv4 = utils.ReverseBytes(remote.IP.To4()) - } - - message.Write(serverIPv4) - - var port [2]byte - - binary.LittleEndian.PutUint16(port[:], uint16(client.Port)) - message.Write(port[:]) - - switch purpose { - case mtprotoCipherPurposeClient: - message.WriteString("CLIENT") - case mtprotoCipherPurposeServer: - message.WriteString("SERVER") - default: - panic("Unexpected cipher purpose") - } - - message.Write(clientIPv4) - binary.LittleEndian.PutUint16(port[:], uint16(remote.Port)) - message.Write(port[:]) - message.Write(secret) - message.Write(resp.Nonce) - - if client.IP.To4() == nil { - message.Write(client.IP.To16()) - message.Write(remote.IP.To16()) - } - - message.Write(req.Nonce) - - data := message.Bytes() - md5sum := md5.Sum(data[1:]) // nolint: gas - sha1sum := sha1.Sum(data) // nolint: gosec - - key := append(md5sum[:12], sha1sum[:]...) - iv := md5.Sum(data[2:]) // nolint: gas - - return key, iv[:] -} - -func mtprotoMakeEncrypterDecrypter(key, iv []byte) (cipher.BlockMode, cipher.BlockMode) { - block, err := aes.NewCipher(key) - if err != nil { - panic(err) - } - - return cipher.NewCBCEncrypter(block, iv), cipher.NewCBCDecrypter(block, iv) -} diff --git a/wrappers/stream/obfuscated2.go b/wrappers/stream/obfuscated2.go deleted file mode 100644 index da2199a..0000000 --- a/wrappers/stream/obfuscated2.go +++ /dev/null @@ -1,93 +0,0 @@ -package stream - -import ( - "bytes" - "crypto/cipher" - "fmt" - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type wrapperObfuscated2 struct { - encryptor cipher.Stream - decryptor cipher.Stream - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperObfuscated2) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - n, err := w.parent.ReadTimeout(p, timeout) - if err != nil { - return 0, fmt.Errorf("cannot read stream ciphered data: %w", err) - } - - w.decryptor.XORKeyStream(p, p[:n]) - - return n, nil -} - -func (w *wrapperObfuscated2) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - if err != nil { - return n, err // nolint: wrapcheck - } - - w.decryptor.XORKeyStream(p, p[:n]) - - return n, nil -} - -func (w *wrapperObfuscated2) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - buffer := bytes.Buffer{} - - buffer.Write(p) - - buf := buffer.Bytes() - - w.encryptor.XORKeyStream(buf, buf) - - return w.parent.WriteTimeout(buf, timeout) -} - -func (w *wrapperObfuscated2) Write(p []byte) (int, error) { - buffer := bytes.Buffer{} - - buffer.Write(p) - - buf := buffer.Bytes() - - w.encryptor.XORKeyStream(buf, buf) - - return w.parent.Write(buf) -} - -func (w *wrapperObfuscated2) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperObfuscated2) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("obfuscated2") -} - -func (w *wrapperObfuscated2) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperObfuscated2) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperObfuscated2) Close() error { - return w.parent.Close() -} - -func NewObfuscated2(socket conntypes.StreamReadWriteCloser, - encryptor, decryptor cipher.Stream) conntypes.StreamReadWriteCloser { - return &wrapperObfuscated2{ - parent: socket, - encryptor: encryptor, - decryptor: decryptor, - } -} diff --git a/wrappers/stream/rewind.go b/wrappers/stream/rewind.go deleted file mode 100644 index d8d7425..0000000 --- a/wrappers/stream/rewind.go +++ /dev/null @@ -1,83 +0,0 @@ -package stream - -import ( - "bytes" - "io" - "net" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -type ReadWriteCloseRewinder interface { - conntypes.StreamReadWriteCloser - Rewind() -} - -type wrapperRewind struct { - parent conntypes.StreamReadWriteCloser - activeReader io.Reader - buf bytes.Buffer - mutex sync.Mutex -} - -func (w *wrapperRewind) Write(p []byte) (int, error) { - return w.parent.Write(p) -} - -func (w *wrapperRewind) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.WriteTimeout(p, timeout) -} - -func (w *wrapperRewind) Read(p []byte) (int, error) { - w.mutex.Lock() - defer w.mutex.Unlock() - - return w.activeReader.Read(p) -} - -func (w *wrapperRewind) ReadTimeout(p []byte, _ time.Duration) (int, error) { - w.mutex.Lock() - defer w.mutex.Unlock() - - return w.activeReader.Read(p) -} - -func (w *wrapperRewind) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperRewind) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("rewinded") -} - -func (w *wrapperRewind) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperRewind) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperRewind) Close() error { - w.buf.Reset() - - return w.parent.Close() -} - -func (w *wrapperRewind) Rewind() { - w.mutex.Lock() - w.activeReader = io.MultiReader(&w.buf, w.parent) - w.mutex.Unlock() -} - -func NewRewind(parent conntypes.StreamReadWriteCloser) ReadWriteCloseRewinder { - rv := &wrapperRewind{ - parent: parent, - } - rv.activeReader = io.TeeReader(parent, &rv.buf) - - return rv -} diff --git a/wrappers/stream/stats_telegram.go b/wrappers/stream/stats_telegram.go deleted file mode 100644 index 9fe6d9e..0000000 --- a/wrappers/stream/stats_telegram.go +++ /dev/null @@ -1,71 +0,0 @@ -package stream - -import ( - "net" - "sync" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/stats" - "go.uber.org/zap" -) - -type wrapperTelegramStats struct { - parent conntypes.StreamReadWriteCloser - dc conntypes.DC - once sync.Once -} - -func (w *wrapperTelegramStats) Write(p []byte) (int, error) { - return w.parent.Write(p) -} - -func (w *wrapperTelegramStats) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.WriteTimeout(p, timeout) -} - -func (w *wrapperTelegramStats) Read(p []byte) (int, error) { - return w.parent.Read(p) -} - -func (w *wrapperTelegramStats) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.ReadTimeout(p, timeout) -} - -func (w *wrapperTelegramStats) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperTelegramStats) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("stats-telegram") -} - -func (w *wrapperTelegramStats) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperTelegramStats) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperTelegramStats) Close() error { - var err error - - w.once.Do(func() { - err = w.parent.Close() - stats.Stats.TelegramDisconnected(w.dc, w.RemoteAddr()) - }) - - return err // nolint: wrapcheck -} - -func NewTelegramStats(dc conntypes.DC, parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - conn := &wrapperTelegramStats{ - parent: parent, - dc: dc, - } - - stats.Stats.TelegramConnected(dc, parent.RemoteAddr()) - - return conn -} diff --git a/wrappers/stream/stats_traffic.go b/wrappers/stream/stats_traffic.go deleted file mode 100644 index cc7d224..0000000 --- a/wrappers/stream/stats_traffic.go +++ /dev/null @@ -1,66 +0,0 @@ -package stream - -import ( - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "github.com/9seconds/mtg/stats" - "go.uber.org/zap" -) - -type wrapperTrafficStats struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperTrafficStats) Write(p []byte) (int, error) { - n, err := w.parent.Write(p) - stats.Stats.EgressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - n, err := w.parent.WriteTimeout(p, timeout) - stats.Stats.EgressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) Read(p []byte) (int, error) { - n, err := w.parent.Read(p) - stats.Stats.IngressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - n, err := w.parent.ReadTimeout(p, timeout) - stats.Stats.IngressTraffic(n) - - return n, err // nolint: wrapcheck -} - -func (w *wrapperTrafficStats) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperTrafficStats) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("stats-traffic") -} - -func (w *wrapperTrafficStats) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperTrafficStats) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func (w *wrapperTrafficStats) Close() error { - return w.parent.Close() -} - -func NewTrafficStats(parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - return &wrapperTrafficStats{parent} -} diff --git a/wrappers/stream/timeout.go b/wrappers/stream/timeout.go deleted file mode 100644 index 2447fe2..0000000 --- a/wrappers/stream/timeout.go +++ /dev/null @@ -1,60 +0,0 @@ -package stream - -import ( - "net" - "time" - - "github.com/9seconds/mtg/conntypes" - "go.uber.org/zap" -) - -const ( - timeoutRead = 2 * time.Minute - timeoutWrite = 2 * time.Minute -) - -type wrapperTimeout struct { - parent conntypes.StreamReadWriteCloser -} - -func (w *wrapperTimeout) WriteTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.WriteTimeout(p, timeout) -} - -func (w *wrapperTimeout) Write(p []byte) (int, error) { - return w.parent.WriteTimeout(p, timeoutWrite) -} - -func (w *wrapperTimeout) ReadTimeout(p []byte, timeout time.Duration) (int, error) { - return w.parent.ReadTimeout(p, timeout) -} - -func (w *wrapperTimeout) Read(p []byte) (int, error) { - return w.parent.ReadTimeout(p, timeoutRead) -} - -func (w *wrapperTimeout) Close() error { - return w.parent.Close() -} - -func (w *wrapperTimeout) Conn() net.Conn { - return w.parent.Conn() -} - -func (w *wrapperTimeout) Logger() *zap.SugaredLogger { - return w.parent.Logger().Named("timeout") -} - -func (w *wrapperTimeout) LocalAddr() *net.TCPAddr { - return w.parent.LocalAddr() -} - -func (w *wrapperTimeout) RemoteAddr() *net.TCPAddr { - return w.parent.RemoteAddr() -} - -func NewTimeout(parent conntypes.StreamReadWriteCloser) conntypes.StreamReadWriteCloser { - return &wrapperTimeout{ - parent: parent, - } -} From 2be900745ffd3bd7a6bf02bfa67bafa31e4c3957 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 4 Mar 2021 14:13:08 +0300 Subject: [PATCH 002/206] Add example of configuration file --- example.config.toml | 127 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 example.config.toml diff --git a/example.config.toml b/example.config.toml new file mode 100644 index 0000000..a94a3cf --- /dev/null +++ b/example.config.toml @@ -0,0 +1,127 @@ +# This is an example of the configuration file for mtg. You actually can +# run mtg with it. It starts a proxy on all interfaces with a secret +# ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d +# +# It has all possible options with default values. So, a real world +# configuration file should contain only those options you are going to +# use. You do not need to enumerate all of them. In other words, each +# option here has a default value. If you comment a key-value pair, it +# should not make any effect. +# +# stats is the only exception. + +# Debug starts application in debug mode. It starts to be quite verbose +# in output. Actually, the idea is that you run it in debug mode only if +# you have any issue. +debug = false + +# A secret. Please remember that mtg supports only FakeTLS mode, legacy +# simple and secured mode are prohibited. For you it means that secret +# should either be base64-encoded or starts with ee. +secret = "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d" + +# Host:port pair to run proxy on. +bind-to = "0.0.0.0:3128" + +# A size of user-space buffer for TCP to use. Since we do 2 connections, +# then we have tcp-buffer * (4 + 2) per each connection: read/write for +# each connection + 2 copy buffers to pump the data between sockets. +tcp-buffer = "4kb" + +# Sometimes you want to enforce mtg to use some types of +# IP connectivity to Telegram. We have 4 modes: +# - prefer-ipv6: +# We can use both ipv4 and ipv6 but ipv6 has a preference +# - prefer-ipv4: +# We can use both ipv4 and ipv6 but ipv4 has a preference +# - only-ipv6: +# Only ipv6 connectivity is used +# - only-ipv4: +# Only ipv4 connectivity is used +prefer-ips = "prefer-ipv6" + +# FakeTLS uses domain fronting protection. So it needs to know a port to +# access. +cloak-port = 443 + +# FakeTLS can compare timestamps to prevent probes. Each message has +# encrypted timestamp. So, mtg can compare this timestamp and decide if +# we need to proceed with connection or not. +# +# Please ensure that you have some ntp active on this host. Otherwise, +# you can endup with badly performing proxy. +[probes.time] +# You can enable/disable that. A good idea is always enable. +enabled = true +# Time can be skewed by many reasons. So, this is a time interval +# when message is cosidered as a good one. +allow-skewness = "5s" + +# Some countries do active probing on Telegram connections. This technique +# allows to protect from such effort. +# +# mtg has a cache of some connection fingerprints. Actually, first bytes +# of each connection. So, it stores them in some in-memory LRU+TTL cache. +# You can configure this cache here. +[probes.anti-replay] +# You can enable/disable this feature. +enabled = true +# max size of such a cache. Please be aware that this number is +# approximate we try hard to store data quite dense but it is possible +# that we can go over this limit for 10-20% under some conditions and +# architectures. +max-size = "16mb" +# TTL for each cache record. +ttl = "8h" + +# public ip addresses of the server. Actually, it is required only to +# generate a correct access file. if you use default values here, mtg +# will try to resolve these IPs on its own. +[public-ip] +ipv4 = "" +ipv6 = "" + +# you can redefine a dialer for mtg. Dialer is how we 'dial' to either +# some external services or telegram. empty string means default +# connectivity. +# +# it is also possible to use socks5 or shadowsocks here +# +# socks5 example: +# socks5://user:password@host:port +# shadowsocks example (SIP002): +# ss://YWVzLTEyOC1nY206dGVzdA@192.168.100.1:8888 +# +# You can define 2 dialers here: telegram and default. Telegram dialer +# is used to connect to Telegram servers only. Default is used for other +# purposes, like accessing ifconfig.co to obtains public address (DNS is +# resolved via DoH) +# +# Please also be aware that dialers are only doing TCP. If UDP is +# required (for statsd for example), then these dialers are going to be +# ignored. +# +# If telegram dialer is not defined, a default one is going to be used. +[dialers] +telegram = "" +default = "" + +# statsd statistics integration. +[stats.statsd] +# enabled/disabled +enabled = false +# host:port for UDP endpoint of statsd +address = "127.0.0.1:8888" +# prefix of metric for statsd +metric-prefix = "mtg" + +# prometheus metrics integration. +[stats.prometheus] +# enabled/disabled +enabled = true +# host:port where to start http server for endpoint +bind-to = "127.0.0.1:3129" +# prefix of http path +http-path = "/" +# prefix for metrics for prometheus +metric-prefix = "mtg" From 46894797453e31c616b710b17705281b9db834b3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 11:51:43 +0300 Subject: [PATCH 003/206] Add base dialers module --- example.config.toml | 7 +++ go.mod | 8 +++ go.sum | 36 +++++++++++ main.go | 19 ++++++ mtglib/dialers/consts.go | 9 +++ mtglib/dialers/default.go | 82 +++++++++++++++++++++++++ mtglib/dialers/dialer.go | 112 ++++++++++++++++++++++++++++++++++ mtglib/dialers/interfaces.go | 11 ++++ mtglib/dialers/shadowsocks.go | 68 +++++++++++++++++++++ mtglib/dialers/socks5.go | 23 +++++++ mtglib/secret.go | 71 +++++++++++++++++++++ raw_config.go | 60 ++++++++++++++++++ 12 files changed, 506 insertions(+) create mode 100644 mtglib/dialers/consts.go create mode 100644 mtglib/dialers/default.go create mode 100644 mtglib/dialers/dialer.go create mode 100644 mtglib/dialers/interfaces.go create mode 100644 mtglib/dialers/shadowsocks.go create mode 100644 mtglib/dialers/socks5.go create mode 100644 mtglib/secret.go create mode 100644 raw_config.go diff --git a/example.config.toml b/example.config.toml index a94a3cf..3f7bea7 100644 --- a/example.config.toml +++ b/example.config.toml @@ -44,6 +44,13 @@ prefer-ips = "prefer-ipv6" # access. cloak-port = 443 +# Path to access file. Each time when proxy starts up, it writes an +# access file. This file contains a JSON with settings how to access +# this proxy. +# +# Pass filepath here or '-' if you want to dump into stdout. +access-file = "-" + # FakeTLS can compare timestamps to prevent probes. Each message has # encrypted timestamp. So, mtg can compare this timestamp and decide if # we need to proceed with connection or not. diff --git a/go.mod b/go.mod index 2b87317..10e8001 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,11 @@ module github.com/9seconds/mtg/v2 go 1.16 + +require ( + github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 + github.com/libp2p/go-reuseport v0.0.2 + github.com/pelletier/go-toml v1.8.1 + github.com/shadowsocks/go-shadowsocks2 v0.1.4 + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 +) diff --git a/go.sum b/go.sum index e69de29..fd68855 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,36 @@ +github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= +github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= +github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr8meMVVGxhp+QBTqY91tM8HjEuMjGg= +github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3/go.mod h1:HgjTstvQsPGkxUsCd2KWxErBblirPizecHcpD3ffK+s= +github.com/shadowsocks/go-shadowsocks2 v0.1.4 h1:4VzajPL7RwwmImysBSvI+lm/UaegDGQq3hr42dYo3gs= +github.com/shadowsocks/go-shadowsocks2 v0.1.4/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 8c4cef3..3281a3f 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,31 @@ package main import ( + "fmt" + "io/ioutil" "math/rand" + "os" "time" + + "github.com/9seconds/mtg/v2/mtglib/dialers" ) var version = "dev" // has to be set by ldflags func main() { rand.Seed(time.Now().UTC().UnixNano()) + + f, _ := os.Open("example.config.toml") + + fmt.Println(parseRawConfig(f)) + + bd, _ := dialers.NewDefaultBaseDialer(0, 0) + d, _ := dialers.MakeDialer(bd, "9.9.9.9", 0) + + r, err := d.HTTP.Get("https://ifconfig.co") + + fmt.Println(err) + body, _ := ioutil.ReadAll(r.Body) + + fmt.Println(string(body)) } diff --git a/mtglib/dialers/consts.go b/mtglib/dialers/consts.go new file mode 100644 index 0000000..40a022d --- /dev/null +++ b/mtglib/dialers/consts.go @@ -0,0 +1,9 @@ +package dialers + +import "time" + +const ( + DefaultTimeout = 10 * time.Second + DefaultHTTPTimeout = DefaultTimeout + DefaultBufferSize = 4096 +) diff --git a/mtglib/dialers/default.go b/mtglib/dialers/default.go new file mode 100644 index 0000000..feeda9d --- /dev/null +++ b/mtglib/dialers/default.go @@ -0,0 +1,82 @@ +package dialers + +import ( + "context" + "fmt" + "net" + "time" + + "github.com/libp2p/go-reuseport" +) + +type defaultBaseDialer struct { + net.Dialer + + bufferSize int +} + +func (d *defaultBaseDialer) Dial(network, address string) (net.Conn, error) { + return d.DialContext(context.Background(), network, address) +} + +func (d *defaultBaseDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + switch network { + case "tcp", "tcp4", "tcp6": + default: + return nil, fmt.Errorf("unsupported network %s", network) + } + + conn, err := d.Dialer.DialContext(ctx, network, address) + if err != nil { + return nil, fmt.Errorf("cannot dial to %s: %w", address, err) + } + + tcpConn := conn.(*net.TCPConn) + + if err := tcpConn.SetNoDelay(true); err != nil { + conn.Close() + return nil, fmt.Errorf("cannot set TCP_NO_DELAY: %w", err) + } + + if err := tcpConn.SetReadBuffer(d.bufferSize); err != nil { + tcpConn.Close() + return nil, fmt.Errorf("cannot set read buffer size: %w", err) + } + + if err := tcpConn.SetWriteBuffer(d.bufferSize); err != nil { + tcpConn.Close() + return nil, fmt.Errorf("cannot set write buffer size: %w", err) + } + + if err := tcpConn.SetKeepAlive(true); err != nil { + tcpConn.Close() + return nil, fmt.Errorf("cannot enable keep-alive: %w", err) + } + + return tcpConn, nil +} + +func NewDefaultBaseDialer(timeout time.Duration, bufferSize int) (BaseDialer, error) { + switch { + case timeout < 0: + return nil, fmt.Errorf("timeout %v should be positive number", timeout) + case bufferSize < 0: + return nil, fmt.Errorf("buffer size %s should be positive number", bufferSize) + } + + if timeout == 0 { + timeout = DefaultTimeout + } + + if bufferSize == 0 { + bufferSize = DefaultBufferSize + } + + return &defaultBaseDialer{ + Dialer: net.Dialer{ + Timeout: timeout, + Control: reuseport.Control, + }, + bufferSize: bufferSize, + }, nil +} diff --git a/mtglib/dialers/dialer.go b/mtglib/dialers/dialer.go new file mode 100644 index 0000000..3f8d1be --- /dev/null +++ b/mtglib/dialers/dialer.go @@ -0,0 +1,112 @@ +package dialers + +import ( + "context" + "fmt" + "math/rand" + "net" + "net/http" + "time" + + doh "github.com/babolivier/go-doh-client" +) + +type Dialer struct { + HTTP http.Client + DNS doh.Resolver + + baseDialer BaseDialer +} + +func (d *Dialer) Dial(network, address string) (net.Conn, error) { + return d.DialContext(context.Background(), network, address) +} + +func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + host, port, _ := net.SplitHostPort(address) + + ips, err := d.resolveIPs(network, host) + if err != nil { + return nil, fmt.Errorf("cannot resolve dns names: %w", err) + } + + rand.Shuffle(len(ips), func(i, j int) { + ips[i], ips[j] = ips[j], ips[i] + }) + + for _, v := range ips { + if conn, err := d.baseDialer.DialContext(ctx, network, net.JoinHostPort(v, port)); err == nil { + return conn, nil + } + } + + return nil, fmt.Errorf("cannot dial to %s:%s", network, address) +} + +func (d *Dialer) resolveIPs(network, address string) ([]string, error) { + if net.ParseIP(address) != nil { + return []string{address}, nil + } + + var ips []string + + switch network { + case "tcp", "tcp4": + if recs, _, err := d.DNS.LookupA(address); err == nil { + for _, v := range recs { + ips = append(ips, v.IP4) + } + } + } + + switch network { + case "tcp", "tcp6": + if recs, _, err := d.DNS.LookupAAAA(address); err == nil { + for _, v := range recs { + ips = append(ips, v.IP6) + } + } + } + + if len(ips) == 0 { + return nil, fmt.Errorf("cannot find any ips for %s:%s", network, address) + } + + return ips, nil +} + +func MakeDialer(base BaseDialer, dohHostname string, httpTimeout time.Duration) (*Dialer, error) { + switch { + case httpTimeout < 0: + return nil, fmt.Errorf("timeout should be positive number %v", httpTimeout) + case httpTimeout == 0: + httpTimeout = DefaultHTTPTimeout + } + + if net.ParseIP(dohHostname) == nil { + return nil, fmt.Errorf("hostname %s should be IP address", dohHostname) + } + + dohHTTPClient := &http.Client{ + Timeout: httpTimeout, + Transport: &http.Transport{ + DialContext: base.DialContext, + }, + } + rv := &Dialer{ + baseDialer: base, + DNS: doh.Resolver{ + Host: dohHostname, + Class: doh.IN, + HTTPClient: dohHTTPClient, + }, + } + rv.HTTP = http.Client{ + Timeout: httpTimeout, + Transport: &http.Transport{ + DialContext: rv.DialContext, + }, + } + + return rv, nil +} diff --git a/mtglib/dialers/interfaces.go b/mtglib/dialers/interfaces.go new file mode 100644 index 0000000..774aa77 --- /dev/null +++ b/mtglib/dialers/interfaces.go @@ -0,0 +1,11 @@ +package dialers + +import ( + "context" + "net" +) + +type BaseDialer interface { + Dial(network, address string) (net.Conn, error) + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} diff --git a/mtglib/dialers/shadowsocks.go b/mtglib/dialers/shadowsocks.go new file mode 100644 index 0000000..1929fa2 --- /dev/null +++ b/mtglib/dialers/shadowsocks.go @@ -0,0 +1,68 @@ +package dialers + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "net/url" + "strings" + "time" + + shadowsocks "github.com/shadowsocks/go-shadowsocks2/core" +) + +type shadowsocksBaseDialer struct { + base BaseDialer + cipher shadowsocks.StreamConnCipher +} + +func (s *shadowsocksBaseDialer) Dial(network, address string) (net.Conn, error) { + conn, err := s.base.Dial(network, address) + if err != nil { + return nil, err + } + + return s.cipher.StreamConn(conn), nil +} + +func (s *shadowsocksBaseDialer) DialContext(ctx context.Context, + network, address string) (net.Conn, error) { + conn, err := s.base.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + + return s.cipher.StreamConn(conn), nil +} + +func NewShadowsocksBaseDialer(proxyUrl *url.URL, + timeout time.Duration, bufferSize int) (BaseDialer, error) { + username := proxyUrl.User.Username() + + decoded, err := base64.RawURLEncoding.DecodeString(username) + if err != nil { + return nil, fmt.Errorf("cannot decode payload: %w", err) + } + + chunks := strings.SplitN(string(decoded), ":", 2) + if len(chunks) != 2 { + return nil, fmt.Errorf("incorrect payload %s", username) + } + + cipher, err := shadowsocks.PickCipher(chunks[0], nil, chunks[1]) + if err != nil { + return nil, fmt.Errorf("cannot initialize shadowsocks cipher: %w", err) + } + + baseDialer, err := NewDefaultBaseDialer(timeout, bufferSize) + if err != nil { + return nil, fmt.Errorf("cannot initialize a base dialer: %w", err) + + } + + return &shadowsocksBaseDialer{ + base: baseDialer, + cipher: cipher, + }, nil +} diff --git a/mtglib/dialers/socks5.go b/mtglib/dialers/socks5.go new file mode 100644 index 0000000..f5f93ca --- /dev/null +++ b/mtglib/dialers/socks5.go @@ -0,0 +1,23 @@ +package dialers + +import ( + "fmt" + "net/url" + "time" + + "golang.org/x/net/proxy" +) + +func NewSocks5BaseDialer(proxyUrl *url.URL, timeout time.Duration, bufferSize int) (BaseDialer, error) { + baseDialer, err := NewDefaultBaseDialer(timeout, bufferSize) + if err != nil { + return nil, fmt.Errorf("cannot initialize base dialer: %w", err) + } + + rv, err := proxy.FromURL(proxyUrl, baseDialer.(*defaultBaseDialer)) + if err != nil { + return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) + } + + return rv.(BaseDialer), nil +} diff --git a/mtglib/secret.go b/mtglib/secret.go new file mode 100644 index 0000000..865ca60 --- /dev/null +++ b/mtglib/secret.go @@ -0,0 +1,71 @@ +package mtglib + +import ( + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strings" +) + +type Secret struct { + Key []byte + Host string +} + +func (s *Secret) MarshalText() ([]byte, error) { + if s == nil { + return nil, nil + } + + return []byte(s.String()), nil +} + +func (s *Secret) UnmarshalText(text []byte) error { + sc, err := ParseSecret(string(text)) + if err != nil { + return err + } + + *s = sc + + return nil +} + +func (s Secret) Base64() string { + return s.String() +} + +func (s Secret) EE() string { + return "ee" + hex.EncodeToString(append(s.Key, s.Host...)) +} + +func (s Secret) String() string { + return base64.StdEncoding.EncodeToString(append(s.Key, s.Host...)) +} + +func ParseSecret(secret string) (Secret, error) { + rv := Secret{} + + if secret == "" { + return rv, errors.New("secret cannot be empty") + } + + decoded, err := base64.RawStdEncoding.DecodeString(secret) + if err != nil && strings.HasPrefix(secret, "ee") { + decoded, err = hex.DecodeString(strings.TrimPrefix(secret, "ee")) + } + + if err != nil { + return rv, fmt.Errorf("incorrect secret format: %w", err) + } + + if len(decoded) < 33 { + return rv, fmt.Errorf("secret %s has incorrect length", secret) + } + + rv.Key = decoded[:32] + rv.Host = string(decoded[32:]) + + return rv, nil +} diff --git a/raw_config.go b/raw_config.go new file mode 100644 index 0000000..f6789d2 --- /dev/null +++ b/raw_config.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "io" + + "github.com/pelletier/go-toml" +) + +type rawConfig struct { + Debug bool `toml:"debug"` + Secret string `toml:"secret"` + BindTo string `toml:"bind-to"` + TCPBuffer string `toml:"tcp-buffer"` + PreferIP string `toml:"prefer-ip"` + CloakPort uint `toml:"cloak-port"` + AccessFile string `toml:"access-file"` + Probes struct { + Time struct { + Enabled bool `toml:"enabled"` + AllowSkewness string `toml:"allow-skewness"` + } `toml:"time"` + AntiReplay struct { + Enabled bool `toml:"enabled"` + MaxSize string `toml:"max-size"` + TTL string `toml:"ttl"` + } `toml:"anti-replay"` + } `toml:"probes"` + PublicIP struct { + IPv4 string `toml:"ipv4"` + IPv6 string `toml:"ipv6"` + } `toml:"public-ip"` + Dialers struct { + Telegram string `toml:"telegram"` + Default string `toml:"default"` + } `toml:"dialers"` + Stats struct { + StatsD struct { + Enabled bool `toml:"enabled"` + Address string `toml:"address"` + MetricPrefix string `toml:"metric-prefix"` + } `toml:"statsd"` + Prometheus struct { + Enabled bool `toml:"enabled"` + BindTo string `toml:"bind-to"` + HttpPath string `toml:"http-path"` + MetricPrefix string `toml:"metric-prefix"` + } `toml:"prometheus"` + } `toml:"stats"` +} + +func parseRawConfig(reader io.Reader) (*rawConfig, error) { + conf := &rawConfig{} + + if err := toml.NewDecoder(reader).Decode(conf); err != nil { + return nil, fmt.Errorf("cannot parse config: %w", err) + } + + return conf, nil +} From d5147f19358c35ae6b136cb03120f2a3b22de98e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 14:12:38 +0300 Subject: [PATCH 004/206] Rename dialers into network --- main.go | 6 ++-- mtglib/dialers/socks5.go | 23 -------------- mtglib/{dialers => network}/consts.go | 2 +- mtglib/{dialers => network}/default.go | 12 ++++---- mtglib/{dialers => network}/interfaces.go | 5 ++-- .../{dialers/dialer.go => network/network.go} | 28 ++++++++--------- mtglib/{dialers => network}/shadowsocks.go | 30 +++++++------------ mtglib/network/socks5.go | 23 ++++++++++++++ 8 files changed, 60 insertions(+), 69 deletions(-) delete mode 100644 mtglib/dialers/socks5.go rename mtglib/{dialers => network}/consts.go (88%) rename mtglib/{dialers => network}/default.go (81%) rename mtglib/{dialers => network}/interfaces.go (54%) rename mtglib/{dialers/dialer.go => network/network.go} (72%) rename mtglib/{dialers => network}/shadowsocks.go (59%) create mode 100644 mtglib/network/socks5.go diff --git a/main.go b/main.go index 3281a3f..5f8b319 100644 --- a/main.go +++ b/main.go @@ -7,7 +7,7 @@ import ( "os" "time" - "github.com/9seconds/mtg/v2/mtglib/dialers" + "github.com/9seconds/mtg/v2/mtglib/network" ) var version = "dev" // has to be set by ldflags @@ -19,8 +19,8 @@ func main() { fmt.Println(parseRawConfig(f)) - bd, _ := dialers.NewDefaultBaseDialer(0, 0) - d, _ := dialers.MakeDialer(bd, "9.9.9.9", 0) + bd, _ := network.NewDefaultDialer(0, 0) + d, _ := network.NewNetwork(bd, "9.9.9.9", 0) r, err := d.HTTP.Get("https://ifconfig.co") diff --git a/mtglib/dialers/socks5.go b/mtglib/dialers/socks5.go deleted file mode 100644 index f5f93ca..0000000 --- a/mtglib/dialers/socks5.go +++ /dev/null @@ -1,23 +0,0 @@ -package dialers - -import ( - "fmt" - "net/url" - "time" - - "golang.org/x/net/proxy" -) - -func NewSocks5BaseDialer(proxyUrl *url.URL, timeout time.Duration, bufferSize int) (BaseDialer, error) { - baseDialer, err := NewDefaultBaseDialer(timeout, bufferSize) - if err != nil { - return nil, fmt.Errorf("cannot initialize base dialer: %w", err) - } - - rv, err := proxy.FromURL(proxyUrl, baseDialer.(*defaultBaseDialer)) - if err != nil { - return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) - } - - return rv.(BaseDialer), nil -} diff --git a/mtglib/dialers/consts.go b/mtglib/network/consts.go similarity index 88% rename from mtglib/dialers/consts.go rename to mtglib/network/consts.go index 40a022d..68f00fc 100644 --- a/mtglib/dialers/consts.go +++ b/mtglib/network/consts.go @@ -1,4 +1,4 @@ -package dialers +package network import "time" diff --git a/mtglib/dialers/default.go b/mtglib/network/default.go similarity index 81% rename from mtglib/dialers/default.go rename to mtglib/network/default.go index feeda9d..67804a8 100644 --- a/mtglib/dialers/default.go +++ b/mtglib/network/default.go @@ -1,4 +1,4 @@ -package dialers +package network import ( "context" @@ -9,17 +9,17 @@ import ( "github.com/libp2p/go-reuseport" ) -type defaultBaseDialer struct { +type defaultDialer struct { net.Dialer bufferSize int } -func (d *defaultBaseDialer) Dial(network, address string) (net.Conn, error) { +func (d *defaultDialer) Dial(network, address string) (net.Conn, error) { return d.DialContext(context.Background(), network, address) } -func (d *defaultBaseDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (d *defaultDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { switch network { case "tcp", "tcp4", "tcp6": default: @@ -56,7 +56,7 @@ func (d *defaultBaseDialer) DialContext(ctx context.Context, network, address st return tcpConn, nil } -func NewDefaultBaseDialer(timeout time.Duration, bufferSize int) (BaseDialer, error) { +func NewDefaultDialer(timeout time.Duration, bufferSize int) (Dialer, error) { switch { case timeout < 0: return nil, fmt.Errorf("timeout %v should be positive number", timeout) @@ -72,7 +72,7 @@ func NewDefaultBaseDialer(timeout time.Duration, bufferSize int) (BaseDialer, er bufferSize = DefaultBufferSize } - return &defaultBaseDialer{ + return &defaultDialer{ Dialer: net.Dialer{ Timeout: timeout, Control: reuseport.Control, diff --git a/mtglib/dialers/interfaces.go b/mtglib/network/interfaces.go similarity index 54% rename from mtglib/dialers/interfaces.go rename to mtglib/network/interfaces.go index 774aa77..4c7f972 100644 --- a/mtglib/dialers/interfaces.go +++ b/mtglib/network/interfaces.go @@ -1,11 +1,10 @@ -package dialers +package network import ( "context" "net" ) -type BaseDialer interface { - Dial(network, address string) (net.Conn, error) +type Dialer interface { DialContext(ctx context.Context, network, address string) (net.Conn, error) } diff --git a/mtglib/dialers/dialer.go b/mtglib/network/network.go similarity index 72% rename from mtglib/dialers/dialer.go rename to mtglib/network/network.go index 3f8d1be..83cb4f7 100644 --- a/mtglib/dialers/dialer.go +++ b/mtglib/network/network.go @@ -1,4 +1,4 @@ -package dialers +package network import ( "context" @@ -11,18 +11,18 @@ import ( doh "github.com/babolivier/go-doh-client" ) -type Dialer struct { +type Network struct { HTTP http.Client DNS doh.Resolver - baseDialer BaseDialer + dialer Dialer } -func (d *Dialer) Dial(network, address string) (net.Conn, error) { +func (d *Network) Dial(network, address string) (net.Conn, error) { return d.DialContext(context.Background(), network, address) } -func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (d *Network) DialContext(ctx context.Context, network, address string) (net.Conn, error) { host, port, _ := net.SplitHostPort(address) ips, err := d.resolveIPs(network, host) @@ -35,7 +35,7 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (net. }) for _, v := range ips { - if conn, err := d.baseDialer.DialContext(ctx, network, net.JoinHostPort(v, port)); err == nil { + if conn, err := d.dialer.DialContext(ctx, network, net.JoinHostPort(v, port)); err == nil { return conn, nil } } @@ -43,7 +43,7 @@ func (d *Dialer) DialContext(ctx context.Context, network, address string) (net. return nil, fmt.Errorf("cannot dial to %s:%s", network, address) } -func (d *Dialer) resolveIPs(network, address string) ([]string, error) { +func (d *Network) resolveIPs(network, address string) ([]string, error) { if net.ParseIP(address) != nil { return []string{address}, nil } @@ -75,7 +75,7 @@ func (d *Dialer) resolveIPs(network, address string) ([]string, error) { return ips, nil } -func MakeDialer(base BaseDialer, dohHostname string, httpTimeout time.Duration) (*Dialer, error) { +func NewNetwork(dialer Dialer, dohHostname string, httpTimeout time.Duration) (*Network, error) { switch { case httpTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %v", httpTimeout) @@ -90,23 +90,23 @@ func MakeDialer(base BaseDialer, dohHostname string, httpTimeout time.Duration) dohHTTPClient := &http.Client{ Timeout: httpTimeout, Transport: &http.Transport{ - DialContext: base.DialContext, + DialContext: dialer.DialContext, }, } - rv := &Dialer{ - baseDialer: base, + network := &Network{ + dialer: dialer, DNS: doh.Resolver{ Host: dohHostname, Class: doh.IN, HTTPClient: dohHTTPClient, }, } - rv.HTTP = http.Client{ + network.HTTP = http.Client{ Timeout: httpTimeout, Transport: &http.Transport{ - DialContext: rv.DialContext, + DialContext: network.DialContext, }, } - return rv, nil + return network, nil } diff --git a/mtglib/dialers/shadowsocks.go b/mtglib/network/shadowsocks.go similarity index 59% rename from mtglib/dialers/shadowsocks.go rename to mtglib/network/shadowsocks.go index 1929fa2..5fff10a 100644 --- a/mtglib/dialers/shadowsocks.go +++ b/mtglib/network/shadowsocks.go @@ -1,4 +1,4 @@ -package dialers +package network import ( "context" @@ -12,23 +12,15 @@ import ( shadowsocks "github.com/shadowsocks/go-shadowsocks2/core" ) -type shadowsocksBaseDialer struct { - base BaseDialer +type shadowsocksDialer struct { + Dialer + cipher shadowsocks.StreamConnCipher } -func (s *shadowsocksBaseDialer) Dial(network, address string) (net.Conn, error) { - conn, err := s.base.Dial(network, address) - if err != nil { - return nil, err - } - - return s.cipher.StreamConn(conn), nil -} - -func (s *shadowsocksBaseDialer) DialContext(ctx context.Context, +func (s *shadowsocksDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - conn, err := s.base.DialContext(ctx, network, address) + conn, err := s.Dialer.DialContext(ctx, network, address) if err != nil { return nil, err } @@ -36,8 +28,8 @@ func (s *shadowsocksBaseDialer) DialContext(ctx context.Context, return s.cipher.StreamConn(conn), nil } -func NewShadowsocksBaseDialer(proxyUrl *url.URL, - timeout time.Duration, bufferSize int) (BaseDialer, error) { +func NewShadowsocksDialer(proxyUrl *url.URL, + timeout time.Duration, bufferSize int) (Dialer, error) { username := proxyUrl.User.Username() decoded, err := base64.RawURLEncoding.DecodeString(username) @@ -55,14 +47,14 @@ func NewShadowsocksBaseDialer(proxyUrl *url.URL, return nil, fmt.Errorf("cannot initialize shadowsocks cipher: %w", err) } - baseDialer, err := NewDefaultBaseDialer(timeout, bufferSize) + dialer, err := NewDefaultDialer(timeout, bufferSize) if err != nil { return nil, fmt.Errorf("cannot initialize a base dialer: %w", err) } - return &shadowsocksBaseDialer{ - base: baseDialer, + return &shadowsocksDialer{ + Dialer: dialer, cipher: cipher, }, nil } diff --git a/mtglib/network/socks5.go b/mtglib/network/socks5.go new file mode 100644 index 0000000..ddb1be3 --- /dev/null +++ b/mtglib/network/socks5.go @@ -0,0 +1,23 @@ +package network + +import ( + "fmt" + "net/url" + "time" + + "golang.org/x/net/proxy" +) + +func NewSocks5Dialer(proxyUrl *url.URL, timeout time.Duration, bufferSize int) (Dialer, error) { + dialer, err := NewDefaultDialer(timeout, bufferSize) + if err != nil { + return nil, fmt.Errorf("cannot initialize base dialer: %w", err) + } + + rv, err := proxy.FromURL(proxyUrl, dialer.(*defaultDialer)) + if err != nil { + return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) + } + + return rv.(Dialer), nil +} From 09f8233bc6f0231436f312cd47cc5997dbe580cc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 14:21:43 +0300 Subject: [PATCH 005/206] Modify config --- example.config.toml | 80 +++++++++++++++++++++++---------------- go.sum | 3 ++ mtglib/network/network.go | 8 ++-- raw_config.go | 19 ++++++---- 4 files changed, 67 insertions(+), 43 deletions(-) diff --git a/example.config.toml b/example.config.toml index 3f7bea7..917d69e 100644 --- a/example.config.toml +++ b/example.config.toml @@ -51,6 +51,54 @@ cloak-port = 443 # Pass filepath here or '-' if you want to dump into stdout. access-file = "-" +# network defines different network-related settings +[network] +# please be aware that mtg needs to do some external requests. For +# example, if you do not pass public ips, it will request your public ip +# address from some external service. +# +# As for 2.0, if you set a public-ip on your own, mtg won't issue any +# network requests except of those required for Telegram. +# +# so, in order of doing them, it needs to do DNS lookup. mtg ignores DNS +# resolver of the operating system and uses DOH instead. This is a host +# it has to access. +# +# By default we use Quad9. +doh-hostname = "9.9.9.9" + +# public ip addresses of the server. Actually, it is required only to +# generate a correct access file. if you use default values here, mtg +# will try to resolve these IPs on its own. +[network.public-ip] +ipv4 = "" +ipv6 = "" + +# you can redefine a dialer for mtg. Dialer is how we 'dial' to either +# some external services or telegram. empty string means default +# connectivity. +# +# it is also possible to use socks5 or shadowsocks here +# +# socks5 example: +# socks5://user:password@host:port +# shadowsocks example (SIP002): +# ss://YWVzLTEyOC1nY206dGVzdA@192.168.100.1:8888 +# +# You can define 2 dialers here: telegram and default. Telegram dialer +# is used to connect to Telegram servers only. Default is used for other +# purposes, like accessing ifconfig.co to obtains public address (DNS is +# resolved via DoH) +# +# Please also be aware that dialers are only doing TCP. If UDP is +# required (for statsd for example), then these dialers are going to be +# ignored. +# +# If telegram dialer is not defined, a default one is going to be used. +[network.dialers] +telegram = "" +default = "" + # FakeTLS can compare timestamps to prevent probes. Each message has # encrypted timestamp. So, mtg can compare this timestamp and decide if # we need to proceed with connection or not. @@ -81,38 +129,6 @@ max-size = "16mb" # TTL for each cache record. ttl = "8h" -# public ip addresses of the server. Actually, it is required only to -# generate a correct access file. if you use default values here, mtg -# will try to resolve these IPs on its own. -[public-ip] -ipv4 = "" -ipv6 = "" - -# you can redefine a dialer for mtg. Dialer is how we 'dial' to either -# some external services or telegram. empty string means default -# connectivity. -# -# it is also possible to use socks5 or shadowsocks here -# -# socks5 example: -# socks5://user:password@host:port -# shadowsocks example (SIP002): -# ss://YWVzLTEyOC1nY206dGVzdA@192.168.100.1:8888 -# -# You can define 2 dialers here: telegram and default. Telegram dialer -# is used to connect to Telegram servers only. Default is used for other -# purposes, like accessing ifconfig.co to obtains public address (DNS is -# resolved via DoH) -# -# Please also be aware that dialers are only doing TCP. If UDP is -# required (for statsd for example), then these dialers are going to be -# ignored. -# -# If telegram dialer is not defined, a default one is going to be used. -[dialers] -telegram = "" -default = "" - # statsd statistics integration. [stats.statsd] # enabled/disabled diff --git a/go.sum b/go.sum index fd68855..684fcb8 100644 --- a/go.sum +++ b/go.sum @@ -9,12 +9,14 @@ github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNC github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr8meMVVGxhp+QBTqY91tM8HjEuMjGg= github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3/go.mod h1:HgjTstvQsPGkxUsCd2KWxErBblirPizecHcpD3ffK+s= github.com/shadowsocks/go-shadowsocks2 v0.1.4 h1:4VzajPL7RwwmImysBSvI+lm/UaegDGQq3hr42dYo3gs= github.com/shadowsocks/go-shadowsocks2 v0.1.4/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= @@ -33,4 +35,5 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/mtglib/network/network.go b/mtglib/network/network.go index 83cb4f7..bc3d84e 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -30,9 +30,11 @@ func (d *Network) DialContext(ctx context.Context, network, address string) (net return nil, fmt.Errorf("cannot resolve dns names: %w", err) } - rand.Shuffle(len(ips), func(i, j int) { - ips[i], ips[j] = ips[j], ips[i] - }) + if len(ips) > 1 { + rand.Shuffle(len(ips), func(i, j int) { + ips[i], ips[j] = ips[j], ips[i] + }) + } for _, v := range ips { if conn, err := d.dialer.DialContext(ctx, network, net.JoinHostPort(v, port)); err == nil { diff --git a/raw_config.go b/raw_config.go index f6789d2..4f98c3e 100644 --- a/raw_config.go +++ b/raw_config.go @@ -26,14 +26,17 @@ type rawConfig struct { TTL string `toml:"ttl"` } `toml:"anti-replay"` } `toml:"probes"` - PublicIP struct { - IPv4 string `toml:"ipv4"` - IPv6 string `toml:"ipv6"` - } `toml:"public-ip"` - Dialers struct { - Telegram string `toml:"telegram"` - Default string `toml:"default"` - } `toml:"dialers"` + Network struct { + PublicIP struct { + IPv4 string `toml:"ipv4"` + IPv6 string `toml:"ipv6"` + } `toml:"public-ip"` + Dialers struct { + Telegram string `toml:"telegram"` + Default string `toml:"default"` + } `toml:"dialers"` + DOHHostname string `toml:"doh-hostname"` + } `toml:"network"` Stats struct { StatsD struct { Enabled bool `toml:"enabled"` From 21ee1e2c6cbe39c606a1f1dd550ca2a4d2c2ae70 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 16:39:09 +0300 Subject: [PATCH 006/206] Liniting for network --- mtglib/network/default.go | 8 +++++-- mtglib/network/network.go | 41 +++++++++++++++++++++++++++-------- mtglib/network/shadowsocks.go | 27 ++++++++++++++++++----- mtglib/network/socks5.go | 4 ++-- 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/mtglib/network/default.go b/mtglib/network/default.go index 67804a8..72b3da5 100644 --- a/mtglib/network/default.go +++ b/mtglib/network/default.go @@ -21,7 +21,7 @@ func (d *defaultDialer) Dial(network, address string) (net.Conn, error) { func (d *defaultDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { switch network { - case "tcp", "tcp4", "tcp6": + case "tcp", "tcp4", "tcp6": // nolint: goconst default: return nil, fmt.Errorf("unsupported network %s", network) } @@ -35,21 +35,25 @@ func (d *defaultDialer) DialContext(ctx context.Context, network, address string if err := tcpConn.SetNoDelay(true); err != nil { conn.Close() + return nil, fmt.Errorf("cannot set TCP_NO_DELAY: %w", err) } if err := tcpConn.SetReadBuffer(d.bufferSize); err != nil { tcpConn.Close() + return nil, fmt.Errorf("cannot set read buffer size: %w", err) } if err := tcpConn.SetWriteBuffer(d.bufferSize); err != nil { tcpConn.Close() + return nil, fmt.Errorf("cannot set write buffer size: %w", err) } if err := tcpConn.SetKeepAlive(true); err != nil { tcpConn.Close() + return nil, fmt.Errorf("cannot enable keep-alive: %w", err) } @@ -61,7 +65,7 @@ func NewDefaultDialer(timeout time.Duration, bufferSize int) (Dialer, error) { case timeout < 0: return nil, fmt.Errorf("timeout %v should be positive number", timeout) case bufferSize < 0: - return nil, fmt.Errorf("buffer size %s should be positive number", bufferSize) + return nil, fmt.Errorf("buffer size %d should be positive number", bufferSize) } if timeout == 0 { diff --git a/mtglib/network/network.go b/mtglib/network/network.go index bc3d84e..ecbf0e7 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -6,6 +6,7 @@ import ( "math/rand" "net" "net/http" + "sync" "time" doh "github.com/babolivier/go-doh-client" @@ -50,26 +51,48 @@ func (d *Network) resolveIPs(network, address string) ([]string, error) { return []string{address}, nil } - var ips []string + ips := []string{} + wg := &sync.WaitGroup{} + mutex := &sync.Mutex{} switch network { case "tcp", "tcp4": - if recs, _, err := d.DNS.LookupA(address); err == nil { - for _, v := range recs { - ips = append(ips, v.IP4) + wg.Add(1) + + go func() { + defer wg.Done() + + if recs, _, err := d.DNS.LookupA(address); err == nil { + mutex.Lock() + defer mutex.Unlock() + + for _, v := range recs { + ips = append(ips, v.IP4) + } } - } + }() } switch network { case "tcp", "tcp6": - if recs, _, err := d.DNS.LookupAAAA(address); err == nil { - for _, v := range recs { - ips = append(ips, v.IP6) + wg.Add(1) + + go func() { + defer wg.Done() + + if recs, _, err := d.DNS.LookupAAAA(address); err == nil { + mutex.Lock() + defer mutex.Unlock() + + for _, v := range recs { + ips = append(ips, v.IP6) + } } - } + }() } + wg.Wait() + if len(ips) == 0 { return nil, fmt.Errorf("cannot find any ips for %s:%s", network, address) } diff --git a/mtglib/network/shadowsocks.go b/mtglib/network/shadowsocks.go index 5fff10a..ffad8aa 100644 --- a/mtglib/network/shadowsocks.go +++ b/mtglib/network/shadowsocks.go @@ -10,6 +10,7 @@ import ( "time" shadowsocks "github.com/shadowsocks/go-shadowsocks2/core" + "golang.org/x/net/proxy" ) type shadowsocksDialer struct { @@ -18,19 +19,23 @@ type shadowsocksDialer struct { cipher shadowsocks.StreamConnCipher } +func (s *shadowsocksDialer) Dial(network, address string) (net.Conn, error) { + return s.DialContext(context.Background(), network, address) +} + func (s *shadowsocksDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { conn, err := s.Dialer.DialContext(ctx, network, address) if err != nil { - return nil, err + return nil, err // nolint: wrapcheck } return s.cipher.StreamConn(conn), nil } -func NewShadowsocksDialer(proxyUrl *url.URL, +func NewShadowsocksDialer(proxyURL *url.URL, timeout time.Duration, bufferSize int) (Dialer, error) { - username := proxyUrl.User.Username() + username := proxyURL.User.Username() decoded, err := base64.RawURLEncoding.DecodeString(username) if err != nil { @@ -47,14 +52,24 @@ func NewShadowsocksDialer(proxyUrl *url.URL, return nil, fmt.Errorf("cannot initialize shadowsocks cipher: %w", err) } + socks5URL := *proxyURL + socks5URL.Scheme = "socks5" + socks5URL.User = nil + dialer, err := NewDefaultDialer(timeout, bufferSize) if err != nil { return nil, fmt.Errorf("cannot initialize a base dialer: %w", err) - } - return &shadowsocksDialer{ + ssDialer := &shadowsocksDialer{ Dialer: dialer, cipher: cipher, - }, nil + } + + rv, err := proxy.FromURL(&socks5URL, ssDialer) + if err != nil { + return nil, fmt.Errorf("cannot initialize ss proxy dialer: %w", err) + } + + return rv.(Dialer), nil } diff --git a/mtglib/network/socks5.go b/mtglib/network/socks5.go index ddb1be3..8ed895c 100644 --- a/mtglib/network/socks5.go +++ b/mtglib/network/socks5.go @@ -8,13 +8,13 @@ import ( "golang.org/x/net/proxy" ) -func NewSocks5Dialer(proxyUrl *url.URL, timeout time.Duration, bufferSize int) (Dialer, error) { +func NewSocks5Dialer(proxyURL *url.URL, timeout time.Duration, bufferSize int) (Dialer, error) { dialer, err := NewDefaultDialer(timeout, bufferSize) if err != nil { return nil, fmt.Errorf("cannot initialize base dialer: %w", err) } - rv, err := proxy.FromURL(proxyUrl, dialer.(*defaultDialer)) + rv, err := proxy.FromURL(proxyURL, dialer.(*defaultDialer)) if err != nil { return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) } From 0ed7e1037904d9d74a24bafe2d473d5aeb23ccfa Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 17:11:54 +0300 Subject: [PATCH 007/206] Add test for default dialer --- go.mod | 2 + go.sum | 7 +++ mtglib/network/default_test.go | 93 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 mtglib/network/default_test.go diff --git a/go.mod b/go.mod index 10e8001..12e986f 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.16 require ( github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/libp2p/go-reuseport v0.0.2 + github.com/mccutchen/go-httpbin v1.1.1 // indirect github.com/pelletier/go-toml v1.8.1 github.com/shadowsocks/go-shadowsocks2 v0.1.4 + github.com/stretchr/testify v1.7.0 // indirect golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 ) diff --git a/go.sum b/go.sum index 684fcb8..616bf5c 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= +github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE= +github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -15,9 +17,12 @@ github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3/go.mod h1:HgjTstvQsPGkxUsCd2KWxErBblirPizecHcpD3ffK+s= github.com/shadowsocks/go-shadowsocks2 v0.1.4 h1:4VzajPL7RwwmImysBSvI+lm/UaegDGQq3hr42dYo3gs= github.com/shadowsocks/go-shadowsocks2 v0.1.4/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= @@ -37,3 +42,5 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mtglib/network/default_test.go b/mtglib/network/default_test.go new file mode 100644 index 0000000..5da940f --- /dev/null +++ b/mtglib/network/default_test.go @@ -0,0 +1,93 @@ +package network_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/mccutchen/go-httpbin/httpbin" + "github.com/stretchr/testify/suite" +) + +type DefaultDialerTestSuite struct { + suite.Suite + + d network.Dialer + srvAddress string + srv *httptest.Server +} + +func (suite *DefaultDialerTestSuite) SetupSuite() { + suite.srv = httptest.NewServer(httpbin.NewHTTPBin().Handler()) + suite.srvAddress = strings.TrimPrefix(suite.srv.URL, "http://") +} + +func (suite *DefaultDialerTestSuite) SetupTest() { + d, err := network.NewDefaultDialer(0, 0) + + suite.NoError(err) + + suite.d = d +} + +func (suite *DefaultDialerTestSuite) TearDownSuite() { + suite.srv.Close() +} + +func (suite *DefaultDialerTestSuite) TestNegativeTimeout() { + _, err := network.NewDefaultDialer(-1, 0) + + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestNegativeBufferSize() { + _, err := network.NewDefaultDialer(0, -1) + + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestUnsupportedProtocol() { + _, err := suite.d.DialContext(context.Background(), "udp", suite.srvAddress) + + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestCannotDial() { + _, err := suite.d.DialContext(context.Background(), + "tcp", + suite.srvAddress+suite.srvAddress) + + suite.Error(err) +} + +func (suite *DefaultDialerTestSuite) TestConnectOk() { + conn, err := suite.d.DialContext(context.Background(), + "tcp", + suite.srvAddress) + + suite.NoError(err) + suite.NotNil(conn) + + conn.Close() +} + +func (suite *DefaultDialerTestSuite) TestRequest() { + httpClient := http.Client{ + Transport: &http.Transport{ + DialContext: suite.d.DialContext, + }, + } + + resp, err := httpClient.Get(suite.srv.URL + "/get") + + suite.NoError(err) + + resp.Body.Close() +} + +func TestDefaultDialer(t *testing.T) { + suite.Run(t, &DefaultDialerTestSuite{}) +} From 88e076a28318c6a2637fde66436ed56cce6547bd Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 17:36:21 +0300 Subject: [PATCH 008/206] Add tests for socks dialer --- go.mod | 1 + go.sum | 2 + mtglib/network/default_test.go | 30 ++++-------- mtglib/network/init_test.go | 27 +++++++++++ mtglib/network/socks5_test.go | 84 ++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 20 deletions(-) create mode 100644 mtglib/network/init_test.go create mode 100644 mtglib/network/socks5_test.go diff --git a/go.mod b/go.mod index 12e986f..da2a623 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/9seconds/mtg/v2 go 1.16 require ( + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 // indirect diff --git a/go.sum b/go.sum index 616bf5c..4da8844 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/mtglib/network/default_test.go b/mtglib/network/default_test.go index 5da940f..203a2bb 100644 --- a/mtglib/network/default_test.go +++ b/mtglib/network/default_test.go @@ -3,29 +3,21 @@ package network_test import ( "context" "net/http" - "net/http/httptest" - "strings" "testing" "github.com/9seconds/mtg/v2/mtglib/network" - "github.com/mccutchen/go-httpbin/httpbin" "github.com/stretchr/testify/suite" ) type DefaultDialerTestSuite struct { - suite.Suite + HTTPServerTestSuite - d network.Dialer - srvAddress string - srv *httptest.Server + d network.Dialer } func (suite *DefaultDialerTestSuite) SetupSuite() { - suite.srv = httptest.NewServer(httpbin.NewHTTPBin().Handler()) - suite.srvAddress = strings.TrimPrefix(suite.srv.URL, "http://") -} + suite.HTTPServerTestSuite.SetupSuite() -func (suite *DefaultDialerTestSuite) SetupTest() { d, err := network.NewDefaultDialer(0, 0) suite.NoError(err) @@ -33,10 +25,6 @@ func (suite *DefaultDialerTestSuite) SetupTest() { suite.d = d } -func (suite *DefaultDialerTestSuite) TearDownSuite() { - suite.srv.Close() -} - func (suite *DefaultDialerTestSuite) TestNegativeTimeout() { _, err := network.NewDefaultDialer(-1, 0) @@ -50,7 +38,9 @@ func (suite *DefaultDialerTestSuite) TestNegativeBufferSize() { } func (suite *DefaultDialerTestSuite) TestUnsupportedProtocol() { - _, err := suite.d.DialContext(context.Background(), "udp", suite.srvAddress) + _, err := suite.d.DialContext(context.Background(), + "udp", + suite.HTTPServerAddress()) suite.Error(err) } @@ -58,7 +48,7 @@ func (suite *DefaultDialerTestSuite) TestUnsupportedProtocol() { func (suite *DefaultDialerTestSuite) TestCannotDial() { _, err := suite.d.DialContext(context.Background(), "tcp", - suite.srvAddress+suite.srvAddress) + suite.HTTPServerAddress()+suite.HTTPServerAddress()) suite.Error(err) } @@ -66,7 +56,7 @@ func (suite *DefaultDialerTestSuite) TestCannotDial() { func (suite *DefaultDialerTestSuite) TestConnectOk() { conn, err := suite.d.DialContext(context.Background(), "tcp", - suite.srvAddress) + suite.HTTPServerAddress()) suite.NoError(err) suite.NotNil(conn) @@ -74,14 +64,14 @@ func (suite *DefaultDialerTestSuite) TestConnectOk() { conn.Close() } -func (suite *DefaultDialerTestSuite) TestRequest() { +func (suite *DefaultDialerTestSuite) TestHTTPRequest() { httpClient := http.Client{ Transport: &http.Transport{ DialContext: suite.d.DialContext, }, } - resp, err := httpClient.Get(suite.srv.URL + "/get") + resp, err := httpClient.Get(suite.httpServer.URL + "/get") suite.NoError(err) diff --git a/mtglib/network/init_test.go b/mtglib/network/init_test.go new file mode 100644 index 0000000..8f8aeff --- /dev/null +++ b/mtglib/network/init_test.go @@ -0,0 +1,27 @@ +package network_test + +import ( + "net/http/httptest" + "strings" + + "github.com/mccutchen/go-httpbin/httpbin" + "github.com/stretchr/testify/suite" +) + +type HTTPServerTestSuite struct { + suite.Suite + + httpServer *httptest.Server +} + +func (suite *HTTPServerTestSuite) SetupSuite() { + suite.httpServer = httptest.NewServer(httpbin.NewHTTPBin().Handler()) +} + +func (suite *HTTPServerTestSuite) TearDownSuite() { + suite.httpServer.Close() +} + +func (suite *HTTPServerTestSuite) HTTPServerAddress() string { + return strings.TrimPrefix(suite.httpServer.URL, "http://") +} diff --git a/mtglib/network/socks5_test.go b/mtglib/network/socks5_test.go new file mode 100644 index 0000000..9f5e285 --- /dev/null +++ b/mtglib/network/socks5_test.go @@ -0,0 +1,84 @@ +package network_test + +import ( + "net" + "net/http" + "net/url" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/network" + socks5 "github.com/armon/go-socks5" + "github.com/stretchr/testify/suite" +) + +type Socks5TestSuite struct { + HTTPServerTestSuite + + socksListener net.Listener + socksProxy *socks5.Server +} + +func (suite *Socks5TestSuite) SetupSuite() { + suite.HTTPServerTestSuite.SetupSuite() + + socksConf := socks5.Config{ + Credentials: socks5.StaticCredentials{ + "user": "password", + }, + } + + suite.socksProxy, _ = socks5.New(&socksConf) + suite.socksListener, _ = net.Listen("tcp", "127.0.0.1:0") + + go suite.socksProxy.Serve(suite.socksListener) +} + +func (suite *Socks5TestSuite) TearDownSuite() { + suite.socksListener.Close() + + suite.HTTPServerTestSuite.TearDownSuite() +} + +func (suite *Socks5TestSuite) TestRequestFailed() { + proxyURL := &url.URL{ + Scheme: "socks5", + User: url.UserPassword("user2", "password"), + Host: suite.socksListener.Addr().String(), + } + dialer, _ := network.NewSocks5Dialer(proxyURL, 0, 0) + + httpClient := http.Client{ + Transport: &http.Transport{ + DialContext: dialer.DialContext, + }, + } + + _, err := httpClient.Get(suite.httpServer.URL + "/get") + + suite.Error(err) +} + +func (suite *Socks5TestSuite) TestRequestOk() { + proxyURL := &url.URL{ + Scheme: "socks5", + User: url.UserPassword("user", "password"), + Host: suite.socksListener.Addr().String(), + } + dialer, _ := network.NewSocks5Dialer(proxyURL, 0, 0) + + httpClient := http.Client{ + Transport: &http.Transport{ + DialContext: dialer.DialContext, + }, + } + + resp, err := httpClient.Get(suite.httpServer.URL + "/get") + + suite.NoError(err) + + resp.Body.Close() +} + +func TestSocks5TestSuite(t *testing.T) { + suite.Run(t, &Socks5TestSuite{}) +} From 5b461529451efdff6f8df2f578037adc4ad6b1c9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 18:16:24 +0300 Subject: [PATCH 009/206] Remove shadowsocks dependency --- go.mod | 7 ++-- go.sum | 16 +------- mtglib/network/shadowsocks.go | 75 ----------------------------------- 3 files changed, 4 insertions(+), 94 deletions(-) delete mode 100644 mtglib/network/shadowsocks.go diff --git a/go.mod b/go.mod index da2a623..e6dd6ce 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,11 @@ module github.com/9seconds/mtg/v2 go 1.16 require ( - github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/libp2p/go-reuseport v0.0.2 - github.com/mccutchen/go-httpbin v1.1.1 // indirect + github.com/mccutchen/go-httpbin v1.1.1 github.com/pelletier/go-toml v1.8.1 - github.com/shadowsocks/go-shadowsocks2 v0.1.4 - github.com/stretchr/testify v1.7.0 // indirect + github.com/stretchr/testify v1.7.0 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 ) diff --git a/go.sum b/go.sum index 4da8844..1563bf5 100644 --- a/go.sum +++ b/go.sum @@ -15,34 +15,20 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 h1:f/FNXud6gA3MNr8meMVVGxhp+QBTqY91tM8HjEuMjGg= -github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3/go.mod h1:HgjTstvQsPGkxUsCd2KWxErBblirPizecHcpD3ffK+s= -github.com/shadowsocks/go-shadowsocks2 v0.1.4 h1:4VzajPL7RwwmImysBSvI+lm/UaegDGQq3hr42dYo3gs= -github.com/shadowsocks/go-shadowsocks2 v0.1.4/go.mod h1:AGGpIoek4HRno4xzyFiAtLHkOpcoznZEkAccaI/rplM= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mtglib/network/shadowsocks.go b/mtglib/network/shadowsocks.go deleted file mode 100644 index ffad8aa..0000000 --- a/mtglib/network/shadowsocks.go +++ /dev/null @@ -1,75 +0,0 @@ -package network - -import ( - "context" - "encoding/base64" - "fmt" - "net" - "net/url" - "strings" - "time" - - shadowsocks "github.com/shadowsocks/go-shadowsocks2/core" - "golang.org/x/net/proxy" -) - -type shadowsocksDialer struct { - Dialer - - cipher shadowsocks.StreamConnCipher -} - -func (s *shadowsocksDialer) Dial(network, address string) (net.Conn, error) { - return s.DialContext(context.Background(), network, address) -} - -func (s *shadowsocksDialer) DialContext(ctx context.Context, - network, address string) (net.Conn, error) { - conn, err := s.Dialer.DialContext(ctx, network, address) - if err != nil { - return nil, err // nolint: wrapcheck - } - - return s.cipher.StreamConn(conn), nil -} - -func NewShadowsocksDialer(proxyURL *url.URL, - timeout time.Duration, bufferSize int) (Dialer, error) { - username := proxyURL.User.Username() - - decoded, err := base64.RawURLEncoding.DecodeString(username) - if err != nil { - return nil, fmt.Errorf("cannot decode payload: %w", err) - } - - chunks := strings.SplitN(string(decoded), ":", 2) - if len(chunks) != 2 { - return nil, fmt.Errorf("incorrect payload %s", username) - } - - cipher, err := shadowsocks.PickCipher(chunks[0], nil, chunks[1]) - if err != nil { - return nil, fmt.Errorf("cannot initialize shadowsocks cipher: %w", err) - } - - socks5URL := *proxyURL - socks5URL.Scheme = "socks5" - socks5URL.User = nil - - dialer, err := NewDefaultDialer(timeout, bufferSize) - if err != nil { - return nil, fmt.Errorf("cannot initialize a base dialer: %w", err) - } - - ssDialer := &shadowsocksDialer{ - Dialer: dialer, - cipher: cipher, - } - - rv, err := proxy.FromURL(&socks5URL, ssDialer) - if err != nil { - return nil, fmt.Errorf("cannot initialize ss proxy dialer: %w", err) - } - - return rv.(Dialer), nil -} From ecfd550a958a0bee74eeb24d5b16f093af22e41e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 5 Mar 2021 18:26:13 +0300 Subject: [PATCH 010/206] Change signature of socks5 dialer --- mtglib/network/interfaces.go | 1 + mtglib/network/socks5.go | 10 ++-------- mtglib/network/socks5_test.go | 6 ++++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/mtglib/network/interfaces.go b/mtglib/network/interfaces.go index 4c7f972..2f450ae 100644 --- a/mtglib/network/interfaces.go +++ b/mtglib/network/interfaces.go @@ -6,5 +6,6 @@ import ( ) type Dialer interface { + Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) } diff --git a/mtglib/network/socks5.go b/mtglib/network/socks5.go index 8ed895c..af4df25 100644 --- a/mtglib/network/socks5.go +++ b/mtglib/network/socks5.go @@ -3,18 +3,12 @@ package network import ( "fmt" "net/url" - "time" "golang.org/x/net/proxy" ) -func NewSocks5Dialer(proxyURL *url.URL, timeout time.Duration, bufferSize int) (Dialer, error) { - dialer, err := NewDefaultDialer(timeout, bufferSize) - if err != nil { - return nil, fmt.Errorf("cannot initialize base dialer: %w", err) - } - - rv, err := proxy.FromURL(proxyURL, dialer.(*defaultDialer)) +func NewSocks5Dialer(proxyURL *url.URL, base Dialer) (Dialer, error) { + rv, err := proxy.FromURL(proxyURL, base) if err != nil { return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) } diff --git a/mtglib/network/socks5_test.go b/mtglib/network/socks5_test.go index 9f5e285..0b1ec34 100644 --- a/mtglib/network/socks5_test.go +++ b/mtglib/network/socks5_test.go @@ -14,6 +14,7 @@ import ( type Socks5TestSuite struct { HTTPServerTestSuite + baseDialer network.Dialer socksListener net.Listener socksProxy *socks5.Server } @@ -29,6 +30,7 @@ func (suite *Socks5TestSuite) SetupSuite() { suite.socksProxy, _ = socks5.New(&socksConf) suite.socksListener, _ = net.Listen("tcp", "127.0.0.1:0") + suite.baseDialer, _ = network.NewDefaultDialer(0, 0) go suite.socksProxy.Serve(suite.socksListener) } @@ -45,7 +47,7 @@ func (suite *Socks5TestSuite) TestRequestFailed() { User: url.UserPassword("user2", "password"), Host: suite.socksListener.Addr().String(), } - dialer, _ := network.NewSocks5Dialer(proxyURL, 0, 0) + dialer, _ := network.NewSocks5Dialer(proxyURL, suite.baseDialer) httpClient := http.Client{ Transport: &http.Transport{ @@ -64,7 +66,7 @@ func (suite *Socks5TestSuite) TestRequestOk() { User: url.UserPassword("user", "password"), Host: suite.socksListener.Addr().String(), } - dialer, _ := network.NewSocks5Dialer(proxyURL, 0, 0) + dialer, _ := network.NewSocks5Dialer(proxyURL, suite.baseDialer) httpClient := http.Client{ Transport: &http.Transport{ From 7002e4cd09b782de66877568b8aa2eeeff4625f1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 11:52:13 +0300 Subject: [PATCH 011/206] Add load balancing network dialer --- mtglib/network/circuit_breaker.go | 195 ++++++++++++++++++++++++++++++ mtglib/network/consts.go | 9 -- mtglib/network/init.go | 25 ++++ mtglib/network/init_test.go | 19 +++ mtglib/network/interfaces.go | 11 -- mtglib/network/load_balanced.go | 55 +++++++++ mtglib/network/network.go | 2 +- mtglib/network/proxy_dialer.go | 43 +++++++ mtglib/network/socks5.go | 4 +- mtglib/network/socks5_test.go | 4 +- 10 files changed, 342 insertions(+), 25 deletions(-) create mode 100644 mtglib/network/circuit_breaker.go delete mode 100644 mtglib/network/consts.go create mode 100644 mtglib/network/init.go delete mode 100644 mtglib/network/interfaces.go create mode 100644 mtglib/network/load_balanced.go create mode 100644 mtglib/network/proxy_dialer.go diff --git a/mtglib/network/circuit_breaker.go b/mtglib/network/circuit_breaker.go new file mode 100644 index 0000000..4581cb7 --- /dev/null +++ b/mtglib/network/circuit_breaker.go @@ -0,0 +1,195 @@ +package network + +import ( + "context" + "net" + "sync/atomic" + "time" +) + +const ( + circuitBreakerStateClosed uint32 = iota + circuitBreakerStateHalfOpened + circuitBreakerStateOpened +) + +type circuitBreakerDialer struct { + Dialer + + stateMutexChan chan bool + + halfOpenTimer *time.Timer + failuresCleanupTimer *time.Timer + + state uint32 + halfOpenAttempts uint32 + failuresCount uint32 + + openThreshold uint32 + halfOpenTimeout time.Duration + resetFailuresTimeout time.Duration +} + +func (c *circuitBreakerDialer) Dial(network, address string) (net.Conn, error) { + return c.DialContext(context.Background(), network, address) +} + +func (c *circuitBreakerDialer) DialContext(ctx context.Context, + network, address string) (net.Conn, error) { + switch atomic.LoadUint32(&c.state) { + case circuitBreakerStateClosed: + return c.doClosed(ctx, network, address) + case circuitBreakerStateHalfOpened: + return c.doHalfOpened(ctx, network, address) + default: + return nil, ErrCircuitBreakerOpened + } +} + +func (c *circuitBreakerDialer) doClosed(ctx context.Context, + network, address string) (net.Conn, error) { + conn, err := c.Dialer.DialContext(ctx, network, address) + + select { + case <-ctx.Done(): + if conn != nil { + conn.Close() + } + + return nil, ctx.Err() + case c.stateMutexChan <- true: + defer func() { + <-c.stateMutexChan + }() + } + + if err == nil { + c.switchState(circuitBreakerStateClosed) + + return conn, err + } + + c.failuresCount++ + + if c.state == circuitBreakerStateClosed && c.failuresCount > c.openThreshold { + c.switchState(circuitBreakerStateOpened) + } + + return conn, err +} + +func (c *circuitBreakerDialer) doHalfOpened(ctx context.Context, network, address string) (net.Conn, error) { + if !atomic.CompareAndSwapUint32(&c.halfOpenAttempts, 0, 1) { + return nil, ErrCircuitBreakerOpened + } + + conn, err := c.Dialer.DialContext(ctx, network, address) + + select { + case <-ctx.Done(): + if conn != nil { + conn.Close() + } + + return nil, ctx.Err() + case c.stateMutexChan <- true: + defer func() { + <-c.stateMutexChan + }() + } + + if c.state != circuitBreakerStateHalfOpened { + return conn, err + } + + if err == nil { + c.switchState(circuitBreakerStateClosed) + } else { + c.switchState(circuitBreakerStateOpened) + } + + return conn, err +} + +func (c *circuitBreakerDialer) switchState(state uint32) { + switch state { + case circuitBreakerStateClosed: + c.stopTimer(&c.halfOpenTimer) + c.ensureTimer(&c.failuresCleanupTimer, c.resetFailuresTimeout, c.resetFailures) + case circuitBreakerStateHalfOpened: + c.stopTimer(&c.failuresCleanupTimer) + c.stopTimer(&c.halfOpenTimer) + case circuitBreakerStateOpened: + c.stopTimer(&c.failuresCleanupTimer) + c.ensureTimer(&c.halfOpenTimer, c.halfOpenTimeout, c.tryHalfOpen) + } + + c.failuresCount = 0 + + atomic.StoreUint32(&c.halfOpenAttempts, 0) + atomic.StoreUint32(&c.state, state) +} + +func (c *circuitBreakerDialer) resetFailures() { + c.stateMutexChan <- true + + defer func() { + <-c.stateMutexChan + }() + + c.stopTimer(&c.failuresCleanupTimer) + + if c.state == circuitBreakerStateClosed { + c.switchState(circuitBreakerStateClosed) + } +} + +func (c *circuitBreakerDialer) tryHalfOpen() { + c.stateMutexChan <- true + + defer func() { + <-c.stateMutexChan + }() + + if c.state == circuitBreakerStateOpened { + c.switchState(circuitBreakerStateHalfOpened) + } +} + +func (c *circuitBreakerDialer) stopTimer(timerRef **time.Timer) { + timer := *timerRef + + if timer == nil { + return + } + + timer.Stop() + + select { + case <-timer.C: + default: + } + + *timerRef = nil +} + +func (c *circuitBreakerDialer) ensureTimer(timerRef **time.Timer, + timeout time.Duration, callback func()) { + if *timerRef == nil { + *timerRef = time.AfterFunc(timeout, callback) + } +} + +func newCircuitBreakerDialer(baseDialer Dialer, + openThreshold uint32, halfOpenTimeout, resetFailuresTimeout time.Duration) Dialer { + cb := &circuitBreakerDialer{ + Dialer: baseDialer, + openThreshold: openThreshold, + halfOpenTimeout: halfOpenTimeout, + resetFailuresTimeout: resetFailuresTimeout, + } + + cb.switchState(circuitBreakerStateClosed) + + return cb +} diff --git a/mtglib/network/consts.go b/mtglib/network/consts.go deleted file mode 100644 index 68f00fc..0000000 --- a/mtglib/network/consts.go +++ /dev/null @@ -1,9 +0,0 @@ -package network - -import "time" - -const ( - DefaultTimeout = 10 * time.Second - DefaultHTTPTimeout = DefaultTimeout - DefaultBufferSize = 4096 -) diff --git a/mtglib/network/init.go b/mtglib/network/init.go new file mode 100644 index 0000000..9d6a9e4 --- /dev/null +++ b/mtglib/network/init.go @@ -0,0 +1,25 @@ +package network + +import ( + "context" + "errors" + "net" + "time" +) + +const ( + DefaultTimeout = 10 * time.Second + DefaultDNSTimeout = time.Second + DefaultHTTPTimeout = DefaultTimeout + DefaultBufferSize = 4096 +) + +var ( + ErrCircuitBreakerOpened = errors.New("circuit breaker is opened") + ErrCannotDialWithAllProxies = errors.New("cannot dial with all proxies") +) + +type Dialer interface { + Dial(network, address string) (net.Conn, error) + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} diff --git a/mtglib/network/init_test.go b/mtglib/network/init_test.go index 8f8aeff..89aba2b 100644 --- a/mtglib/network/init_test.go +++ b/mtglib/network/init_test.go @@ -1,13 +1,32 @@ package network_test import ( + "context" + "net" "net/http/httptest" "strings" "github.com/mccutchen/go-httpbin/httpbin" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) +type DialerMock struct { + mock.Mock +} + +func (d *DialerMock) Dial(network, address string) (net.Conn, error) { + args := d.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := d.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + type HTTPServerTestSuite struct { suite.Suite diff --git a/mtglib/network/interfaces.go b/mtglib/network/interfaces.go deleted file mode 100644 index 2f450ae..0000000 --- a/mtglib/network/interfaces.go +++ /dev/null @@ -1,11 +0,0 @@ -package network - -import ( - "context" - "net" -) - -type Dialer interface { - Dial(network, address string) (net.Conn, error) - DialContext(ctx context.Context, network, address string) (net.Conn, error) -} diff --git a/mtglib/network/load_balanced.go b/mtglib/network/load_balanced.go new file mode 100644 index 0000000..9ffbd82 --- /dev/null +++ b/mtglib/network/load_balanced.go @@ -0,0 +1,55 @@ +package network + +import ( + "context" + "math/rand" + "net" + "net/url" +) + +type loadBalancedDialer struct { + dialers []Dialer +} + +func (l loadBalancedDialer) Dial(network, address string) (net.Conn, error) { + return l.DialContext(context.Background(), network, address) +} + +func (l loadBalancedDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + length := len(l.dialers) + start := rand.Intn(length) + moved := false + + for i := start; i != start || !moved; i = (i + 1) % length { + moved = true + if conn, err := l.dialers[i].DialContext(ctx, network, address); err == nil { + return conn, nil + } + } + + return nil, ErrCannotDialWithAllProxies +} + +func NewLoadBalancedDialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { + switch len(proxyURLs) { + case 0: + return baseDialer, nil + case 1: + return NewSocks5Dialer(baseDialer, proxyURLs[0]) + } + + dialers := []Dialer{} + + for _, u := range proxyURLs { + dialer, err := NewSocks5Dialer(newProxyDialer(baseDialer, u), u) + if err != nil { + return nil, err + } + + dialers = append(dialers, dialer) + } + + return loadBalancedDialer{ + dialers: dialers, + }, nil +} diff --git a/mtglib/network/network.go b/mtglib/network/network.go index ecbf0e7..31289ff 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -113,7 +113,7 @@ func NewNetwork(dialer Dialer, dohHostname string, httpTimeout time.Duration) (* } dohHTTPClient := &http.Client{ - Timeout: httpTimeout, + Timeout: DefaultDNSTimeout, Transport: &http.Transport{ DialContext: dialer.DialContext, }, diff --git a/mtglib/network/proxy_dialer.go b/mtglib/network/proxy_dialer.go new file mode 100644 index 0000000..0087819 --- /dev/null +++ b/mtglib/network/proxy_dialer.go @@ -0,0 +1,43 @@ +package network + +import ( + "net/url" + "strconv" + "time" +) + +const ( + ProxyDialerOpenThreshold = 5 + ProxyDialerHalfOpenTimeout = time.Minute + ProxyDialerResetFailuresTimeout = 10 * time.Second +) + +func newProxyDialer(baseDialer Dialer, proxyURL *url.URL) Dialer { + params := proxyURL.Query() + + var ( + openThreshold uint32 = ProxyDialerOpenThreshold + halfOpenTimeout = ProxyDialerHalfOpenTimeout + resetFailuresTimeout = ProxyDialerResetFailuresTimeout + ) + + if param := params.Get("open_threshold"); param != "" { + if intNum, err := strconv.ParseUint(param, 10, 32); err == nil { + openThreshold = uint32(intNum) + } + } + + if param := params.Get("half_open_timeout"); param != "" { + if dur, err := time.ParseDuration(param); err == nil && dur > 0 { + halfOpenTimeout = dur + } + } + + if param := params.Get("reset_failures_timeout"); param != "" { + if dur, err := time.ParseDuration(param); err == nil && dur > 0 { + resetFailuresTimeout = dur + } + } + + return newCircuitBreakerDialer(baseDialer, openThreshold, halfOpenTimeout, resetFailuresTimeout) +} diff --git a/mtglib/network/socks5.go b/mtglib/network/socks5.go index af4df25..91c9c61 100644 --- a/mtglib/network/socks5.go +++ b/mtglib/network/socks5.go @@ -7,8 +7,8 @@ import ( "golang.org/x/net/proxy" ) -func NewSocks5Dialer(proxyURL *url.URL, base Dialer) (Dialer, error) { - rv, err := proxy.FromURL(proxyURL, base) +func NewSocks5Dialer(baseDialer Dialer, proxyURL *url.URL) (Dialer, error) { + rv, err := proxy.FromURL(proxyURL, baseDialer) if err != nil { return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) } diff --git a/mtglib/network/socks5_test.go b/mtglib/network/socks5_test.go index 0b1ec34..f419a69 100644 --- a/mtglib/network/socks5_test.go +++ b/mtglib/network/socks5_test.go @@ -47,7 +47,7 @@ func (suite *Socks5TestSuite) TestRequestFailed() { User: url.UserPassword("user2", "password"), Host: suite.socksListener.Addr().String(), } - dialer, _ := network.NewSocks5Dialer(proxyURL, suite.baseDialer) + dialer, _ := network.NewSocks5Dialer(suite.baseDialer, proxyURL) httpClient := http.Client{ Transport: &http.Transport{ @@ -66,7 +66,7 @@ func (suite *Socks5TestSuite) TestRequestOk() { User: url.UserPassword("user", "password"), Host: suite.socksListener.Addr().String(), } - dialer, _ := network.NewSocks5Dialer(proxyURL, suite.baseDialer) + dialer, _ := network.NewSocks5Dialer(suite.baseDialer, proxyURL) httpClient := http.Client{ Transport: &http.Transport{ From b5346668f85cba1f40171256374c322d6a9fcbd5 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 11:57:55 +0300 Subject: [PATCH 012/206] Add mock for net conn --- go.mod | 1 + go.sum | 3 +++ mtglib/network/init_test.go | 41 +++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/go.mod b/go.mod index e6dd6ce..79b8059 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 github.com/pelletier/go-toml v1.8.1 + github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 ) diff --git a/go.sum b/go.sum index 1563bf5..aea74ac 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,9 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/mtglib/network/init_test.go b/mtglib/network/init_test.go index 89aba2b..d44ff5b 100644 --- a/mtglib/network/init_test.go +++ b/mtglib/network/init_test.go @@ -5,12 +5,53 @@ import ( "net" "net/http/httptest" "strings" + "time" "github.com/mccutchen/go-httpbin/httpbin" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) +type ConnMock struct { + mock.Mock +} + +func (c *ConnMock) Read(b []byte) (int, error) { + args := c.Called(b) + + return args.Int(0), args.Error(1) +} + +func (c *ConnMock) Write(b []byte) (int, error) { + args := c.Called(b) + + return args.Int(0), args.Error(1) +} + +func (c *ConnMock) Close() error { + return c.Called().Error(0) +} + +func (c *ConnMock) LocalAddr() net.Addr { + return c.Called().Get(0).(net.Addr) +} + +func (c *ConnMock) RemoteAddr() net.Addr { + return c.Called().Get(0).(net.Addr) +} + +func (c *ConnMock) SetDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +func (c *ConnMock) SetReadDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +func (c *ConnMock) SetWriteDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + type DialerMock struct { mock.Mock } From e386ae0daf3c212b215ca4acb6161618fa63309d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 12:00:06 +0300 Subject: [PATCH 013/206] Move proxy consts into init file --- mtglib/network/init.go | 4 ++++ mtglib/network/proxy_dialer.go | 6 ------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/mtglib/network/init.go b/mtglib/network/init.go index 9d6a9e4..d07f2b0 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -12,6 +12,10 @@ const ( DefaultDNSTimeout = time.Second DefaultHTTPTimeout = DefaultTimeout DefaultBufferSize = 4096 + + ProxyDialerOpenThreshold = 5 + ProxyDialerHalfOpenTimeout = time.Minute + ProxyDialerResetFailuresTimeout = 10 * time.Second ) var ( diff --git a/mtglib/network/proxy_dialer.go b/mtglib/network/proxy_dialer.go index 0087819..9c3f0f7 100644 --- a/mtglib/network/proxy_dialer.go +++ b/mtglib/network/proxy_dialer.go @@ -6,12 +6,6 @@ import ( "time" ) -const ( - ProxyDialerOpenThreshold = 5 - ProxyDialerHalfOpenTimeout = time.Minute - ProxyDialerResetFailuresTimeout = 10 * time.Second -) - func newProxyDialer(baseDialer Dialer, proxyURL *url.URL) Dialer { params := proxyURL.Query() From 7c43a4b0b75b3ee0cfb27ea38640e862b9f7954b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 18:20:21 +0300 Subject: [PATCH 014/206] Add tests for circuit breaker --- mtglib/network/circuit_breaker.go | 3 +- .../network/circuit_breaker_internal_test.go | 139 ++++++++++++++++++ mtglib/network/init_internal_test.go | 65 ++++++++ mtglib/network/init_test.go | 60 -------- 4 files changed, 206 insertions(+), 61 deletions(-) create mode 100644 mtglib/network/circuit_breaker_internal_test.go create mode 100644 mtglib/network/init_internal_test.go diff --git a/mtglib/network/circuit_breaker.go b/mtglib/network/circuit_breaker.go index 4581cb7..cd3d3c9 100644 --- a/mtglib/network/circuit_breaker.go +++ b/mtglib/network/circuit_breaker.go @@ -71,7 +71,7 @@ func (c *circuitBreakerDialer) doClosed(ctx context.Context, c.failuresCount++ - if c.state == circuitBreakerStateClosed && c.failuresCount > c.openThreshold { + if c.state == circuitBreakerStateClosed && c.failuresCount >= c.openThreshold { c.switchState(circuitBreakerStateOpened) } @@ -184,6 +184,7 @@ func newCircuitBreakerDialer(baseDialer Dialer, openThreshold uint32, halfOpenTimeout, resetFailuresTimeout time.Duration) Dialer { cb := &circuitBreakerDialer{ Dialer: baseDialer, + stateMutexChan: make(chan bool, 1), openThreshold: openThreshold, halfOpenTimeout: halfOpenTimeout, resetFailuresTimeout: resetFailuresTimeout, diff --git a/mtglib/network/circuit_breaker_internal_test.go b/mtglib/network/circuit_breaker_internal_test.go new file mode 100644 index 0000000..abb33df --- /dev/null +++ b/mtglib/network/circuit_breaker_internal_test.go @@ -0,0 +1,139 @@ +package network + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type CircuitBreakerTestSuite struct { + suite.Suite + + d Dialer + mutex sync.Mutex + ctx context.Context + ctxCancel context.CancelFunc + connMock *ConnMock + baseDialerMock *DialerMock +} + +func (suite *CircuitBreakerTestSuite) SetupTest() { + suite.mutex = sync.Mutex{} + suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) + suite.baseDialerMock = &DialerMock{} + suite.connMock = &ConnMock{} + suite.d = newCircuitBreakerDialer(suite.baseDialerMock, + 3, 100*time.Millisecond, 50*time.Millisecond) +} + +func (suite *CircuitBreakerTestSuite) TearDownTest() { + suite.ctxCancel() + suite.baseDialerMock.AssertExpectations(suite.T()) + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *CircuitBreakerTestSuite) TestMultipleRunsOk() { + suite.connMock.On("RemoteAddr"). + Times(5). + Return(&net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 3128, + }) + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.1"). + Times(5). + Return(suite.connMock, nil) + + wg := &sync.WaitGroup{} + + wg.Add(5) + + go func() { + wg.Wait() + suite.ctxCancel() + }() + + for i := 0; i < 5; i++ { + go func() { + defer wg.Done() + + conn, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + + suite.mutex.Lock() + defer suite.mutex.Unlock() + + suite.NoError(err) + suite.Equal("127.0.0.1:3128", conn.RemoteAddr().String()) + }() + } + + suite.Eventually(func() bool { + _, ok := <-suite.ctx.Done() + + return !ok + }, time.Second, 10*time.Millisecond) +} + +func (suite *CircuitBreakerTestSuite) TestFromClosedToOpen() { + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.1"). + Times(3). + Return(&net.TCPConn{}, io.EOF) + + _, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, ErrCircuitBreakerOpened)) +} + +func (suite *CircuitBreakerTestSuite) TestHalfOpen() { + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.1"). + Times(4). + Return(&net.TCPConn{}, io.EOF) + suite.baseDialerMock.On("DialContext", mock.Anything, "tcp", "127.0.0.2"). + Twice(). + Return(suite.connMock, nil) + suite.connMock.On("RemoteAddr").Return(&net.TCPAddr{ + IP: net.ParseIP("10.0.0.10"), + Port: 80, + }) + + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + + time.Sleep(500 * time.Millisecond) + + _, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, io.EOF)) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.True(errors.Is(err, ErrCircuitBreakerOpened)) + + time.Sleep(500 * time.Millisecond) + + conn, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.2") + + suite.NoError(err) + suite.Equal("10.0.0.10:80", conn.RemoteAddr().String()) + + _, err = suite.d.DialContext(suite.ctx, "tcp", "127.0.0.2") + suite.NoError(err) +} + +func TestCircuitBreaker(t *testing.T) { + suite.Run(t, &CircuitBreakerTestSuite{}) +} diff --git a/mtglib/network/init_internal_test.go b/mtglib/network/init_internal_test.go new file mode 100644 index 0000000..d40233d --- /dev/null +++ b/mtglib/network/init_internal_test.go @@ -0,0 +1,65 @@ +package network + +import ( + "context" + "net" + "time" + + "github.com/stretchr/testify/mock" +) + +type ConnMock struct { + mock.Mock +} + +func (c *ConnMock) Read(b []byte) (int, error) { + args := c.Called(b) + + return args.Int(0), args.Error(1) +} + +func (c *ConnMock) Write(b []byte) (int, error) { + args := c.Called(b) + + return args.Int(0), args.Error(1) +} + +func (c *ConnMock) Close() error { + return c.Called().Error(0) +} + +func (c *ConnMock) LocalAddr() net.Addr { + return c.Called().Get(0).(net.Addr) +} + +func (c *ConnMock) RemoteAddr() net.Addr { + return c.Called().Get(0).(net.Addr) +} + +func (c *ConnMock) SetDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +func (c *ConnMock) SetReadDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +func (c *ConnMock) SetWriteDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +type DialerMock struct { + mock.Mock +} + +func (d *DialerMock) Dial(network, address string) (net.Conn, error) { + args := d.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := d.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} diff --git a/mtglib/network/init_test.go b/mtglib/network/init_test.go index d44ff5b..8f8aeff 100644 --- a/mtglib/network/init_test.go +++ b/mtglib/network/init_test.go @@ -1,73 +1,13 @@ package network_test import ( - "context" - "net" "net/http/httptest" "strings" - "time" "github.com/mccutchen/go-httpbin/httpbin" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) -type ConnMock struct { - mock.Mock -} - -func (c *ConnMock) Read(b []byte) (int, error) { - args := c.Called(b) - - return args.Int(0), args.Error(1) -} - -func (c *ConnMock) Write(b []byte) (int, error) { - args := c.Called(b) - - return args.Int(0), args.Error(1) -} - -func (c *ConnMock) Close() error { - return c.Called().Error(0) -} - -func (c *ConnMock) LocalAddr() net.Addr { - return c.Called().Get(0).(net.Addr) -} - -func (c *ConnMock) RemoteAddr() net.Addr { - return c.Called().Get(0).(net.Addr) -} - -func (c *ConnMock) SetDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -func (c *ConnMock) SetReadDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -func (c *ConnMock) SetWriteDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -type DialerMock struct { - mock.Mock -} - -func (d *DialerMock) Dial(network, address string) (net.Conn, error) { - args := d.Called(network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - -func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - args := d.Called(ctx, network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - type HTTPServerTestSuite struct { suite.Suite From c8914f90db293339f5a0fd89dc689c83be7187f1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 18:21:16 +0300 Subject: [PATCH 015/206] Add tests for circuit breaker --- mtglib/network/load_balanced.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/mtglib/network/load_balanced.go b/mtglib/network/load_balanced.go index 9ffbd82..09e554d 100644 --- a/mtglib/network/load_balanced.go +++ b/mtglib/network/load_balanced.go @@ -2,6 +2,7 @@ package network import ( "context" + "fmt" "math/rand" "net" "net/url" @@ -31,19 +32,12 @@ func (l loadBalancedDialer) DialContext(ctx context.Context, network, address st } func NewLoadBalancedDialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { - switch len(proxyURLs) { - case 0: - return baseDialer, nil - case 1: - return NewSocks5Dialer(baseDialer, proxyURLs[0]) - } - - dialers := []Dialer{} + var dialers []Dialer for _, u := range proxyURLs { dialer, err := NewSocks5Dialer(newProxyDialer(baseDialer, u), u) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot build dialer for %s: %w", u.String(), err) } dialers = append(dialers, dialer) From 9124ffabf12bf707ef6c1237b349096b5b77712d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 18:23:39 +0300 Subject: [PATCH 016/206] Rename load balanced dialer to loadBalancedSocks5Dialer --- .../{load_balanced.go => load_balanced_socks5.go} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename mtglib/network/{load_balanced.go => load_balanced_socks5.go} (67%) diff --git a/mtglib/network/load_balanced.go b/mtglib/network/load_balanced_socks5.go similarity index 67% rename from mtglib/network/load_balanced.go rename to mtglib/network/load_balanced_socks5.go index 09e554d..adb7ef5 100644 --- a/mtglib/network/load_balanced.go +++ b/mtglib/network/load_balanced_socks5.go @@ -8,15 +8,15 @@ import ( "net/url" ) -type loadBalancedDialer struct { +type loadBalancedSocks5Dialer struct { dialers []Dialer } -func (l loadBalancedDialer) Dial(network, address string) (net.Conn, error) { +func (l loadBalancedSocks5Dialer) Dial(network, address string) (net.Conn, error) { return l.DialContext(context.Background(), network, address) } -func (l loadBalancedDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { length := len(l.dialers) start := rand.Intn(length) moved := false @@ -31,7 +31,7 @@ func (l loadBalancedDialer) DialContext(ctx context.Context, network, address st return nil, ErrCannotDialWithAllProxies } -func NewLoadBalancedDialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { +func NewLoadBalancedSocks5Dialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { var dialers []Dialer for _, u := range proxyURLs { @@ -43,7 +43,7 @@ func NewLoadBalancedDialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, err dialers = append(dialers, dialer) } - return loadBalancedDialer{ + return loadBalancedSocks5Dialer{ dialers: dialers, }, nil } From 8f7e79862b60c6cd4df81708111e2b48b21e04f1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 9 Mar 2021 21:36:40 +0300 Subject: [PATCH 017/206] Add tests for proxy dialers --- mtglib/network/proxy_dialer_internal_test.go | 96 ++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 mtglib/network/proxy_dialer_internal_test.go diff --git a/mtglib/network/proxy_dialer_internal_test.go b/mtglib/network/proxy_dialer_internal_test.go new file mode 100644 index 0000000..42c18fb --- /dev/null +++ b/mtglib/network/proxy_dialer_internal_test.go @@ -0,0 +1,96 @@ +package network + +import ( + "fmt" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type ProxyDialerTestSuite struct { + suite.Suite + + u *url.URL +} + +func (suite *ProxyDialerTestSuite) SetupSuite() { + u, _ := url.Parse("socks5://hello:world@10.0.0.10:3128") + suite.u = u +} + +func (suite *ProxyDialerTestSuite) TestSetupDefaults() { + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + + suite.EqualValues(ProxyDialerOpenThreshold, d.openThreshold) + suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + suite.EqualValues(ProxyDialerResetFailuresTimeout, d.resetFailuresTimeout) +} + +func (suite *ProxyDialerTestSuite) TestSetupValuesAllOk() { + query := url.Values{} + + query.Set("open_threshold", "30") + query.Set("reset_failures_timeout", "1s") + query.Set("half_open_timeout", "2s") + + suite.u.RawQuery = query.Encode() + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + + suite.EqualValues(30, d.openThreshold) + suite.EqualValues(2*time.Second, d.halfOpenTimeout) + suite.EqualValues(time.Second, d.resetFailuresTimeout) +} + +func (suite *ProxyDialerTestSuite) TestOpenThreshold() { + query := url.Values{} + params := []string{"-30", "aaa", "1.0", "-1.0"} + + for _, v := range params { + suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { + query.Set("open_threshold", v) + + suite.u.RawQuery = query.Encode() + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + + suite.EqualValues(ProxyDialerOpenThreshold, d.openThreshold) + }) + } +} + +func (suite *ProxyDialerTestSuite) TestHalfOpenTimeout() { + query := url.Values{} + params := []string{"-30", "30", "aaa", "-3.0", "3.0"} + + for _, v := range params { + suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { + query.Set("half_open_timeout", v) + + suite.u.RawQuery = query.Encode() + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + + suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + }) + } +} + +func (suite *ProxyDialerTestSuite) TestResetFailuresTimeout() { + query := url.Values{} + params := []string{"-30", "30", "aaa", "-3.0", "3.0"} + + for _, v := range params { + suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { + query.Set("reset_failures_timeout", v) + + suite.u.RawQuery = query.Encode() + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + + suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + }) + } +} + +func TestProxyDialer(t *testing.T) { + suite.Run(t, &ProxyDialerTestSuite{}) +} From ba875da9b5e64f46d92714c9168f93787ca5d03a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 10:25:24 +0300 Subject: [PATCH 018/206] Make tests for load balanced client --- mtglib/network/default_test.go | 12 +-- mtglib/network/init_test.go | 107 +++++++++++++++++++- mtglib/network/load_balanced_socks5_test.go | 88 ++++++++++++++++ mtglib/network/socks5_test.go | 62 +++--------- 4 files changed, 211 insertions(+), 58 deletions(-) create mode 100644 mtglib/network/load_balanced_socks5_test.go diff --git a/mtglib/network/default_test.go b/mtglib/network/default_test.go index 203a2bb..95048cf 100644 --- a/mtglib/network/default_test.go +++ b/mtglib/network/default_test.go @@ -10,6 +10,7 @@ import ( ) type DefaultDialerTestSuite struct { + suite.Suite HTTPServerTestSuite d network.Dialer @@ -65,17 +66,12 @@ func (suite *DefaultDialerTestSuite) TestConnectOk() { } func (suite *DefaultDialerTestSuite) TestHTTPRequest() { - httpClient := http.Client{ - Transport: &http.Transport{ - DialContext: suite.d.DialContext, - }, - } + httpClient := suite.MakeHTTPClient(suite.d) - resp, err := httpClient.Get(suite.httpServer.URL + "/get") + resp, err := httpClient.Get(suite.MakeURL("/get")) suite.NoError(err) - - resp.Body.Close() + suite.Equal(http.StatusOK, resp.StatusCode) } func TestDefaultDialer(t *testing.T) { diff --git a/mtglib/network/init_test.go b/mtglib/network/init_test.go index 8f8aeff..5ac8cdd 100644 --- a/mtglib/network/init_test.go +++ b/mtglib/network/init_test.go @@ -1,16 +1,77 @@ package network_test import ( + "context" + "net" + "net/http" "net/http/httptest" + "net/url" "strings" + "time" + "github.com/9seconds/mtg/v2/mtglib/network" + socks5 "github.com/armon/go-socks5" "github.com/mccutchen/go-httpbin/httpbin" - "github.com/stretchr/testify/suite" + "github.com/stretchr/testify/mock" ) -type HTTPServerTestSuite struct { - suite.Suite +type ConnMock struct { + mock.Mock +} +func (c *ConnMock) Read(b []byte) (int, error) { + args := c.Called(b) + + return args.Int(0), args.Error(1) +} + +func (c *ConnMock) Write(b []byte) (int, error) { + args := c.Called(b) + + return args.Int(0), args.Error(1) +} + +func (c *ConnMock) Close() error { + return c.Called().Error(0) +} + +func (c *ConnMock) LocalAddr() net.Addr { + return c.Called().Get(0).(net.Addr) +} + +func (c *ConnMock) RemoteAddr() net.Addr { + return c.Called().Get(0).(net.Addr) +} + +func (c *ConnMock) SetDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +func (c *ConnMock) SetReadDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +func (c *ConnMock) SetWriteDeadline(t time.Time) error { + return c.Called(t).Error(0) +} + +type DialerMock struct { + mock.Mock +} + +func (d *DialerMock) Dial(network, address string) (net.Conn, error) { + args := d.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := d.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +type HTTPServerTestSuite struct { httpServer *httptest.Server } @@ -25,3 +86,43 @@ func (suite *HTTPServerTestSuite) TearDownSuite() { func (suite *HTTPServerTestSuite) HTTPServerAddress() string { return strings.TrimPrefix(suite.httpServer.URL, "http://") } + +func (suite *HTTPServerTestSuite) MakeURL(path string) string { + return suite.httpServer.URL + path +} + +func (suite *HTTPServerTestSuite) MakeHTTPClient(dialer network.Dialer) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DialContext: dialer.DialContext, + }, + } +} + +type Socks5ServerTestSuite struct { + socks5Listener net.Listener + socks5Server *socks5.Server +} + +func (suite *Socks5ServerTestSuite) SetupSuite() { + suite.socks5Listener, _ = net.Listen("tcp", "127.0.0.1:0") + suite.socks5Server, _ = socks5.New(&socks5.Config{ + Credentials: socks5.StaticCredentials{ + "user": "password", + }, + }) + + go suite.socks5Server.Serve(suite.socks5Listener) +} + +func (suite *Socks5ServerTestSuite) TearDownSuite() { + suite.socks5Listener.Close() +} + +func (suite *Socks5ServerTestSuite) MakeSocks5URL(user, password string) *url.URL { + return &url.URL{ + Scheme: "socks5", + User: url.UserPassword(user, password), + Host: suite.socks5Listener.Addr().String(), + } +} diff --git a/mtglib/network/load_balanced_socks5_test.go b/mtglib/network/load_balanced_socks5_test.go new file mode 100644 index 0000000..129597b --- /dev/null +++ b/mtglib/network/load_balanced_socks5_test.go @@ -0,0 +1,88 @@ +package network_test + +import ( + "errors" + "io" + "net" + "net/http" + "net/url" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type LoadBalancedSocks5TestSuite struct { + suite.Suite + HTTPServerTestSuite + Socks5ServerTestSuite + + httpClient *http.Client +} + +func (suite *LoadBalancedSocks5TestSuite) SetupSuite() { + suite.HTTPServerTestSuite.SetupSuite() + suite.Socks5ServerTestSuite.SetupSuite() +} + +func (suite *LoadBalancedSocks5TestSuite) SetupTest() { + baseDialer, _ := network.NewDefaultDialer(0, 0) + + lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ + suite.MakeSocks5URL("user", "password"), + suite.MakeSocks5URL("user2", "password"), + }) + + suite.NoError(err) + + suite.httpClient = suite.MakeHTTPClient(lbDialer) +} + +func (suite *LoadBalancedSocks5TestSuite) TearDownSuite() { + suite.Socks5ServerTestSuite.SetupSuite() + suite.HTTPServerTestSuite.SetupSuite() +} + +func (suite *LoadBalancedSocks5TestSuite) TestIncorrectURL() { + _, err := network.NewLoadBalancedSocks5Dialer(&DialerMock{}, []*url.URL{ + {Scheme: "http"}, + }) + + suite.Error(err) +} + +func (suite *LoadBalancedSocks5TestSuite) TestCannotDial() { + baseDialer := &DialerMock{} + + baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.1:1080"). + Times(network.ProxyDialerOpenThreshold). + Return(&net.TCPConn{}, io.EOF) + baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.2:1080"). + Times(network.ProxyDialerOpenThreshold). + Return(&net.TCPConn{}, io.EOF) + + lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ + {Scheme: "socks5", User: url.UserPassword("user", "password"), Host: "127.0.0.1:1080"}, + {Scheme: "socks5", User: url.UserPassword("user", "password"), Host: "127.0.0.2:1080"}, + }) + suite.NoError(err) + + for i := 0; i < network.ProxyDialerOpenThreshold*2; i++ { + _, err = lbDialer.Dial("tcp", "127.1.1.1:80") + suite.True(errors.Is(err, network.ErrCannotDialWithAllProxies)) + } + + baseDialer.AssertExpectations(suite.T()) +} + +func (suite *LoadBalancedSocks5TestSuite) TestDialOk() { + resp, err := suite.httpClient.Get(suite.MakeURL("/get")) + + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) +} + +func TestLoadBalancedSocks5(t *testing.T) { + suite.Run(t, &LoadBalancedSocks5TestSuite{}) +} diff --git a/mtglib/network/socks5_test.go b/mtglib/network/socks5_test.go index f419a69..5499af4 100644 --- a/mtglib/network/socks5_test.go +++ b/mtglib/network/socks5_test.go @@ -1,84 +1,52 @@ package network_test import ( - "net" "net/http" - "net/url" "testing" "github.com/9seconds/mtg/v2/mtglib/network" - socks5 "github.com/armon/go-socks5" "github.com/stretchr/testify/suite" ) type Socks5TestSuite struct { + suite.Suite HTTPServerTestSuite + Socks5ServerTestSuite - baseDialer network.Dialer - socksListener net.Listener - socksProxy *socks5.Server + d network.Dialer } func (suite *Socks5TestSuite) SetupSuite() { suite.HTTPServerTestSuite.SetupSuite() + suite.Socks5ServerTestSuite.SetupSuite() - socksConf := socks5.Config{ - Credentials: socks5.StaticCredentials{ - "user": "password", - }, - } - - suite.socksProxy, _ = socks5.New(&socksConf) - suite.socksListener, _ = net.Listen("tcp", "127.0.0.1:0") - suite.baseDialer, _ = network.NewDefaultDialer(0, 0) - - go suite.socksProxy.Serve(suite.socksListener) + suite.d, _ = network.NewDefaultDialer(0, 0) } func (suite *Socks5TestSuite) TearDownSuite() { - suite.socksListener.Close() - + suite.Socks5ServerTestSuite.TearDownSuite() suite.HTTPServerTestSuite.TearDownSuite() } func (suite *Socks5TestSuite) TestRequestFailed() { - proxyURL := &url.URL{ - Scheme: "socks5", - User: url.UserPassword("user2", "password"), - Host: suite.socksListener.Addr().String(), - } - dialer, _ := network.NewSocks5Dialer(suite.baseDialer, proxyURL) + proxyURL := suite.MakeSocks5URL("user2", "password") + dialer, _ := network.NewSocks5Dialer(suite.d, proxyURL) + httpClient := suite.MakeHTTPClient(dialer) - httpClient := http.Client{ - Transport: &http.Transport{ - DialContext: dialer.DialContext, - }, - } - - _, err := httpClient.Get(suite.httpServer.URL + "/get") + _, err := httpClient.Get(suite.MakeURL("/get")) suite.Error(err) } func (suite *Socks5TestSuite) TestRequestOk() { - proxyURL := &url.URL{ - Scheme: "socks5", - User: url.UserPassword("user", "password"), - Host: suite.socksListener.Addr().String(), - } - dialer, _ := network.NewSocks5Dialer(suite.baseDialer, proxyURL) + proxyURL := suite.MakeSocks5URL("user", "password") + dialer, _ := network.NewSocks5Dialer(suite.d, proxyURL) + httpClient := suite.MakeHTTPClient(dialer) - httpClient := http.Client{ - Transport: &http.Transport{ - DialContext: dialer.DialContext, - }, - } - - resp, err := httpClient.Get(suite.httpServer.URL + "/get") + resp, err := httpClient.Get(suite.MakeURL("/get")) suite.NoError(err) - - resp.Body.Close() + suite.Equal(http.StatusOK, resp.StatusCode) } func TestSocks5TestSuite(t *testing.T) { From ef87d031eddd0cb8728f369c44d9828840502339 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 10:35:17 +0300 Subject: [PATCH 019/206] Add gofumpt as formatter --- Makefile | 11 ++++++++++- go.mod | 3 +++ go.sum | 11 +++++++++-- mtglib/network/default_test.go | 2 +- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3f84fba..b11dccd 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,10 @@ ccbuilds: vendor: go.mod go.sum @$(MOD_ON) go mod vendor +.PHONY: fmt +fmt: + @$(GOTOOL) gofumpt -w -s -extra "$(ROOT_DIR)" + .PHONY: test test: @go test -v ./... @@ -74,7 +78,7 @@ doc: @$(GOTOOL) godoc -http 0.0.0.0:10000 .PHONY: install-tools -install-tools: install-tools-lint install-tools-godoc +install-tools: install-tools-lint install-tools-godoc install-tools-gofumpt .PHONY: install-tools-lint install-tools-lint: @@ -87,6 +91,11 @@ install-tools-godoc: @mkdir -p "$(GOBIN)" || true && \ $(GOTOOL) go get -u golang.org/x/tools/cmd/godoc +.PHONY: install-tools-gofumpt +install-tools-gofumpt: + @mkdir -p "$(GOBIN)" || true && \ + $(GOTOOL) go get -u mvdan.cc/gofumpt + .PHONY: update-deps upgrade-deps: $go get -u && go mod tidy diff --git a/go.mod b/go.mod index 79b8059..f201e45 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,13 @@ go 1.16 require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 + github.com/kr/pretty v0.1.0 // indirect github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 github.com/pelletier/go-toml v1.8.1 github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index aea74ac..817b6fb 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,11 @@ github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE= @@ -25,13 +30,15 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mtglib/network/default_test.go b/mtglib/network/default_test.go index 95048cf..3f1deb9 100644 --- a/mtglib/network/default_test.go +++ b/mtglib/network/default_test.go @@ -71,7 +71,7 @@ func (suite *DefaultDialerTestSuite) TestHTTPRequest() { resp, err := httpClient.Get(suite.MakeURL("/get")) suite.NoError(err) - suite.Equal(http.StatusOK, resp.StatusCode) + suite.Equal(http.StatusOK, resp.StatusCode) } func TestDefaultDialer(t *testing.T) { From 0395ce8d1af489da0078a25f155c13ae3d3694f8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 10:45:32 +0300 Subject: [PATCH 020/206] Fix lint issues --- mtglib/network/circuit_breaker.go | 8 ++-- .../network/circuit_breaker_internal_test.go | 9 ++-- mtglib/network/default_test.go | 6 ++- mtglib/network/init_test.go | 43 +------------------ mtglib/network/load_balanced_socks5.go | 3 +- mtglib/network/load_balanced_socks5_test.go | 6 ++- mtglib/network/proxy_dialer_internal_test.go | 10 +++-- mtglib/network/socks5_test.go | 11 ++++- 8 files changed, 38 insertions(+), 58 deletions(-) diff --git a/mtglib/network/circuit_breaker.go b/mtglib/network/circuit_breaker.go index cd3d3c9..6c08b33 100644 --- a/mtglib/network/circuit_breaker.go +++ b/mtglib/network/circuit_breaker.go @@ -66,7 +66,7 @@ func (c *circuitBreakerDialer) doClosed(ctx context.Context, if err == nil { c.switchState(circuitBreakerStateClosed) - return conn, err + return conn, err // nolint: wrapcheck } c.failuresCount++ @@ -75,7 +75,7 @@ func (c *circuitBreakerDialer) doClosed(ctx context.Context, c.switchState(circuitBreakerStateOpened) } - return conn, err + return conn, err // nolint: wrapcheck } func (c *circuitBreakerDialer) doHalfOpened(ctx context.Context, network, address string) (net.Conn, error) { @@ -99,7 +99,7 @@ func (c *circuitBreakerDialer) doHalfOpened(ctx context.Context, network, addres } if c.state != circuitBreakerStateHalfOpened { - return conn, err + return conn, err // nolint: wrapcheck } if err == nil { @@ -108,7 +108,7 @@ func (c *circuitBreakerDialer) doHalfOpened(ctx context.Context, network, addres c.switchState(circuitBreakerStateOpened) } - return conn, err + return conn, err // nolint: wrapcheck } func (c *circuitBreakerDialer) switchState(state uint32) { diff --git a/mtglib/network/circuit_breaker_internal_test.go b/mtglib/network/circuit_breaker_internal_test.go index abb33df..849addd 100644 --- a/mtglib/network/circuit_breaker_internal_test.go +++ b/mtglib/network/circuit_breaker_internal_test.go @@ -110,10 +110,10 @@ func (suite *CircuitBreakerTestSuite) TestHalfOpen() { Port: 80, }) - suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") - suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") - suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") - suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck + suite.d.DialContext(suite.ctx, "tcp", "127.0.0.1") // nolint: errcheck time.Sleep(500 * time.Millisecond) @@ -135,5 +135,6 @@ func (suite *CircuitBreakerTestSuite) TestHalfOpen() { } func TestCircuitBreaker(t *testing.T) { + t.Parallel() suite.Run(t, &CircuitBreakerTestSuite{}) } diff --git a/mtglib/network/default_test.go b/mtglib/network/default_test.go index 3f1deb9..ba0d779 100644 --- a/mtglib/network/default_test.go +++ b/mtglib/network/default_test.go @@ -68,12 +68,16 @@ func (suite *DefaultDialerTestSuite) TestConnectOk() { func (suite *DefaultDialerTestSuite) TestHTTPRequest() { httpClient := suite.MakeHTTPClient(suite.d) - resp, err := httpClient.Get(suite.MakeURL("/get")) + resp, err := httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } suite.NoError(err) suite.Equal(http.StatusOK, resp.StatusCode) } func TestDefaultDialer(t *testing.T) { + t.Parallel() suite.Run(t, &DefaultDialerTestSuite{}) } diff --git a/mtglib/network/init_test.go b/mtglib/network/init_test.go index 5ac8cdd..8f648d9 100644 --- a/mtglib/network/init_test.go +++ b/mtglib/network/init_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "net/url" "strings" - "time" "github.com/9seconds/mtg/v2/mtglib/network" socks5 "github.com/armon/go-socks5" @@ -15,46 +14,6 @@ import ( "github.com/stretchr/testify/mock" ) -type ConnMock struct { - mock.Mock -} - -func (c *ConnMock) Read(b []byte) (int, error) { - args := c.Called(b) - - return args.Int(0), args.Error(1) -} - -func (c *ConnMock) Write(b []byte) (int, error) { - args := c.Called(b) - - return args.Int(0), args.Error(1) -} - -func (c *ConnMock) Close() error { - return c.Called().Error(0) -} - -func (c *ConnMock) LocalAddr() net.Addr { - return c.Called().Get(0).(net.Addr) -} - -func (c *ConnMock) RemoteAddr() net.Addr { - return c.Called().Get(0).(net.Addr) -} - -func (c *ConnMock) SetDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -func (c *ConnMock) SetReadDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -func (c *ConnMock) SetWriteDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - type DialerMock struct { mock.Mock } @@ -112,7 +71,7 @@ func (suite *Socks5ServerTestSuite) SetupSuite() { }, }) - go suite.socks5Server.Serve(suite.socks5Listener) + go suite.socks5Server.Serve(suite.socks5Listener) // nolint: errcheck } func (suite *Socks5ServerTestSuite) TearDownSuite() { diff --git a/mtglib/network/load_balanced_socks5.go b/mtglib/network/load_balanced_socks5.go index adb7ef5..2a60d69 100644 --- a/mtglib/network/load_balanced_socks5.go +++ b/mtglib/network/load_balanced_socks5.go @@ -23,6 +23,7 @@ func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, addr for i := start; i != start || !moved; i = (i + 1) % length { moved = true + if conn, err := l.dialers[i].DialContext(ctx, network, address); err == nil { return conn, nil } @@ -32,7 +33,7 @@ func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, addr } func NewLoadBalancedSocks5Dialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { - var dialers []Dialer + dialers := make([]Dialer, 0, len(proxyURLs)) for _, u := range proxyURLs { dialer, err := NewSocks5Dialer(newProxyDialer(baseDialer, u), u) diff --git a/mtglib/network/load_balanced_socks5_test.go b/mtglib/network/load_balanced_socks5_test.go index 129597b..f86e3f1 100644 --- a/mtglib/network/load_balanced_socks5_test.go +++ b/mtglib/network/load_balanced_socks5_test.go @@ -77,12 +77,16 @@ func (suite *LoadBalancedSocks5TestSuite) TestCannotDial() { } func (suite *LoadBalancedSocks5TestSuite) TestDialOk() { - resp, err := suite.httpClient.Get(suite.MakeURL("/get")) + resp, err := suite.httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } suite.NoError(err) suite.Equal(http.StatusOK, resp.StatusCode) } func TestLoadBalancedSocks5(t *testing.T) { + t.Parallel() suite.Run(t, &LoadBalancedSocks5TestSuite{}) } diff --git a/mtglib/network/proxy_dialer_internal_test.go b/mtglib/network/proxy_dialer_internal_test.go index 42c18fb..1183908 100644 --- a/mtglib/network/proxy_dialer_internal_test.go +++ b/mtglib/network/proxy_dialer_internal_test.go @@ -48,8 +48,9 @@ func (suite *ProxyDialerTestSuite) TestOpenThreshold() { params := []string{"-30", "aaa", "1.0", "-1.0"} for _, v := range params { + param := v suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { - query.Set("open_threshold", v) + query.Set("open_threshold", param) suite.u.RawQuery = query.Encode() d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) @@ -64,8 +65,9 @@ func (suite *ProxyDialerTestSuite) TestHalfOpenTimeout() { params := []string{"-30", "30", "aaa", "-3.0", "3.0"} for _, v := range params { + param := v suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { - query.Set("half_open_timeout", v) + query.Set("half_open_timeout", param) suite.u.RawQuery = query.Encode() d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) @@ -80,8 +82,9 @@ func (suite *ProxyDialerTestSuite) TestResetFailuresTimeout() { params := []string{"-30", "30", "aaa", "-3.0", "3.0"} for _, v := range params { + param := v suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { - query.Set("reset_failures_timeout", v) + query.Set("reset_failures_timeout", param) suite.u.RawQuery = query.Encode() d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) @@ -92,5 +95,6 @@ func (suite *ProxyDialerTestSuite) TestResetFailuresTimeout() { } func TestProxyDialer(t *testing.T) { + t.Parallel() suite.Run(t, &ProxyDialerTestSuite{}) } diff --git a/mtglib/network/socks5_test.go b/mtglib/network/socks5_test.go index 5499af4..82b471f 100644 --- a/mtglib/network/socks5_test.go +++ b/mtglib/network/socks5_test.go @@ -33,7 +33,10 @@ func (suite *Socks5TestSuite) TestRequestFailed() { dialer, _ := network.NewSocks5Dialer(suite.d, proxyURL) httpClient := suite.MakeHTTPClient(dialer) - _, err := httpClient.Get(suite.MakeURL("/get")) + resp, err := httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } suite.Error(err) } @@ -43,12 +46,16 @@ func (suite *Socks5TestSuite) TestRequestOk() { dialer, _ := network.NewSocks5Dialer(suite.d, proxyURL) httpClient := suite.MakeHTTPClient(dialer) - resp, err := httpClient.Get(suite.MakeURL("/get")) + resp, err := httpClient.Get(suite.MakeURL("/get")) // nolint: noctx + if err == nil { + defer resp.Body.Close() + } suite.NoError(err) suite.Equal(http.StatusOK, resp.StatusCode) } func TestSocks5TestSuite(t *testing.T) { + t.Parallel() suite.Run(t, &Socks5TestSuite{}) } From f89f5f8469615e8e1090fe447dffa2f352dc639a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 11:03:23 +0300 Subject: [PATCH 021/206] Remove access-file --- example.config.toml | 7 ------- raw_config.go | 1 - 2 files changed, 8 deletions(-) diff --git a/example.config.toml b/example.config.toml index 917d69e..da7c108 100644 --- a/example.config.toml +++ b/example.config.toml @@ -44,13 +44,6 @@ prefer-ips = "prefer-ipv6" # access. cloak-port = 443 -# Path to access file. Each time when proxy starts up, it writes an -# access file. This file contains a JSON with settings how to access -# this proxy. -# -# Pass filepath here or '-' if you want to dump into stdout. -access-file = "-" - # network defines different network-related settings [network] # please be aware that mtg needs to do some external requests. For diff --git a/raw_config.go b/raw_config.go index 4f98c3e..7de66d9 100644 --- a/raw_config.go +++ b/raw_config.go @@ -14,7 +14,6 @@ type rawConfig struct { TCPBuffer string `toml:"tcp-buffer"` PreferIP string `toml:"prefer-ip"` CloakPort uint `toml:"cloak-port"` - AccessFile string `toml:"access-file"` Probes struct { Time struct { Enabled bool `toml:"enabled"` From e0833e86d91d40a39b20818515a7edc13fdae9f7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 11:12:10 +0300 Subject: [PATCH 022/206] Update configuration file with proxies specification --- example.config.toml | 35 +++++++++++++++++++++++++++++++++++ raw_config.go | 17 +++++++++-------- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/example.config.toml b/example.config.toml index da7c108..3397220 100644 --- a/example.config.toml +++ b/example.config.toml @@ -60,6 +60,41 @@ cloak-port = 443 # By default we use Quad9. doh-hostname = "9.9.9.9" +# mtg can work via proxies (for now, we support only socks5). Proxy +# configuration is done via list. So, you can specify many proxies +# there. +# +# Actually, if you supply an empty list, then no proxies are going to be +# used. If you supply a single proxy, then mtg will use it exclusively. +# If you supply >= 2, then mtg will load balance between them. +# +# If you add an empty string here, this is an equivalent of 'plain network', +# with no proxy usage. +# +# Proxy configuration is done via ordinary URI schema: +# +# socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s +# + +# Only socks5 proxy is used. user/password is optional. As you can +# see, you can specify some parameters in GET query. These parameters +# configure circuit breaker. +# +# open_threshold means a number of errors which should happen so we stop +# use a proxy. +# +# half_open_timeout means a time period (in Golang duration notation) +# after which we can retry with this proxy +# +# reset_failures_timeout means a time period when we flush out errors +# when circuit breaker in closed state. +# +# Please see https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker +# on details about circuit breakers. +proxies = [ + # "socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s" +] + # public ip addresses of the server. Actually, it is required only to # generate a correct access file. if you use default values here, mtg # will try to resolve these IPs on its own. diff --git a/raw_config.go b/raw_config.go index 7de66d9..7e1b3d8 100644 --- a/raw_config.go +++ b/raw_config.go @@ -8,13 +8,13 @@ import ( ) type rawConfig struct { - Debug bool `toml:"debug"` - Secret string `toml:"secret"` - BindTo string `toml:"bind-to"` - TCPBuffer string `toml:"tcp-buffer"` - PreferIP string `toml:"prefer-ip"` - CloakPort uint `toml:"cloak-port"` - Probes struct { + Debug bool `toml:"debug"` + Secret string `toml:"secret"` + BindTo string `toml:"bind-to"` + TCPBuffer string `toml:"tcp-buffer"` + PreferIP string `toml:"prefer-ip"` + CloakPort uint `toml:"cloak-port"` + Probes struct { Time struct { Enabled bool `toml:"enabled"` AllowSkewness string `toml:"allow-skewness"` @@ -34,7 +34,8 @@ type rawConfig struct { Telegram string `toml:"telegram"` Default string `toml:"default"` } `toml:"dialers"` - DOHHostname string `toml:"doh-hostname"` + DOHHostname string `toml:"doh-hostname"` + Proxies []string `toml:"proxies"` } `toml:"network"` Stats struct { StatsD struct { From 015f02c077f485ec08ffbb813991f0c03871b21f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 11:48:44 +0300 Subject: [PATCH 023/206] Cleanups of secret module --- mtglib/secret.go | 66 +++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/mtglib/secret.go b/mtglib/secret.go index 865ca60..4981707 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -1,13 +1,15 @@ package mtglib import ( + "crypto/rand" "encoding/base64" "encoding/hex" - "errors" "fmt" "strings" ) +const SecretKeyLength = 32 + type Secret struct { Key []byte Host string @@ -21,51 +23,53 @@ func (s *Secret) MarshalText() ([]byte, error) { return []byte(s.String()), nil } -func (s *Secret) UnmarshalText(text []byte) error { - sc, err := ParseSecret(string(text)) - if err != nil { - return err +func (s *Secret) UnmarshalText(data []byte) error { + text := string(data) + + if text == "" { + return ErrSecretEmpty } - *s = sc + decoded, err := base64.RawStdEncoding.DecodeString(text) + if err != nil && strings.HasPrefix(text, "ee") { + decoded, err = hex.DecodeString(strings.TrimPrefix(text, "ee")) + } + + if err != nil { + return fmt.Errorf("incorrect secret format: %w", err) + } + + if len(decoded) <= SecretKeyLength { + return fmt.Errorf("secret has incorrect length %d", len(text)) + } + + s.Key = decoded[:SecretKeyLength] + s.Host = string(decoded[SecretKeyLength:]) return nil } func (s Secret) Base64() string { - return s.String() -} - -func (s Secret) EE() string { - return "ee" + hex.EncodeToString(append(s.Key, s.Host...)) + return base64.StdEncoding.EncodeToString(append(s.Key[:], s.Host...)) } func (s Secret) String() string { - return base64.StdEncoding.EncodeToString(append(s.Key, s.Host...)) + return s.Base64() } -func ParseSecret(secret string) (Secret, error) { - rv := Secret{} +func (s Secret) EE() string { + return "ee" + hex.EncodeToString(append(s.Key[:], s.Host...)) +} - if secret == "" { - return rv, errors.New("secret cannot be empty") +func GenerateSecret(hostname string) Secret { + s := Secret{ + Key: make([]byte, SecretKeyLength), + Host: hostname, } - decoded, err := base64.RawStdEncoding.DecodeString(secret) - if err != nil && strings.HasPrefix(secret, "ee") { - decoded, err = hex.DecodeString(strings.TrimPrefix(secret, "ee")) + if _, err := rand.Read(s.Key); err != nil { + panic(err) } - if err != nil { - return rv, fmt.Errorf("incorrect secret format: %w", err) - } - - if len(decoded) < 33 { - return rv, fmt.Errorf("secret %s has incorrect length", secret) - } - - rv.Key = decoded[:32] - rv.Host = string(decoded[32:]) - - return rv, nil + return s } From 783c49db37007ce1f3b073be2a422e981ad54653 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 12:26:07 +0300 Subject: [PATCH 024/206] Correct secret parsing --- config.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 15 +-------- mtglib/init.go | 7 ++++ mtglib/secret.go | 21 +++++++++--- raw_config.go | 63 ------------------------------------ 5 files changed, 107 insertions(+), 82 deletions(-) create mode 100644 config.go create mode 100644 mtglib/init.go delete mode 100644 raw_config.go diff --git a/config.go b/config.go new file mode 100644 index 0000000..7734886 --- /dev/null +++ b/config.go @@ -0,0 +1,83 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/pelletier/go-toml" +) + +type config struct { + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` +} + +type configRaw struct { + Debug bool `toml:"debug" json:"debug"` + Secret string `toml:"secret" json:"secret"` + BindTo string `toml:"bind-to" json:"bind-to"` + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip"` + CloakPort uint `toml:"cloak-port" json:"cloak-port"` + Probes struct { + Time struct { + Enabled bool `toml:"enabled" json:"enabled"` + AllowSkewness string `toml:"allow-skewness" json:"allow-skewness"` + } `toml:"time" json:"time"` + AntiReplay struct { + Enabled bool `toml:"enabled" json:"enabled"` + MaxSize string `toml:"max-size" json:"max-size"` + TTL string `toml:"ttl" json:"ttl"` + } `toml:"anti-replay" json:"anti-replay"` + } `toml:"probes" json:"probes"` + Network struct { + PublicIP struct { + IPv4 string `toml:"ipv4" json:"ipv4"` + IPv6 string `toml:"ipv6" json:"ipv6"` + } `toml:"public-ip" json:"public-ip"` + DOHHostname string `toml:"doh-hostname" json:"doh-hostname"` + Proxies []string `toml:"proxies" json:"proxies"` + } `toml:"network" json:"network"` + Stats struct { + StatsD struct { + Enabled bool `toml:"enabled" json:"enabled"` + Address string `toml:"address" json:"address"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` + } `toml:"statsd" json:"statsd"` + Prometheus struct { + Enabled bool `toml:"enabled" json:"enabled"` + BindTo string `toml:"bind-to" json:"bind-to"` + HTTPPath string `toml:"http-path" json:"http-path"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` + } `toml:"prometheus" json:"prometheus"` + } `toml:"stats" json:"stats"` +} + +func parseConfig(reader io.Reader) (*config, error) { + rawConf := &configRaw{} + + if err := toml.NewDecoder(reader).Decode(rawConf); err != nil { + return nil, fmt.Errorf("cannot parse toml config: %w", err) + } + + jsonBuf := &bytes.Buffer{} + jsonEncoder := json.NewEncoder(jsonBuf) + + jsonEncoder.SetEscapeHTML(false) + jsonEncoder.SetIndent("", "") + + if err := jsonEncoder.Encode(rawConf); err != nil { + return nil, fmt.Errorf("cannot dump into interim format: %w", err) + } + + conf := &config{} + + if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { + return nil, fmt.Errorf("cannot parse final config: %w", err) + } + + return conf, nil +} diff --git a/main.go b/main.go index 5f8b319..61e5db8 100644 --- a/main.go +++ b/main.go @@ -2,12 +2,9 @@ package main import ( "fmt" - "io/ioutil" "math/rand" "os" "time" - - "github.com/9seconds/mtg/v2/mtglib/network" ) var version = "dev" // has to be set by ldflags @@ -17,15 +14,5 @@ func main() { f, _ := os.Open("example.config.toml") - fmt.Println(parseRawConfig(f)) - - bd, _ := network.NewDefaultDialer(0, 0) - d, _ := network.NewNetwork(bd, "9.9.9.9", 0) - - r, err := d.HTTP.Get("https://ifconfig.co") - - fmt.Println(err) - body, _ := ioutil.ReadAll(r.Body) - - fmt.Println(string(body)) + fmt.Println(parseConfig(f)) } diff --git a/mtglib/init.go b/mtglib/init.go new file mode 100644 index 0000000..1fa135a --- /dev/null +++ b/mtglib/init.go @@ -0,0 +1,7 @@ +package mtglib + +import "errors" + +var ( + ErrSecretEmpty = errors.New("secret is empty") +) diff --git a/mtglib/secret.go b/mtglib/secret.go index 4981707..8fe6a90 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -8,7 +8,7 @@ import ( "strings" ) -const SecretKeyLength = 32 +const SecretKeyLength = 16 type Secret struct { Key []byte @@ -30,11 +30,19 @@ func (s *Secret) UnmarshalText(data []byte) error { return ErrSecretEmpty } - decoded, err := base64.RawStdEncoding.DecodeString(text) - if err != nil && strings.HasPrefix(text, "ee") { + var ( + decoded []byte + err error + ) + + if strings.HasPrefix(text, "ee") { decoded, err = hex.DecodeString(strings.TrimPrefix(text, "ee")) } + if err != nil || len(decoded) <= SecretKeyLength { + decoded, err = base64.RawURLEncoding.DecodeString(text) + } + if err != nil { return fmt.Errorf("incorrect secret format: %w", err) } @@ -50,7 +58,10 @@ func (s *Secret) UnmarshalText(data []byte) error { } func (s Secret) Base64() string { - return base64.StdEncoding.EncodeToString(append(s.Key[:], s.Host...)) + data := append([]byte{238}, s.Key...) // 238 = hex ee + data = append(data, s.Host...) + + return base64.RawURLEncoding.EncodeToString(data) } func (s Secret) String() string { @@ -58,7 +69,7 @@ func (s Secret) String() string { } func (s Secret) EE() string { - return "ee" + hex.EncodeToString(append(s.Key[:], s.Host...)) + return "ee" + hex.EncodeToString(append(s.Key, s.Host...)) } func GenerateSecret(hostname string) Secret { diff --git a/raw_config.go b/raw_config.go deleted file mode 100644 index 7e1b3d8..0000000 --- a/raw_config.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "fmt" - "io" - - "github.com/pelletier/go-toml" -) - -type rawConfig struct { - Debug bool `toml:"debug"` - Secret string `toml:"secret"` - BindTo string `toml:"bind-to"` - TCPBuffer string `toml:"tcp-buffer"` - PreferIP string `toml:"prefer-ip"` - CloakPort uint `toml:"cloak-port"` - Probes struct { - Time struct { - Enabled bool `toml:"enabled"` - AllowSkewness string `toml:"allow-skewness"` - } `toml:"time"` - AntiReplay struct { - Enabled bool `toml:"enabled"` - MaxSize string `toml:"max-size"` - TTL string `toml:"ttl"` - } `toml:"anti-replay"` - } `toml:"probes"` - Network struct { - PublicIP struct { - IPv4 string `toml:"ipv4"` - IPv6 string `toml:"ipv6"` - } `toml:"public-ip"` - Dialers struct { - Telegram string `toml:"telegram"` - Default string `toml:"default"` - } `toml:"dialers"` - DOHHostname string `toml:"doh-hostname"` - Proxies []string `toml:"proxies"` - } `toml:"network"` - Stats struct { - StatsD struct { - Enabled bool `toml:"enabled"` - Address string `toml:"address"` - MetricPrefix string `toml:"metric-prefix"` - } `toml:"statsd"` - Prometheus struct { - Enabled bool `toml:"enabled"` - BindTo string `toml:"bind-to"` - HttpPath string `toml:"http-path"` - MetricPrefix string `toml:"metric-prefix"` - } `toml:"prometheus"` - } `toml:"stats"` -} - -func parseRawConfig(reader io.Reader) (*rawConfig, error) { - conf := &rawConfig{} - - if err := toml.NewDecoder(reader).Decode(conf); err != nil { - return nil, fmt.Errorf("cannot parse config: %w", err) - } - - return conf, nil -} From 59f328b804176a7afc25b691ac15f5f36b063ed8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 12:29:13 +0300 Subject: [PATCH 025/206] Validate config --- config.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/config.go b/config.go index 7734886..51f43fe 100644 --- a/config.go +++ b/config.go @@ -15,6 +15,14 @@ type config struct { Secret mtglib.Secret `json:"secret"` } +func (c *config) Validate() error { + if len(c.Secret.Key) == 0 || c.Secret.Host == "" { + return fmt.Errorf("incorrect secret %s", c.Secret.String()) + } + + return nil +} + type configRaw struct { Debug bool `toml:"debug" json:"debug"` Secret string `toml:"secret" json:"secret"` @@ -79,5 +87,9 @@ func parseConfig(reader io.Reader) (*config, error) { return nil, fmt.Errorf("cannot parse final config: %w", err) } + if err := conf.Validate(); err != nil { + return nil, fmt.Errorf("cannot validate config: %w", err) + } + return conf, nil } From d1dd56550f32b81948f64ebba7d7d5e8581872d3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 14:41:21 +0300 Subject: [PATCH 026/206] Add correct configuration --- config.go | 383 +++++++++++++++++++++++++++++++++++++++++++- example.config.toml | 35 +--- go.mod | 1 + go.sum | 3 + mtglib/init.go | 4 +- mtglib/secret.go | 6 +- 6 files changed, 389 insertions(+), 43 deletions(-) diff --git a/config.go b/config.go index 51f43fe..5ef956d 100644 --- a/config.go +++ b/config.go @@ -5,14 +5,383 @@ import ( "encoding/json" "fmt" "io" + "net" + "net/url" + "regexp" + "strconv" + "strings" + "time" "github.com/9seconds/mtg/v2/mtglib" + "github.com/alecthomas/units" "github.com/pelletier/go-toml" ) +type configTypeHostPort struct { + host configTypeIP + port configTypePort +} + +func (c *configTypeHostPort) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + host, port, err := net.SplitHostPort(string(data)) + if err != nil { + return fmt.Errorf("incorrect host:port syntax: %w", err) + } + + if err := c.port.UnmarshalJSON([]byte(port)); err != nil { + return fmt.Errorf("incorrect port in host:port: %w", err) + } + + if err := c.host.UnmarshalText([]byte(host)); err != nil { + return fmt.Errorf("incorrect host: %w", err) + } + + return nil +} + +func (c configTypeHostPort) String() string { + return c.Value(net.IP{}, 0) +} + +func (c configTypeHostPort) Value(defaultHostValue net.IP, defaultPortValue uint) string { + return net.JoinHostPort(c.host.Value(defaultHostValue).String(), + strconv.Itoa(int(c.port.Value(defaultPortValue)))) +} + +type configTypePort struct { + value uint +} + +func (c *configTypePort) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + return nil + } + + intValue, err := strconv.ParseUint(string(data), 10, 16) + if err != nil { + return fmt.Errorf("port number is not a number: %w", err) + } + + if intValue == 0 || intValue > 65536 { + return fmt.Errorf("port number should be 0 < portNo < 65536: %d", intValue) + } + + c.value = uint(intValue) + + return nil +} + +func (c configTypePort) String() string { + return strconv.Itoa(int(c.value)) +} + +func (c configTypePort) Value(defaultValue uint) uint { + if c.value == 0 { + return defaultValue + } + + return c.value +} + +type configTypeBytes struct { + value uint +} + +func (c *configTypeBytes) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + value, err := units.ParseStrictBytes(strings.ToUpper(string(data))) + if err != nil { + return fmt.Errorf("incorrect bytes value: %w", err) + } + + if value < 0 { + return fmt.Errorf("%d should be positive number", value) + } + + c.value = uint(value) + + return nil +} + +func (c configTypeBytes) String() string { + return units.ToString(int64(c.value), 1024, "ib", "b") +} + +func (c configTypeBytes) Value(defaultValue uint) uint { + if c.value == 0 { + return defaultValue + } + + return c.value +} + +type configTypePreferIP struct { + value string +} + +func (c *configTypePreferIP) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := strings.ToLower(string(data)) + + switch text { + case "prefer-ipv4", "prefer-ipv6", "only-ipv4", "only-ipv6": + c.value = text + default: + return fmt.Errorf("incorrect prefer-ip value: %s", string(data)) + } + + return nil +} + +func (c *configTypePreferIP) String() string { + return c.value +} + +func (c *configTypePreferIP) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} + +type configTypeDuration struct { + value time.Duration +} + +func (c *configTypeDuration) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + dur, err := time.ParseDuration(strings.ToLower(string(data))) + if err != nil { + return fmt.Errorf("incorrect duration: %w", err) + } + + if dur < 0 { + return fmt.Errorf("%s should be positive duration", dur) + } + + c.value = dur + + return nil +} + +func (c configTypeDuration) String() string { + return c.value.String() +} + +func (c configTypeDuration) Value(defaultValue time.Duration) time.Duration { + if c.value == 0 { + return defaultValue + } + + return c.value +} + +type configTypeFloat struct { + value float64 +} + +func (c *configTypeFloat) UnmarshalJSON(data []byte) error { + value, err := strconv.ParseFloat(string(data), 64) + if err != nil { + return fmt.Errorf("incorrect float value: %w", err) + } + + if value < 0 { + return fmt.Errorf("%f should be positive", value) + } + + c.value = value + + return nil +} + +func (c configTypeFloat) String() string { + return strconv.FormatFloat(c.value, 'f', -1, 64) +} + +func (c configTypeFloat) Value(defaultValue float64) float64 { + if c.value < 0.00001 { + return defaultValue + } + + return c.value +} + +type configTypeIP struct { + value net.IP +} + +func (c *configTypeIP) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + ip := net.ParseIP(string(data)) + if ip == nil { + return fmt.Errorf("incorrect ip address: %s", string(data)) + } + + c.value = ip + + return nil +} + +func (c configTypeIP) String() string { + return c.value.String() +} + +func (c configTypeIP) Value(defaultValue net.IP) net.IP { + if c.value == nil { + return defaultValue + } + + return c.value +} + +type configTypeURL struct { + value *url.URL +} + +func (c *configTypeURL) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + value, err := url.Parse(string(data)) + if err != nil { + return fmt.Errorf("incorrect URL: %w", err) + } + + c.value = value + + return nil +} + +func (c configTypeURL) String() string { + if c.value == nil { + return "" + } + + return c.value.String() +} + +func (c configTypeURL) Value(defaultValue *url.URL) *url.URL { + if c.value == nil { + return defaultValue + } + + return c.value +} + +type configTypeMetricPrefix struct { + value string +} + +func (c *configTypeMetricPrefix) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + prefix := string(data) + + if ok, err := regexp.MatchString("^[a-z0-9]+$", prefix); !ok || err != nil { + return fmt.Errorf("incorrect metric prefix: %s", prefix) + } + + c.value = prefix + + return nil +} + +func (c configTypeMetricPrefix) String() string { + return c.value +} + +func (c configTypeMetricPrefix) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} + +type configTypeHTTPPath struct { + value string +} + +func (c *configTypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unparam + if len(data) > 0 { + c.value = "/" + strings.Trim(string(data), "/") + } + + return nil +} + +func (c configTypeHTTPPath) String() string { + return c.value +} + +func (c configTypeHTTPPath) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} + type config struct { - Debug bool `json:"debug"` - Secret mtglib.Secret `json:"secret"` + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` + BindTo configTypeHostPort `json:"bind-to"` + TCPBuffer configTypeBytes `json:"tcp-buffer"` + PreferIP configTypePreferIP `json:"prefer-ip"` + CloakPort configTypePort `json:"cloak-port"` + Probes struct { + Time struct { + Enabled bool `json:"enabled"` + AllowSkewness configTypeDuration `json:"allow-skewness"` + } `json:"time"` + AntiReplay struct { + Enabled bool `json:"enabled"` + MaxSize configTypeBytes `json:"max-size"` + ErrorRate configTypeFloat `json:"error-rate"` + } `json:"anti-replay"` + } `json:"probes"` + Network struct { + PublicIP struct { + IPv4 configTypeIP `json:"ipv4"` + IPv6 configTypeIP `json:"ipv6"` + } `json:"public-ip"` + DOHIP configTypeIP `json:"doh-ip"` + Proxies []configTypeURL `json:"proxies"` + } `json:"network"` + Stats struct { + StatsD struct { + Enabled bool `json:"enabled"` + Address configTypeHostPort `json:"address"` + MetricPrefix configTypeMetricPrefix `json:"metric-prefix"` + } `json:"statsd"` + Prometheus struct { + Enabled bool `json:"enabled"` + BindTo configTypeHostPort `json:"bind-to"` + HTTPPath configTypeHTTPPath `json:"http-path"` + MetricPrefix configTypeMetricPrefix `json:"metric-prefix"` + } `json:"prometheus"` + } `json:"stats"` } func (c *config) Validate() error { @@ -36,9 +405,9 @@ type configRaw struct { AllowSkewness string `toml:"allow-skewness" json:"allow-skewness"` } `toml:"time" json:"time"` AntiReplay struct { - Enabled bool `toml:"enabled" json:"enabled"` - MaxSize string `toml:"max-size" json:"max-size"` - TTL string `toml:"ttl" json:"ttl"` + Enabled bool `toml:"enabled" json:"enabled"` + MaxSize string `toml:"max-size" json:"max-size"` + ErrorRate float64 `toml:"error-rate" json:"error-rate"` } `toml:"anti-replay" json:"anti-replay"` } `toml:"probes" json:"probes"` Network struct { @@ -46,8 +415,8 @@ type configRaw struct { IPv4 string `toml:"ipv4" json:"ipv4"` IPv6 string `toml:"ipv6" json:"ipv6"` } `toml:"public-ip" json:"public-ip"` - DOHHostname string `toml:"doh-hostname" json:"doh-hostname"` - Proxies []string `toml:"proxies" json:"proxies"` + DOHIP string `toml:"doh-ip" json:"doh-ip"` + Proxies []string `toml:"proxies" json:"proxies"` } `toml:"network" json:"network"` Stats struct { StatsD struct { diff --git a/example.config.toml b/example.config.toml index 3397220..ee896d0 100644 --- a/example.config.toml +++ b/example.config.toml @@ -38,7 +38,7 @@ tcp-buffer = "4kb" # Only ipv6 connectivity is used # - only-ipv4: # Only ipv4 connectivity is used -prefer-ips = "prefer-ipv6" +prefer-ip = "prefer-ipv6" # FakeTLS uses domain fronting protection. So it needs to know a port to # access. @@ -58,7 +58,7 @@ cloak-port = 443 # it has to access. # # By default we use Quad9. -doh-hostname = "9.9.9.9" +doh-ip = "9.9.9.9" # mtg can work via proxies (for now, we support only socks5). Proxy # configuration is done via list. So, you can specify many proxies @@ -75,7 +75,6 @@ doh-hostname = "9.9.9.9" # # socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s # - # Only socks5 proxy is used. user/password is optional. As you can # see, you can specify some parameters in GET query. These parameters # configure circuit breaker. @@ -102,31 +101,6 @@ proxies = [ ipv4 = "" ipv6 = "" -# you can redefine a dialer for mtg. Dialer is how we 'dial' to either -# some external services or telegram. empty string means default -# connectivity. -# -# it is also possible to use socks5 or shadowsocks here -# -# socks5 example: -# socks5://user:password@host:port -# shadowsocks example (SIP002): -# ss://YWVzLTEyOC1nY206dGVzdA@192.168.100.1:8888 -# -# You can define 2 dialers here: telegram and default. Telegram dialer -# is used to connect to Telegram servers only. Default is used for other -# purposes, like accessing ifconfig.co to obtains public address (DNS is -# resolved via DoH) -# -# Please also be aware that dialers are only doing TCP. If UDP is -# required (for statsd for example), then these dialers are going to be -# ignored. -# -# If telegram dialer is not defined, a default one is going to be used. -[network.dialers] -telegram = "" -default = "" - # FakeTLS can compare timestamps to prevent probes. Each message has # encrypted timestamp. So, mtg can compare this timestamp and decide if # we need to proceed with connection or not. @@ -154,8 +128,9 @@ enabled = true # that we can go over this limit for 10-20% under some conditions and # architectures. max-size = "16mb" -# TTL for each cache record. -ttl = "8h" +# we use stable bloom filters for anti-replay cache. This helps +# to maintain a desired error ratio. +error-rate = 0.0001 # statsd statistics integration. [stats.statsd] diff --git a/go.mod b/go.mod index f201e45..910aef8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/9seconds/mtg/v2 go 1.16 require ( + github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/kr/pretty v0.1.0 // indirect diff --git a/go.sum b/go.sum index 817b6fb..497262e 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= @@ -24,6 +26,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/mtglib/init.go b/mtglib/init.go index 1fa135a..e5e1a0d 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -2,6 +2,4 @@ package mtglib import "errors" -var ( - ErrSecretEmpty = errors.New("secret is empty") -) +var ErrSecretEmpty = errors.New("secret is empty") diff --git a/mtglib/secret.go b/mtglib/secret.go index 8fe6a90..4a4cd80 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -58,10 +58,10 @@ func (s *Secret) UnmarshalText(data []byte) error { } func (s Secret) Base64() string { - data := append([]byte{238}, s.Key...) // 238 = hex ee - data = append(data, s.Host...) + data := append([]byte{238}, s.Key...) // 238 = hex ee + data = append(data, s.Host...) - return base64.RawURLEncoding.EncodeToString(data) + return base64.RawURLEncoding.EncodeToString(data) } func (s Secret) String() string { From 098a9c411a1c54e6b1106d42b750370050877b35 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 15:28:06 +0300 Subject: [PATCH 027/206] Marshalling of config to string --- config.go | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/config.go b/config.go index 5ef956d..c496453 100644 --- a/config.go +++ b/config.go @@ -43,6 +43,10 @@ func (c *configTypeHostPort) UnmarshalText(data []byte) error { return nil } +func (c configTypeHostPort) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeHostPort) String() string { return c.Value(net.IP{}, 0) } @@ -75,6 +79,10 @@ func (c *configTypePort) UnmarshalJSON(data []byte) error { return nil } +func (c *configTypePort) MarshalJSON() ([]byte, error) { + return json.Marshal(c.value) +} + func (c configTypePort) String() string { return strconv.Itoa(int(c.value)) } @@ -110,6 +118,10 @@ func (c *configTypeBytes) UnmarshalText(data []byte) error { return nil } +func (c configTypeBytes) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeBytes) String() string { return units.ToString(int64(c.value), 1024, "ib", "b") } @@ -143,6 +155,10 @@ func (c *configTypePreferIP) UnmarshalText(data []byte) error { return nil } +func (c configTypePreferIP) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + func (c *configTypePreferIP) String() string { return c.value } @@ -178,6 +194,10 @@ func (c *configTypeDuration) UnmarshalText(data []byte) error { return nil } +func (c configTypeDuration) MarshalText() ([]byte, error) { + return []byte(c.value.String()), nil +} + func (c configTypeDuration) String() string { return c.value.String() } @@ -209,6 +229,10 @@ func (c *configTypeFloat) UnmarshalJSON(data []byte) error { return nil } +func (c *configTypeFloat) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeFloat) String() string { return strconv.FormatFloat(c.value, 'f', -1, 64) } @@ -240,7 +264,15 @@ func (c *configTypeIP) UnmarshalText(data []byte) error { return nil } +func (c *configTypeIP) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeIP) String() string { + if c.value == nil { + return "" + } + return c.value.String() } @@ -271,6 +303,10 @@ func (c *configTypeURL) UnmarshalText(data []byte) error { return nil } +func (c *configTypeURL) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeURL) String() string { if c.value == nil { return "" @@ -307,6 +343,10 @@ func (c *configTypeMetricPrefix) UnmarshalText(data []byte) error { return nil } +func (c configTypeMetricPrefix) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeMetricPrefix) String() string { return c.value } @@ -331,6 +371,10 @@ func (c *configTypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unpar return nil } +func (c configTypeHTTPPath) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + func (c configTypeHTTPPath) String() string { return c.value } @@ -392,6 +436,19 @@ func (c *config) Validate() error { return nil } +func (c *config) String() string { + buf := &bytes.Buffer{} + encoder := json.NewEncoder(buf) + + encoder.SetEscapeHTML(false) + + if err := encoder.Encode(c); err != nil { + panic(err) + } + + return buf.String() +} + type configRaw struct { Debug bool `toml:"debug" json:"debug"` Secret string `toml:"secret" json:"secret"` From 6f0b0e0c32f40125c8fd5dd1516e899d321cf519 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 15:59:48 +0300 Subject: [PATCH 028/206] Add command for generating secrets --- cli_generate_secret.go | 17 +++++++++++++++++ go.mod | 1 + go.sum | 4 ++++ main.go | 31 +++++++++++++++++++++++++++---- 4 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 cli_generate_secret.go diff --git a/cli_generate_secret.go b/cli_generate_secret.go new file mode 100644 index 0000000..d0edfa3 --- /dev/null +++ b/cli_generate_secret.go @@ -0,0 +1,17 @@ +package main + +import ( + "fmt" + + "github.com/9seconds/mtg/v2/mtglib" +) + +func runGenerateSecret(cli *CLI) { + secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) + + if cli.GenerateSecret.Hex { + fmt.Println(secret.EE()) + } else { + fmt.Println(secret.Base64()) + } +} diff --git a/go.mod b/go.mod index 910aef8..20cbeb9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/9seconds/mtg/v2 go 1.16 require ( + github.com/alecthomas/kong v0.2.16 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 diff --git a/go.sum b/go.sum index 497262e..73e7830 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alecthomas/kong v0.2.16 h1:F232CiYSn54Tnl1sJGTeHmx4vJDNLVP2b9yCVMOQwHQ= +github.com/alecthomas/kong v0.2.16/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -18,6 +20,7 @@ github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDy github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -25,6 +28,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/main.go b/main.go index 61e5db8..a45e0a8 100644 --- a/main.go +++ b/main.go @@ -1,18 +1,41 @@ package main import ( - "fmt" "math/rand" - "os" "time" + + "github.com/alecthomas/kong" ) var version = "dev" // has to be set by ldflags +type CLI struct { + GenerateSecret struct { + HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` + Hex bool `help:"Print secret in hex encoding."` + } `cmd help:"Generate new proxy secret."` + Access struct { + ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` + } `cmd help:"Print access information."` + Run struct { + ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` + } `cmd help:"Run proxy."` +} + func main() { rand.Seed(time.Now().UTC().UnixNano()) - f, _ := os.Open("example.config.toml") + cli := &CLI{} + ctx := kong.Parse(cli, kong.Vars{ + "domain_front": "amazonaws.com", + "config_path": "/etc/mtg.toml", + }) - fmt.Println(parseConfig(f)) + switch ctx.Command() { + case "generate-secret": + runGenerateSecret(cli) + case "access": + case "run": + panic("not implemented yet") + } } From cbcc113e41c644edfc0fc3e74691d7615e506c0a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 16:59:19 +0300 Subject: [PATCH 029/206] Add cli command for access --- cli_access.go | 156 +++++++++++++++++++++++++++++++++++++++++ config.go | 8 +++ example.config.toml | 6 ++ main.go | 8 ++- mtglib/network/init.go | 2 + utils.go | 64 +++++++++++++++++ 6 files changed, 242 insertions(+), 2 deletions(-) create mode 100644 cli_access.go create mode 100644 utils.go diff --git a/cli_access.go b/cli_access.go new file mode 100644 index 0000000..ab356a8 --- /dev/null +++ b/cli_access.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "github.com/9seconds/mtg/v2/mtglib/network" +) + +type runAccessResponse struct { + IPv4 *runAccessResponseURLs `json:"ipv4,omitempty"` + IPv6 *runAccessResponseURLs `json:"ipv6,omitempty"` + Secret struct { + Hex string `json:"hex"` + Base64 string `json:"base64"` + } `json:"secret"` +} + +type runAccessResponseURLs struct { + IP net.IP `json:"ip"` + TgURL string `json:"tg_url"` + TgQrCode string `json:"tg_qrcode"` + TmeURL string `json:"tme_url"` + TmeQrCode string `json:"tme_qrcode"` +} + +func runAccess(cli *CLI) { + filefp, err := os.Open(cli.Access.ConfigPath) + if err != nil { + exit(err) + } + + defer filefp.Close() + + conf, err := parseConfig(filefp) + if err != nil { + exit(err) + } + + ntw, err := makeNetwork(conf) + if err != nil { + exit(err) + } + + ipv4 := conf.Network.PublicIP.IPv4.Value(nil) + ipv6 := conf.Network.PublicIP.IPv6.Value(nil) + + if ipv4 == nil { + ipv4 = runAccessGetIP(ntw, "tcp4") + } + + if ipv6 == nil { + ipv6 = runAccessGetIP(ntw, "tcp6") + } + + resp := runAccessResponse{ + IPv4: runMakeAccessResponseURLs(ipv4, conf, cli), + IPv6: runMakeAccessResponseURLs(ipv6, conf, cli), + } + resp.Secret.Base64 = conf.Secret.Base64() + resp.Secret.Hex = conf.Secret.EE() + + encoder := json.NewEncoder(os.Stdout) + + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + + if err := encoder.Encode(resp); err != nil { + exit(err) + } +} + +func runAccessGetIP(ntw *network.Network, protocol string) net.IP { + client := &http.Client{ + Timeout: ntw.HTTP.Timeout, + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return ntw.DialContext(ctx, protocol, address) + }, + }, + } + + resp, err := client.Get("https://ifconfig.co") + if err != nil { + return nil + } + + defer exhaustResponse(resp) + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil + } + + return net.ParseIP(strings.TrimSpace(string(data))) +} + +func runMakeAccessResponseURLs(ip net.IP, conf *config, cli *CLI) *runAccessResponseURLs { + if ip == nil { + return nil + } + + values := url.Values{} + + values.Set("server", ip.String()) + values.Set("port", strconv.Itoa(int(conf.BindTo.port.Value(0)))) + values.Set("secret", conf.Secret.Base64()) + + if cli.Access.Hex { + values.Set("secret", conf.Secret.EE()) + } + + urlQuery := values.Encode() + + rv := &runAccessResponseURLs{ + IP: ip, + TgURL: (&url.URL{ + Scheme: "tg", + Host: "proxy", + RawQuery: urlQuery, + }).String(), + TmeURL: (&url.URL{ + Scheme: "https", + Host: "t.me", + Path: "proxy", + RawQuery: urlQuery, + }).String(), + } + + rv.TgQrCode = runMakeAccessResponseURLsQRCode(rv.TgURL) + rv.TmeQrCode = runMakeAccessResponseURLsQRCode(rv.TmeURL) + + return rv +} + +func runMakeAccessResponseURLsQRCode(data string) string { + values := url.Values{} + + values.Set("qzone", "4") + values.Set("format", "svg") + values.Set("data", data) + + return (&url.URL{ + Scheme: "https", + Host: "api.qrserver.com", + Path: "v1/create-qr-code", + RawQuery: values.Encode(), + }).String() +} diff --git a/config.go b/config.go index c496453..4a11b50 100644 --- a/config.go +++ b/config.go @@ -410,6 +410,10 @@ type config struct { IPv4 configTypeIP `json:"ipv4"` IPv6 configTypeIP `json:"ipv6"` } `json:"public-ip"` + Timeout struct { + TCP configTypeDuration `json:"tcp"` + HTTP configTypeDuration `json:"http"` + } `json:"timeout"` DOHIP configTypeIP `json:"doh-ip"` Proxies []configTypeURL `json:"proxies"` } `json:"network"` @@ -472,6 +476,10 @@ type configRaw struct { IPv4 string `toml:"ipv4" json:"ipv4"` IPv6 string `toml:"ipv6" json:"ipv6"` } `toml:"public-ip" json:"public-ip"` + Timeout struct { + TCP string `toml:"tcp" json:"tcp"` + HTTP string `toml:"http" json:"http"` + } `toml:"timeout" json:"timeout"` DOHIP string `toml:"doh-ip" json:"doh-ip"` Proxies []string `toml:"proxies" json:"proxies"` } `toml:"network" json:"network"` diff --git a/example.config.toml b/example.config.toml index ee896d0..0f74c79 100644 --- a/example.config.toml +++ b/example.config.toml @@ -101,6 +101,12 @@ proxies = [ ipv4 = "" ipv6 = "" +# network timeouts define different settings for timeouts. HTTP timeout +# is required only for DOH. +[network.timeout] +tcp = "5s" +http = "10s" + # FakeTLS can compare timestamps to prevent probes. Each message has # encrypted timestamp. So, mtg can compare this timestamp and decide if # we need to proceed with connection or not. diff --git a/main.go b/main.go index a45e0a8..184b198 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ type CLI struct { } `cmd help:"Generate new proxy secret."` Access struct { ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` + Hex bool `help:"Print secret in hex encoding."` } `cmd help:"Print access information."` Run struct { ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` @@ -34,8 +35,11 @@ func main() { switch ctx.Command() { case "generate-secret": runGenerateSecret(cli) - case "access": - case "run": + case "access ": + runAccess(cli) + case "run ": panic("not implemented yet") + default: + panic(ctx.Command()) } } diff --git a/mtglib/network/init.go b/mtglib/network/init.go index d07f2b0..593b935 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -16,6 +16,8 @@ const ( ProxyDialerOpenThreshold = 5 ProxyDialerHalfOpenTimeout = time.Minute ProxyDialerResetFailuresTimeout = 10 * time.Second + + DefaultDOHHostname = "9.9.9.9" ) var ( diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..7299518 --- /dev/null +++ b/utils.go @@ -0,0 +1,64 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + + "github.com/9seconds/mtg/v2/mtglib/network" +) + +func exit(err error) { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) +} + +func makeNetwork(conf *config) (*network.Network, error) { + tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) + httpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultHTTPTimeout) + dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() + bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) + + baseDialer, err := network.NewDefaultDialer(tcpTimeout, int(bufferSize)) + if err != nil { + return nil, fmt.Errorf("cannot build a default dialer: %w", err) + } + + proxyURLs := make([]*url.URL, len(conf.Network.Proxies)) + + for _, v := range conf.Network.Proxies { + if value := v.Value(nil); value != nil { + proxyURLs = append(proxyURLs, v.Value(nil)) + } + } + + switch len(proxyURLs) { + case 0: + return network.NewNetwork(baseDialer, dohIP, httpTimeout) + case 1: + socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) + + if err != nil { + return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) + } + + return network.NewNetwork(socksDialer, dohIP, httpTimeout) + } + + socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) + if err != nil { + + return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) + } + + return network.NewNetwork(socksDialer, dohIP, httpTimeout) +} + +func exhaustResponse(response *http.Response) { + io.Copy(ioutil.Discard, response.Body) + response.Body.Close() +} From a34949f15c4c486643b38b50403e812e22da26cc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 17:03:56 +0300 Subject: [PATCH 030/206] Rename secret ee to hex --- cli_access.go | 13 +++++++++---- cli_generate_secret.go | 2 +- mtglib/secret.go | 22 +++++++++++++--------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/cli_access.go b/cli_access.go index ab356a8..d63e5b1 100644 --- a/cli_access.go +++ b/cli_access.go @@ -64,8 +64,8 @@ func runAccess(cli *CLI) { IPv4: runMakeAccessResponseURLs(ipv4, conf, cli), IPv6: runMakeAccessResponseURLs(ipv6, conf, cli), } - resp.Secret.Base64 = conf.Secret.Base64() - resp.Secret.Hex = conf.Secret.EE() + resp.Secret.Base64 = conf.Secret.Base64() + resp.Secret.Hex = conf.Secret.Hex() encoder := json.NewEncoder(os.Stdout) @@ -92,6 +92,10 @@ func runAccessGetIP(ntw *network.Network, protocol string) net.IP { return nil } + if resp.StatusCode != http.StatusOK { + return nil + } + defer exhaustResponse(resp) data, err := ioutil.ReadAll(resp.Body) @@ -111,10 +115,11 @@ func runMakeAccessResponseURLs(ip net.IP, conf *config, cli *CLI) *runAccessResp values.Set("server", ip.String()) values.Set("port", strconv.Itoa(int(conf.BindTo.port.Value(0)))) - values.Set("secret", conf.Secret.Base64()) if cli.Access.Hex { - values.Set("secret", conf.Secret.EE()) + values.Set("secret", conf.Secret.Hex()) + } else { + values.Set("secret", conf.Secret.Base64()) } urlQuery := values.Encode() diff --git a/cli_generate_secret.go b/cli_generate_secret.go index d0edfa3..b88d36f 100644 --- a/cli_generate_secret.go +++ b/cli_generate_secret.go @@ -10,7 +10,7 @@ func runGenerateSecret(cli *CLI) { secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) if cli.GenerateSecret.Hex { - fmt.Println(secret.EE()) + fmt.Println(secret.Hex()) } else { fmt.Println(secret.Base64()) } diff --git a/mtglib/secret.go b/mtglib/secret.go index 4a4cd80..bc0aed8 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -57,19 +57,23 @@ func (s *Secret) UnmarshalText(data []byte) error { return nil } -func (s Secret) Base64() string { - data := append([]byte{238}, s.Key...) // 238 = hex ee - data = append(data, s.Host...) - - return base64.RawURLEncoding.EncodeToString(data) -} - func (s Secret) String() string { return s.Base64() } -func (s Secret) EE() string { - return "ee" + hex.EncodeToString(append(s.Key, s.Host...)) +func (s Secret) Base64() string { + return base64.RawURLEncoding.EncodeToString(s.makeBytes()) +} + +func (s Secret) Hex() string { + return hex.EncodeToString(s.makeBytes()) +} + +func (s *Secret) makeBytes() []byte { + data := append([]byte{238}, s.Key...) // hex 'ee' = 238 + data = append(data, s.Host...) + + return data } func GenerateSecret(hostname string) Secret { From 9aa56191f951315111044e9272b7c370558cf148 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 17:16:33 +0300 Subject: [PATCH 031/206] Linting --- cli_access.go | 2 +- cli_generate_secret.go | 4 ++-- config.go | 18 +++++++++--------- main.go | 14 ++++++++------ mtglib/secret.go | 4 ++-- utils.go | 6 ++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/cli_access.go b/cli_access.go index d63e5b1..dc54bc1 100644 --- a/cli_access.go +++ b/cli_access.go @@ -87,7 +87,7 @@ func runAccessGetIP(ntw *network.Network, protocol string) net.IP { }, } - resp, err := client.Get("https://ifconfig.co") + resp, err := client.Get("https://ifconfig.co") // nolint: bodyclose, noctx if err != nil { return nil } diff --git a/cli_generate_secret.go b/cli_generate_secret.go index b88d36f..1c8af9a 100644 --- a/cli_generate_secret.go +++ b/cli_generate_secret.go @@ -10,8 +10,8 @@ func runGenerateSecret(cli *CLI) { secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) if cli.GenerateSecret.Hex { - fmt.Println(secret.Hex()) + fmt.Println(secret.Hex()) // nolint: forbidigo } else { - fmt.Println(secret.Base64()) + fmt.Println(secret.Base64()) // nolint: forbidigo } } diff --git a/config.go b/config.go index 4a11b50..e0c8442 100644 --- a/config.go +++ b/config.go @@ -43,7 +43,7 @@ func (c *configTypeHostPort) UnmarshalText(data []byte) error { return nil } -func (c configTypeHostPort) MarshalText() ([]byte, error) { +func (c configTypeHostPort) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } @@ -118,7 +118,7 @@ func (c *configTypeBytes) UnmarshalText(data []byte) error { return nil } -func (c configTypeBytes) MarshalText() ([]byte, error) { +func (c configTypeBytes) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } @@ -155,7 +155,7 @@ func (c *configTypePreferIP) UnmarshalText(data []byte) error { return nil } -func (c configTypePreferIP) MarshalText() ([]byte, error) { +func (c configTypePreferIP) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.value), nil } @@ -194,7 +194,7 @@ func (c *configTypeDuration) UnmarshalText(data []byte) error { return nil } -func (c configTypeDuration) MarshalText() ([]byte, error) { +func (c configTypeDuration) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.value.String()), nil } @@ -229,7 +229,7 @@ func (c *configTypeFloat) UnmarshalJSON(data []byte) error { return nil } -func (c *configTypeFloat) MarshalText() ([]byte, error) { +func (c *configTypeFloat) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } @@ -264,7 +264,7 @@ func (c *configTypeIP) UnmarshalText(data []byte) error { return nil } -func (c *configTypeIP) MarshalText() ([]byte, error) { +func (c *configTypeIP) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } @@ -303,7 +303,7 @@ func (c *configTypeURL) UnmarshalText(data []byte) error { return nil } -func (c *configTypeURL) MarshalText() ([]byte, error) { +func (c *configTypeURL) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } @@ -343,7 +343,7 @@ func (c *configTypeMetricPrefix) UnmarshalText(data []byte) error { return nil } -func (c configTypeMetricPrefix) MarshalText() ([]byte, error) { +func (c configTypeMetricPrefix) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } @@ -371,7 +371,7 @@ func (c *configTypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unpar return nil } -func (c configTypeHTTPPath) MarshalText() ([]byte, error) { +func (c configTypeHTTPPath) MarshalText() ([]byte, error) { // nolint: unparam return []byte(c.String()), nil } diff --git a/main.go b/main.go index 184b198..0b78ac4 100644 --- a/main.go +++ b/main.go @@ -10,17 +10,18 @@ import ( var version = "dev" // has to be set by ldflags type CLI struct { - GenerateSecret struct { - HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` + GenerateSecret struct { // nolint: govet + HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` } `cmd help:"Generate new proxy secret."` - Access struct { - ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` + Access struct { // nolint: govet + ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` } `cmd help:"Print access information."` - Run struct { - ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` + Run struct { // nolint: govet + ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet } `cmd help:"Run proxy."` + Version kong.VersionFlag `help:"Print version."` } func main() { @@ -30,6 +31,7 @@ func main() { ctx := kong.Parse(cli, kong.Vars{ "domain_front": "amazonaws.com", "config_path": "/etc/mtg.toml", + "version": version, }) switch ctx.Command() { diff --git a/mtglib/secret.go b/mtglib/secret.go index bc0aed8..b35fc7a 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -15,8 +15,8 @@ type Secret struct { Host string } -func (s *Secret) MarshalText() ([]byte, error) { - if s == nil { +func (s Secret) MarshalText() ([]byte, error) { + if s.Key == nil { return nil, nil } diff --git a/utils.go b/utils.go index 7299518..c2a11ae 100644 --- a/utils.go +++ b/utils.go @@ -28,7 +28,7 @@ func makeNetwork(conf *config) (*network.Network, error) { return nil, fmt.Errorf("cannot build a default dialer: %w", err) } - proxyURLs := make([]*url.URL, len(conf.Network.Proxies)) + proxyURLs := make([]*url.URL, 0, len(conf.Network.Proxies)) for _, v := range conf.Network.Proxies { if value := v.Value(nil); value != nil { @@ -41,7 +41,6 @@ func makeNetwork(conf *config) (*network.Network, error) { return network.NewNetwork(baseDialer, dohIP, httpTimeout) case 1: socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) - if err != nil { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } @@ -51,7 +50,6 @@ func makeNetwork(conf *config) (*network.Network, error) { socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) if err != nil { - return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } @@ -59,6 +57,6 @@ func makeNetwork(conf *config) (*network.Network, error) { } func exhaustResponse(response *http.Response) { - io.Copy(ioutil.Discard, response.Body) + io.Copy(ioutil.Discard, response.Body) // nolint: errcheck response.Body.Close() } From 94084674c055c369868c7a309a64983a5df86e81 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 18:08:25 +0300 Subject: [PATCH 032/206] Use custom cli classes --- cli_access.go | 36 ++++++++++++++++++++++-------------- cli_generate_secret.go | 9 ++++++++- main.go | 27 ++++----------------------- utils.go | 6 ------ 4 files changed, 34 insertions(+), 44 deletions(-) diff --git a/cli_access.go b/cli_access.go index dc54bc1..ed9c7b8 100644 --- a/cli_access.go +++ b/cli_access.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "fmt" "io/ioutil" "net" "net/http" @@ -31,38 +32,43 @@ type runAccessResponseURLs struct { TmeQrCode string `json:"tme_qrcode"` } -func runAccess(cli *CLI) { +type cliCommandAccess struct { + ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet + Hex bool `help:"Print secret in hex encoding."` +} + +func (c *cliCommandAccess) Run(cli *CLI) error { filefp, err := os.Open(cli.Access.ConfigPath) if err != nil { - exit(err) + return fmt.Errorf("cannot open config file: %w", err) } defer filefp.Close() conf, err := parseConfig(filefp) if err != nil { - exit(err) + return fmt.Errorf("cannot parse config: %w", err) } ntw, err := makeNetwork(conf) if err != nil { - exit(err) + return fmt.Errorf("cannot build a network: %w", err) } ipv4 := conf.Network.PublicIP.IPv4.Value(nil) ipv6 := conf.Network.PublicIP.IPv6.Value(nil) if ipv4 == nil { - ipv4 = runAccessGetIP(ntw, "tcp4") + ipv4 = c.getIP(ntw, "tcp4") } if ipv6 == nil { - ipv6 = runAccessGetIP(ntw, "tcp6") + ipv6 = c.getIP(ntw, "tcp6") } resp := runAccessResponse{ - IPv4: runMakeAccessResponseURLs(ipv4, conf, cli), - IPv6: runMakeAccessResponseURLs(ipv6, conf, cli), + IPv4: c.makeResponseURLs(ipv4, conf, cli), + IPv6: c.makeResponseURLs(ipv6, conf, cli), } resp.Secret.Base64 = conf.Secret.Base64() resp.Secret.Hex = conf.Secret.Hex() @@ -73,11 +79,13 @@ func runAccess(cli *CLI) { encoder.SetIndent("", " ") if err := encoder.Encode(resp); err != nil { - exit(err) + return fmt.Errorf("cannot dump access json: %w", err) } + + return nil } -func runAccessGetIP(ntw *network.Network, protocol string) net.IP { +func (c *cliCommandAccess) getIP(ntw *network.Network, protocol string) net.IP { client := &http.Client{ Timeout: ntw.HTTP.Timeout, Transport: &http.Transport{ @@ -106,7 +114,7 @@ func runAccessGetIP(ntw *network.Network, protocol string) net.IP { return net.ParseIP(strings.TrimSpace(string(data))) } -func runMakeAccessResponseURLs(ip net.IP, conf *config, cli *CLI) *runAccessResponseURLs { +func (c *cliCommandAccess) makeResponseURLs(ip net.IP, conf *config, cli *CLI) *runAccessResponseURLs { if ip == nil { return nil } @@ -139,13 +147,13 @@ func runMakeAccessResponseURLs(ip net.IP, conf *config, cli *CLI) *runAccessResp }).String(), } - rv.TgQrCode = runMakeAccessResponseURLsQRCode(rv.TgURL) - rv.TmeQrCode = runMakeAccessResponseURLsQRCode(rv.TmeURL) + rv.TgQrCode = c.makeResponseQRCode(rv.TgURL) + rv.TmeQrCode = c.makeResponseQRCode(rv.TmeURL) return rv } -func runMakeAccessResponseURLsQRCode(data string) string { +func (c *cliCommandAccess) makeResponseQRCode(data string) string { values := url.Values{} values.Set("qzone", "4") diff --git a/cli_generate_secret.go b/cli_generate_secret.go index 1c8af9a..d84466e 100644 --- a/cli_generate_secret.go +++ b/cli_generate_secret.go @@ -6,7 +6,12 @@ import ( "github.com/9seconds/mtg/v2/mtglib" ) -func runGenerateSecret(cli *CLI) { +type cliCommandGenerateSecret struct { + HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet + Hex bool `help:"Print secret in hex encoding."` +} + +func (c *cliCommandGenerateSecret) Run(cli *CLI) error { // nolint: unparam secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) if cli.GenerateSecret.Hex { @@ -14,4 +19,6 @@ func runGenerateSecret(cli *CLI) { } else { fmt.Println(secret.Base64()) // nolint: forbidigo } + + return nil } diff --git a/main.go b/main.go index 0b78ac4..b4cf3f6 100644 --- a/main.go +++ b/main.go @@ -10,18 +10,9 @@ import ( var version = "dev" // has to be set by ldflags type CLI struct { - GenerateSecret struct { // nolint: govet - HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet - Hex bool `help:"Print secret in hex encoding."` - } `cmd help:"Generate new proxy secret."` - Access struct { // nolint: govet - ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet - Hex bool `help:"Print secret in hex encoding."` - } `cmd help:"Print access information."` - Run struct { // nolint: govet - ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet - } `cmd help:"Run proxy."` - Version kong.VersionFlag `help:"Print version."` + GenerateSecret cliCommandGenerateSecret `cmd help:"Generate new proxy secret"` // nolint: govet + Access cliCommandAccess `cmd help:"Print access information."` // nolint: govet + Version kong.VersionFlag `help:"Print version."` } func main() { @@ -30,18 +21,8 @@ func main() { cli := &CLI{} ctx := kong.Parse(cli, kong.Vars{ "domain_front": "amazonaws.com", - "config_path": "/etc/mtg.toml", "version": version, }) - switch ctx.Command() { - case "generate-secret": - runGenerateSecret(cli) - case "access ": - runAccess(cli) - case "run ": - panic("not implemented yet") - default: - panic(ctx.Command()) - } + ctx.FatalIfErrorf(ctx.Run(cli)) } diff --git a/utils.go b/utils.go index c2a11ae..1c46d27 100644 --- a/utils.go +++ b/utils.go @@ -7,16 +7,10 @@ import ( "net" "net/http" "net/url" - "os" "github.com/9seconds/mtg/v2/mtglib/network" ) -func exit(err error) { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) -} - func makeNetwork(conf *config) (*network.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) httpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultHTTPTimeout) From ca0a550f121d000a5f03e116099b4ca36d0e9433 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 18:26:36 +0300 Subject: [PATCH 033/206] Use array of fixed length for secret --- mtglib/secret.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mtglib/secret.go b/mtglib/secret.go index b35fc7a..ea2d51e 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -10,13 +10,15 @@ import ( const SecretKeyLength = 16 +var secretEmptyKey [SecretKeyLength]byte + type Secret struct { - Key []byte + Key [SecretKeyLength]byte Host string } func (s Secret) MarshalText() ([]byte, error) { - if s.Key == nil { + if s.Key == secretEmptyKey { return nil, nil } @@ -51,7 +53,7 @@ func (s *Secret) UnmarshalText(data []byte) error { return fmt.Errorf("secret has incorrect length %d", len(text)) } - s.Key = decoded[:SecretKeyLength] + copy(s.Key[:], decoded[:SecretKeyLength]) s.Host = string(decoded[SecretKeyLength:]) return nil @@ -70,7 +72,7 @@ func (s Secret) Hex() string { } func (s *Secret) makeBytes() []byte { - data := append([]byte{238}, s.Key...) // hex 'ee' = 238 + data := append([]byte{238}, s.Key[:]...) // hex 'ee' = 238 data = append(data, s.Host...) return data @@ -78,13 +80,18 @@ func (s *Secret) makeBytes() []byte { func GenerateSecret(hostname string) Secret { s := Secret{ - Key: make([]byte, SecretKeyLength), Host: hostname, } - if _, err := rand.Read(s.Key); err != nil { + if _, err := rand.Read(s.Key[:]); err != nil { panic(err) } return s } + +func ParseSecret(secret string) (Secret, error) { + s := Secret{} + + return s, s.UnmarshalText([]byte(secret)) +} From d5b0a47d522237d4bb7442753f6f0de6b955d54b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 18:34:08 +0300 Subject: [PATCH 034/206] Fix race detector tests --- mtglib/network/circuit_breaker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mtglib/network/circuit_breaker.go b/mtglib/network/circuit_breaker.go index 6c08b33..e2640fd 100644 --- a/mtglib/network/circuit_breaker.go +++ b/mtglib/network/circuit_breaker.go @@ -190,7 +190,9 @@ func newCircuitBreakerDialer(baseDialer Dialer, resetFailuresTimeout: resetFailuresTimeout, } + cb.stateMutexChan <- true // to convince race detector we are good cb.switchState(circuitBreakerStateClosed) + <-cb.stateMutexChan return cb } From ddd3d608fab8edf8fd1f53869e8c1c0bf42d3655 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 21:37:52 +0300 Subject: [PATCH 035/206] Add tests for secret --- mtglib/secret.go | 30 +++++++++--- mtglib/secret_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 mtglib/secret_test.go diff --git a/mtglib/secret.go b/mtglib/secret.go index ea2d51e..b2e59da 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -8,7 +8,11 @@ import ( "strings" ) -const SecretKeyLength = 16 +const ( + SecretKeyLength = 16 + + secretFakeTLSFirstByte byte = 238 +) var secretEmptyKey [SecretKeyLength]byte @@ -43,19 +47,29 @@ func (s *Secret) UnmarshalText(data []byte) error { if err != nil || len(decoded) <= SecretKeyLength { decoded, err = base64.RawURLEncoding.DecodeString(text) - } - if err != nil { - return fmt.Errorf("incorrect secret format: %w", err) - } + if err != nil { + return fmt.Errorf("incorrect secret format: %w", err) + } - if len(decoded) <= SecretKeyLength { - return fmt.Errorf("secret has incorrect length %d", len(text)) + if len(decoded) <= SecretKeyLength { + return fmt.Errorf("secret has incorrect length %d", len(text)) + } + + if decoded[0] != secretFakeTLSFirstByte { + return fmt.Errorf("incorrect first byte: %v", decoded[0]) + } + + decoded = decoded[1:] } copy(s.Key[:], decoded[:SecretKeyLength]) s.Host = string(decoded[SecretKeyLength:]) + if s.Host == "" { + return fmt.Errorf("hostname cannot be empty: %s", text) + } + return nil } @@ -72,7 +86,7 @@ func (s Secret) Hex() string { } func (s *Secret) makeBytes() []byte { - data := append([]byte{238}, s.Key[:]...) // hex 'ee' = 238 + data := append([]byte{secretFakeTLSFirstByte}, s.Key[:]...) data = append(data, s.Host...) return data diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go new file mode 100644 index 0000000..568abf8 --- /dev/null +++ b/mtglib/secret_test.go @@ -0,0 +1,110 @@ +package mtglib_test + +import ( + "encoding/hex" + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type SecretTestSuite struct { + suite.Suite +} + +func (suite *SecretTestSuite) TestParseSecret() { + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") + s := mtglib.Secret{ + Host: "google.com", + } + + copy(s.Key[:], secretData) + + testData := map[string]string{ + "hex": "eed11c6cbbd9efe7fed5bc0db220b09665676f6f676c652e636f6d", + "base64": "7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t", + } + + for name, value := range testData { + param := value + + suite.T().Run(name, func(t *testing.T) { + parsed, err := mtglib.ParseSecret(param) + + suite.NoError(err) + suite.Equal(s.Key, parsed.Key) + suite.Equal(s.Host, parsed.Host) + + newSecret := mtglib.Secret{} + + suite.NoError(newSecret.UnmarshalText([]byte(param))) + + suite.Equal(s.Key, newSecret.Key) + suite.Equal(s.Host, newSecret.Host) + }) + } +} + +func (suite *SecretTestSuite) TestSerialize() { + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") + s := mtglib.Secret{ + Host: "google.com", + } + + copy(s.Key[:], secretData) + + suite.Equal("eed11c6cbbd9efe7fed5bc0db220b09665676f6f676c652e636f6d", s.Hex()) + suite.Equal("7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t", s.Base64()) +} + +func (suite *SecretTestSuite) TestMarshalData() { + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") + s := mtglib.Secret{ + Host: "google.com", + } + + copy(s.Key[:], secretData) + + data, err := json.Marshal(&s) + + suite.NoError(err) + suite.Equal(string(data), `"7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t"`) +} + +func (suite *SecretTestSuite) TestIncorrectSecret() { + testData := []string{ + "aaa", + "d11c6cbbd9efe7fed5bc0db220b09665", + "ddd11c6cbbd9efe7fed5bc0db220b09665", + "+ueJ0q91t5XOnFYP8Xac3A", + "eed11c6cbbd9efe7fed5bc0db220b09665", + "ed11c6cbbd9efe7fed5bc0db220b09665", + } + + for _, v := range testData { + param := v + + suite.T().Run(param, func(t *testing.T) { + _, err := mtglib.ParseSecret(param) + + suite.Error(err) + }) + } +} + +func (suite *SecretTestSuite) TestInvariant() { + generated := mtglib.GenerateSecret("google.com") + + parsed, err := mtglib.ParseSecret(generated.Hex()) + + suite.NoError(err) + suite.Equal(generated.Key, parsed.Key) + suite.Equal(generated.Host, parsed.Host) + suite.Equal("google.com", parsed.Host) +} + +func TestSecret(t *testing.T) { + t.Parallel() + suite.Run(t, &SecretTestSuite{}) +} From 299c6478c258a03e8c2067897976fc42b28550de Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 10 Mar 2021 21:59:01 +0300 Subject: [PATCH 036/206] Create base structure to allow patching --- cli.go | 37 ++++++++++++++++++++++++++++++ cli_access.go | 51 ++++++++++++++++-------------------------- cli_generate_secret.go | 2 ++ 3 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 cli.go diff --git a/cli.go b/cli.go new file mode 100644 index 0000000..076b381 --- /dev/null +++ b/cli.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "os" + + "github.com/9seconds/mtg/v2/mtglib/network" +) + +type cli struct { + conf *config + network *network.Network +} + +func (c *cli) ReadConfig(path string) error { + filefp, err := os.Open(path) + if err != nil { + return fmt.Errorf("cannot open config file: %w", err) + } + + defer filefp.Close() + + conf, err := parseConfig(filefp) + if err != nil { + return fmt.Errorf("cannot parse config: %w", err) + } + + ntw, err := makeNetwork(conf) + if err != nil { + return fmt.Errorf("cannot build a network: %w", err) + } + + c.conf = conf + c.network = ntw + + return nil +} diff --git a/cli_access.go b/cli_access.go index ed9c7b8..e969700 100644 --- a/cli_access.go +++ b/cli_access.go @@ -11,8 +11,6 @@ import ( "os" "strconv" "strings" - - "github.com/9seconds/mtg/v2/mtglib/network" ) type runAccessResponse struct { @@ -33,45 +31,34 @@ type runAccessResponseURLs struct { } type cliCommandAccess struct { + cli + ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` } func (c *cliCommandAccess) Run(cli *CLI) error { - filefp, err := os.Open(cli.Access.ConfigPath) - if err != nil { - return fmt.Errorf("cannot open config file: %w", err) + if err := c.ReadConfig(cli.Access.ConfigPath); err != nil { + return fmt.Errorf("cannot init config: %w", err) } - defer filefp.Close() - - conf, err := parseConfig(filefp) - if err != nil { - return fmt.Errorf("cannot parse config: %w", err) - } - - ntw, err := makeNetwork(conf) - if err != nil { - return fmt.Errorf("cannot build a network: %w", err) - } - - ipv4 := conf.Network.PublicIP.IPv4.Value(nil) - ipv6 := conf.Network.PublicIP.IPv6.Value(nil) + ipv4 := c.conf.Network.PublicIP.IPv4.Value(nil) + ipv6 := c.conf.Network.PublicIP.IPv6.Value(nil) if ipv4 == nil { - ipv4 = c.getIP(ntw, "tcp4") + ipv4 = c.getIP("tcp4") } if ipv6 == nil { - ipv6 = c.getIP(ntw, "tcp6") + ipv6 = c.getIP("tcp6") } resp := runAccessResponse{ - IPv4: c.makeResponseURLs(ipv4, conf, cli), - IPv6: c.makeResponseURLs(ipv6, conf, cli), + IPv4: c.makeResponseURLs(ipv4, cli), + IPv6: c.makeResponseURLs(ipv6, cli), } - resp.Secret.Base64 = conf.Secret.Base64() - resp.Secret.Hex = conf.Secret.Hex() + resp.Secret.Base64 = c.conf.Secret.Base64() + resp.Secret.Hex = c.conf.Secret.Hex() encoder := json.NewEncoder(os.Stdout) @@ -85,12 +72,12 @@ func (c *cliCommandAccess) Run(cli *CLI) error { return nil } -func (c *cliCommandAccess) getIP(ntw *network.Network, protocol string) net.IP { +func (c *cliCommandAccess) getIP(protocol string) net.IP { client := &http.Client{ - Timeout: ntw.HTTP.Timeout, + Timeout: c.network.HTTP.Timeout, Transport: &http.Transport{ DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return ntw.DialContext(ctx, protocol, address) + return c.network.DialContext(ctx, protocol, address) }, }, } @@ -114,7 +101,7 @@ func (c *cliCommandAccess) getIP(ntw *network.Network, protocol string) net.IP { return net.ParseIP(strings.TrimSpace(string(data))) } -func (c *cliCommandAccess) makeResponseURLs(ip net.IP, conf *config, cli *CLI) *runAccessResponseURLs { +func (c *cliCommandAccess) makeResponseURLs(ip net.IP, cli *CLI) *runAccessResponseURLs { if ip == nil { return nil } @@ -122,12 +109,12 @@ func (c *cliCommandAccess) makeResponseURLs(ip net.IP, conf *config, cli *CLI) * values := url.Values{} values.Set("server", ip.String()) - values.Set("port", strconv.Itoa(int(conf.BindTo.port.Value(0)))) + values.Set("port", strconv.Itoa(int(c.conf.BindTo.port.Value(0)))) if cli.Access.Hex { - values.Set("secret", conf.Secret.Hex()) + values.Set("secret", c.conf.Secret.Hex()) } else { - values.Set("secret", conf.Secret.Base64()) + values.Set("secret", c.conf.Secret.Base64()) } urlQuery := values.Encode() diff --git a/cli_generate_secret.go b/cli_generate_secret.go index d84466e..786e74c 100644 --- a/cli_generate_secret.go +++ b/cli_generate_secret.go @@ -7,6 +7,8 @@ import ( ) type cliCommandGenerateSecret struct { + cli + HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` } From 24add0dce402868d9246bfd72001c453411c0bab Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 04:57:07 +0300 Subject: [PATCH 037/206] Make network as a separae interface --- cli.go | 2 +- cli_access.go | 10 ++-- cli_generate_secret.go | 2 +- config.go | 4 +- example.config.toml | 12 +++-- mtglib/network/init.go | 15 +++++- mtglib/network/network.go | 98 ++++++++++++++++++++++----------------- utils.go | 10 ++-- 8 files changed, 90 insertions(+), 63 deletions(-) diff --git a/cli.go b/cli.go index 076b381..d57b159 100644 --- a/cli.go +++ b/cli.go @@ -8,8 +8,8 @@ import ( ) type cli struct { + network network.Network conf *config - network *network.Network } func (c *cli) ReadConfig(path string) error { diff --git a/cli_access.go b/cli_access.go index e969700..d78ffca 100644 --- a/cli_access.go +++ b/cli_access.go @@ -73,12 +73,10 @@ func (c *cliCommandAccess) Run(cli *CLI) error { } func (c *cliCommandAccess) getIP(protocol string) net.IP { - client := &http.Client{ - Timeout: c.network.HTTP.Timeout, - Transport: &http.Transport{ - DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return c.network.DialContext(ctx, protocol, address) - }, + client := c.network.MakeHTTPClient(0) + client.Transport = &http.Transport{ + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return c.network.DialContext(ctx, protocol, address) }, } diff --git a/cli_generate_secret.go b/cli_generate_secret.go index 786e74c..41f8b04 100644 --- a/cli_generate_secret.go +++ b/cli_generate_secret.go @@ -7,7 +7,7 @@ import ( ) type cliCommandGenerateSecret struct { - cli + cli HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` diff --git a/config.go b/config.go index e0c8442..b0c309a 100644 --- a/config.go +++ b/config.go @@ -412,7 +412,7 @@ type config struct { } `json:"public-ip"` Timeout struct { TCP configTypeDuration `json:"tcp"` - HTTP configTypeDuration `json:"http"` + Idle configTypeDuration `json:"idle"` } `json:"timeout"` DOHIP configTypeIP `json:"doh-ip"` Proxies []configTypeURL `json:"proxies"` @@ -478,7 +478,7 @@ type configRaw struct { } `toml:"public-ip" json:"public-ip"` Timeout struct { TCP string `toml:"tcp" json:"tcp"` - HTTP string `toml:"http" json:"http"` + Idle string `toml:"idle" json:"idle"` } `toml:"timeout" json:"timeout"` DOHIP string `toml:"doh-ip" json:"doh-ip"` Proxies []string `toml:"proxies" json:"proxies"` diff --git a/example.config.toml b/example.config.toml index 0f74c79..59bc550 100644 --- a/example.config.toml +++ b/example.config.toml @@ -101,11 +101,17 @@ proxies = [ ipv4 = "" ipv6 = "" -# network timeouts define different settings for timeouts. HTTP timeout -# is required only for DOH. +# network timeouts define different settings for timeouts. tcp timeout +# define a global timeout on establishing of network connections. idle +# means a timeout on pumping data between sockset when nothing is +# happening. +# +# please be noticed that handshakes have no timeouts intentionally. You can +# find a reasoning here: +# https://www.ndss-symposium.org/wp-content/uploads/2020/02/23087-paper.pdf [network.timeout] tcp = "5s" -http = "10s" +idle = "1m" # FakeTLS can compare timestamps to prevent probes. Each message has # encrypted timestamp. So, mtg can compare this timestamp and decide if diff --git a/mtglib/network/init.go b/mtglib/network/init.go index 593b935..d3aa8dc 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -4,13 +4,14 @@ import ( "context" "errors" "net" + "net/http" "time" ) const ( DefaultTimeout = 10 * time.Second - DefaultDNSTimeout = time.Second - DefaultHTTPTimeout = DefaultTimeout + DefaultIdleTimeout = time.Minute + DefaultHTTPTimeout = 5 * time.Second DefaultBufferSize = 4096 ProxyDialerOpenThreshold = 5 @@ -18,6 +19,8 @@ const ( ProxyDialerResetFailuresTimeout = 10 * time.Second DefaultDOHHostname = "9.9.9.9" + + DNSTimeout = 5 * time.Second ) var ( @@ -29,3 +32,11 @@ type Dialer interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) } + +type Network interface { + Dialer + + DNSResolve(network, hostname string) (ips []string, err error) + MakeHTTPClient(timeout time.Duration) *http.Client + IdleTimeout() time.Duration +} diff --git a/mtglib/network/network.go b/mtglib/network/network.go index 31289ff..7d82878 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -12,21 +12,20 @@ import ( doh "github.com/babolivier/go-doh-client" ) -type Network struct { - HTTP http.Client - DNS doh.Resolver - - dialer Dialer +type network struct { + idleTimeout time.Duration + dialer Dialer + dns doh.Resolver } -func (d *Network) Dial(network, address string) (net.Conn, error) { - return d.DialContext(context.Background(), network, address) +func (n *network) Dial(protocol, address string) (net.Conn, error) { + return n.DialContext(context.Background(), protocol, address) } -func (d *Network) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (n *network) DialContext(ctx context.Context, protocol, address string) (net.Conn, error) { host, port, _ := net.SplitHostPort(address) - ips, err := d.resolveIPs(network, host) + ips, err := n.DNSResolve(protocol, host) if err != nil { return nil, fmt.Errorf("cannot resolve dns names: %w", err) } @@ -37,16 +36,20 @@ func (d *Network) DialContext(ctx context.Context, network, address string) (net }) } + var conn net.Conn + for _, v := range ips { - if conn, err := d.dialer.DialContext(ctx, network, net.JoinHostPort(v, port)); err == nil { + conn, err = n.dialer.DialContext(ctx, protocol, net.JoinHostPort(v, port)) + + if err == nil { return conn, nil } } - return nil, fmt.Errorf("cannot dial to %s:%s", network, address) + return nil, fmt.Errorf("cannot dial to %s:%s: %w", protocol, address, err) } -func (d *Network) resolveIPs(network, address string) ([]string, error) { +func (n *network) DNSResolve(protocol, address string) ([]string, error) { if net.ParseIP(address) != nil { return []string{address}, nil } @@ -55,14 +58,14 @@ func (d *Network) resolveIPs(network, address string) ([]string, error) { wg := &sync.WaitGroup{} mutex := &sync.Mutex{} - switch network { + switch protocol { case "tcp", "tcp4": wg.Add(1) go func() { defer wg.Done() - if recs, _, err := d.DNS.LookupA(address); err == nil { + if recs, _, err := n.dns.LookupA(address); err == nil { mutex.Lock() defer mutex.Unlock() @@ -73,14 +76,14 @@ func (d *Network) resolveIPs(network, address string) ([]string, error) { }() } - switch network { + switch protocol { case "tcp", "tcp6": wg.Add(1) go func() { defer wg.Done() - if recs, _, err := d.DNS.LookupAAAA(address); err == nil { + if recs, _, err := n.dns.LookupAAAA(address); err == nil { mutex.Lock() defer mutex.Unlock() @@ -94,44 +97,53 @@ func (d *Network) resolveIPs(network, address string) ([]string, error) { wg.Wait() if len(ips) == 0 { - return nil, fmt.Errorf("cannot find any ips for %s:%s", network, address) + return nil, fmt.Errorf("cannot find any ips for %s:%s", protocol, address) } return ips, nil } -func NewNetwork(dialer Dialer, dohHostname string, httpTimeout time.Duration) (*Network, error) { +func (n *network) IdleTimeout() time.Duration { + return n.idleTimeout +} + +func (n *network) MakeHTTPClient(timeout time.Duration) *http.Client { + if timeout <= 0 { + timeout = DefaultHTTPTimeout + } + + return &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + DialContext: n.DialContext, + }, + } +} + +func NewNetwork(dialer Dialer, dohHostname string, idleTimeout time.Duration) (Network, error) { switch { - case httpTimeout < 0: - return nil, fmt.Errorf("timeout should be positive number %v", httpTimeout) - case httpTimeout == 0: - httpTimeout = DefaultHTTPTimeout + case idleTimeout < 0: + return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout) + case idleTimeout == 0: + idleTimeout = DefaultIdleTimeout } if net.ParseIP(dohHostname) == nil { return nil, fmt.Errorf("hostname %s should be IP address", dohHostname) } - dohHTTPClient := &http.Client{ - Timeout: DefaultDNSTimeout, - Transport: &http.Transport{ - DialContext: dialer.DialContext, + return &network{ + dialer: dialer, + idleTimeout: idleTimeout, + dns: doh.Resolver{ + Host: dohHostname, + Class: doh.IN, + HTTPClient: &http.Client{ + Timeout: DNSTimeout, + Transport: &http.Transport{ + DialContext: dialer.DialContext, + }, + }, }, - } - network := &Network{ - dialer: dialer, - DNS: doh.Resolver{ - Host: dohHostname, - Class: doh.IN, - HTTPClient: dohHTTPClient, - }, - } - network.HTTP = http.Client{ - Timeout: httpTimeout, - Transport: &http.Transport{ - DialContext: network.DialContext, - }, - } - - return network, nil + }, nil } diff --git a/utils.go b/utils.go index 1c46d27..1411635 100644 --- a/utils.go +++ b/utils.go @@ -11,9 +11,9 @@ import ( "github.com/9seconds/mtg/v2/mtglib/network" ) -func makeNetwork(conf *config) (*network.Network, error) { +func makeNetwork(conf *config) (network.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) - httpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultHTTPTimeout) + idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) @@ -32,14 +32,14 @@ func makeNetwork(conf *config) (*network.Network, error) { switch len(proxyURLs) { case 0: - return network.NewNetwork(baseDialer, dohIP, httpTimeout) + return network.NewNetwork(baseDialer, dohIP, idleTimeout) case 1: socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) if err != nil { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, dohIP, httpTimeout) + return network.NewNetwork(socksDialer, dohIP, idleTimeout) } socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) @@ -47,7 +47,7 @@ func makeNetwork(conf *config) (*network.Network, error) { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, dohIP, httpTimeout) + return network.NewNetwork(socksDialer, dohIP, idleTimeout) } func exhaustResponse(response *http.Response) { From 757ea5b63cacd1d1a08459658a6ae0aa127b7b42 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 05:00:56 +0300 Subject: [PATCH 038/206] Add PatchHTTPClient method This method is mostly required for testing --- cli_access.go | 2 ++ mtglib/network/init.go | 1 + mtglib/network/network.go | 3 +++ 3 files changed, 6 insertions(+) diff --git a/cli_access.go b/cli_access.go index d78ffca..e0eea15 100644 --- a/cli_access.go +++ b/cli_access.go @@ -80,6 +80,8 @@ func (c *cliCommandAccess) getIP(protocol string) net.IP { }, } + c.network.PatchHTTPClient(client) + resp, err := client.Get("https://ifconfig.co") // nolint: bodyclose, noctx if err != nil { return nil diff --git a/mtglib/network/init.go b/mtglib/network/init.go index d3aa8dc..d922ce1 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -39,4 +39,5 @@ type Network interface { DNSResolve(network, hostname string) (ips []string, err error) MakeHTTPClient(timeout time.Duration) *http.Client IdleTimeout() time.Duration + PatchHTTPClient(*http.Client) } diff --git a/mtglib/network/network.go b/mtglib/network/network.go index 7d82878..d256dd0 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -120,6 +120,9 @@ func (n *network) MakeHTTPClient(timeout time.Duration) *http.Client { } } +func (n *network) PatchHTTPClient(_ *http.Client) { +} + func NewNetwork(dialer Dialer, dohHostname string, idleTimeout time.Duration) (Network, error) { switch { case idleTimeout < 0: From 1a02511afe4b305dcc85339f7726a6d3f7d084c9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 05:37:18 +0300 Subject: [PATCH 039/206] Move config into separate package --- cli.go | 17 +- cli_access.go | 26 +- config.go | 529 ----------------------------------- config/config.go | 151 ++++++++++ config/type_bytes.go | 47 ++++ config/type_duration.go | 46 +++ config/type_float.go | 41 +++ config/type_hostport.go | 56 ++++ config/type_http_path.go | 31 ++ config/type_ip.go | 45 +++ config/type_metric_prefix.go | 42 +++ config/type_port.go | 45 +++ config/type_prefer_ip.go | 43 +++ config/type_url.go | 45 +++ utils.go | 3 +- 15 files changed, 615 insertions(+), 552 deletions(-) delete mode 100644 config.go create mode 100644 config/config.go create mode 100644 config/type_bytes.go create mode 100644 config/type_duration.go create mode 100644 config/type_float.go create mode 100644 config/type_hostport.go create mode 100644 config/type_http_path.go create mode 100644 config/type_ip.go create mode 100644 config/type_metric_prefix.go create mode 100644 config/type_port.go create mode 100644 config/type_prefer_ip.go create mode 100644 config/type_url.go diff --git a/cli.go b/cli.go index d57b159..78373d3 100644 --- a/cli.go +++ b/cli.go @@ -2,25 +2,24 @@ package main import ( "fmt" - "os" + "io/ioutil" + "github.com/9seconds/mtg/v2/config" "github.com/9seconds/mtg/v2/mtglib/network" ) type cli struct { network network.Network - conf *config + conf *config.Config } func (c *cli) ReadConfig(path string) error { - filefp, err := os.Open(path) - if err != nil { - return fmt.Errorf("cannot open config file: %w", err) - } + content, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("cannot read config file: %w", err) + } - defer filefp.Close() - - conf, err := parseConfig(filefp) + conf, err := config.Parse(content) if err != nil { return fmt.Errorf("cannot parse config: %w", err) } diff --git a/cli_access.go b/cli_access.go index e0eea15..7036231 100644 --- a/cli_access.go +++ b/cli_access.go @@ -13,16 +13,16 @@ import ( "strings" ) -type runAccessResponse struct { - IPv4 *runAccessResponseURLs `json:"ipv4,omitempty"` - IPv6 *runAccessResponseURLs `json:"ipv6,omitempty"` +type accessResponse struct { + IPv4 *accessResponseURLs `json:"ipv4,omitempty"` + IPv6 *accessResponseURLs `json:"ipv6,omitempty"` Secret struct { Hex string `json:"hex"` Base64 string `json:"base64"` } `json:"secret"` } -type runAccessResponseURLs struct { +type accessResponseURLs struct { IP net.IP `json:"ip"` TgURL string `json:"tg_url"` TgQrCode string `json:"tg_qrcode"` @@ -53,9 +53,9 @@ func (c *cliCommandAccess) Run(cli *CLI) error { ipv6 = c.getIP("tcp6") } - resp := runAccessResponse{ - IPv4: c.makeResponseURLs(ipv4, cli), - IPv6: c.makeResponseURLs(ipv6, cli), + resp := accessResponse{ + IPv4: c.makeURLs(ipv4, cli), + IPv6: c.makeURLs(ipv6, cli), } resp.Secret.Base64 = c.conf.Secret.Base64() resp.Secret.Hex = c.conf.Secret.Hex() @@ -101,7 +101,7 @@ func (c *cliCommandAccess) getIP(protocol string) net.IP { return net.ParseIP(strings.TrimSpace(string(data))) } -func (c *cliCommandAccess) makeResponseURLs(ip net.IP, cli *CLI) *runAccessResponseURLs { +func (c *cliCommandAccess) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { if ip == nil { return nil } @@ -109,7 +109,7 @@ func (c *cliCommandAccess) makeResponseURLs(ip net.IP, cli *CLI) *runAccessRespo values := url.Values{} values.Set("server", ip.String()) - values.Set("port", strconv.Itoa(int(c.conf.BindTo.port.Value(0)))) + values.Set("port", strconv.Itoa(int(c.conf.BindTo.PortValue(0)))) if cli.Access.Hex { values.Set("secret", c.conf.Secret.Hex()) @@ -119,7 +119,7 @@ func (c *cliCommandAccess) makeResponseURLs(ip net.IP, cli *CLI) *runAccessRespo urlQuery := values.Encode() - rv := &runAccessResponseURLs{ + rv := &accessResponseURLs{ IP: ip, TgURL: (&url.URL{ Scheme: "tg", @@ -134,13 +134,13 @@ func (c *cliCommandAccess) makeResponseURLs(ip net.IP, cli *CLI) *runAccessRespo }).String(), } - rv.TgQrCode = c.makeResponseQRCode(rv.TgURL) - rv.TmeQrCode = c.makeResponseQRCode(rv.TmeURL) + rv.TgQrCode = c.makeQRCode(rv.TgURL) + rv.TmeQrCode = c.makeQRCode(rv.TmeURL) return rv } -func (c *cliCommandAccess) makeResponseQRCode(data string) string { +func (c *cliCommandAccess) makeQRCode(data string) string { values := url.Values{} values.Set("qzone", "4") diff --git a/config.go b/config.go deleted file mode 100644 index b0c309a..0000000 --- a/config.go +++ /dev/null @@ -1,529 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net" - "net/url" - "regexp" - "strconv" - "strings" - "time" - - "github.com/9seconds/mtg/v2/mtglib" - "github.com/alecthomas/units" - "github.com/pelletier/go-toml" -) - -type configTypeHostPort struct { - host configTypeIP - port configTypePort -} - -func (c *configTypeHostPort) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - host, port, err := net.SplitHostPort(string(data)) - if err != nil { - return fmt.Errorf("incorrect host:port syntax: %w", err) - } - - if err := c.port.UnmarshalJSON([]byte(port)); err != nil { - return fmt.Errorf("incorrect port in host:port: %w", err) - } - - if err := c.host.UnmarshalText([]byte(host)); err != nil { - return fmt.Errorf("incorrect host: %w", err) - } - - return nil -} - -func (c configTypeHostPort) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeHostPort) String() string { - return c.Value(net.IP{}, 0) -} - -func (c configTypeHostPort) Value(defaultHostValue net.IP, defaultPortValue uint) string { - return net.JoinHostPort(c.host.Value(defaultHostValue).String(), - strconv.Itoa(int(c.port.Value(defaultPortValue)))) -} - -type configTypePort struct { - value uint -} - -func (c *configTypePort) UnmarshalJSON(data []byte) error { - if len(data) == 0 { - return nil - } - - intValue, err := strconv.ParseUint(string(data), 10, 16) - if err != nil { - return fmt.Errorf("port number is not a number: %w", err) - } - - if intValue == 0 || intValue > 65536 { - return fmt.Errorf("port number should be 0 < portNo < 65536: %d", intValue) - } - - c.value = uint(intValue) - - return nil -} - -func (c *configTypePort) MarshalJSON() ([]byte, error) { - return json.Marshal(c.value) -} - -func (c configTypePort) String() string { - return strconv.Itoa(int(c.value)) -} - -func (c configTypePort) Value(defaultValue uint) uint { - if c.value == 0 { - return defaultValue - } - - return c.value -} - -type configTypeBytes struct { - value uint -} - -func (c *configTypeBytes) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - value, err := units.ParseStrictBytes(strings.ToUpper(string(data))) - if err != nil { - return fmt.Errorf("incorrect bytes value: %w", err) - } - - if value < 0 { - return fmt.Errorf("%d should be positive number", value) - } - - c.value = uint(value) - - return nil -} - -func (c configTypeBytes) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeBytes) String() string { - return units.ToString(int64(c.value), 1024, "ib", "b") -} - -func (c configTypeBytes) Value(defaultValue uint) uint { - if c.value == 0 { - return defaultValue - } - - return c.value -} - -type configTypePreferIP struct { - value string -} - -func (c *configTypePreferIP) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - text := strings.ToLower(string(data)) - - switch text { - case "prefer-ipv4", "prefer-ipv6", "only-ipv4", "only-ipv6": - c.value = text - default: - return fmt.Errorf("incorrect prefer-ip value: %s", string(data)) - } - - return nil -} - -func (c configTypePreferIP) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.value), nil -} - -func (c *configTypePreferIP) String() string { - return c.value -} - -func (c *configTypePreferIP) Value(defaultValue string) string { - if c.value == "" { - return defaultValue - } - - return c.value -} - -type configTypeDuration struct { - value time.Duration -} - -func (c *configTypeDuration) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - dur, err := time.ParseDuration(strings.ToLower(string(data))) - if err != nil { - return fmt.Errorf("incorrect duration: %w", err) - } - - if dur < 0 { - return fmt.Errorf("%s should be positive duration", dur) - } - - c.value = dur - - return nil -} - -func (c configTypeDuration) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.value.String()), nil -} - -func (c configTypeDuration) String() string { - return c.value.String() -} - -func (c configTypeDuration) Value(defaultValue time.Duration) time.Duration { - if c.value == 0 { - return defaultValue - } - - return c.value -} - -type configTypeFloat struct { - value float64 -} - -func (c *configTypeFloat) UnmarshalJSON(data []byte) error { - value, err := strconv.ParseFloat(string(data), 64) - if err != nil { - return fmt.Errorf("incorrect float value: %w", err) - } - - if value < 0 { - return fmt.Errorf("%f should be positive", value) - } - - c.value = value - - return nil -} - -func (c *configTypeFloat) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeFloat) String() string { - return strconv.FormatFloat(c.value, 'f', -1, 64) -} - -func (c configTypeFloat) Value(defaultValue float64) float64 { - if c.value < 0.00001 { - return defaultValue - } - - return c.value -} - -type configTypeIP struct { - value net.IP -} - -func (c *configTypeIP) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - ip := net.ParseIP(string(data)) - if ip == nil { - return fmt.Errorf("incorrect ip address: %s", string(data)) - } - - c.value = ip - - return nil -} - -func (c *configTypeIP) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeIP) String() string { - if c.value == nil { - return "" - } - - return c.value.String() -} - -func (c configTypeIP) Value(defaultValue net.IP) net.IP { - if c.value == nil { - return defaultValue - } - - return c.value -} - -type configTypeURL struct { - value *url.URL -} - -func (c *configTypeURL) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - value, err := url.Parse(string(data)) - if err != nil { - return fmt.Errorf("incorrect URL: %w", err) - } - - c.value = value - - return nil -} - -func (c *configTypeURL) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeURL) String() string { - if c.value == nil { - return "" - } - - return c.value.String() -} - -func (c configTypeURL) Value(defaultValue *url.URL) *url.URL { - if c.value == nil { - return defaultValue - } - - return c.value -} - -type configTypeMetricPrefix struct { - value string -} - -func (c *configTypeMetricPrefix) UnmarshalText(data []byte) error { - if len(data) == 0 { - return nil - } - - prefix := string(data) - - if ok, err := regexp.MatchString("^[a-z0-9]+$", prefix); !ok || err != nil { - return fmt.Errorf("incorrect metric prefix: %s", prefix) - } - - c.value = prefix - - return nil -} - -func (c configTypeMetricPrefix) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeMetricPrefix) String() string { - return c.value -} - -func (c configTypeMetricPrefix) Value(defaultValue string) string { - if c.value == "" { - return defaultValue - } - - return c.value -} - -type configTypeHTTPPath struct { - value string -} - -func (c *configTypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unparam - if len(data) > 0 { - c.value = "/" + strings.Trim(string(data), "/") - } - - return nil -} - -func (c configTypeHTTPPath) MarshalText() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil -} - -func (c configTypeHTTPPath) String() string { - return c.value -} - -func (c configTypeHTTPPath) Value(defaultValue string) string { - if c.value == "" { - return defaultValue - } - - return c.value -} - -type config struct { - Debug bool `json:"debug"` - Secret mtglib.Secret `json:"secret"` - BindTo configTypeHostPort `json:"bind-to"` - TCPBuffer configTypeBytes `json:"tcp-buffer"` - PreferIP configTypePreferIP `json:"prefer-ip"` - CloakPort configTypePort `json:"cloak-port"` - Probes struct { - Time struct { - Enabled bool `json:"enabled"` - AllowSkewness configTypeDuration `json:"allow-skewness"` - } `json:"time"` - AntiReplay struct { - Enabled bool `json:"enabled"` - MaxSize configTypeBytes `json:"max-size"` - ErrorRate configTypeFloat `json:"error-rate"` - } `json:"anti-replay"` - } `json:"probes"` - Network struct { - PublicIP struct { - IPv4 configTypeIP `json:"ipv4"` - IPv6 configTypeIP `json:"ipv6"` - } `json:"public-ip"` - Timeout struct { - TCP configTypeDuration `json:"tcp"` - Idle configTypeDuration `json:"idle"` - } `json:"timeout"` - DOHIP configTypeIP `json:"doh-ip"` - Proxies []configTypeURL `json:"proxies"` - } `json:"network"` - Stats struct { - StatsD struct { - Enabled bool `json:"enabled"` - Address configTypeHostPort `json:"address"` - MetricPrefix configTypeMetricPrefix `json:"metric-prefix"` - } `json:"statsd"` - Prometheus struct { - Enabled bool `json:"enabled"` - BindTo configTypeHostPort `json:"bind-to"` - HTTPPath configTypeHTTPPath `json:"http-path"` - MetricPrefix configTypeMetricPrefix `json:"metric-prefix"` - } `json:"prometheus"` - } `json:"stats"` -} - -func (c *config) Validate() error { - if len(c.Secret.Key) == 0 || c.Secret.Host == "" { - return fmt.Errorf("incorrect secret %s", c.Secret.String()) - } - - return nil -} - -func (c *config) String() string { - buf := &bytes.Buffer{} - encoder := json.NewEncoder(buf) - - encoder.SetEscapeHTML(false) - - if err := encoder.Encode(c); err != nil { - panic(err) - } - - return buf.String() -} - -type configRaw struct { - Debug bool `toml:"debug" json:"debug"` - Secret string `toml:"secret" json:"secret"` - BindTo string `toml:"bind-to" json:"bind-to"` - TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer"` - PreferIP string `toml:"prefer-ip" json:"prefer-ip"` - CloakPort uint `toml:"cloak-port" json:"cloak-port"` - Probes struct { - Time struct { - Enabled bool `toml:"enabled" json:"enabled"` - AllowSkewness string `toml:"allow-skewness" json:"allow-skewness"` - } `toml:"time" json:"time"` - AntiReplay struct { - Enabled bool `toml:"enabled" json:"enabled"` - MaxSize string `toml:"max-size" json:"max-size"` - ErrorRate float64 `toml:"error-rate" json:"error-rate"` - } `toml:"anti-replay" json:"anti-replay"` - } `toml:"probes" json:"probes"` - Network struct { - PublicIP struct { - IPv4 string `toml:"ipv4" json:"ipv4"` - IPv6 string `toml:"ipv6" json:"ipv6"` - } `toml:"public-ip" json:"public-ip"` - Timeout struct { - TCP string `toml:"tcp" json:"tcp"` - Idle string `toml:"idle" json:"idle"` - } `toml:"timeout" json:"timeout"` - DOHIP string `toml:"doh-ip" json:"doh-ip"` - Proxies []string `toml:"proxies" json:"proxies"` - } `toml:"network" json:"network"` - Stats struct { - StatsD struct { - Enabled bool `toml:"enabled" json:"enabled"` - Address string `toml:"address" json:"address"` - MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` - } `toml:"statsd" json:"statsd"` - Prometheus struct { - Enabled bool `toml:"enabled" json:"enabled"` - BindTo string `toml:"bind-to" json:"bind-to"` - HTTPPath string `toml:"http-path" json:"http-path"` - MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` - } `toml:"prometheus" json:"prometheus"` - } `toml:"stats" json:"stats"` -} - -func parseConfig(reader io.Reader) (*config, error) { - rawConf := &configRaw{} - - if err := toml.NewDecoder(reader).Decode(rawConf); err != nil { - return nil, fmt.Errorf("cannot parse toml config: %w", err) - } - - jsonBuf := &bytes.Buffer{} - jsonEncoder := json.NewEncoder(jsonBuf) - - jsonEncoder.SetEscapeHTML(false) - jsonEncoder.SetIndent("", "") - - if err := jsonEncoder.Encode(rawConf); err != nil { - return nil, fmt.Errorf("cannot dump into interim format: %w", err) - } - - conf := &config{} - - if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { - return nil, fmt.Errorf("cannot parse final config: %w", err) - } - - if err := conf.Validate(); err != nil { - return nil, fmt.Errorf("cannot validate config: %w", err) - } - - return conf, nil -} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..ab4f5fe --- /dev/null +++ b/config/config.go @@ -0,0 +1,151 @@ +package config + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/pelletier/go-toml" +) + +type Config struct { + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` + BindTo TypeHostPort `json:"bind-to"` + TCPBuffer TypeBytes `json:"tcp-buffer"` + PreferIP TypePreferIP `json:"prefer-ip"` + CloakPort TypePort `json:"cloak-port"` + Probes struct { + Time struct { + Enabled bool `json:"enabled"` + AllowSkewness TypeDuration `json:"allow-skewness"` + } `json:"time"` + AntiReplay struct { + Enabled bool `json:"enabled"` + MaxSize TypeBytes `json:"max-size"` + ErrorRate TypeFloat `json:"error-rate"` + } `json:"anti-replay"` + } `json:"probes"` + Network struct { + PublicIP struct { + IPv4 TypeIP `json:"ipv4"` + IPv6 TypeIP `json:"ipv6"` + } `json:"public-ip"` + Timeout struct { + TCP TypeDuration `json:"tcp"` + Idle TypeDuration `json:"idle"` + } `json:"timeout"` + DOHIP TypeIP `json:"doh-ip"` + Proxies []TypeURL `json:"proxies"` + } `json:"network"` + Stats struct { + StatsD struct { + Enabled bool `json:"enabled"` + Address TypeHostPort `json:"address"` + MetricPrefix TypeMetricPrefix `json:"metric-prefix"` + } `json:"statsd"` + Prometheus struct { + Enabled bool `json:"enabled"` + BindTo TypeHostPort `json:"bind-to"` + HTTPPath TypeHTTPPath `json:"http-path"` + MetricPrefix TypeMetricPrefix `json:"metric-prefix"` + } `json:"prometheus"` + } `json:"stats"` +} + +func (c *Config) Validate() error { + if len(c.Secret.Key) == 0 || c.Secret.Host == "" { + return fmt.Errorf("incorrect secret %s", c.Secret.String()) + } + + return nil +} + +func (c *Config) String() string { + buf := &bytes.Buffer{} + encoder := json.NewEncoder(buf) + + encoder.SetEscapeHTML(false) + + if err := encoder.Encode(c); err != nil { + panic(err) + } + + return buf.String() +} + +type configRaw struct { + Debug bool `toml:"debug" json:"debug"` + Secret string `toml:"secret" json:"secret"` + BindTo string `toml:"bind-to" json:"bind-to"` + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip"` + CloakPort uint `toml:"cloak-port" json:"cloak-port"` + Probes struct { + Time struct { + Enabled bool `toml:"enabled" json:"enabled"` + AllowSkewness string `toml:"allow-skewness" json:"allow-skewness"` + } `toml:"time" json:"time"` + AntiReplay struct { + Enabled bool `toml:"enabled" json:"enabled"` + MaxSize string `toml:"max-size" json:"max-size"` + ErrorRate float64 `toml:"error-rate" json:"error-rate"` + } `toml:"anti-replay" json:"anti-replay"` + } `toml:"probes" json:"probes"` + Network struct { + PublicIP struct { + IPv4 string `toml:"ipv4" json:"ipv4"` + IPv6 string `toml:"ipv6" json:"ipv6"` + } `toml:"public-ip" json:"public-ip"` + Timeout struct { + TCP string `toml:"tcp" json:"tcp"` + Idle string `toml:"idle" json:"idle"` + } `toml:"timeout" json:"timeout"` + DOHIP string `toml:"doh-ip" json:"doh-ip"` + Proxies []string `toml:"proxies" json:"proxies"` + } `toml:"network" json:"network"` + Stats struct { + StatsD struct { + Enabled bool `toml:"enabled" json:"enabled"` + Address string `toml:"address" json:"address"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` + } `toml:"statsd" json:"statsd"` + Prometheus struct { + Enabled bool `toml:"enabled" json:"enabled"` + BindTo string `toml:"bind-to" json:"bind-to"` + HTTPPath string `toml:"http-path" json:"http-path"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` + } `toml:"prometheus" json:"prometheus"` + } `toml:"stats" json:"stats"` +} + +func Parse(rawData []byte) (*Config, error) { + rawConf := &configRaw{} + + if err := toml.Unmarshal(rawData, rawConf); err != nil { + return nil, fmt.Errorf("cannot parse toml config: %w", err) + } + + jsonBuf := &bytes.Buffer{} + jsonEncoder := json.NewEncoder(jsonBuf) + + jsonEncoder.SetEscapeHTML(false) + jsonEncoder.SetIndent("", "") + + if err := jsonEncoder.Encode(rawConf); err != nil { + return nil, fmt.Errorf("cannot dump into interim format: %w", err) + } + + conf := &Config{} + + if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { + return nil, fmt.Errorf("cannot parse final config: %w", err) + } + + if err := conf.Validate(); err != nil { + return nil, fmt.Errorf("cannot validate config: %w", err) + } + + return conf, nil +} diff --git a/config/type_bytes.go b/config/type_bytes.go new file mode 100644 index 0000000..585ee86 --- /dev/null +++ b/config/type_bytes.go @@ -0,0 +1,47 @@ +package config + +import ( + "fmt" + "strings" + + "github.com/alecthomas/units" +) + +type TypeBytes struct { + value uint +} + +func (c *TypeBytes) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + value, err := units.ParseStrictBytes(strings.ToUpper(string(data))) + if err != nil { + return fmt.Errorf("incorrect bytes value: %w", err) + } + + if value < 0 { + return fmt.Errorf("%d should be positive number", value) + } + + c.value = uint(value) + + return nil +} + +func (c TypeBytes) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeBytes) String() string { + return units.ToString(int64(c.value), 1024, "ib", "b") +} + +func (c TypeBytes) Value(defaultValue uint) uint { + if c.value == 0 { + return defaultValue + } + + return c.value +} diff --git a/config/type_duration.go b/config/type_duration.go new file mode 100644 index 0000000..f5dbbab --- /dev/null +++ b/config/type_duration.go @@ -0,0 +1,46 @@ +package config + +import ( + "fmt" + "strings" + "time" +) + +type TypeDuration struct { + value time.Duration +} + +func (c *TypeDuration) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + dur, err := time.ParseDuration(strings.ToLower(string(data))) + if err != nil { + return fmt.Errorf("incorrect duration: %w", err) + } + + if dur < 0 { + return fmt.Errorf("%s should be positive duration", dur) + } + + c.value = dur + + return nil +} + +func (c TypeDuration) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.value.String()), nil +} + +func (c TypeDuration) String() string { + return c.value.String() +} + +func (c TypeDuration) Value(defaultValue time.Duration) time.Duration { + if c.value == 0 { + return defaultValue + } + + return c.value +} diff --git a/config/type_float.go b/config/type_float.go new file mode 100644 index 0000000..bf5aff6 --- /dev/null +++ b/config/type_float.go @@ -0,0 +1,41 @@ +package config + +import ( + "fmt" + "strconv" +) + +type TypeFloat struct { + value float64 +} + +func (c *TypeFloat) UnmarshalJSON(data []byte) error { + value, err := strconv.ParseFloat(string(data), 64) + if err != nil { + return fmt.Errorf("incorrect float value: %w", err) + } + + if value < 0 { + return fmt.Errorf("%f should be positive", value) + } + + c.value = value + + return nil +} + +func (c *TypeFloat) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeFloat) String() string { + return strconv.FormatFloat(c.value, 'f', -1, 64) +} + +func (c TypeFloat) Value(defaultValue float64) float64 { + if c.value < 0.00001 { + return defaultValue + } + + return c.value +} diff --git a/config/type_hostport.go b/config/type_hostport.go new file mode 100644 index 0000000..42a9fc7 --- /dev/null +++ b/config/type_hostport.go @@ -0,0 +1,56 @@ +package config + +import ( + "fmt" + "net" + "strconv" +) + +type TypeHostPort struct { + host TypeIP + port TypePort +} + +func (c *TypeHostPort) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + host, port, err := net.SplitHostPort(string(data)) + if err != nil { + return fmt.Errorf("incorrect host:port syntax: %w", err) + } + + if err := c.port.UnmarshalJSON([]byte(port)); err != nil { + return fmt.Errorf("incorrect port in host:port: %w", err) + } + + if err := c.host.UnmarshalText([]byte(host)); err != nil { + return fmt.Errorf("incorrect host: %w", err) + } + + return nil +} + +func (c TypeHostPort) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeHostPort) String() string { + return c.Value(net.IP{}, 0) +} + +func (c TypeHostPort) HostValue(defaultValue net.IP) net.IP { + return c.host.Value(defaultValue) +} + +func (c TypeHostPort) PortValue(defaultValue uint) uint { + return c.port.Value(defaultValue) +} + +func (c TypeHostPort) Value(defaultHostValue net.IP, defaultPortValue uint) string { + host := c.HostValue(defaultHostValue) + port := c.PortValue(defaultPortValue) + + return net.JoinHostPort(host.String(), strconv.Itoa(int(port))) +} diff --git a/config/type_http_path.go b/config/type_http_path.go new file mode 100644 index 0000000..79d311a --- /dev/null +++ b/config/type_http_path.go @@ -0,0 +1,31 @@ +package config + +import "strings" + +type TypeHTTPPath struct { + value string +} + +func (c *TypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unparam + if len(data) > 0 { + c.value = "/" + strings.Trim(string(data), "/") + } + + return nil +} + +func (c TypeHTTPPath) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeHTTPPath) String() string { + return c.value +} + +func (c TypeHTTPPath) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/config/type_ip.go b/config/type_ip.go new file mode 100644 index 0000000..e249a94 --- /dev/null +++ b/config/type_ip.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "net" +) + +type TypeIP struct { + value net.IP +} + +func (c *TypeIP) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + ip := net.ParseIP(string(data)) + if ip == nil { + return fmt.Errorf("incorrect ip address: %s", string(data)) + } + + c.value = ip + + return nil +} + +func (c *TypeIP) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeIP) String() string { + if c.value == nil { + return "" + } + + return c.value.String() +} + +func (c TypeIP) Value(defaultValue net.IP) net.IP { + if c.value == nil { + return defaultValue + } + + return c.value +} diff --git a/config/type_metric_prefix.go b/config/type_metric_prefix.go new file mode 100644 index 0000000..adc1029 --- /dev/null +++ b/config/type_metric_prefix.go @@ -0,0 +1,42 @@ +package config + +import ( + "fmt" + "regexp" +) + +type TypeMetricPrefix struct { + value string +} + +func (c *TypeMetricPrefix) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + prefix := string(data) + + if ok, err := regexp.MatchString("^[a-z0-9]+$", prefix); !ok || err != nil { + return fmt.Errorf("incorrect metric prefix: %s", prefix) + } + + c.value = prefix + + return nil +} + +func (c TypeMetricPrefix) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeMetricPrefix) String() string { + return c.value +} + +func (c TypeMetricPrefix) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/config/type_port.go b/config/type_port.go new file mode 100644 index 0000000..41963e5 --- /dev/null +++ b/config/type_port.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "strconv" +) + +type TypePort struct { + value uint +} + +func (c *TypePort) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + return nil + } + + intValue, err := strconv.ParseUint(string(data), 10, 16) + if err != nil { + return fmt.Errorf("port number is not a number: %w", err) + } + + if intValue == 0 || intValue > 65536 { + return fmt.Errorf("port number should be 0 < portNo < 65536: %d", intValue) + } + + c.value = uint(intValue) + + return nil +} + +func (c *TypePort) MarshalJSON() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypePort) String() string { + return strconv.Itoa(int(c.value)) +} + +func (c TypePort) Value(defaultValue uint) uint { + if c.value == 0 { + return defaultValue + } + + return c.value +} diff --git a/config/type_prefer_ip.go b/config/type_prefer_ip.go new file mode 100644 index 0000000..fc5e034 --- /dev/null +++ b/config/type_prefer_ip.go @@ -0,0 +1,43 @@ +package config + +import ( + "fmt" + "strings" +) + +type TypePreferIP struct { + value string +} + +func (c *TypePreferIP) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := strings.ToLower(string(data)) + + switch text { + case "prefer-ipv4", "prefer-ipv6", "only-ipv4", "only-ipv6": + c.value = text + default: + return fmt.Errorf("incorrect prefer-ip value: %s", string(data)) + } + + return nil +} + +func (c TypePreferIP) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.value), nil +} + +func (c *TypePreferIP) String() string { + return c.value +} + +func (c *TypePreferIP) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/config/type_url.go b/config/type_url.go new file mode 100644 index 0000000..38378e2 --- /dev/null +++ b/config/type_url.go @@ -0,0 +1,45 @@ +package config + +import ( + "fmt" + "net/url" +) + +type TypeURL struct { + value *url.URL +} + +func (c *TypeURL) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + value, err := url.Parse(string(data)) + if err != nil { + return fmt.Errorf("incorrect URL: %w", err) + } + + c.value = value + + return nil +} + +func (c *TypeURL) MarshalText() ([]byte, error) { // nolint: unparam + return []byte(c.String()), nil +} + +func (c TypeURL) String() string { + if c.value == nil { + return "" + } + + return c.value.String() +} + +func (c TypeURL) Value(defaultValue *url.URL) *url.URL { + if c.value == nil { + return defaultValue + } + + return c.value +} diff --git a/utils.go b/utils.go index 1411635..84b008d 100644 --- a/utils.go +++ b/utils.go @@ -8,10 +8,11 @@ import ( "net/http" "net/url" + "github.com/9seconds/mtg/v2/config" "github.com/9seconds/mtg/v2/mtglib/network" ) -func makeNetwork(conf *config) (network.Network, error) { +func makeNetwork(conf *config.Config) (network.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() From 6b28488fbdfe8f3dc359b35a35751f285a7582e2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 05:50:04 +0300 Subject: [PATCH 040/206] Move cli to separate package --- cli.go | 36 --------- cli_access.go => cli/access.go | 29 ++++--- cli/base.go | 77 +++++++++++++++++++ cli/cli.go | 11 +++ .../generate_secret.go | 8 +- main.go | 9 +-- mtglib/network/init.go | 2 +- 7 files changed, 114 insertions(+), 58 deletions(-) delete mode 100644 cli.go rename cli_access.go => cli/access.go (84%) create mode 100644 cli/base.go create mode 100644 cli/cli.go rename cli_generate_secret.go => cli/generate_secret.go (79%) diff --git a/cli.go b/cli.go deleted file mode 100644 index 78373d3..0000000 --- a/cli.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "fmt" - "io/ioutil" - - "github.com/9seconds/mtg/v2/config" - "github.com/9seconds/mtg/v2/mtglib/network" -) - -type cli struct { - network network.Network - conf *config.Config -} - -func (c *cli) ReadConfig(path string) error { - content, err := ioutil.ReadFile(path) - if err != nil { - return fmt.Errorf("cannot read config file: %w", err) - } - - conf, err := config.Parse(content) - if err != nil { - return fmt.Errorf("cannot parse config: %w", err) - } - - ntw, err := makeNetwork(conf) - if err != nil { - return fmt.Errorf("cannot build a network: %w", err) - } - - c.conf = conf - c.network = ntw - - return nil -} diff --git a/cli_access.go b/cli/access.go similarity index 84% rename from cli_access.go rename to cli/access.go index 7036231..b8fdc97 100644 --- a/cli_access.go +++ b/cli/access.go @@ -1,9 +1,10 @@ -package main +package cli import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -30,14 +31,14 @@ type accessResponseURLs struct { TmeQrCode string `json:"tme_qrcode"` } -type cliCommandAccess struct { - cli +type Access struct { + base ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` } -func (c *cliCommandAccess) Run(cli *CLI) error { +func (c *Access) Run(cli *CLI) error { if err := c.ReadConfig(cli.Access.ConfigPath); err != nil { return fmt.Errorf("cannot init config: %w", err) } @@ -72,7 +73,7 @@ func (c *cliCommandAccess) Run(cli *CLI) error { return nil } -func (c *cliCommandAccess) getIP(protocol string) net.IP { +func (c *Access) getIP(protocol string) net.IP { client := c.network.MakeHTTPClient(0) client.Transport = &http.Transport{ DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { @@ -80,9 +81,14 @@ func (c *cliCommandAccess) getIP(protocol string) net.IP { }, } - c.network.PatchHTTPClient(client) + c.network.PrepareHTTPClient(client) - resp, err := client.Get("https://ifconfig.co") // nolint: bodyclose, noctx + req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) + if err != nil { + panic(err) + } + + resp, err := client.Do(req) if err != nil { return nil } @@ -91,7 +97,10 @@ func (c *cliCommandAccess) getIP(protocol string) net.IP { return nil } - defer exhaustResponse(resp) + defer func() { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + }() data, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -101,7 +110,7 @@ func (c *cliCommandAccess) getIP(protocol string) net.IP { return net.ParseIP(strings.TrimSpace(string(data))) } -func (c *cliCommandAccess) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { +func (c *Access) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { if ip == nil { return nil } @@ -140,7 +149,7 @@ func (c *cliCommandAccess) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { return rv } -func (c *cliCommandAccess) makeQRCode(data string) string { +func (c *Access) makeQRCode(data string) string { values := url.Values{} values.Set("qzone", "4") diff --git a/cli/base.go b/cli/base.go new file mode 100644 index 0000000..7736592 --- /dev/null +++ b/cli/base.go @@ -0,0 +1,77 @@ +package cli + +import ( + "fmt" + "io/ioutil" + "net" + "net/url" + + "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/mtglib/network" +) + +type base struct { + network network.Network + conf *config.Config +} + +func (b *base) ReadConfig(path string) error { + content, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("cannot read config file: %w", err) + } + + conf, err := config.Parse(content) + if err != nil { + return fmt.Errorf("cannot parse config: %w", err) + } + + ntw, err := b.makeNetwork(conf) + if err != nil { + return fmt.Errorf("cannot build a network: %w", err) + } + + b.conf = conf + b.network = ntw + + return nil +} + +func (b *base) makeNetwork(conf *config.Config) (network.Network, error) { + tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) + idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) + dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() + bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) + + baseDialer, err := network.NewDefaultDialer(tcpTimeout, int(bufferSize)) + if err != nil { + return nil, fmt.Errorf("cannot build a default dialer: %w", err) + } + + proxyURLs := make([]*url.URL, 0, len(conf.Network.Proxies)) + + for _, v := range conf.Network.Proxies { + if value := v.Value(nil); value != nil { + proxyURLs = append(proxyURLs, v.Value(nil)) + } + } + + switch len(proxyURLs) { + case 0: + return network.NewNetwork(baseDialer, dohIP, idleTimeout) + case 1: + socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) + if err != nil { + return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) + } + + return network.NewNetwork(socksDialer, dohIP, idleTimeout) + } + + socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) + if err != nil { + return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) + } + + return network.NewNetwork(socksDialer, dohIP, idleTimeout) +} diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 0000000..9f27d72 --- /dev/null +++ b/cli/cli.go @@ -0,0 +1,11 @@ +package cli + +import ( + "github.com/alecthomas/kong" +) + +type CLI struct { + GenerateSecret GenerateSecret `cmd help:"Generate new proxy secret"` // nolint: govet + Access Access `cmd help:"Print access information."` // nolint: govet + Version kong.VersionFlag `help:"Print version."` +} diff --git a/cli_generate_secret.go b/cli/generate_secret.go similarity index 79% rename from cli_generate_secret.go rename to cli/generate_secret.go index 41f8b04..a92e9cc 100644 --- a/cli_generate_secret.go +++ b/cli/generate_secret.go @@ -1,4 +1,4 @@ -package main +package cli import ( "fmt" @@ -6,14 +6,14 @@ import ( "github.com/9seconds/mtg/v2/mtglib" ) -type cliCommandGenerateSecret struct { - cli +type GenerateSecret struct { + base HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet Hex bool `help:"Print secret in hex encoding."` } -func (c *cliCommandGenerateSecret) Run(cli *CLI) error { // nolint: unparam +func (c *GenerateSecret) Run(cli *CLI) error { // nolint: unparam secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) if cli.GenerateSecret.Hex { diff --git a/main.go b/main.go index b4cf3f6..622ed5c 100644 --- a/main.go +++ b/main.go @@ -4,21 +4,16 @@ import ( "math/rand" "time" + "github.com/9seconds/mtg/v2/cli" "github.com/alecthomas/kong" ) var version = "dev" // has to be set by ldflags -type CLI struct { - GenerateSecret cliCommandGenerateSecret `cmd help:"Generate new proxy secret"` // nolint: govet - Access cliCommandAccess `cmd help:"Print access information."` // nolint: govet - Version kong.VersionFlag `help:"Print version."` -} - func main() { rand.Seed(time.Now().UTC().UnixNano()) - cli := &CLI{} + cli := &cli.CLI{} ctx := kong.Parse(cli, kong.Vars{ "domain_front": "amazonaws.com", "version": version, diff --git a/mtglib/network/init.go b/mtglib/network/init.go index d922ce1..9be2dcc 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -39,5 +39,5 @@ type Network interface { DNSResolve(network, hostname string) (ips []string, err error) MakeHTTPClient(timeout time.Duration) *http.Client IdleTimeout() time.Duration - PatchHTTPClient(*http.Client) + PrepareHTTPClient(*http.Client) } From d6566e5dfbaf4ec7ccf5fdda22d8bb641bcc19db Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 06:08:15 +0300 Subject: [PATCH 041/206] Change network to accept DialFunc --- cli/access.go | 24 +++++---------- cli/base.go | 13 ++++---- cli/cli.go | 4 +-- cli/generate_secret.go | 2 +- config/type_bytes.go | 2 +- config/type_duration.go | 2 +- config/type_float.go | 2 +- config/type_hostport.go | 6 ++-- config/type_http_path.go | 4 +-- config/type_ip.go | 2 +- config/type_metric_prefix.go | 2 +- config/type_port.go | 4 +-- config/type_prefer_ip.go | 2 +- config/type_url.go | 2 +- main.go | 2 +- mtglib/network/init.go | 9 +++--- mtglib/network/network.go | 45 +++++++++++++++++++--------- utils.go | 57 ------------------------------------ 18 files changed, 67 insertions(+), 117 deletions(-) delete mode 100644 utils.go diff --git a/cli/access.go b/cli/access.go index b8fdc97..4e13f89 100644 --- a/cli/access.go +++ b/cli/access.go @@ -38,8 +38,8 @@ type Access struct { Hex bool `help:"Print secret in hex encoding."` } -func (c *Access) Run(cli *CLI) error { - if err := c.ReadConfig(cli.Access.ConfigPath); err != nil { +func (c *Access) Run(cli *CLI, version string) error { + if err := c.ReadConfig(cli.Access.ConfigPath, version); err != nil { return fmt.Errorf("cannot init config: %w", err) } @@ -74,21 +74,11 @@ func (c *Access) Run(cli *CLI) error { } func (c *Access) getIP(protocol string) net.IP { - client := c.network.MakeHTTPClient(0) - client.Transport = &http.Transport{ - DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return c.network.DialContext(ctx, protocol, address) - }, - } + client := c.network.MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error) { + return c.network.DialContext(ctx, protocol, address) + }) - c.network.PrepareHTTPClient(client) - - req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) - if err != nil { - panic(err) - } - - resp, err := client.Do(req) + resp, err := client.Get("https://ifconfig.co") // nolint: noctx if err != nil { return nil } @@ -98,7 +88,7 @@ func (c *Access) getIP(protocol string) net.IP { } defer func() { - io.Copy(ioutil.Discard, resp.Body) + io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck resp.Body.Close() }() diff --git a/cli/base.go b/cli/base.go index 7736592..3f508b1 100644 --- a/cli/base.go +++ b/cli/base.go @@ -15,7 +15,7 @@ type base struct { conf *config.Config } -func (b *base) ReadConfig(path string) error { +func (b *base) ReadConfig(path, version string) error { content, err := ioutil.ReadFile(path) if err != nil { return fmt.Errorf("cannot read config file: %w", err) @@ -26,7 +26,7 @@ func (b *base) ReadConfig(path string) error { return fmt.Errorf("cannot parse config: %w", err) } - ntw, err := b.makeNetwork(conf) + ntw, err := b.makeNetwork(conf, version) if err != nil { return fmt.Errorf("cannot build a network: %w", err) } @@ -37,11 +37,12 @@ func (b *base) ReadConfig(path string) error { return nil } -func (b *base) makeNetwork(conf *config.Config) (network.Network, error) { +func (b *base) makeNetwork(conf *config.Config, version string) (network.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) + userAgent := "mtg/" + version baseDialer, err := network.NewDefaultDialer(tcpTimeout, int(bufferSize)) if err != nil { @@ -58,14 +59,14 @@ func (b *base) makeNetwork(conf *config.Config) (network.Network, error) { switch len(proxyURLs) { case 0: - return network.NewNetwork(baseDialer, dohIP, idleTimeout) + return network.NewNetwork(baseDialer, userAgent, dohIP, idleTimeout) case 1: socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) if err != nil { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, dohIP, idleTimeout) + return network.NewNetwork(socksDialer, userAgent, dohIP, idleTimeout) } socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) @@ -73,5 +74,5 @@ func (b *base) makeNetwork(conf *config.Config) (network.Network, error) { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, dohIP, idleTimeout) + return network.NewNetwork(socksDialer, userAgent, dohIP, idleTimeout) } diff --git a/cli/cli.go b/cli/cli.go index 9f27d72..9892043 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -1,8 +1,6 @@ package cli -import ( - "github.com/alecthomas/kong" -) +import "github.com/alecthomas/kong" type CLI struct { GenerateSecret GenerateSecret `cmd help:"Generate new proxy secret"` // nolint: govet diff --git a/cli/generate_secret.go b/cli/generate_secret.go index a92e9cc..f0f8f69 100644 --- a/cli/generate_secret.go +++ b/cli/generate_secret.go @@ -13,7 +13,7 @@ type GenerateSecret struct { Hex bool `help:"Print secret in hex encoding."` } -func (c *GenerateSecret) Run(cli *CLI) error { // nolint: unparam +func (c *GenerateSecret) Run(cli *CLI, _ string) error { secret := mtglib.GenerateSecret(cli.GenerateSecret.HostName) if cli.GenerateSecret.Hex { diff --git a/config/type_bytes.go b/config/type_bytes.go index 585ee86..49d0b06 100644 --- a/config/type_bytes.go +++ b/config/type_bytes.go @@ -30,7 +30,7 @@ func (c *TypeBytes) UnmarshalText(data []byte) error { return nil } -func (c TypeBytes) MarshalText() ([]byte, error) { // nolint: unparam +func (c TypeBytes) MarshalText() ([]byte, error) { return []byte(c.String()), nil } diff --git a/config/type_duration.go b/config/type_duration.go index f5dbbab..8971a25 100644 --- a/config/type_duration.go +++ b/config/type_duration.go @@ -29,7 +29,7 @@ func (c *TypeDuration) UnmarshalText(data []byte) error { return nil } -func (c TypeDuration) MarshalText() ([]byte, error) { // nolint: unparam +func (c TypeDuration) MarshalText() ([]byte, error) { return []byte(c.value.String()), nil } diff --git a/config/type_float.go b/config/type_float.go index bf5aff6..f30ccb4 100644 --- a/config/type_float.go +++ b/config/type_float.go @@ -24,7 +24,7 @@ func (c *TypeFloat) UnmarshalJSON(data []byte) error { return nil } -func (c *TypeFloat) MarshalText() ([]byte, error) { // nolint: unparam +func (c *TypeFloat) MarshalText() ([]byte, error) { return []byte(c.String()), nil } diff --git a/config/type_hostport.go b/config/type_hostport.go index 42a9fc7..e86149d 100644 --- a/config/type_hostport.go +++ b/config/type_hostport.go @@ -32,7 +32,7 @@ func (c *TypeHostPort) UnmarshalText(data []byte) error { return nil } -func (c TypeHostPort) MarshalText() ([]byte, error) { // nolint: unparam +func (c TypeHostPort) MarshalText() ([]byte, error) { return []byte(c.String()), nil } @@ -41,11 +41,11 @@ func (c TypeHostPort) String() string { } func (c TypeHostPort) HostValue(defaultValue net.IP) net.IP { - return c.host.Value(defaultValue) + return c.host.Value(defaultValue) } func (c TypeHostPort) PortValue(defaultValue uint) uint { - return c.port.Value(defaultValue) + return c.port.Value(defaultValue) } func (c TypeHostPort) Value(defaultHostValue net.IP, defaultPortValue uint) string { diff --git a/config/type_http_path.go b/config/type_http_path.go index 79d311a..fea406f 100644 --- a/config/type_http_path.go +++ b/config/type_http_path.go @@ -6,7 +6,7 @@ type TypeHTTPPath struct { value string } -func (c *TypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unparam +func (c *TypeHTTPPath) UnmarshalText(data []byte) error { if len(data) > 0 { c.value = "/" + strings.Trim(string(data), "/") } @@ -14,7 +14,7 @@ func (c *TypeHTTPPath) UnmarshalText(data []byte) error { // nolint: unparam return nil } -func (c TypeHTTPPath) MarshalText() ([]byte, error) { // nolint: unparam +func (c TypeHTTPPath) MarshalText() ([]byte, error) { return []byte(c.String()), nil } diff --git a/config/type_ip.go b/config/type_ip.go index e249a94..76d61b0 100644 --- a/config/type_ip.go +++ b/config/type_ip.go @@ -24,7 +24,7 @@ func (c *TypeIP) UnmarshalText(data []byte) error { return nil } -func (c *TypeIP) MarshalText() ([]byte, error) { // nolint: unparam +func (c *TypeIP) MarshalText() ([]byte, error) { return []byte(c.String()), nil } diff --git a/config/type_metric_prefix.go b/config/type_metric_prefix.go index adc1029..947099e 100644 --- a/config/type_metric_prefix.go +++ b/config/type_metric_prefix.go @@ -25,7 +25,7 @@ func (c *TypeMetricPrefix) UnmarshalText(data []byte) error { return nil } -func (c TypeMetricPrefix) MarshalText() ([]byte, error) { // nolint: unparam +func (c TypeMetricPrefix) MarshalText() ([]byte, error) { return []byte(c.String()), nil } diff --git a/config/type_port.go b/config/type_port.go index 41963e5..3c5ec15 100644 --- a/config/type_port.go +++ b/config/type_port.go @@ -28,8 +28,8 @@ func (c *TypePort) UnmarshalJSON(data []byte) error { return nil } -func (c *TypePort) MarshalJSON() ([]byte, error) { // nolint: unparam - return []byte(c.String()), nil +func (c *TypePort) MarshalJSON() ([]byte, error) { + return []byte(c.String()), nil } func (c TypePort) String() string { diff --git a/config/type_prefer_ip.go b/config/type_prefer_ip.go index fc5e034..b0763db 100644 --- a/config/type_prefer_ip.go +++ b/config/type_prefer_ip.go @@ -26,7 +26,7 @@ func (c *TypePreferIP) UnmarshalText(data []byte) error { return nil } -func (c TypePreferIP) MarshalText() ([]byte, error) { // nolint: unparam +func (c TypePreferIP) MarshalText() ([]byte, error) { return []byte(c.value), nil } diff --git a/config/type_url.go b/config/type_url.go index 38378e2..6452b2c 100644 --- a/config/type_url.go +++ b/config/type_url.go @@ -24,7 +24,7 @@ func (c *TypeURL) UnmarshalText(data []byte) error { return nil } -func (c *TypeURL) MarshalText() ([]byte, error) { // nolint: unparam +func (c *TypeURL) MarshalText() ([]byte, error) { return []byte(c.String()), nil } diff --git a/main.go b/main.go index 622ed5c..fc51371 100644 --- a/main.go +++ b/main.go @@ -19,5 +19,5 @@ func main() { "version": version, }) - ctx.FatalIfErrorf(ctx.Run(cli)) + ctx.FatalIfErrorf(ctx.Run(cli, version)) } diff --git a/mtglib/network/init.go b/mtglib/network/init.go index 9be2dcc..e66ce3b 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -11,7 +11,6 @@ import ( const ( DefaultTimeout = 10 * time.Second DefaultIdleTimeout = time.Minute - DefaultHTTPTimeout = 5 * time.Second DefaultBufferSize = 4096 ProxyDialerOpenThreshold = 5 @@ -20,7 +19,8 @@ const ( DefaultDOHHostname = "9.9.9.9" - DNSTimeout = 5 * time.Second + DNSTimeout = 5 * time.Second + HTTPTimeout = 10 * time.Second ) var ( @@ -28,6 +28,8 @@ var ( ErrCannotDialWithAllProxies = errors.New("cannot dial with all proxies") ) +type DialFunc func(ctx context.Context, protocol, address string) (net.Conn, error) + type Dialer interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) @@ -37,7 +39,6 @@ type Network interface { Dialer DNSResolve(network, hostname string) (ips []string, err error) - MakeHTTPClient(timeout time.Duration) *http.Client + MakeHTTPClient(DialFunc) *http.Client IdleTimeout() time.Duration - PrepareHTTPClient(*http.Client) } diff --git a/mtglib/network/network.go b/mtglib/network/network.go index d256dd0..853511b 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -12,10 +12,22 @@ import ( doh "github.com/babolivier/go-doh-client" ) +type networkHTTPTransport struct { + userAgent string + next http.RoundTripper +} + +func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", n.userAgent) + + return n.next.RoundTrip(req) +} + type network struct { - idleTimeout time.Duration dialer Dialer dns doh.Resolver + idleTimeout time.Duration + userAgent string } func (n *network) Dial(protocol, address string) (net.Conn, error) { @@ -107,23 +119,15 @@ func (n *network) IdleTimeout() time.Duration { return n.idleTimeout } -func (n *network) MakeHTTPClient(timeout time.Duration) *http.Client { - if timeout <= 0 { - timeout = DefaultHTTPTimeout +func (n *network) MakeHTTPClient(dialFunc DialFunc) *http.Client { + if dialFunc == nil { + dialFunc = n.DialContext } - return &http.Client{ - Timeout: timeout, - Transport: &http.Transport{ - DialContext: n.DialContext, - }, - } + return makeHTTPClient(n.userAgent, dialFunc) } -func (n *network) PatchHTTPClient(_ *http.Client) { -} - -func NewNetwork(dialer Dialer, dohHostname string, idleTimeout time.Duration) (Network, error) { +func NewNetwork(dialer Dialer, userAgent, dohHostname string, idleTimeout time.Duration) (Network, error) { switch { case idleTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout) @@ -138,6 +142,7 @@ func NewNetwork(dialer Dialer, dohHostname string, idleTimeout time.Duration) (N return &network{ dialer: dialer, idleTimeout: idleTimeout, + userAgent: userAgent, dns: doh.Resolver{ Host: dohHostname, Class: doh.IN, @@ -150,3 +155,15 @@ func NewNetwork(dialer Dialer, dohHostname string, idleTimeout time.Duration) (N }, }, nil } + +func makeHTTPClient(userAgent string, dialFunc DialFunc) *http.Client { + return &http.Client{ + Timeout: HTTPTimeout, + Transport: networkHTTPTransport{ + userAgent: userAgent, + next: &http.Transport{ + DialContext: dialFunc, + }, + }, + } +} diff --git a/utils.go b/utils.go deleted file mode 100644 index 84b008d..0000000 --- a/utils.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "net/url" - - "github.com/9seconds/mtg/v2/config" - "github.com/9seconds/mtg/v2/mtglib/network" -) - -func makeNetwork(conf *config.Config) (network.Network, error) { - tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) - idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) - dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() - bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) - - baseDialer, err := network.NewDefaultDialer(tcpTimeout, int(bufferSize)) - if err != nil { - return nil, fmt.Errorf("cannot build a default dialer: %w", err) - } - - proxyURLs := make([]*url.URL, 0, len(conf.Network.Proxies)) - - for _, v := range conf.Network.Proxies { - if value := v.Value(nil); value != nil { - proxyURLs = append(proxyURLs, v.Value(nil)) - } - } - - switch len(proxyURLs) { - case 0: - return network.NewNetwork(baseDialer, dohIP, idleTimeout) - case 1: - socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) - if err != nil { - return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) - } - - return network.NewNetwork(socksDialer, dohIP, idleTimeout) - } - - socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) - if err != nil { - return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) - } - - return network.NewNetwork(socksDialer, dohIP, idleTimeout) -} - -func exhaustResponse(response *http.Response) { - io.Copy(ioutil.Discard, response.Body) // nolint: errcheck - response.Body.Close() -} From 6b9b437a5a98eb58e415eb10117b355579252979 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 06:22:01 +0300 Subject: [PATCH 042/206] Correct usage of ifconfig.co --- cli/access.go | 63 +++++++++++++++++++++++++++++---------- mtglib/network/network.go | 17 ++++------- 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/cli/access.go b/cli/access.go index 4e13f89..43e13b9 100644 --- a/cli/access.go +++ b/cli/access.go @@ -12,6 +12,7 @@ import ( "os" "strconv" "strings" + "sync" ) type accessResponse struct { @@ -43,24 +44,47 @@ func (c *Access) Run(cli *CLI, version string) error { return fmt.Errorf("cannot init config: %w", err) } - ipv4 := c.conf.Network.PublicIP.IPv4.Value(nil) - ipv6 := c.conf.Network.PublicIP.IPv6.Value(nil) - - if ipv4 == nil { - ipv4 = c.getIP("tcp4") - } - - if ipv6 == nil { - ipv6 = c.getIP("tcp6") - } - - resp := accessResponse{ - IPv4: c.makeURLs(ipv4, cli), - IPv6: c.makeURLs(ipv6, cli), - } + wg := &sync.WaitGroup{} + resp := &accessResponse{} resp.Secret.Base64 = c.conf.Secret.Base64() resp.Secret.Hex = c.conf.Secret.Hex() + wg.Add(2) + + go func() { + defer wg.Done() + + ip := c.conf.Network.PublicIP.IPv4.Value(nil) + + if ip == nil { + ip = c.getIP("tcp4") + } + + if ip != nil { + ip = ip.To4() + } + + resp.IPv4 = c.makeURLs(ip, cli) + }() + + go func() { + defer wg.Done() + + ip := c.conf.Network.PublicIP.IPv4.Value(nil) + + if ip == nil { + ip = c.getIP("tcp6") + } + + if ip != nil { + ip = ip.To16() + } + + resp.IPv6 = c.makeURLs(ip, cli) + }() + + wg.Wait() + encoder := json.NewEncoder(os.Stdout) encoder.SetEscapeHTML(false) @@ -78,7 +102,14 @@ func (c *Access) getIP(protocol string) net.IP { return c.network.DialContext(ctx, protocol, address) }) - resp, err := client.Get("https://ifconfig.co") // nolint: noctx + req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) // nolint: noctx + if err != nil { + panic(err) + } + + req.Header.Add("Accept", "text/plain") + + resp, err := client.Do(req) if err != nil { return nil } diff --git a/mtglib/network/network.go b/mtglib/network/network.go index 853511b..5e29be7 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -124,7 +124,7 @@ func (n *network) MakeHTTPClient(dialFunc DialFunc) *http.Client { dialFunc = n.DialContext } - return makeHTTPClient(n.userAgent, dialFunc) + return makeHTTPClient(n.userAgent, HTTPTimeout, dialFunc) } func NewNetwork(dialer Dialer, userAgent, dohHostname string, idleTimeout time.Duration) (Network, error) { @@ -144,21 +144,16 @@ func NewNetwork(dialer Dialer, userAgent, dohHostname string, idleTimeout time.D idleTimeout: idleTimeout, userAgent: userAgent, dns: doh.Resolver{ - Host: dohHostname, - Class: doh.IN, - HTTPClient: &http.Client{ - Timeout: DNSTimeout, - Transport: &http.Transport{ - DialContext: dialer.DialContext, - }, - }, + Host: dohHostname, + Class: doh.IN, + HTTPClient: makeHTTPClient(userAgent, DNSTimeout, dialer.DialContext), }, }, nil } -func makeHTTPClient(userAgent string, dialFunc DialFunc) *http.Client { +func makeHTTPClient(userAgent string, timeout time.Duration, dialFunc DialFunc) *http.Client { return &http.Client{ - Timeout: HTTPTimeout, + Timeout: timeout, Transport: networkHTTPTransport{ userAgent: userAgent, next: &http.Transport{ From bc4b14c83d70b8af0050cb20a3e209d255a6e97c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 16:33:29 +0300 Subject: [PATCH 043/206] Add tests for type bytes --- config/type_bytes.go | 13 ++- config/type_bytes_test.go | 114 +++++++++++++++++++ mtglib/network/proxy_dialer_internal_test.go | 14 +-- mtglib/secret_test.go | 16 +-- 4 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 config/type_bytes_test.go diff --git a/config/type_bytes.go b/config/type_bytes.go index 49d0b06..deed46e 100644 --- a/config/type_bytes.go +++ b/config/type_bytes.go @@ -8,7 +8,7 @@ import ( ) type TypeBytes struct { - value uint + value units.Base2Bytes } func (c *TypeBytes) UnmarshalText(data []byte) error { @@ -16,7 +16,10 @@ func (c *TypeBytes) UnmarshalText(data []byte) error { return nil } - value, err := units.ParseStrictBytes(strings.ToUpper(string(data))) + normalizedData := strings.ToUpper(string(data)) + normalizedData = strings.ReplaceAll(normalizedData, "IB", "iB") + + value, err := units.ParseBase2Bytes(normalizedData) if err != nil { return fmt.Errorf("incorrect bytes value: %w", err) } @@ -25,7 +28,7 @@ func (c *TypeBytes) UnmarshalText(data []byte) error { return fmt.Errorf("%d should be positive number", value) } - c.value = uint(value) + c.value = value return nil } @@ -35,7 +38,7 @@ func (c TypeBytes) MarshalText() ([]byte, error) { } func (c TypeBytes) String() string { - return units.ToString(int64(c.value), 1024, "ib", "b") + return strings.ToLower(c.value.String()) } func (c TypeBytes) Value(defaultValue uint) uint { @@ -43,5 +46,5 @@ func (c TypeBytes) Value(defaultValue uint) uint { return defaultValue } - return c.value + return uint(c.value) } diff --git a/config/type_bytes_test.go b/config/type_bytes_test.go new file mode 100644 index 0000000..15428fb --- /dev/null +++ b/config/type_bytes_test.go @@ -0,0 +1,114 @@ +package config_test + +import ( + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeBytesTestStruct struct { + Value config.TypeBytes `json:"value"` +} + +type TypeBytesTestSuite struct { + suite.Suite +} + +func (suite *TypeBytesTestSuite) TestUnmarshalFail() { + testData := []string{ + "1m", + "1", + "-1kb", + "-1kib", + "-1QB", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeBytesTestStruct{})) + }) + } +} + +func (suite *TypeBytesTestSuite) TestUnmarshalOk() { + testData := map[string]uint{ + "1b": 1, + "1kb": 1024, + "1kib": 1024, + "2mb": 2 * 1024 * 1024, + "2mib": 2 * 1024 * 1024, + } + + for k, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": k, + }) + suite.NoError(err) + + suite.T().Run(k, func(t *testing.T) { + testStruct := &typeBytesTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, int(value), int(testStruct.Value.Value(0))) + }) + } +} + +func (suite *TypeBytesTestSuite) TestMarshalOk() { + testData := []string{ + "1b", + "1kib", + "2mib", + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeBytesTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, string(marshalled), name) + }) + } +} + +func (suite *TypeBytesTestSuite) TestValue() { + testStruct := &typeBytesTestStruct{} + + suite.EqualValues(0, testStruct.Value.Value(0)) + suite.EqualValues(1, testStruct.Value.Value(1)) + + data, err := json.Marshal(map[string]string{ + "value": "1kb", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(1024, testStruct.Value.Value(0)) + suite.EqualValues(1024, testStruct.Value.Value(1)) +} + +func TestTypeBytes(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeBytesTestSuite{}) +} diff --git a/mtglib/network/proxy_dialer_internal_test.go b/mtglib/network/proxy_dialer_internal_test.go index 1183908..1c4f4cc 100644 --- a/mtglib/network/proxy_dialer_internal_test.go +++ b/mtglib/network/proxy_dialer_internal_test.go @@ -1,11 +1,11 @@ package network import ( - "fmt" "net/url" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -49,13 +49,13 @@ func (suite *ProxyDialerTestSuite) TestOpenThreshold() { for _, v := range params { param := v - suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { + suite.T().Run(v, func(t *testing.T) { query.Set("open_threshold", param) suite.u.RawQuery = query.Encode() d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) - suite.EqualValues(ProxyDialerOpenThreshold, d.openThreshold) + assert.EqualValues(t, ProxyDialerOpenThreshold, d.openThreshold) }) } } @@ -66,13 +66,13 @@ func (suite *ProxyDialerTestSuite) TestHalfOpenTimeout() { for _, v := range params { param := v - suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { + suite.T().Run(v, func(t *testing.T) { query.Set("half_open_timeout", param) suite.u.RawQuery = query.Encode() d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) - suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + assert.EqualValues(t, ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) }) } } @@ -83,13 +83,13 @@ func (suite *ProxyDialerTestSuite) TestResetFailuresTimeout() { for _, v := range params { param := v - suite.T().Run(fmt.Sprintf("param=%s", v), func(t *testing.T) { + suite.T().Run(v, func(t *testing.T) { query.Set("reset_failures_timeout", param) suite.u.RawQuery = query.Encode() d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) - suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) + assert.EqualValues(t, ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) }) } } diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go index 568abf8..2019d7e 100644 --- a/mtglib/secret_test.go +++ b/mtglib/secret_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -32,16 +33,15 @@ func (suite *SecretTestSuite) TestParseSecret() { suite.T().Run(name, func(t *testing.T) { parsed, err := mtglib.ParseSecret(param) - suite.NoError(err) - suite.Equal(s.Key, parsed.Key) - suite.Equal(s.Host, parsed.Host) + assert.NoError(t, err) + assert.Equal(t, s.Key, parsed.Key) + assert.Equal(t, s.Host, parsed.Host) newSecret := mtglib.Secret{} - suite.NoError(newSecret.UnmarshalText([]byte(param))) - - suite.Equal(s.Key, newSecret.Key) - suite.Equal(s.Host, newSecret.Host) + assert.NoError(t, newSecret.UnmarshalText([]byte(param))) + assert.Equal(t, s.Key, newSecret.Key) + assert.Equal(t, s.Host, newSecret.Host) }) } } @@ -88,7 +88,7 @@ func (suite *SecretTestSuite) TestIncorrectSecret() { suite.T().Run(param, func(t *testing.T) { _, err := mtglib.ParseSecret(param) - suite.Error(err) + assert.Error(t, err) }) } } From d48e82be7c8eb32080b79bc210657544bf20686e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 17:24:57 +0300 Subject: [PATCH 044/206] Add tests for duration config type --- config/type_bytes_test.go | 2 +- config/type_duration_test.go | 112 +++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 config/type_duration_test.go diff --git a/config/type_bytes_test.go b/config/type_bytes_test.go index 15428fb..c6a8c40 100644 --- a/config/type_bytes_test.go +++ b/config/type_bytes_test.go @@ -59,7 +59,7 @@ func (suite *TypeBytesTestSuite) TestUnmarshalOk() { testStruct := &typeBytesTestStruct{} assert.NoError(t, json.Unmarshal(data, testStruct)) - assert.Equal(t, int(value), int(testStruct.Value.Value(0))) + assert.EqualValues(t, value, testStruct.Value.Value(0)) }) } } diff --git a/config/type_duration_test.go b/config/type_duration_test.go new file mode 100644 index 0000000..004ad52 --- /dev/null +++ b/config/type_duration_test.go @@ -0,0 +1,112 @@ +package config_test + +import ( + "encoding/json" + "testing" + "time" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeDurationTestStruct struct { + Value config.TypeDuration `json:"value"` +} + +type TypeDurationTestSuite struct { + suite.Suite +} + +func (suite *TypeDurationTestSuite) TestUnmarshalFail() { + testData := []string{ + "1t", + "1", + "-1s", + "-1h", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeDurationTestStruct{})) + }) + } +} + +func (suite *TypeDurationTestSuite) TestUnmarshalOk() { + testData := map[string]time.Duration{ + "1s": time.Second, + "1m": time.Minute, + "2h1s": 2*time.Hour + time.Second, + } + + for k, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": k, + }) + suite.NoError(err) + + suite.T().Run(k, func(t *testing.T) { + testStruct := &typeDurationTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.Value(0)) + }) + } +} + +func (suite *TypeDurationTestSuite) TestMarshalOk() { + testData := []string{ + "1s", + "1m0s", + "2h0m1s", + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeDurationTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, string(marshalled), name) + }) + } +} + +func (suite *TypeDurationTestSuite) TestValue() { + testStruct := &typeDurationTestStruct{} + + suite.EqualValues(0, testStruct.Value.Value(0)) + suite.Equal(time.Second, testStruct.Value.Value(time.Second)) + + data, err := json.Marshal(map[string]string{ + "value": "1s", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal(time.Second, testStruct.Value.Value(0)) + suite.Equal(time.Second, testStruct.Value.Value(time.Minute)) +} + +func TestTypeDuration(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeDurationTestSuite{}) +} From 5c4536c5911d8951375f62357005a912b824577a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 21:30:43 +0300 Subject: [PATCH 045/206] Add tests for config error rate type --- config/config.go | 6 +- config/{type_float.go => type_error_rate.go} | 16 +-- config/type_error_rate_test.go | 119 +++++++++++++++++++ 3 files changed, 130 insertions(+), 11 deletions(-) rename config/{type_float.go => type_error_rate.go} (50%) create mode 100644 config/type_error_rate_test.go diff --git a/config/config.go b/config/config.go index ab4f5fe..78626cb 100644 --- a/config/config.go +++ b/config/config.go @@ -22,9 +22,9 @@ type Config struct { AllowSkewness TypeDuration `json:"allow-skewness"` } `json:"time"` AntiReplay struct { - Enabled bool `json:"enabled"` - MaxSize TypeBytes `json:"max-size"` - ErrorRate TypeFloat `json:"error-rate"` + Enabled bool `json:"enabled"` + MaxSize TypeBytes `json:"max-size"` + ErrorRate TypeErrorRate `json:"error-rate"` } `json:"anti-replay"` } `json:"probes"` Network struct { diff --git a/config/type_float.go b/config/type_error_rate.go similarity index 50% rename from config/type_float.go rename to config/type_error_rate.go index f30ccb4..690d85e 100644 --- a/config/type_float.go +++ b/config/type_error_rate.go @@ -5,18 +5,18 @@ import ( "strconv" ) -type TypeFloat struct { +type TypeErrorRate struct { value float64 } -func (c *TypeFloat) UnmarshalJSON(data []byte) error { +func (c *TypeErrorRate) UnmarshalJSON(data []byte) error { value, err := strconv.ParseFloat(string(data), 64) if err != nil { return fmt.Errorf("incorrect float value: %w", err) } - if value < 0 { - return fmt.Errorf("%f should be positive", value) + if value <= 0 || value >= 100 { + return fmt.Errorf("%f should be 0 < x < 100", value) } c.value = value @@ -24,16 +24,16 @@ func (c *TypeFloat) UnmarshalJSON(data []byte) error { return nil } -func (c *TypeFloat) MarshalText() ([]byte, error) { +func (c *TypeErrorRate) MarshalText() ([]byte, error) { return []byte(c.String()), nil } -func (c TypeFloat) String() string { +func (c TypeErrorRate) String() string { return strconv.FormatFloat(c.value, 'f', -1, 64) } -func (c TypeFloat) Value(defaultValue float64) float64 { - if c.value < 0.00001 { +func (c TypeErrorRate) Value(defaultValue float64) float64 { + if c.value < 1e-8 { return defaultValue } diff --git a/config/type_error_rate_test.go b/config/type_error_rate_test.go new file mode 100644 index 0000000..971b787 --- /dev/null +++ b/config/type_error_rate_test.go @@ -0,0 +1,119 @@ +package config_test + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeErrorRateTestStruct struct { + Value config.TypeErrorRate `json:"value"` +} + +type TypeErrorRateTestSuite struct { + suite.Suite +} + +func (suite *TypeErrorRateTestSuite) TestUnmarshalFail() { + testData := []float64{ + 1000, + -100, + -0.0001, + } + + for _, v := range testData { + data, err := json.Marshal(map[string]float64{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatFloat(v, 'f', -1, 64), func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeErrorRateTestStruct{})) + }) + } +} + +func (suite *TypeErrorRateTestSuite) TestUnmarshalOk() { + testData := []float64{ + 1, + 55.5, + 0.0001, + 1e-6, + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]float64{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatFloat(v, 'f', -1, 64), func(t *testing.T) { + testStruct := &typeErrorRateTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.InEpsilon(t, value, testStruct.Value.Value(0), 1e-10) + }) + } +} + +func (suite *TypeErrorRateTestSuite) TestMarshalOk() { + testData := []float64{ + 1, + 55.5, + 0.0001, + 1e-6, + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]float64{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatFloat(v, 'f', -1, 64), func(t *testing.T) { + testStruct := &typeErrorRateTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + + parsed, err := strconv.ParseFloat(testStruct.Value.String(), 64) + assert.NoError(t, err) + assert.InEpsilon(t, value, parsed, 1e-10) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + + parsed, err = strconv.ParseFloat(string(marshalled), 64) + assert.NoError(t, err) + assert.InEpsilon(t, value, parsed, 1e-10) + }) + } +} + +func (suite *TypeErrorRateTestSuite) TestValue() { + testStruct := &typeErrorRateTestStruct{} + + suite.InEpsilon(1, testStruct.Value.Value(1), 1e-10) + suite.InEpsilon(2, testStruct.Value.Value(2), 1e-10) + + data, err := json.Marshal(map[string]float64{ + "value": 1, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.InEpsilon(1, testStruct.Value.Value(2), 1e-10) + suite.InEpsilon(1, testStruct.Value.Value(3), 1e-10) +} + +func TestTypeErrorRate(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeErrorRateTestSuite{}) +} From c0ab254acf37e25816ae1186686a0093c78528b2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 21:50:47 +0300 Subject: [PATCH 046/206] Minor formatting issues --- .golangci.toml | 2 +- cli/access.go | 10 ++-------- config/config.go | 14 ++++++-------- config/type_error_rate.go | 4 +++- config/type_metric_prefix.go | 1 - mtglib/network/circuit_breaker.go | 2 -- mtglib/network/circuit_breaker_internal_test.go | 2 -- mtglib/network/default_test.go | 6 ------ mtglib/network/load_balanced_socks5_test.go | 4 ---- mtglib/network/network.go | 1 - mtglib/network/proxy_dialer_internal_test.go | 14 ++++---------- mtglib/secret.go | 1 - mtglib/secret_test.go | 5 ----- 13 files changed, 16 insertions(+), 50 deletions(-) diff --git a/.golangci.toml b/.golangci.toml index 04dcc43..40560ac 100644 --- a/.golangci.toml +++ b/.golangci.toml @@ -10,4 +10,4 @@ format = "colored-line-number" [linters] enable-all = true -disable = ["gochecknoglobals", "gas", "gomnd", "goerr113", "exhaustivestruct"] +disable = ["gochecknoglobals", "gas", "goerr113", "exhaustivestruct"] diff --git a/cli/access.go b/cli/access.go index 43e13b9..6dc98b6 100644 --- a/cli/access.go +++ b/cli/access.go @@ -44,18 +44,17 @@ func (c *Access) Run(cli *CLI, version string) error { return fmt.Errorf("cannot init config: %w", err) } - wg := &sync.WaitGroup{} resp := &accessResponse{} resp.Secret.Base64 = c.conf.Secret.Base64() resp.Secret.Hex = c.conf.Secret.Hex() - wg.Add(2) + wg := &sync.WaitGroup{} + wg.Add(2) // nolint: gomnd go func() { defer wg.Done() ip := c.conf.Network.PublicIP.IPv4.Value(nil) - if ip == nil { ip = c.getIP("tcp4") } @@ -71,7 +70,6 @@ func (c *Access) Run(cli *CLI, version string) error { defer wg.Done() ip := c.conf.Network.PublicIP.IPv4.Value(nil) - if ip == nil { ip = c.getIP("tcp6") } @@ -86,7 +84,6 @@ func (c *Access) Run(cli *CLI, version string) error { wg.Wait() encoder := json.NewEncoder(os.Stdout) - encoder.SetEscapeHTML(false) encoder.SetIndent("", " ") @@ -137,7 +134,6 @@ func (c *Access) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { } values := url.Values{} - values.Set("server", ip.String()) values.Set("port", strconv.Itoa(int(c.conf.BindTo.PortValue(0)))) @@ -163,7 +159,6 @@ func (c *Access) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { RawQuery: urlQuery, }).String(), } - rv.TgQrCode = c.makeQRCode(rv.TgURL) rv.TmeQrCode = c.makeQRCode(rv.TmeURL) @@ -172,7 +167,6 @@ func (c *Access) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { func (c *Access) makeQRCode(data string) string { values := url.Values{} - values.Set("qzone", "4") values.Set("format", "svg") values.Set("data", data) diff --git a/config/config.go b/config/config.go index 78626cb..db7bf51 100644 --- a/config/config.go +++ b/config/config.go @@ -122,23 +122,21 @@ type configRaw struct { func Parse(rawData []byte) (*Config, error) { rawConf := &configRaw{} + jsonBuf := &bytes.Buffer{} + conf := &Config{} + + jsonEncoder := json.NewEncoder(jsonBuf) + jsonEncoder.SetEscapeHTML(false) + jsonEncoder.SetIndent("", "") if err := toml.Unmarshal(rawData, rawConf); err != nil { return nil, fmt.Errorf("cannot parse toml config: %w", err) } - jsonBuf := &bytes.Buffer{} - jsonEncoder := json.NewEncoder(jsonBuf) - - jsonEncoder.SetEscapeHTML(false) - jsonEncoder.SetIndent("", "") - if err := jsonEncoder.Encode(rawConf); err != nil { return nil, fmt.Errorf("cannot dump into interim format: %w", err) } - conf := &Config{} - if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { return nil, fmt.Errorf("cannot parse final config: %w", err) } diff --git a/config/type_error_rate.go b/config/type_error_rate.go index 690d85e..0214956 100644 --- a/config/type_error_rate.go +++ b/config/type_error_rate.go @@ -5,6 +5,8 @@ import ( "strconv" ) +const typeErrorRateIgnoreLess = 1e-8 + type TypeErrorRate struct { value float64 } @@ -33,7 +35,7 @@ func (c TypeErrorRate) String() string { } func (c TypeErrorRate) Value(defaultValue float64) float64 { - if c.value < 1e-8 { + if c.value < typeErrorRateIgnoreLess { return defaultValue } diff --git a/config/type_metric_prefix.go b/config/type_metric_prefix.go index 947099e..64b5f5b 100644 --- a/config/type_metric_prefix.go +++ b/config/type_metric_prefix.go @@ -15,7 +15,6 @@ func (c *TypeMetricPrefix) UnmarshalText(data []byte) error { } prefix := string(data) - if ok, err := regexp.MatchString("^[a-z0-9]+$", prefix); !ok || err != nil { return fmt.Errorf("incorrect metric prefix: %s", prefix) } diff --git a/mtglib/network/circuit_breaker.go b/mtglib/network/circuit_breaker.go index e2640fd..4a02bda 100644 --- a/mtglib/network/circuit_breaker.go +++ b/mtglib/network/circuit_breaker.go @@ -125,7 +125,6 @@ func (c *circuitBreakerDialer) switchState(state uint32) { } c.failuresCount = 0 - atomic.StoreUint32(&c.halfOpenAttempts, 0) atomic.StoreUint32(&c.state, state) } @@ -158,7 +157,6 @@ func (c *circuitBreakerDialer) tryHalfOpen() { func (c *circuitBreakerDialer) stopTimer(timerRef **time.Timer) { timer := *timerRef - if timer == nil { return } diff --git a/mtglib/network/circuit_breaker_internal_test.go b/mtglib/network/circuit_breaker_internal_test.go index 849addd..cc26064 100644 --- a/mtglib/network/circuit_breaker_internal_test.go +++ b/mtglib/network/circuit_breaker_internal_test.go @@ -51,7 +51,6 @@ func (suite *CircuitBreakerTestSuite) TestMultipleRunsOk() { Return(suite.connMock, nil) wg := &sync.WaitGroup{} - wg.Add(5) go func() { @@ -126,7 +125,6 @@ func (suite *CircuitBreakerTestSuite) TestHalfOpen() { time.Sleep(500 * time.Millisecond) conn, err := suite.d.DialContext(suite.ctx, "tcp", "127.0.0.2") - suite.NoError(err) suite.Equal("10.0.0.10:80", conn.RemoteAddr().String()) diff --git a/mtglib/network/default_test.go b/mtglib/network/default_test.go index ba0d779..c954456 100644 --- a/mtglib/network/default_test.go +++ b/mtglib/network/default_test.go @@ -20,7 +20,6 @@ func (suite *DefaultDialerTestSuite) SetupSuite() { suite.HTTPServerTestSuite.SetupSuite() d, err := network.NewDefaultDialer(0, 0) - suite.NoError(err) suite.d = d @@ -28,13 +27,11 @@ func (suite *DefaultDialerTestSuite) SetupSuite() { func (suite *DefaultDialerTestSuite) TestNegativeTimeout() { _, err := network.NewDefaultDialer(-1, 0) - suite.Error(err) } func (suite *DefaultDialerTestSuite) TestNegativeBufferSize() { _, err := network.NewDefaultDialer(0, -1) - suite.Error(err) } @@ -42,7 +39,6 @@ func (suite *DefaultDialerTestSuite) TestUnsupportedProtocol() { _, err := suite.d.DialContext(context.Background(), "udp", suite.HTTPServerAddress()) - suite.Error(err) } @@ -50,7 +46,6 @@ func (suite *DefaultDialerTestSuite) TestCannotDial() { _, err := suite.d.DialContext(context.Background(), "tcp", suite.HTTPServerAddress()+suite.HTTPServerAddress()) - suite.Error(err) } @@ -58,7 +53,6 @@ func (suite *DefaultDialerTestSuite) TestConnectOk() { conn, err := suite.d.DialContext(context.Background(), "tcp", suite.HTTPServerAddress()) - suite.NoError(err) suite.NotNil(conn) diff --git a/mtglib/network/load_balanced_socks5_test.go b/mtglib/network/load_balanced_socks5_test.go index f86e3f1..03dd6f5 100644 --- a/mtglib/network/load_balanced_socks5_test.go +++ b/mtglib/network/load_balanced_socks5_test.go @@ -28,12 +28,10 @@ func (suite *LoadBalancedSocks5TestSuite) SetupSuite() { func (suite *LoadBalancedSocks5TestSuite) SetupTest() { baseDialer, _ := network.NewDefaultDialer(0, 0) - lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ suite.MakeSocks5URL("user", "password"), suite.MakeSocks5URL("user2", "password"), }) - suite.NoError(err) suite.httpClient = suite.MakeHTTPClient(lbDialer) @@ -48,13 +46,11 @@ func (suite *LoadBalancedSocks5TestSuite) TestIncorrectURL() { _, err := network.NewLoadBalancedSocks5Dialer(&DialerMock{}, []*url.URL{ {Scheme: "http"}, }) - suite.Error(err) } func (suite *LoadBalancedSocks5TestSuite) TestCannotDial() { baseDialer := &DialerMock{} - baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.1:1080"). Times(network.ProxyDialerOpenThreshold). Return(&net.TCPConn{}, io.EOF) diff --git a/mtglib/network/network.go b/mtglib/network/network.go index 5e29be7..b4b0b71 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -49,7 +49,6 @@ func (n *network) DialContext(ctx context.Context, protocol, address string) (ne } var conn net.Conn - for _, v := range ips { conn, err = n.dialer.DialContext(ctx, protocol, net.JoinHostPort(v, port)) diff --git a/mtglib/network/proxy_dialer_internal_test.go b/mtglib/network/proxy_dialer_internal_test.go index 1c4f4cc..33aa2b7 100644 --- a/mtglib/network/proxy_dialer_internal_test.go +++ b/mtglib/network/proxy_dialer_internal_test.go @@ -22,7 +22,6 @@ func (suite *ProxyDialerTestSuite) SetupSuite() { func (suite *ProxyDialerTestSuite) TestSetupDefaults() { d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) - suite.EqualValues(ProxyDialerOpenThreshold, d.openThreshold) suite.EqualValues(ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) suite.EqualValues(ProxyDialerResetFailuresTimeout, d.resetFailuresTimeout) @@ -30,14 +29,12 @@ func (suite *ProxyDialerTestSuite) TestSetupDefaults() { func (suite *ProxyDialerTestSuite) TestSetupValuesAllOk() { query := url.Values{} - query.Set("open_threshold", "30") query.Set("reset_failures_timeout", "1s") query.Set("half_open_timeout", "2s") - suite.u.RawQuery = query.Encode() - d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) suite.EqualValues(30, d.openThreshold) suite.EqualValues(2*time.Second, d.halfOpenTimeout) suite.EqualValues(time.Second, d.resetFailuresTimeout) @@ -51,10 +48,9 @@ func (suite *ProxyDialerTestSuite) TestOpenThreshold() { param := v suite.T().Run(v, func(t *testing.T) { query.Set("open_threshold", param) - suite.u.RawQuery = query.Encode() - d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) assert.EqualValues(t, ProxyDialerOpenThreshold, d.openThreshold) }) } @@ -68,10 +64,9 @@ func (suite *ProxyDialerTestSuite) TestHalfOpenTimeout() { param := v suite.T().Run(v, func(t *testing.T) { query.Set("half_open_timeout", param) - suite.u.RawQuery = query.Encode() - d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) assert.EqualValues(t, ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) }) } @@ -85,10 +80,9 @@ func (suite *ProxyDialerTestSuite) TestResetFailuresTimeout() { param := v suite.T().Run(v, func(t *testing.T) { query.Set("reset_failures_timeout", param) - suite.u.RawQuery = query.Encode() - d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) + d := newProxyDialer(&DialerMock{}, suite.u).(*circuitBreakerDialer) assert.EqualValues(t, ProxyDialerHalfOpenTimeout, d.halfOpenTimeout) }) } diff --git a/mtglib/secret.go b/mtglib/secret.go index b2e59da..2c60e30 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -31,7 +31,6 @@ func (s Secret) MarshalText() ([]byte, error) { func (s *Secret) UnmarshalText(data []byte) error { text := string(data) - if text == "" { return ErrSecretEmpty } diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go index 2019d7e..1bb3f7b 100644 --- a/mtglib/secret_test.go +++ b/mtglib/secret_test.go @@ -32,13 +32,11 @@ func (suite *SecretTestSuite) TestParseSecret() { suite.T().Run(name, func(t *testing.T) { parsed, err := mtglib.ParseSecret(param) - assert.NoError(t, err) assert.Equal(t, s.Key, parsed.Key) assert.Equal(t, s.Host, parsed.Host) newSecret := mtglib.Secret{} - assert.NoError(t, newSecret.UnmarshalText([]byte(param))) assert.Equal(t, s.Key, newSecret.Key) assert.Equal(t, s.Host, newSecret.Host) @@ -67,7 +65,6 @@ func (suite *SecretTestSuite) TestMarshalData() { copy(s.Key[:], secretData) data, err := json.Marshal(&s) - suite.NoError(err) suite.Equal(string(data), `"7tEcbLvZ7-f-1bwNsiCwlmVnb29nbGUuY29t"`) } @@ -87,7 +84,6 @@ func (suite *SecretTestSuite) TestIncorrectSecret() { suite.T().Run(param, func(t *testing.T) { _, err := mtglib.ParseSecret(param) - assert.Error(t, err) }) } @@ -97,7 +93,6 @@ func (suite *SecretTestSuite) TestInvariant() { generated := mtglib.GenerateSecret("google.com") parsed, err := mtglib.ParseSecret(generated.Hex()) - suite.NoError(err) suite.Equal(generated.Key, parsed.Key) suite.Equal(generated.Host, parsed.Host) From f7dc54b652b035df7f8452581df53950150cd36a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 11 Mar 2021 22:07:01 +0300 Subject: [PATCH 047/206] Add tests for config type http path --- config/type_bytes_test.go | 2 +- config/type_duration_test.go | 2 +- config/type_http_path.go | 4 ++ config/type_http_path_test.go | 91 +++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 config/type_http_path_test.go diff --git a/config/type_bytes_test.go b/config/type_bytes_test.go index c6a8c40..6baec1c 100644 --- a/config/type_bytes_test.go +++ b/config/type_bytes_test.go @@ -87,7 +87,7 @@ func (suite *TypeBytesTestSuite) TestMarshalOk() { marshalled, err := testStruct.Value.MarshalText() assert.NoError(t, err) - assert.Equal(t, string(marshalled), name) + assert.Equal(t, name, string(marshalled)) }) } } diff --git a/config/type_duration_test.go b/config/type_duration_test.go index 004ad52..be9a8fe 100644 --- a/config/type_duration_test.go +++ b/config/type_duration_test.go @@ -85,7 +85,7 @@ func (suite *TypeDurationTestSuite) TestMarshalOk() { marshalled, err := testStruct.Value.MarshalText() assert.NoError(t, err) - assert.Equal(t, string(marshalled), name) + assert.Equal(t, name, string(marshalled)) }) } } diff --git a/config/type_http_path.go b/config/type_http_path.go index fea406f..56a7a4e 100644 --- a/config/type_http_path.go +++ b/config/type_http_path.go @@ -19,6 +19,10 @@ func (c TypeHTTPPath) MarshalText() ([]byte, error) { } func (c TypeHTTPPath) String() string { + if c.value == "" { + return "/" + } + return c.value } diff --git a/config/type_http_path_test.go b/config/type_http_path_test.go new file mode 100644 index 0000000..4c456ef --- /dev/null +++ b/config/type_http_path_test.go @@ -0,0 +1,91 @@ +package config_test + +import ( + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeHTTPPathTestStruct struct { + Value config.TypeHTTPPath `json:"value"` +} + +type TypeHTTPPathTestSuite struct { + suite.Suite +} + +func (suite *TypeHTTPPathTestSuite) TestUnmarshal() { + testData := []string{ + "/hello", + "hello", + "hello/", + "/hello/", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeHTTPPathTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, "/hello", testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeHTTPPathTestSuite) TestMarshalOk() { + testData := map[string]string{ + "": "/", + "/hello": "/hello", + "/hello/": "/hello", + "hello/": "/hello", + "hello": "/hello", + } + + for k, v := range testData { + toPass := k + compareWith := v + + data, err := json.Marshal(map[string]string{ + "value": toPass, + }) + suite.NoError(err) + + suite.T().Run(toPass, func(t *testing.T) { + testStruct := &typeHTTPPathTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, compareWith, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, compareWith, string(marshalled)) + }) + } +} + +func (suite *TypeHTTPPathTestSuite) TestValue() { + testStruct := &typeHTTPPathTestStruct{} + + suite.Equal("/hello", testStruct.Value.Value("/hello")) + + data, err := json.Marshal(map[string]string{ + "value": "/map", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("/map", testStruct.Value.Value("/hello")) +} + +func TestTypeHTTPPath(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeHTTPPathTestSuite{}) +} From dc81740bda9c7a2f65672b7c9ea525f8dd80519a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 09:53:27 +0300 Subject: [PATCH 048/206] Add tests for hostport type --- config/type_hostport.go | 15 ++++- config/type_hostport_test.go | 115 +++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 config/type_hostport_test.go diff --git a/config/type_hostport.go b/config/type_hostport.go index e86149d..4bc58f1 100644 --- a/config/type_hostport.go +++ b/config/type_hostport.go @@ -16,11 +16,17 @@ func (c *TypeHostPort) UnmarshalText(data []byte) error { return nil } - host, port, err := net.SplitHostPort(string(data)) + text := string(data) + + host, port, err := net.SplitHostPort(text) if err != nil { return fmt.Errorf("incorrect host:port syntax: %w", err) } + if port == "" { + return fmt.Errorf("port in %s host:port pair cannot be empty", text) + } + if err := c.port.UnmarshalJSON([]byte(port)); err != nil { return fmt.Errorf("incorrect port in host:port: %w", err) } @@ -52,5 +58,10 @@ func (c TypeHostPort) Value(defaultHostValue net.IP, defaultPortValue uint) stri host := c.HostValue(defaultHostValue) port := c.PortValue(defaultPortValue) - return net.JoinHostPort(host.String(), strconv.Itoa(int(port))) + hostStr := "" + if len(host) > 0 { + hostStr = host.String() + } + + return net.JoinHostPort(hostStr, strconv.Itoa(int(port))) } diff --git a/config/type_hostport_test.go b/config/type_hostport_test.go new file mode 100644 index 0000000..5cf1994 --- /dev/null +++ b/config/type_hostport_test.go @@ -0,0 +1,115 @@ +package config_test + +import ( + "encoding/json" + "net" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeHostPortTestStruct struct { + Value config.TypeHostPort `json:"value"` +} + +type TypeHostPortTestSuite struct { + suite.Suite +} + +func (suite *TypeHostPortTestSuite) TestUnmarshalFail() { + testData := []string{ + "10.0.0.10:aaa", + "10.0.0.10:", + ":", + "xxx", + "xxx:80", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeHostPortTestStruct{})) + }) + } +} + +func (suite *TypeHostPortTestSuite) TestUnmarshalOk() { + testData := []string{ + "10.0.0.10:80", + "0.0.0.0:80", + ":8000", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeHostPortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value(nil, 0)) + }) + } +} + +func (suite *TypeHostPortTestSuite) TestMarshalOk() { + testData := []string{ + "10.0.0.10:80", + "0.0.0.0:80", + ":8000", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeHostPortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, value, string(marshalled)) + }) + } +} + +func (suite *TypeHostPortTestSuite) TestValue() { + testStruct := &typeHostPortTestStruct{} + + suite.EqualValues("127.0.0.1:80", + testStruct.Value.Value(net.ParseIP("127.0.0.1"), 80)) + suite.EqualValues("127.1.0.1:80", + testStruct.Value.Value(net.ParseIP("127.1.0.1"), 80)) + + data, err := json.Marshal(map[string]string{ + "value": "127.0.0.1:80", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues("127.0.0.1:80", testStruct.Value.Value(nil, 0)) + suite.EqualValues("127.0.0.1:80", testStruct.Value.Value(net.ParseIP("10.0.0.10"), 3000)) +} + +func TestTypeHostPort(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeHostPortTestSuite{}) +} From 428010880e9adf761feb6d029301e29c3b97e534 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 10:28:08 +0300 Subject: [PATCH 049/206] Add tests for prefer ip config type --- config/type_ip.go | 6 +- config/type_prefer_ip.go | 9 ++- config/type_prefer_ip_test.go | 136 ++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 config/type_prefer_ip_test.go diff --git a/config/type_ip.go b/config/type_ip.go index 76d61b0..0fe61e7 100644 --- a/config/type_ip.go +++ b/config/type_ip.go @@ -29,11 +29,11 @@ func (c *TypeIP) MarshalText() ([]byte, error) { } func (c TypeIP) String() string { - if c.value == nil { - return "" + if len(c.value) > 0 { + return c.value.String() } - return c.value.String() + return "" } func (c TypeIP) Value(defaultValue net.IP) net.IP { diff --git a/config/type_prefer_ip.go b/config/type_prefer_ip.go index b0763db..3d150d4 100644 --- a/config/type_prefer_ip.go +++ b/config/type_prefer_ip.go @@ -5,6 +5,13 @@ import ( "strings" ) +const ( + TypePreferIPPreferIPv4 = "prefer-ipv4" + TypePreferIPPreferIPv6 = "prefer-ipv6" + TypePreferOnlyIPv4 = "only-ipv4" + TypePreferOnlyIPv6 = "only-ipv6" +) + type TypePreferIP struct { value string } @@ -17,7 +24,7 @@ func (c *TypePreferIP) UnmarshalText(data []byte) error { text := strings.ToLower(string(data)) switch text { - case "prefer-ipv4", "prefer-ipv6", "only-ipv4", "only-ipv6": + case TypePreferIPPreferIPv4, TypePreferIPPreferIPv6, TypePreferOnlyIPv4, TypePreferOnlyIPv6: c.value = text default: return fmt.Errorf("incorrect prefer-ip value: %s", string(data)) diff --git a/config/type_prefer_ip_test.go b/config/type_prefer_ip_test.go new file mode 100644 index 0000000..30eb5ea --- /dev/null +++ b/config/type_prefer_ip_test.go @@ -0,0 +1,136 @@ +package config_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typePreferIPTestStruct struct { + Value config.TypePreferIP `json:"value"` +} + +type TypePreferIPTestSuite struct { + suite.Suite +} + +func (suite *TypePreferIPTestSuite) TestUnmarshalFail() { + testData := []string{ + "p", + "ipv4", + "onlyipv4", + "ipv6prefer", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typePreferIPTestStruct{})) + }) + } +} + +func (suite *TypePreferIPTestSuite) TestUnmarshalOk() { + testData := []string{ + config.TypePreferIPPreferIPv4, + config.TypePreferIPPreferIPv6, + config.TypePreferOnlyIPv4, + config.TypePreferOnlyIPv6, + strings.ToUpper(config.TypePreferIPPreferIPv4), + strings.ToUpper(config.TypePreferIPPreferIPv6), + strings.ToUpper(config.TypePreferOnlyIPv4), + strings.ToUpper(config.TypePreferOnlyIPv6), + strings.ToLower(config.TypePreferIPPreferIPv4), + strings.ToLower(config.TypePreferIPPreferIPv6), + strings.ToLower(config.TypePreferOnlyIPv4), + strings.ToLower(config.TypePreferOnlyIPv6), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typePreferIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, + strings.ToLower(value), + testStruct.Value.Value(config.TypePreferIPPreferIPv4)) + }) + } +} + +func (suite *TypePreferIPTestSuite) TestMarshalOk() { + testData := []string{ + config.TypePreferIPPreferIPv4, + config.TypePreferIPPreferIPv6, + config.TypePreferOnlyIPv4, + config.TypePreferOnlyIPv6, + strings.ToUpper(config.TypePreferIPPreferIPv4), + strings.ToUpper(config.TypePreferIPPreferIPv6), + strings.ToUpper(config.TypePreferOnlyIPv4), + strings.ToUpper(config.TypePreferOnlyIPv6), + strings.ToLower(config.TypePreferIPPreferIPv4), + strings.ToLower(config.TypePreferIPPreferIPv6), + strings.ToLower(config.TypePreferOnlyIPv4), + strings.ToLower(config.TypePreferOnlyIPv6), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typePreferIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, strings.ToLower(value), testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, strings.ToLower(value), string(marshalled)) + }) + } +} + +func (suite *TypePreferIPTestSuite) TestValue() { + testStruct := &typePreferIPTestStruct{} + + suite.EqualValues(config.TypePreferIPPreferIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv4)) + suite.EqualValues(config.TypePreferIPPreferIPv6, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) + + data, err := json.Marshal(map[string]string{ + "value": config.TypePreferOnlyIPv4, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferOnlyIPv6)) + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) +} + +func TestTypePreferIP(t *testing.T) { + t.Parallel() + suite.Run(t, &TypePreferIPTestSuite{}) +} From 7ca143d35255319de7da135ff178a61c445ad97a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 12:16:35 +0300 Subject: [PATCH 050/206] Add tests for config type url --- config/type_url.go | 26 +++++++++++ config/type_url_test.go | 97 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 config/type_url_test.go diff --git a/config/type_url.go b/config/type_url.go index 6452b2c..ec19012 100644 --- a/config/type_url.go +++ b/config/type_url.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "net" "net/url" ) @@ -19,6 +20,31 @@ func (c *TypeURL) UnmarshalText(data []byte) error { return fmt.Errorf("incorrect URL: %w", err) } + switch value.Scheme { + case "http", "https", "socks5": + case "": + return fmt.Errorf("url %s has to have a schema", value) + default: + return fmt.Errorf("unsupported schema %s", value.Scheme) + } + + if value.Host == "" { + return fmt.Errorf("url %s has to have a host", value) + } + + if _, _, err := net.SplitHostPort(value.Host); err != nil { + switch value.Scheme { + case "http": + value.Host = net.JoinHostPort(value.Host, "80") + case "https": + value.Host = net.JoinHostPort(value.Host, "443") + case "socks5": + value.Host = net.JoinHostPort(value.Host, "1080") + default: + return fmt.Errorf("cannot set a default port for %s", value) + } + } + c.value = value return nil diff --git a/config/type_url_test.go b/config/type_url_test.go new file mode 100644 index 0000000..0ab8e5a --- /dev/null +++ b/config/type_url_test.go @@ -0,0 +1,97 @@ +package config_test + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeURLTestStruct struct { + Value config.TypeURL `json:"value"` +} + +type TypeURLTestSuite struct { + suite.Suite +} + +func (suite *TypeURLTestSuite) TestUnmarshalFail() { + testData := []string{ + "http:/aaa.com", + "ipv4", + "111", + "://111", + "http://aaa.com:xxx", + "gopher://aaa.com:888", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeURLTestStruct{})) + }) + } +} + +func (suite *TypeURLTestSuite) TestUnmarshalOk() { + testData := map[string]string{ + "https://10.0.0.10:80": "https://10.0.0.10:80", + "https://10.0.0.10:443": "https://10.0.0.10", + "http://10.0.0.10:8": "http://10.0.0.10:8", + "http://10.0.0.10:80": "http://10.0.0.10", + "socks5://10.0.0.10:1080": "socks5://10.0.0.10", + "socks5://10.0.0.10:888": "socks5://10.0.0.10:888", + } + + for k, v := range testData { + expected := k + actual := v + + data, err := json.Marshal(map[string]string{ + "value": actual, + }) + suite.NoError(err) + + suite.T().Run(actual, func(t *testing.T) { + testStruct := &typeURLTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, expected, testStruct.Value.Value(nil).String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, expected, string(marshalled)) + }) + } +} + +func (suite *TypeURLTestSuite) TestValue() { + testStruct := &typeURLTestStruct{} + + u1, _ := url.Parse("https://10.0.0.10:80") + u2, _ := url.Parse("https://10.1.0.10:80") + + suite.Equal("https://10.0.0.10:80", testStruct.Value.Value(u1).String()) + suite.Equal("https://10.1.0.10:80", testStruct.Value.Value(u2).String()) + + data, err := json.Marshal(map[string]string{ + "value": "http://127.0.0.1:80", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("http://127.0.0.1:80", testStruct.Value.Value(u1).String()) + suite.Equal("http://127.0.0.1:80", testStruct.Value.Value(u2).String()) +} + +func TestTypeURL(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeURLTestSuite{}) +} From 1218f18af977b57bd9db8956e821dc5ca54c96a8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 14:02:02 +0300 Subject: [PATCH 051/206] Add tests for config type port --- config/type_metric_prefix_test.go | 109 +++++++++++++++++++++++++++++ config/type_port.go | 2 +- config/type_port_test.go | 111 ++++++++++++++++++++++++++++++ config/type_url.go | 2 +- 4 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 config/type_metric_prefix_test.go create mode 100644 config/type_port_test.go diff --git a/config/type_metric_prefix_test.go b/config/type_metric_prefix_test.go new file mode 100644 index 0000000..1954f62 --- /dev/null +++ b/config/type_metric_prefix_test.go @@ -0,0 +1,109 @@ +package config_test + +import ( + "encoding/json" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeMetricPrefixTestStruct struct { + Value config.TypeMetricPrefix `json:"value"` +} + +type TypeMetricPrefixTestSuite struct { + suite.Suite +} + +func (suite *TypeMetricPrefixTestSuite) TestUnmarshalFail() { + testData := []string{ + "aaa.aaa", + "aaa-bbb", + "aaa:ccc", + "metric prefix", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeMetricPrefixTestStruct{})) + }) + } +} + +func (suite *TypeMetricPrefixTestSuite) TestUnmarshalOk() { + testData := []string{ + "mtg", + "mtg111", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeMetricPrefixTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeMetricPrefixTestSuite) TestMarshalOk() { + testData := []string{ + "mtg", + "mtg111", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeMetricPrefixTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, value, string(marshalled)) + }) + } +} + +func (suite *TypeMetricPrefixTestSuite) TestValue() { + testStruct := &typeMetricPrefixTestStruct{} + + suite.Equal("mtg", testStruct.Value.Value("mtg")) + suite.Equal("vvv", testStruct.Value.Value("vvv")) + + data, err := json.Marshal(map[string]string{ + "value": "aaa", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("aaa", testStruct.Value.Value("mtg")) + suite.Equal("aaa", testStruct.Value.Value("vvv")) +} + +func TestTypeMetricPrefix(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeMetricPrefixTestSuite{}) +} diff --git a/config/type_port.go b/config/type_port.go index 3c5ec15..ea02fbd 100644 --- a/config/type_port.go +++ b/config/type_port.go @@ -19,7 +19,7 @@ func (c *TypePort) UnmarshalJSON(data []byte) error { return fmt.Errorf("port number is not a number: %w", err) } - if intValue == 0 || intValue > 65536 { + if intValue == 0 || intValue >= 65536 { return fmt.Errorf("port number should be 0 < portNo < 65536: %d", intValue) } diff --git a/config/type_port_test.go b/config/type_port_test.go new file mode 100644 index 0000000..0bc4bfc --- /dev/null +++ b/config/type_port_test.go @@ -0,0 +1,111 @@ +package config_test + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typePortTestStruct struct { + Value config.TypePort `json:"value"` +} + +type TypePortTestSuite struct { + suite.Suite +} + +func (suite *TypePortTestSuite) TestUnmarshalFail() { + testData := []int{ + -1, + 1_000_000, + } + + for _, v := range testData { + data, err := json.Marshal(map[string]int{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.Itoa(v), func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typePortTestStruct{})) + }) + } +} + +func (suite *TypePortTestSuite) TestUnmarshalOk() { + testData := []int{ + 1, + 1_000, + 65535, + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]int{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.Itoa(v), func(t *testing.T) { + testStruct := &typePortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value(0)) + }) + } +} + +func (suite *TypePortTestSuite) TestMarshalOk() { + testData := map[string]int{ + "1": 1, + "1000": 1000, + "65535": 65535, + } + + for k, v := range testData { + name := k + value := v + + data, err := json.Marshal(map[string]int{ + "value": value, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typePortTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalJSON() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypePortTestSuite) TestValue() { + testStruct := &typePortTestStruct{} + + suite.EqualValues(0, testStruct.Value.Value(0)) + suite.EqualValues(1, testStruct.Value.Value(1)) + + data, err := json.Marshal(map[string]int{ + "value": 5, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(5, testStruct.Value.Value(0)) + suite.EqualValues(5, testStruct.Value.Value(1)) +} + +func TestTypePort(t *testing.T) { + t.Parallel() + suite.Run(t, &TypePortTestSuite{}) +} diff --git a/config/type_url.go b/config/type_url.go index ec19012..9250492 100644 --- a/config/type_url.go +++ b/config/type_url.go @@ -10,7 +10,7 @@ type TypeURL struct { value *url.URL } -func (c *TypeURL) UnmarshalText(data []byte) error { +func (c *TypeURL) UnmarshalText(data []byte) error { // nolint: cyclop if len(data) == 0 { return nil } From 4daf927a72fa00e32211e82dc54de8ccc41ad791 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 14:39:09 +0300 Subject: [PATCH 052/206] Add method for validation of the secret --- config/config.go | 4 ++-- mtglib/secret.go | 10 +++++++--- mtglib/secret_test.go | 11 +++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index db7bf51..8631cb2 100644 --- a/config/config.go +++ b/config/config.go @@ -55,8 +55,8 @@ type Config struct { } func (c *Config) Validate() error { - if len(c.Secret.Key) == 0 || c.Secret.Host == "" { - return fmt.Errorf("incorrect secret %s", c.Secret.String()) + if !c.Secret.Valid() { + return fmt.Errorf("invalid secret %s", c.Secret.String()) } return nil diff --git a/mtglib/secret.go b/mtglib/secret.go index 2c60e30..a5b438f 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -22,11 +22,11 @@ type Secret struct { } func (s Secret) MarshalText() ([]byte, error) { - if s.Key == secretEmptyKey { - return nil, nil + if s.Valid() { + return []byte(s.String()), nil } - return []byte(s.String()), nil + return nil, nil } func (s *Secret) UnmarshalText(data []byte) error { @@ -72,6 +72,10 @@ func (s *Secret) UnmarshalText(data []byte) error { return nil } +func (s Secret) Valid() bool { + return s.Key != secretEmptyKey && s.Host != "" +} + func (s Secret) String() string { return s.Base64() } diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go index 1bb3f7b..49f80cd 100644 --- a/mtglib/secret_test.go +++ b/mtglib/secret_test.go @@ -99,6 +99,17 @@ func (suite *SecretTestSuite) TestInvariant() { suite.Equal("google.com", parsed.Host) } +func (suite *SecretTestSuite) TestValid() { + s := mtglib.Secret{} + suite.False(s.Valid()) + + s.Key[0] = 1 + suite.False(s.Valid()) + + s.Host = "11" + suite.True(s.Valid()) +} + func TestSecret(t *testing.T) { t.Parallel() suite.Run(t, &SecretTestSuite{}) From 113b5ecbe006eb1403964dc78665429064c18896 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 15:08:36 +0300 Subject: [PATCH 053/206] Add tests for config type ip --- config/type_ip_test.go | 114 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 config/type_ip_test.go diff --git a/config/type_ip_test.go b/config/type_ip_test.go new file mode 100644 index 0000000..8b23fc8 --- /dev/null +++ b/config/type_ip_test.go @@ -0,0 +1,114 @@ +package config_test + +import ( + "encoding/json" + "net" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeIPTestStruct struct { + Value config.TypeIP `json:"value"` +} + +type TypeIPTestSuite struct { + suite.Suite +} + +func (suite *TypeIPTestSuite) TestUnmarshalFail() { + testData := []string{ + "0.0.10", + "10.0.0.10:", + "xxx:80", + "2001:0db8:85a3:0000:0000:8a2e:4", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeIPTestStruct{})) + }) + } +} + +func (suite *TypeIPTestSuite) TestUnmarshalOk() { + testData := []string{ + "0.0.0.0", + "10.0.0.10", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, + net.ParseIP(value).String(), + testStruct.Value.Value(nil).String()) + }) + } +} + +func (suite *TypeIPTestSuite) TestMarshalOk() { + testData := []string{ + "0.0.0.0", + "10.0.0.10", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + } + + for _, v := range testData { + value := net.ParseIP(v).String() + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeIPTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, value, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, value, string(marshalled)) + }) + } +} + +func (suite *TypeIPTestSuite) TestValue() { + testStruct := &typeIPTestStruct{} + + suite.Nil(testStruct.Value.Value(nil)) + suite.Equal("127.1.0.1", testStruct.Value.Value(net.ParseIP("127.1.0.1")).String()) + + data, err := json.Marshal(map[string]string{ + "value": "127.0.0.1", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("127.0.0.1", testStruct.Value.Value(nil).String()) + suite.Equal("127.0.0.1", testStruct.Value.Value(net.ParseIP("10.0.0.10")).String()) +} + +func TestTypeIP(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeIPTestSuite{}) +} From c0b6124d94118e181871b5615b4f89156c3fbbf9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 15:34:32 +0300 Subject: [PATCH 054/206] Add tests for configuration parsing --- config/config.go | 71 +++++++++++++++++--------------- config/config_test.go | 48 +++++++++++++++++++++ config/testdata/broken.toml | 1 + config/testdata/minimal.toml | 2 + config/testdata/only_secret.toml | 1 + example.config.toml | 4 +- 6 files changed, 91 insertions(+), 36 deletions(-) create mode 100644 config/config_test.go create mode 100644 config/testdata/broken.toml create mode 100644 config/testdata/minimal.toml create mode 100644 config/testdata/only_secret.toml diff --git a/config/config.go b/config/config.go index 8631cb2..0dd64ed 100644 --- a/config/config.go +++ b/config/config.go @@ -16,7 +16,7 @@ type Config struct { TCPBuffer TypeBytes `json:"tcp-buffer"` PreferIP TypePreferIP `json:"prefer-ip"` CloakPort TypePort `json:"cloak-port"` - Probes struct { + Defense struct { Time struct { Enabled bool `json:"enabled"` AllowSkewness TypeDuration `json:"allow-skewness"` @@ -26,7 +26,7 @@ type Config struct { MaxSize TypeBytes `json:"max-size"` ErrorRate TypeErrorRate `json:"error-rate"` } `json:"anti-replay"` - } `json:"probes"` + } `json:"defense"` Network struct { PublicIP struct { IPv4 TypeIP `json:"ipv4"` @@ -58,6 +58,9 @@ func (c *Config) Validate() error { if !c.Secret.Valid() { return fmt.Errorf("invalid secret %s", c.Secret.String()) } + if len(c.BindTo.HostValue(nil)) == 0 || c.BindTo.PortValue(0) == 0 { + return fmt.Errorf("incorrect bind-to parameter %s", c.BindTo.String()) + } return nil } @@ -76,48 +79,48 @@ func (c *Config) String() string { } type configRaw struct { - Debug bool `toml:"debug" json:"debug"` + Debug bool `toml:"debug" json:"debug,omitempty"` Secret string `toml:"secret" json:"secret"` BindTo string `toml:"bind-to" json:"bind-to"` - TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer"` - PreferIP string `toml:"prefer-ip" json:"prefer-ip"` - CloakPort uint `toml:"cloak-port" json:"cloak-port"` - Probes struct { + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` + CloakPort uint `toml:"cloak-port" json:"cloak-port,omitempty"` + Defense struct { Time struct { - Enabled bool `toml:"enabled" json:"enabled"` - AllowSkewness string `toml:"allow-skewness" json:"allow-skewness"` - } `toml:"time" json:"time"` + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + AllowSkewness string `toml:"allow-skewness" json:"allow-skewness,omitempty"` + } `toml:"time" json:"time,omitempty"` AntiReplay struct { - Enabled bool `toml:"enabled" json:"enabled"` - MaxSize string `toml:"max-size" json:"max-size"` - ErrorRate float64 `toml:"error-rate" json:"error-rate"` - } `toml:"anti-replay" json:"anti-replay"` - } `toml:"probes" json:"probes"` + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + MaxSize string `toml:"max-size" json:"max-size,omitempty"` + ErrorRate float64 `toml:"error-rate" json:"error-rate,omitempty"` + } `toml:"anti-replay" json:"anti-replay,omitempty"` + } `toml:"defense" json:"defense,omitempty"` Network struct { PublicIP struct { - IPv4 string `toml:"ipv4" json:"ipv4"` - IPv6 string `toml:"ipv6" json:"ipv6"` - } `toml:"public-ip" json:"public-ip"` + IPv4 string `toml:"ipv4" json:"ipv4,omitempty"` + IPv6 string `toml:"ipv6" json:"ipv6,omitempty"` + } `toml:"public-ip" json:"public-ip,omitempty"` Timeout struct { - TCP string `toml:"tcp" json:"tcp"` - Idle string `toml:"idle" json:"idle"` - } `toml:"timeout" json:"timeout"` - DOHIP string `toml:"doh-ip" json:"doh-ip"` - Proxies []string `toml:"proxies" json:"proxies"` - } `toml:"network" json:"network"` + TCP string `toml:"tcp" json:"tcp,omitempty"` + Idle string `toml:"idle" json:"idle,omitempty"` + } `toml:"timeout" json:"timeout,omitempty"` + DOHIP string `toml:"doh-ip" json:"doh-ip,omitempty"` + Proxies []string `toml:"proxies" json:"proxies,omitempty"` + } `toml:"network" json:"network,omitempty"` Stats struct { StatsD struct { - Enabled bool `toml:"enabled" json:"enabled"` - Address string `toml:"address" json:"address"` - MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` - } `toml:"statsd" json:"statsd"` + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + Address string `toml:"address" json:"address,omitempty"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix,omitempty"` + } `toml:"statsd" json:"statsd,omitempty"` Prometheus struct { - Enabled bool `toml:"enabled" json:"enabled"` - BindTo string `toml:"bind-to" json:"bind-to"` - HTTPPath string `toml:"http-path" json:"http-path"` - MetricPrefix string `toml:"metric-prefix" json:"metric-prefix"` - } `toml:"prometheus" json:"prometheus"` - } `toml:"stats" json:"stats"` + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + BindTo string `toml:"bind-to" json:"bind-to,omitempty"` + HTTPPath string `toml:"http-path" json:"http-path,omitempty"` + MetricPrefix string `toml:"metric-prefix" json:"metric-prefix,omitempty"` + } `toml:"prometheus" json:"prometheus,omitempty"` + } `toml:"stats" json:"stats,omitempty"` } func Parse(rawData []byte) (*Config, error) { diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..513ad7b --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,48 @@ +package config_test + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/suite" +) + +type ConfigTestSuite struct { + suite.Suite +} + +func (suite *ConfigTestSuite) ReadConfig(filename string) []byte { + data, err := ioutil.ReadFile(filepath.Join("testdata", filename)) + suite.NoError(err) + + return data +} + +func (suite *ConfigTestSuite) TestParseEmpty() { + _, err := config.Parse([]byte{}) + suite.Error(err) +} + +func (suite *ConfigTestSuite) TestParseBrokenToml() { + _, err := config.Parse(suite.ReadConfig("broken.toml")) + suite.Error(err) +} + +func (suite *ConfigTestSuite) TestParseOnlySecret() { + _, err := config.Parse(suite.ReadConfig("only_secret.toml")) + suite.Error(err) +} + +func (suite *ConfigTestSuite) TestParseMinimalConfig() { + conf, err := config.Parse(suite.ReadConfig("minimal.toml")) + suite.NoError(err) + suite.Equal("7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t", conf.Secret.Base64()) + suite.Equal("0.0.0.0:3128", conf.BindTo.String()) +} + +func TestConfig(t *testing.T) { + t.Parallel() + suite.Run(t, &ConfigTestSuite{}) +} diff --git a/config/testdata/broken.toml b/config/testdata/broken.toml new file mode 100644 index 0000000..d95f791 --- /dev/null +++ b/config/testdata/broken.toml @@ -0,0 +1 @@ +s = sdfsdfds diff --git a/config/testdata/minimal.toml b/config/testdata/minimal.toml new file mode 100644 index 0000000..9d0961a --- /dev/null +++ b/config/testdata/minimal.toml @@ -0,0 +1,2 @@ +secret = "7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t" +bind-to = "0.0.0.0:3128" diff --git a/config/testdata/only_secret.toml b/config/testdata/only_secret.toml new file mode 100644 index 0000000..f6b0bee --- /dev/null +++ b/config/testdata/only_secret.toml @@ -0,0 +1 @@ +secret = "7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t" diff --git a/example.config.toml b/example.config.toml index 59bc550..de9eeb6 100644 --- a/example.config.toml +++ b/example.config.toml @@ -119,7 +119,7 @@ idle = "1m" # # Please ensure that you have some ntp active on this host. Otherwise, # you can endup with badly performing proxy. -[probes.time] +[defense.time] # You can enable/disable that. A good idea is always enable. enabled = true # Time can be skewed by many reasons. So, this is a time interval @@ -132,7 +132,7 @@ allow-skewness = "5s" # mtg has a cache of some connection fingerprints. Actually, first bytes # of each connection. So, it stores them in some in-memory LRU+TTL cache. # You can configure this cache here. -[probes.anti-replay] +[defense.anti-replay] # You can enable/disable this feature. enabled = true # max size of such a cache. Please be aware that this number is From e167b85d3bc55a9ae9cea6617ed5ba59910e8390 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 16:19:52 +0300 Subject: [PATCH 055/206] Add HTTP timeout --- config/config.go | 2 ++ example.config.toml | 1 + 2 files changed, 3 insertions(+) diff --git a/config/config.go b/config/config.go index 0dd64ed..c97075b 100644 --- a/config/config.go +++ b/config/config.go @@ -34,6 +34,7 @@ type Config struct { } `json:"public-ip"` Timeout struct { TCP TypeDuration `json:"tcp"` + HTTP TypeDuration `json:"http"` Idle TypeDuration `json:"idle"` } `json:"timeout"` DOHIP TypeIP `json:"doh-ip"` @@ -103,6 +104,7 @@ type configRaw struct { } `toml:"public-ip" json:"public-ip,omitempty"` Timeout struct { TCP string `toml:"tcp" json:"tcp,omitempty"` + HTTP string `toml:"http" json:"http,omitempty"` Idle string `toml:"idle" json:"idle,omitempty"` } `toml:"timeout" json:"timeout,omitempty"` DOHIP string `toml:"doh-ip" json:"doh-ip,omitempty"` diff --git a/example.config.toml b/example.config.toml index de9eeb6..cf18a31 100644 --- a/example.config.toml +++ b/example.config.toml @@ -111,6 +111,7 @@ ipv6 = "" # https://www.ndss-symposium.org/wp-content/uploads/2020/02/23087-paper.pdf [network.timeout] tcp = "5s" +http = "10s" idle = "1m" # FakeTLS can compare timestamps to prevent probes. Each message has From 02643fb6bbac8e0da4cc46ce2fd48acac76eb570 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 16:21:52 +0300 Subject: [PATCH 056/206] Add concurrency parameter --- config/config.go | 30 ++++++++++++++++-------------- example.config.toml | 4 ++++ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/config/config.go b/config/config.go index c97075b..702c5f7 100644 --- a/config/config.go +++ b/config/config.go @@ -10,13 +10,14 @@ import ( ) type Config struct { - Debug bool `json:"debug"` - Secret mtglib.Secret `json:"secret"` - BindTo TypeHostPort `json:"bind-to"` - TCPBuffer TypeBytes `json:"tcp-buffer"` - PreferIP TypePreferIP `json:"prefer-ip"` - CloakPort TypePort `json:"cloak-port"` - Defense struct { + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` + BindTo TypeHostPort `json:"bind-to"` + TCPBuffer TypeBytes `json:"tcp-buffer"` + PreferIP TypePreferIP `json:"prefer-ip"` + CloakPort TypePort `json:"cloak-port"` + Concurrency uint `json:"concurrency"` + Defense struct { Time struct { Enabled bool `json:"enabled"` AllowSkewness TypeDuration `json:"allow-skewness"` @@ -80,13 +81,14 @@ func (c *Config) String() string { } type configRaw struct { - Debug bool `toml:"debug" json:"debug,omitempty"` - Secret string `toml:"secret" json:"secret"` - BindTo string `toml:"bind-to" json:"bind-to"` - TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` - PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` - CloakPort uint `toml:"cloak-port" json:"cloak-port,omitempty"` - Defense struct { + Debug bool `toml:"debug" json:"debug,omitempty"` + Secret string `toml:"secret" json:"secret"` + BindTo string `toml:"bind-to" json:"bind-to"` + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` + CloakPort uint `toml:"cloak-port" json:"cloak-port,omitempty"` + Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"` + Defense struct { Time struct { Enabled bool `toml:"enabled" json:"enabled,omitempty"` AllowSkewness string `toml:"allow-skewness" json:"allow-skewness,omitempty"` diff --git a/example.config.toml b/example.config.toml index cf18a31..a9ff45b 100644 --- a/example.config.toml +++ b/example.config.toml @@ -23,6 +23,10 @@ secret = "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c6561706973 # Host:port pair to run proxy on. bind-to = "0.0.0.0:3128" +# Defines how many concurrent connections are allowed to this proxy. +# All other incoming connections are going to be dropped. +concurrency = 8192 + # A size of user-space buffer for TCP to use. Since we do 2 connections, # then we have tcp-buffer * (4 + 2) per each connection: read/write for # each connection + 2 copy buffers to pump the data between sockets. From 1f862027af98d3f6dd669746fc9ab02afb05f01f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 16:53:15 +0300 Subject: [PATCH 057/206] Add blocklist to config --- config/config.go | 13 +++ config/config_test.go | 2 +- config/type_blocklist_uri.go | 68 +++++++++++++ config/type_blocklist_uri_test.go | 156 ++++++++++++++++++++++++++++++ example.config.toml | 24 +++++ 5 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 config/type_blocklist_uri.go create mode 100644 config/type_blocklist_uri_test.go diff --git a/config/config.go b/config/config.go index 702c5f7..3176867 100644 --- a/config/config.go +++ b/config/config.go @@ -27,6 +27,12 @@ type Config struct { MaxSize TypeBytes `json:"max-size"` ErrorRate TypeErrorRate `json:"error-rate"` } `json:"anti-replay"` + Blocklist struct { + Enabled bool `json:"enabled"` + DownloadConcurrency uint `json:"download-concurrency"` + URLs []TypeBlocklistURI `json:"urls"` + UpdateEach TypeDuration `json:"update-each"` + } `json:"blocklist"` } `json:"defense"` Network struct { PublicIP struct { @@ -60,6 +66,7 @@ func (c *Config) Validate() error { if !c.Secret.Valid() { return fmt.Errorf("invalid secret %s", c.Secret.String()) } + if len(c.BindTo.HostValue(nil)) == 0 || c.BindTo.PortValue(0) == 0 { return fmt.Errorf("incorrect bind-to parameter %s", c.BindTo.String()) } @@ -98,6 +105,12 @@ type configRaw struct { MaxSize string `toml:"max-size" json:"max-size,omitempty"` ErrorRate float64 `toml:"error-rate" json:"error-rate,omitempty"` } `toml:"anti-replay" json:"anti-replay,omitempty"` + Blocklist struct { + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + DownloadConcurrency uint `toml:"download-concurrency" json:"download-concurrency,omitempty"` + URLs []string `toml:"urls" json:"urls,omitempty"` + UpdateEach string `toml:"update-each" json:"update-each,omitempty"` + } `toml:"blocklist" json:"blocklist,omitempty"` } `toml:"defense" json:"defense,omitempty"` Network struct { PublicIP struct { diff --git a/config/config_test.go b/config/config_test.go index 513ad7b..3233c13 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,7 +39,7 @@ func (suite *ConfigTestSuite) TestParseMinimalConfig() { conf, err := config.Parse(suite.ReadConfig("minimal.toml")) suite.NoError(err) suite.Equal("7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t", conf.Secret.Base64()) - suite.Equal("0.0.0.0:3128", conf.BindTo.String()) + suite.Equal("0.0.0.0:3128", conf.BindTo.String()) } func TestConfig(t *testing.T) { diff --git a/config/type_blocklist_uri.go b/config/type_blocklist_uri.go new file mode 100644 index 0000000..6bcf6b5 --- /dev/null +++ b/config/type_blocklist_uri.go @@ -0,0 +1,68 @@ +package config + +import ( + "fmt" + "net/url" + "os" + "path/filepath" +) + +type TypeBlocklistURI struct { + value string +} + +func (c *TypeBlocklistURI) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := string(data) + if filepath.IsAbs(text) { + if _, err := os.Stat(text); os.IsNotExist(err) { + return fmt.Errorf("filepath %s does not exist", text) + } + + c.value = text + + return nil + } + + parsedURL, err := url.Parse(text) + if err != nil { + return fmt.Errorf("incorrect url: %w", err) + } + + switch parsedURL.Scheme { + case "http", "https": // nolint: goconst + default: + return fmt.Errorf("unknown schema %s", parsedURL.Scheme) + } + + if parsedURL.Host == "" { + return fmt.Errorf("incorrect url %s", text) + } + + c.value = parsedURL.String() + + return nil +} + +func (c TypeBlocklistURI) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + +func (c TypeBlocklistURI) String() string { + return c.value +} + +func (c TypeBlocklistURI) IsRemote() bool { + return !filepath.IsAbs(c.value) +} + +func (c TypeBlocklistURI) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/config/type_blocklist_uri_test.go b/config/type_blocklist_uri_test.go new file mode 100644 index 0000000..e0b8974 --- /dev/null +++ b/config/type_blocklist_uri_test.go @@ -0,0 +1,156 @@ +package config_test + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeBlocklistURITestStruct struct { + Value config.TypeBlocklistURI `json:"value"` +} + +type TypeBlocklistURITestSuite struct { + suite.Suite +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() { + rnd := make([]byte, 48) + + rand.Read(rnd) // nolint: errcheck + + unknownPath := base64.StdEncoding.EncodeToString(rnd) + + testData := []string{ + "1", + unknownPath, + "/" + unknownPath, + "http:/", + "gopher://lalalal", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeBlocklistURITestStruct{})) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalOk() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := []string{ + "http://lalala", + filepath.Join(dir, "config.go"), + "https://lalala", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestMarshalOk() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := []string{ + "http://lalalal", + filepath.Join(dir, "config.go"), + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestValue() { + testStruct := &typeBlocklistURITestStruct{} + + suite.Equal("http://lalala", testStruct.Value.Value("http://lalala")) + + data, err := json.Marshal(map[string]string{ + "value": "http://blablabla", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("http://blablabla", testStruct.Value.Value("")) +} + +func (suite *TypeBlocklistURITestSuite) TestIsRemote() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := map[bool]string{ + true: "http://lalalal", + false: filepath.Join(dir, "config.go"), + } + + for k, v := range testData { + ok := k + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatBool(ok), func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + assert.NoError(t, json.Unmarshal(data, testStruct)) + + if ok { + assert.True(t, testStruct.Value.IsRemote()) + } else { + assert.False(t, testStruct.Value.IsRemote()) + } + }) + } +} + +func TestTypeBlocklistURI(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeBlocklistURITestSuite{}) +} diff --git a/example.config.toml b/example.config.toml index a9ff45b..0522909 100644 --- a/example.config.toml +++ b/example.config.toml @@ -149,6 +149,30 @@ max-size = "16mb" # to maintain a desired error ratio. error-rate = 0.0001 +# You can protect proxies by using different blocklists. If client has +# ip from the given range, we do not try to do a proper handshake. We +# actually route it to fronting domain. So, this client will never ever +# have a chance to use mtg to access Telegram. +# +# Please remember that blocklists are initialized in async way. So, +# when you start a proxy, blocklists are empty, they are populated and +# processed in backgrounds. An error in any URL is ignored. +[defense.blocklist] +# You can enable/disable this feature. +enabled = true +# This is a limiter for concurrency. In order to protect website +# from overloading, we download files in this number of threads. +download-concurrency = 2 +# A list of URLs in FireHOL format (https://iplists.firehol.org/) +# You can provider links here (starts with https:// or http://) or +# path to a local file, but in this case it should be absolute. +urls = [ + # "https://iplists.firehol.org/files/firehol_level1.netset", + # "/local.file" +] +# How often do we need to update a blocklist set. +update-each = "1d" + # statsd statistics integration. [stats.statsd] # enabled/disabled From 2cc81219b179f7bf0bb582850a0e7568ffda0346 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 17:00:52 +0300 Subject: [PATCH 058/206] Use HTTP timeout from config --- cli/base.go | 7 ++++--- mtglib/network/init.go | 5 ++--- mtglib/network/network.go | 8 ++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cli/base.go b/cli/base.go index 3f508b1..4cf9d6d 100644 --- a/cli/base.go +++ b/cli/base.go @@ -40,6 +40,7 @@ func (b *base) ReadConfig(path, version string) error { func (b *base) makeNetwork(conf *config.Config, version string) (network.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) + httpTimeout := conf.Network.Timeout.HTTP.Value(network.DefaultHTTPTimeout) dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) userAgent := "mtg/" + version @@ -59,14 +60,14 @@ func (b *base) makeNetwork(conf *config.Config, version string) (network.Network switch len(proxyURLs) { case 0: - return network.NewNetwork(baseDialer, userAgent, dohIP, idleTimeout) + return network.NewNetwork(baseDialer, userAgent, dohIP, httpTimeout, idleTimeout) case 1: socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) if err != nil { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, userAgent, dohIP, idleTimeout) + return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout, idleTimeout) } socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) @@ -74,5 +75,5 @@ func (b *base) makeNetwork(conf *config.Config, version string) (network.Network return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, userAgent, dohIP, idleTimeout) + return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout, idleTimeout) } diff --git a/mtglib/network/init.go b/mtglib/network/init.go index e66ce3b..9378435 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -11,6 +11,7 @@ import ( const ( DefaultTimeout = 10 * time.Second DefaultIdleTimeout = time.Minute + DefaultHTTPTimeout = 10 * time.Second DefaultBufferSize = 4096 ProxyDialerOpenThreshold = 5 @@ -18,9 +19,7 @@ const ( ProxyDialerResetFailuresTimeout = 10 * time.Second DefaultDOHHostname = "9.9.9.9" - - DNSTimeout = 5 * time.Second - HTTPTimeout = 10 * time.Second + DNSTimeout = 5 * time.Second ) var ( diff --git a/mtglib/network/network.go b/mtglib/network/network.go index b4b0b71..f5fc8ed 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -27,6 +27,7 @@ type network struct { dialer Dialer dns doh.Resolver idleTimeout time.Duration + httpTimeout time.Duration userAgent string } @@ -123,10 +124,12 @@ func (n *network) MakeHTTPClient(dialFunc DialFunc) *http.Client { dialFunc = n.DialContext } - return makeHTTPClient(n.userAgent, HTTPTimeout, dialFunc) + return makeHTTPClient(n.userAgent, n.httpTimeout, dialFunc) } -func NewNetwork(dialer Dialer, userAgent, dohHostname string, idleTimeout time.Duration) (Network, error) { +func NewNetwork(dialer Dialer, + userAgent, dohHostname string, + httpTimeout, idleTimeout time.Duration) (Network, error) { switch { case idleTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout) @@ -141,6 +144,7 @@ func NewNetwork(dialer Dialer, userAgent, dohHostname string, idleTimeout time.D return &network{ dialer: dialer, idleTimeout: idleTimeout, + httpTimeout: httpTimeout, userAgent: userAgent, dns: doh.Resolver{ Host: dohHostname, From 546a5849f3b20797c7fab981a93ade287950889c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 20:07:18 +0300 Subject: [PATCH 059/206] Add test for generate-secret command --- cli/base_internal_test.go | 31 ++++++++++ cli/generate_secret_test.go | 50 ++++++++++++++++ cli/init_test.go | 112 ++++++++++++++++++++++++++++++++++++ cli/testdata/minimal.toml | 2 + go.mod | 1 + go.sum | 2 + mtglib/network/init.go | 1 + mtglib/network/network.go | 12 ++-- 8 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 cli/base_internal_test.go create mode 100644 cli/generate_secret_test.go create mode 100644 cli/init_test.go create mode 100644 cli/testdata/minimal.toml diff --git a/cli/base_internal_test.go b/cli/base_internal_test.go new file mode 100644 index 0000000..1cd0b97 --- /dev/null +++ b/cli/base_internal_test.go @@ -0,0 +1,31 @@ +package cli + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/suite" +) + +type BaseTestSuite struct { + suite.Suite + + b base +} + +func (suite *BaseTestSuite) SetupTest() { + suite.b = base{} +} + +func (suite *BaseTestSuite) TestReadConfigNok() { + suite.Error(suite.b.ReadConfig(filepath.Join("testdata", "unknown"), "dev")) +} + +func (suite *BaseTestSuite) TestReadConfig() { + suite.NoError(suite.b.ReadConfig(filepath.Join("testdata", "minimal.toml"), "dev")) +} + +func TestBase(t *testing.T) { + t.Parallel() + suite.Run(t, &BaseTestSuite{}) +} diff --git a/cli/generate_secret_test.go b/cli/generate_secret_test.go new file mode 100644 index 0000000..7e32775 --- /dev/null +++ b/cli/generate_secret_test.go @@ -0,0 +1,50 @@ +package cli_test + +import ( + "strings" + "testing" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type GenerateSecretTestSuite struct { + CommonTestSuite +} + +func (suite *GenerateSecretTestSuite) SetupTest() { + suite.CommonTestSuite.SetupTest() + + suite.cli.GenerateSecret.HostName = "google.com" +} + +func (suite *GenerateSecretTestSuite) TestDefault() { + output := suite.CaptureStdout(func() { + suite.NoError(suite.cli.GenerateSecret.Run(suite.cli, "dev")) + }) + suite.True(strings.HasPrefix(output, "7")) + + secret, err := mtglib.ParseSecret(output) + suite.NoError(err) + suite.True(secret.Valid()) + suite.Equal("google.com", secret.Host) +} + +func (suite *GenerateSecretTestSuite) TestHex() { + suite.cli.GenerateSecret.Hex = true + + output := suite.CaptureStdout(func() { + suite.NoError(suite.cli.GenerateSecret.Run(suite.cli, "dev")) + }) + suite.True(strings.HasPrefix(output, "ee")) + + secret, err := mtglib.ParseSecret(output) + suite.NoError(err) + suite.True(secret.Valid()) + suite.Equal("google.com", secret.Host) +} + +func TestGenerateSecret(t *testing.T) { + t.Parallel() + suite.Run(t, &GenerateSecretTestSuite{}) +} diff --git a/cli/init_test.go b/cli/init_test.go new file mode 100644 index 0000000..a1894b6 --- /dev/null +++ b/cli/init_test.go @@ -0,0 +1,112 @@ +package cli_test + +import ( + "bytes" + "context" + "io" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/9seconds/mtg/v2/cli" + "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type NetworkMock struct { + mock.Mock +} + +func (n *NetworkMock) Dial(network, address string) (net.Conn, error) { + args := n.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (n *NetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := n.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (n *NetworkMock) DNSResolve(network, hostname string) ([]string, error) { + args := n.Called(network, hostname) + + return args.Get(0).([]string), args.Error(1) +} + +func (n *NetworkMock) MakeHTTPClient(dialFunc network.DialFunc) *http.Client { + return n.Called(dialFunc).Get(0).(*http.Client) +} + +func (n *NetworkMock) IdleTimeout() time.Duration { + return n.Called().Get(0).(time.Duration) +} + +func (n *NetworkMock) HTTPTimeout() time.Duration { + return n.Called().Get(0).(time.Duration) +} + +type CommonTestSuite struct { + suite.Suite + + cli *cli.CLI + networkMock *NetworkMock + httpClient *http.Client +} + +func (suite *CommonTestSuite) SetupTest() { + suite.networkMock = &NetworkMock{} + suite.httpClient = &http.Client{} + suite.cli = &cli.CLI{} + + httpmock.ActivateNonDefault(suite.httpClient) + + suite.networkMock. + On("MakeHTTPClient", mock.Anything). + Maybe(). + Return(suite.httpClient) +} + +func (suite *CommonTestSuite) TearDownTest() { + suite.networkMock.AssertExpectations(suite.T()) + httpmock.DeactivateAndReset() +} + +func (suite *CommonTestSuite) CaptureStdout(callback func()) string { + return suite.captureOutput(&os.Stdout, callback) +} + +func (suite *CommonTestSuite) CaptureStderr(callback func()) string { + return suite.captureOutput(&os.Stderr, callback) +} + +func (suite *CommonTestSuite) captureOutput(filefp **os.File, callback func()) string { + oldFp := *filefp + + defer func() { + *filefp = oldFp + }() + + reader, writer, _ := os.Pipe() + buf := &bytes.Buffer{} + closeChan := make(chan bool) + + go func() { + io.Copy(buf, reader) // nolint: errcheck + close(closeChan) + }() + + *filefp = writer + + callback() + + writer.Close() + <-closeChan + + return strings.TrimSpace(buf.String()) +} diff --git a/cli/testdata/minimal.toml b/cli/testdata/minimal.toml new file mode 100644 index 0000000..ea35f78 --- /dev/null +++ b/cli/testdata/minimal.toml @@ -0,0 +1,2 @@ +secret = "7mqFMMq3P2Tvvt_rPx5qhmFnb29nbGUuY29t" +bind-to = "0.0.0.0:80" diff --git a/go.mod b/go.mod index 20cbeb9..6486b6b 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 + github.com/jarcoal/httpmock v1.0.8 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 diff --git a/go.sum b/go.sum index 73e7830..17aca99 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= +github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/mtglib/network/init.go b/mtglib/network/init.go index 9378435..82416af 100644 --- a/mtglib/network/init.go +++ b/mtglib/network/init.go @@ -40,4 +40,5 @@ type Network interface { DNSResolve(network, hostname string) (ips []string, err error) MakeHTTPClient(DialFunc) *http.Client IdleTimeout() time.Duration + HTTPTimeout() time.Duration } diff --git a/mtglib/network/network.go b/mtglib/network/network.go index f5fc8ed..1835eb0 100644 --- a/mtglib/network/network.go +++ b/mtglib/network/network.go @@ -115,10 +115,6 @@ func (n *network) DNSResolve(protocol, address string) ([]string, error) { return ips, nil } -func (n *network) IdleTimeout() time.Duration { - return n.idleTimeout -} - func (n *network) MakeHTTPClient(dialFunc DialFunc) *http.Client { if dialFunc == nil { dialFunc = n.DialContext @@ -127,6 +123,14 @@ func (n *network) MakeHTTPClient(dialFunc DialFunc) *http.Client { return makeHTTPClient(n.userAgent, n.httpTimeout, dialFunc) } +func (n *network) IdleTimeout() time.Duration { + return n.idleTimeout +} + +func (n *network) HTTPTimeout() time.Duration { + return n.httpTimeout +} + func NewNetwork(dialer Dialer, userAgent, dohHostname string, httpTimeout, idleTimeout time.Duration) (Network, error) { From 185baf6bc908dc9b8bc25b44a1df39a9239b8217 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 21:57:53 +0300 Subject: [PATCH 060/206] Add tests for access command --- cli/access.go | 38 +++++--- cli/access_test.go | 205 +++++++++++++++++++++++++++++++++++++++++ cli/base.go | 8 +- cli/cli.go | 6 +- cli/generate_secret.go | 6 +- go.mod | 5 +- go.sum | 10 +- main.go | 3 +- 8 files changed, 252 insertions(+), 29 deletions(-) create mode 100644 cli/access_test.go diff --git a/cli/access.go b/cli/access.go index 6dc98b6..9ae7af5 100644 --- a/cli/access.go +++ b/cli/access.go @@ -26,6 +26,7 @@ type accessResponse struct { type accessResponseURLs struct { IP net.IP `json:"ip"` + Port uint `json:"port"` TgURL string `json:"tg_url"` TgQrCode string `json:"tg_qrcode"` TmeURL string `json:"tme_url"` @@ -33,10 +34,11 @@ type accessResponseURLs struct { } type Access struct { - base + base `kong:"-"` - ConfigPath string `arg required type:"existingfile" help:"Path to the configuration file." name:"config-path"` // nolint: lll, govet - Hex bool `help:"Print secret in hex encoding."` + ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll + Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint'"` + Hex bool `kong:"help='Print secret in hex encoding.'"` } func (c *Access) Run(cli *CLI, version string) error { @@ -44,9 +46,13 @@ func (c *Access) Run(cli *CLI, version string) error { return fmt.Errorf("cannot init config: %w", err) } + return c.Execute(cli) +} + +func (c *Access) Execute(cli *CLI) error { resp := &accessResponse{} - resp.Secret.Base64 = c.conf.Secret.Base64() - resp.Secret.Hex = c.conf.Secret.Hex() + resp.Secret.Base64 = c.Config.Secret.Base64() + resp.Secret.Hex = c.Config.Secret.Hex() wg := &sync.WaitGroup{} wg.Add(2) // nolint: gomnd @@ -54,7 +60,7 @@ func (c *Access) Run(cli *CLI, version string) error { go func() { defer wg.Done() - ip := c.conf.Network.PublicIP.IPv4.Value(nil) + ip := c.Config.Network.PublicIP.IPv4.Value(nil) if ip == nil { ip = c.getIP("tcp4") } @@ -69,7 +75,7 @@ func (c *Access) Run(cli *CLI, version string) error { go func() { defer wg.Done() - ip := c.conf.Network.PublicIP.IPv4.Value(nil) + ip := c.Config.Network.PublicIP.IPv6.Value(nil) if ip == nil { ip = c.getIP("tcp6") } @@ -95,8 +101,8 @@ func (c *Access) Run(cli *CLI, version string) error { } func (c *Access) getIP(protocol string) net.IP { - client := c.network.MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error) { - return c.network.DialContext(ctx, protocol, address) + client := c.Network.MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error) { + return c.Network.DialContext(ctx, protocol, address) }) req, err := http.NewRequest(http.MethodGet, "https://ifconfig.co", nil) // nolint: noctx @@ -133,20 +139,26 @@ func (c *Access) makeURLs(ip net.IP, cli *CLI) *accessResponseURLs { return nil } + portNo := cli.Access.Port + if portNo == 0 { + portNo = c.Config.BindTo.PortValue(0) + } + values := url.Values{} values.Set("server", ip.String()) - values.Set("port", strconv.Itoa(int(c.conf.BindTo.PortValue(0)))) + values.Set("port", strconv.Itoa(int(portNo))) if cli.Access.Hex { - values.Set("secret", c.conf.Secret.Hex()) + values.Set("secret", c.Config.Secret.Hex()) } else { - values.Set("secret", c.conf.Secret.Base64()) + values.Set("secret", c.Config.Secret.Base64()) } urlQuery := values.Encode() rv := &accessResponseURLs{ - IP: ip, + IP: ip, + Port: portNo, TgURL: (&url.URL{ Scheme: "tg", Host: "proxy", diff --git a/cli/access_test.go b/cli/access_test.go new file mode 100644 index 0000000..1a06351 --- /dev/null +++ b/cli/access_test.go @@ -0,0 +1,205 @@ +package cli_test + +import ( + "net/http" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/suite" + "github.com/xeipuuv/gojsonschema" +) + +var accressResponseJSONSchema = func() *gojsonschema.Schema { + schema, err := gojsonschema.NewSchema(gojsonschema.NewStringLoader(` +{ + "type": "object", + "required": ["secret"], + "additionalProperties": true, + "properties": { + "secret": { + "type": "object", + "required": [ + "hex", + "base64" + ], + "additionalProperties": false, + "properties": { + "hex": { + "type": "string", + "minLength": 34 + }, + "base64": { + "type": "string", + "minLength": 10 + } + } + }, + "ipv4": { + "$ref": "#/definitions/ip" + }, + "ipv6": { + "$ref": "#/definitions/ip" + } + }, + "definitions": { + "ip": { + "type": "object", + "required": [ + "ip", + "port", + "tg_url", + "tg_qrcode", + "tme_url", + "tme_qrcode" + ], + "additionalProperties": false, + "properties": { + "ip": { + "type": "string", + "minLength": 1, + "anyOf": [ + { + "format": "ipv4" + }, + { + "format": "ipv6" + } + ] + }, + "port": { + "type": "integer", + "multipleOf": 1.0, + "exclusiveMinimum": 0, + "exclusiveMaximum": 65536 + }, + "tg_url": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "tg_qrcode": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "tme_url": { + "type": "string", + "minLength": 1, + "format": "uri" + }, + "tme_qrcode": { + "type": "string", + "minLength": 1, + "format": "uri" + } + } + } + } +} + `)) + + if err != nil { + panic(err) + } + + return schema +}() + +type AccessTestSuite struct { + CommonTestSuite +} + +func (suite *AccessTestSuite) SetupTest() { + suite.CommonTestSuite.SetupTest() + + suite.cli.Access.Config = &config.Config{} + suite.cli.Access.Config.Secret = mtglib.GenerateSecret("google.com") + suite.cli.Access.Network = suite.networkMock + + suite.NoError( + suite.cli.Access.Config.BindTo.UnmarshalText([]byte("0.0.0.0:80"))) +} + +func (suite *AccessTestSuite) TestGenerateNoCalls() { + suite.NoError( + suite.cli.Access.Config.Network.PublicIP.IPv4.UnmarshalText( + []byte("10.0.0.10"))) + suite.NoError( + suite.cli.Access.Config.Network.PublicIP.IPv6.UnmarshalText( + []byte("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))) + + output := suite.CaptureStdout(func() { + suite.NoError(suite.cli.Access.Execute(suite.cli)) + }) + + validated, err := accressResponseJSONSchema.Validate( + gojsonschema.NewStringLoader(output)) + suite.NoError(err) + suite.Empty(validated.Errors()) + suite.True(validated.Valid()) + + suite.Contains(output, "10.0.0.10") + suite.Contains(output, "2001:db8:85a3::8a2e:370:7334") + suite.Contains(output, "ipv4") + suite.Contains(output, "ipv6") + suite.Contains(output, suite.cli.Access.Config.Secret.Base64()) + suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) +} + +func (suite *AccessTestSuite) TestGenerateIPv4Call() { + suite.NoError( + suite.cli.Access.Config.Network.PublicIP.IPv6.UnmarshalText( + []byte("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))) + + httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", + httpmock.NewStringResponder(http.StatusOK, "10.11.12.13")) + + output := suite.CaptureStdout(func() { + suite.NoError(suite.cli.Access.Execute(suite.cli)) + }) + + validated, err := accressResponseJSONSchema.Validate( + gojsonschema.NewStringLoader(output)) + suite.NoError(err) + suite.Empty(validated.Errors()) + suite.True(validated.Valid()) + + suite.Contains(output, "10.11.12.13") + suite.Contains(output, "2001:db8:85a3::8a2e:370:7334") + suite.Contains(output, "ipv4") + suite.Contains(output, "ipv6") + suite.Contains(output, suite.cli.Access.Config.Secret.Base64()) + suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) +} + +func (suite *AccessTestSuite) TestIPv4CallFail() { + suite.NoError( + suite.cli.Access.Config.Network.PublicIP.IPv6.UnmarshalText( + []byte("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))) + + httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", + httpmock.NewStringResponder(http.StatusForbidden, "")) + + output := suite.CaptureStdout(func() { + suite.NoError(suite.cli.Access.Execute(suite.cli)) + }) + + validated, err := accressResponseJSONSchema.Validate( + gojsonschema.NewStringLoader(output)) + suite.NoError(err) + suite.Empty(validated.Errors()) + suite.True(validated.Valid()) + + suite.Contains(output, "2001:db8:85a3::8a2e:370:7334") + suite.NotContains(output, "ipv4") + suite.Contains(output, "ipv6") + suite.Contains(output, suite.cli.Access.Config.Secret.Base64()) + suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) +} + +func TestAccess(t *testing.T) { + t.Parallel() + suite.Run(t, &AccessTestSuite{}) +} diff --git a/cli/base.go b/cli/base.go index 4cf9d6d..40348c9 100644 --- a/cli/base.go +++ b/cli/base.go @@ -11,8 +11,8 @@ import ( ) type base struct { - network network.Network - conf *config.Config + Network network.Network + Config *config.Config } func (b *base) ReadConfig(path, version string) error { @@ -31,8 +31,8 @@ func (b *base) ReadConfig(path, version string) error { return fmt.Errorf("cannot build a network: %w", err) } - b.conf = conf - b.network = ntw + b.Config = conf + b.Network = ntw return nil } diff --git a/cli/cli.go b/cli/cli.go index 9892043..19ff668 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -3,7 +3,7 @@ package cli import "github.com/alecthomas/kong" type CLI struct { - GenerateSecret GenerateSecret `cmd help:"Generate new proxy secret"` // nolint: govet - Access Access `cmd help:"Print access information."` // nolint: govet - Version kong.VersionFlag `help:"Print version."` + GenerateSecret GenerateSecret `kong:"cmd,help='Generate new proxy secret'"` // nolint: govet + Access Access `kong:"cmd,help='Print access information.'"` // nolint: govet + Version kong.VersionFlag `kong:"help='Print version.'"` } diff --git a/cli/generate_secret.go b/cli/generate_secret.go index f0f8f69..8a57251 100644 --- a/cli/generate_secret.go +++ b/cli/generate_secret.go @@ -7,10 +7,10 @@ import ( ) type GenerateSecret struct { - base + base `kong:"-"` - HostName string `arg optional help:"Hostname to use for domain fronting. Default is '${domain_front}'." name:"hostname" default:"${domain_front}"` // nolint: lll, govet - Hex bool `help:"Print secret in hex encoding."` + HostName string `kong:"arg,required,help='Hostname to use for domain fronting.',name='hostname'"` // nolint: lll, govet + Hex bool `kong:"help='Print secret in hex encoding.'"` } func (c *GenerateSecret) Run(cli *CLI, _ string) error { diff --git a/go.mod b/go.mod index 6486b6b..c59047b 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,15 @@ require ( github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 - github.com/jarcoal/httpmock v1.0.8 // indirect + github.com/jarcoal/httpmock v1.0.8 github.com/kr/pretty v0.1.0 // indirect github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 github.com/pelletier/go-toml v1.8.1 github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 + github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 - golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect + golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index 17aca99..dafe96e 100644 --- a/go.sum +++ b/go.sum @@ -36,12 +36,18 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go index fc51371..e74a760 100644 --- a/main.go +++ b/main.go @@ -15,8 +15,7 @@ func main() { cli := &cli.CLI{} ctx := kong.Parse(cli, kong.Vars{ - "domain_front": "amazonaws.com", - "version": version, + "version": version, }) ctx.FatalIfErrorf(ctx.Run(cli, version)) From 18e3ca77de9d0be5e8597ee60223628b54223587 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 12 Mar 2021 21:59:05 +0300 Subject: [PATCH 061/206] Fix lint issues --- cli/access.go | 4 ++-- cli/access_test.go | 1 - cli/cli.go | 4 ++-- cli/generate_secret.go | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cli/access.go b/cli/access.go index 9ae7af5..95e02cc 100644 --- a/cli/access.go +++ b/cli/access.go @@ -36,8 +36,8 @@ type accessResponseURLs struct { type Access struct { base `kong:"-"` - ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll - Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint'"` + ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll + Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint'"` // nolint: lll Hex bool `kong:"help='Print secret in hex encoding.'"` } diff --git a/cli/access_test.go b/cli/access_test.go index 1a06351..d442d4a 100644 --- a/cli/access_test.go +++ b/cli/access_test.go @@ -99,7 +99,6 @@ var accressResponseJSONSchema = func() *gojsonschema.Schema { } } `)) - if err != nil { panic(err) } diff --git a/cli/cli.go b/cli/cli.go index 19ff668..6cfb2b4 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -3,7 +3,7 @@ package cli import "github.com/alecthomas/kong" type CLI struct { - GenerateSecret GenerateSecret `kong:"cmd,help='Generate new proxy secret'"` // nolint: govet - Access Access `kong:"cmd,help='Print access information.'"` // nolint: govet + GenerateSecret GenerateSecret `kong:"cmd,help='Generate new proxy secret'"` + Access Access `kong:"cmd,help='Print access information.'"` Version kong.VersionFlag `kong:"help='Print version.'"` } diff --git a/cli/generate_secret.go b/cli/generate_secret.go index 8a57251..224ba85 100644 --- a/cli/generate_secret.go +++ b/cli/generate_secret.go @@ -7,9 +7,9 @@ import ( ) type GenerateSecret struct { - base `kong:"-"` + base `kong:"-"` - HostName string `kong:"arg,required,help='Hostname to use for domain fronting.',name='hostname'"` // nolint: lll, govet + HostName string `kong:"arg,required,help='Hostname to use for domain fronting.',name='hostname'"` Hex bool `kong:"help='Print secret in hex encoding.'"` } From 37a78bd1c3ca1cb6884c9a24ae0f43222120c483 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 13 Mar 2021 21:30:57 +0300 Subject: [PATCH 062/206] More reasonable cli for access --- cli/access.go | 13 ++++++++----- cli/access_test.go | 17 +++++------------ cli/cli.go | 2 +- cli/generate_secret.go | 2 +- config/config.go | 8 -------- example.config.toml | 9 +-------- 6 files changed, 16 insertions(+), 35 deletions(-) diff --git a/cli/access.go b/cli/access.go index 95e02cc..d6212b6 100644 --- a/cli/access.go +++ b/cli/access.go @@ -36,9 +36,12 @@ type accessResponseURLs struct { type Access struct { base `kong:"-"` - ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll - Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint'"` // nolint: lll - Hex bool `kong:"help='Print secret in hex encoding.'"` + PublicIPv4 net.IP `kong:"help='Public IPv4 address for proxy. By default it is resolved via remote website',name='ipv4',short='i'"` // nolint: lll + PublicIPv6 net.IP `kong:"help='Public IPv6 address for proxy. By default it is resolved via remote website',name='ipv6',short='I'"` // nolint: lll + Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint',short='p'"` // nolint: lll + Hex bool `kong:"help='Print secret in hex encoding.',short='x'"` + + ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll } func (c *Access) Run(cli *CLI, version string) error { @@ -60,7 +63,7 @@ func (c *Access) Execute(cli *CLI) error { go func() { defer wg.Done() - ip := c.Config.Network.PublicIP.IPv4.Value(nil) + ip := cli.Access.PublicIPv4 if ip == nil { ip = c.getIP("tcp4") } @@ -75,7 +78,7 @@ func (c *Access) Execute(cli *CLI) error { go func() { defer wg.Done() - ip := c.Config.Network.PublicIP.IPv6.Value(nil) + ip := cli.Access.PublicIPv6 if ip == nil { ip = c.getIP("tcp6") } diff --git a/cli/access_test.go b/cli/access_test.go index d442d4a..f1c1435 100644 --- a/cli/access_test.go +++ b/cli/access_test.go @@ -1,6 +1,7 @@ package cli_test import ( + "net" "net/http" "testing" @@ -122,12 +123,8 @@ func (suite *AccessTestSuite) SetupTest() { } func (suite *AccessTestSuite) TestGenerateNoCalls() { - suite.NoError( - suite.cli.Access.Config.Network.PublicIP.IPv4.UnmarshalText( - []byte("10.0.0.10"))) - suite.NoError( - suite.cli.Access.Config.Network.PublicIP.IPv6.UnmarshalText( - []byte("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))) + suite.cli.Access.PublicIPv4 = net.ParseIP("10.0.0.10") + suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") output := suite.CaptureStdout(func() { suite.NoError(suite.cli.Access.Execute(suite.cli)) @@ -148,9 +145,7 @@ func (suite *AccessTestSuite) TestGenerateNoCalls() { } func (suite *AccessTestSuite) TestGenerateIPv4Call() { - suite.NoError( - suite.cli.Access.Config.Network.PublicIP.IPv6.UnmarshalText( - []byte("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))) + suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", httpmock.NewStringResponder(http.StatusOK, "10.11.12.13")) @@ -174,9 +169,7 @@ func (suite *AccessTestSuite) TestGenerateIPv4Call() { } func (suite *AccessTestSuite) TestIPv4CallFail() { - suite.NoError( - suite.cli.Access.Config.Network.PublicIP.IPv6.UnmarshalText( - []byte("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))) + suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", httpmock.NewStringResponder(http.StatusForbidden, "")) diff --git a/cli/cli.go b/cli/cli.go index 6cfb2b4..f620e21 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -5,5 +5,5 @@ import "github.com/alecthomas/kong" type CLI struct { GenerateSecret GenerateSecret `kong:"cmd,help='Generate new proxy secret'"` Access Access `kong:"cmd,help='Print access information.'"` - Version kong.VersionFlag `kong:"help='Print version.'"` + Version kong.VersionFlag `kong:"help='Print version.',short='v'"` } diff --git a/cli/generate_secret.go b/cli/generate_secret.go index 224ba85..e618dfd 100644 --- a/cli/generate_secret.go +++ b/cli/generate_secret.go @@ -10,7 +10,7 @@ type GenerateSecret struct { base `kong:"-"` HostName string `kong:"arg,required,help='Hostname to use for domain fronting.',name='hostname'"` - Hex bool `kong:"help='Print secret in hex encoding.'"` + Hex bool `kong:"help='Print secret in hex encoding.',short='x'"` } func (c *GenerateSecret) Run(cli *CLI, _ string) error { diff --git a/config/config.go b/config/config.go index 3176867..dd7f9c8 100644 --- a/config/config.go +++ b/config/config.go @@ -35,10 +35,6 @@ type Config struct { } `json:"blocklist"` } `json:"defense"` Network struct { - PublicIP struct { - IPv4 TypeIP `json:"ipv4"` - IPv6 TypeIP `json:"ipv6"` - } `json:"public-ip"` Timeout struct { TCP TypeDuration `json:"tcp"` HTTP TypeDuration `json:"http"` @@ -113,10 +109,6 @@ type configRaw struct { } `toml:"blocklist" json:"blocklist,omitempty"` } `toml:"defense" json:"defense,omitempty"` Network struct { - PublicIP struct { - IPv4 string `toml:"ipv4" json:"ipv4,omitempty"` - IPv6 string `toml:"ipv6" json:"ipv6,omitempty"` - } `toml:"public-ip" json:"public-ip,omitempty"` Timeout struct { TCP string `toml:"tcp" json:"tcp,omitempty"` HTTP string `toml:"http" json:"http,omitempty"` diff --git a/example.config.toml b/example.config.toml index 0522909..c3d3fbb 100644 --- a/example.config.toml +++ b/example.config.toml @@ -98,13 +98,6 @@ proxies = [ # "socks5://user:password@host:port?open_threshold=5&half_open_timeout=1m&reset_failures_timeout=10s" ] -# public ip addresses of the server. Actually, it is required only to -# generate a correct access file. if you use default values here, mtg -# will try to resolve these IPs on its own. -[network.public-ip] -ipv4 = "" -ipv6 = "" - # network timeouts define different settings for timeouts. tcp timeout # define a global timeout on establishing of network connections. idle # means a timeout on pumping data between sockset when nothing is @@ -171,7 +164,7 @@ urls = [ # "/local.file" ] # How often do we need to update a blocklist set. -update-each = "1d" +update-each = "24h" # statsd statistics integration. [stats.statsd] From f8ad90c845b901ac59e60ef73673ad2ab35278e7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 14 Mar 2021 21:42:52 +0300 Subject: [PATCH 063/206] Refactor network to a top-level module --- cli/access_test.go | 10 +-- cli/base.go | 7 +- cli/generate_secret_test.go | 5 +- cli/init_test.go | 81 +------------------ go.mod | 1 + go.sum | 11 +++ mtglib/init.go | 29 ++++++- mtglib/network/init_internal_test.go | 65 --------------- .../network => network}/circuit_breaker.go | 0 .../circuit_breaker_internal_test.go | 5 +- {mtglib/network => network}/default.go | 0 {mtglib/network => network}/default_test.go | 2 +- {mtglib/network => network}/init.go | 12 --- network/init_internal_test.go | 24 ++++++ {mtglib/network => network}/init_test.go | 2 +- .../load_balanced_socks5.go | 0 .../load_balanced_socks5_test.go | 2 +- {mtglib/network => network}/network.go | 40 ++++----- {mtglib/network => network}/proxy_dialer.go | 0 .../proxy_dialer_internal_test.go | 0 {mtglib/network => network}/socks5.go | 0 {mtglib/network => network}/socks5_test.go | 2 +- testlib/capture_output.go | 42 ++++++++++ testlib/net_conn_mock.go | 48 +++++++++++ testlib/network_mock.go | 35 ++++++++ 25 files changed, 231 insertions(+), 192 deletions(-) delete mode 100644 mtglib/network/init_internal_test.go rename {mtglib/network => network}/circuit_breaker.go (100%) rename {mtglib/network => network}/circuit_breaker_internal_test.go (96%) rename {mtglib/network => network}/default.go (100%) rename {mtglib/network => network}/default_test.go (97%) rename {mtglib/network => network}/init.go (71%) create mode 100644 network/init_internal_test.go rename {mtglib/network => network}/init_test.go (97%) rename {mtglib/network => network}/load_balanced_socks5.go (100%) rename {mtglib/network => network}/load_balanced_socks5_test.go (98%) rename {mtglib/network => network}/network.go (86%) rename {mtglib/network => network}/proxy_dialer.go (100%) rename {mtglib/network => network}/proxy_dialer_internal_test.go (100%) rename {mtglib/network => network}/socks5.go (100%) rename {mtglib/network => network}/socks5_test.go (96%) create mode 100644 testlib/capture_output.go create mode 100644 testlib/net_conn_mock.go create mode 100644 testlib/network_mock.go diff --git a/cli/access_test.go b/cli/access_test.go index f1c1435..0cf0579 100644 --- a/cli/access_test.go +++ b/cli/access_test.go @@ -7,6 +7,7 @@ import ( "github.com/9seconds/mtg/v2/config" "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/testlib" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/suite" "github.com/xeipuuv/gojsonschema" @@ -126,7 +127,7 @@ func (suite *AccessTestSuite) TestGenerateNoCalls() { suite.cli.Access.PublicIPv4 = net.ParseIP("10.0.0.10") suite.cli.Access.PublicIPv6 = net.ParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334") - output := suite.CaptureStdout(func() { + output := testlib.CaptureStdout(func() { suite.NoError(suite.cli.Access.Execute(suite.cli)) }) @@ -150,7 +151,7 @@ func (suite *AccessTestSuite) TestGenerateIPv4Call() { httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", httpmock.NewStringResponder(http.StatusOK, "10.11.12.13")) - output := suite.CaptureStdout(func() { + output := testlib.CaptureStdout(func() { suite.NoError(suite.cli.Access.Execute(suite.cli)) }) @@ -174,7 +175,7 @@ func (suite *AccessTestSuite) TestIPv4CallFail() { httpmock.RegisterResponder(http.MethodGet, "https://ifconfig.co", httpmock.NewStringResponder(http.StatusForbidden, "")) - output := suite.CaptureStdout(func() { + output := testlib.CaptureStdout(func() { suite.NoError(suite.cli.Access.Execute(suite.cli)) }) @@ -191,7 +192,6 @@ func (suite *AccessTestSuite) TestIPv4CallFail() { suite.Contains(output, suite.cli.Access.Config.Secret.Hex()) } -func TestAccess(t *testing.T) { - t.Parallel() +func TestAccess(t *testing.T) { // nolint: paralleltest suite.Run(t, &AccessTestSuite{}) } diff --git a/cli/base.go b/cli/base.go index 40348c9..def25d8 100644 --- a/cli/base.go +++ b/cli/base.go @@ -7,11 +7,12 @@ import ( "net/url" "github.com/9seconds/mtg/v2/config" - "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/network" ) type base struct { - Network network.Network + Network mtglib.Network Config *config.Config } @@ -37,7 +38,7 @@ func (b *base) ReadConfig(path, version string) error { return nil } -func (b *base) makeNetwork(conf *config.Config, version string) (network.Network, error) { +func (b *base) makeNetwork(conf *config.Config, version string) (mtglib.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) httpTimeout := conf.Network.Timeout.HTTP.Value(network.DefaultHTTPTimeout) diff --git a/cli/generate_secret_test.go b/cli/generate_secret_test.go index 7e32775..5cbe18d 100644 --- a/cli/generate_secret_test.go +++ b/cli/generate_secret_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/suite" ) @@ -19,7 +20,7 @@ func (suite *GenerateSecretTestSuite) SetupTest() { } func (suite *GenerateSecretTestSuite) TestDefault() { - output := suite.CaptureStdout(func() { + output := testlib.CaptureStdout(func() { suite.NoError(suite.cli.GenerateSecret.Run(suite.cli, "dev")) }) suite.True(strings.HasPrefix(output, "7")) @@ -33,7 +34,7 @@ func (suite *GenerateSecretTestSuite) TestDefault() { func (suite *GenerateSecretTestSuite) TestHex() { suite.cli.GenerateSecret.Hex = true - output := suite.CaptureStdout(func() { + output := testlib.CaptureStdout(func() { suite.NoError(suite.cli.GenerateSecret.Run(suite.cli, "dev")) }) suite.True(strings.HasPrefix(output, "ee")) diff --git a/cli/init_test.go b/cli/init_test.go index a1894b6..197d6d6 100644 --- a/cli/init_test.go +++ b/cli/init_test.go @@ -1,66 +1,25 @@ package cli_test import ( - "bytes" - "context" - "io" - "net" "net/http" - "os" - "strings" - "time" "github.com/9seconds/mtg/v2/cli" - "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/9seconds/mtg/v2/testlib" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) -type NetworkMock struct { - mock.Mock -} - -func (n *NetworkMock) Dial(network, address string) (net.Conn, error) { - args := n.Called(network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - -func (n *NetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - args := n.Called(ctx, network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - -func (n *NetworkMock) DNSResolve(network, hostname string) ([]string, error) { - args := n.Called(network, hostname) - - return args.Get(0).([]string), args.Error(1) -} - -func (n *NetworkMock) MakeHTTPClient(dialFunc network.DialFunc) *http.Client { - return n.Called(dialFunc).Get(0).(*http.Client) -} - -func (n *NetworkMock) IdleTimeout() time.Duration { - return n.Called().Get(0).(time.Duration) -} - -func (n *NetworkMock) HTTPTimeout() time.Duration { - return n.Called().Get(0).(time.Duration) -} - type CommonTestSuite struct { suite.Suite cli *cli.CLI - networkMock *NetworkMock + networkMock *testlib.NetworkMock httpClient *http.Client } func (suite *CommonTestSuite) SetupTest() { - suite.networkMock = &NetworkMock{} + suite.networkMock = &testlib.NetworkMock{} suite.httpClient = &http.Client{} suite.cli = &cli.CLI{} @@ -76,37 +35,3 @@ func (suite *CommonTestSuite) TearDownTest() { suite.networkMock.AssertExpectations(suite.T()) httpmock.DeactivateAndReset() } - -func (suite *CommonTestSuite) CaptureStdout(callback func()) string { - return suite.captureOutput(&os.Stdout, callback) -} - -func (suite *CommonTestSuite) CaptureStderr(callback func()) string { - return suite.captureOutput(&os.Stderr, callback) -} - -func (suite *CommonTestSuite) captureOutput(filefp **os.File, callback func()) string { - oldFp := *filefp - - defer func() { - *filefp = oldFp - }() - - reader, writer, _ := os.Pipe() - buf := &bytes.Buffer{} - closeChan := make(chan bool) - - go func() { - io.Copy(buf, reader) // nolint: errcheck - close(closeChan) - }() - - *filefp = writer - - callback() - - writer.Close() - <-closeChan - - return strings.TrimSpace(buf.String()) -} diff --git a/go.mod b/go.mod index c59047b..3f1b78b 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 github.com/pelletier/go-toml v1.8.1 + github.com/rs/zerolog v1.20.0 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index dafe96e..e6d0a7f 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -27,6 +28,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= +github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -42,15 +46,22 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/mtglib/init.go b/mtglib/init.go index e5e1a0d..1c50d61 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -1,5 +1,32 @@ package mtglib -import "errors" +import ( + "context" + "errors" + "net" + "net/http" + "time" +) var ErrSecretEmpty = errors.New("secret is empty") + +type Network interface { + Dial(network, address string) (net.Conn, error) + DialContext(ctx context.Context, network, address string) (net.Conn, error) + MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client + IdleTimeout() time.Duration +} + +type Logger interface { + Named(name string) Logger + + BindInt(name string, value int) Logger + BindStr(name, value string) Logger + + Info(msg string) + InfoError(msg string, err error) + Warning(msg string) + WarningError(msg string, err error) + Debug(msg string) + DebugError(msg string, err error) +} diff --git a/mtglib/network/init_internal_test.go b/mtglib/network/init_internal_test.go deleted file mode 100644 index d40233d..0000000 --- a/mtglib/network/init_internal_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package network - -import ( - "context" - "net" - "time" - - "github.com/stretchr/testify/mock" -) - -type ConnMock struct { - mock.Mock -} - -func (c *ConnMock) Read(b []byte) (int, error) { - args := c.Called(b) - - return args.Int(0), args.Error(1) -} - -func (c *ConnMock) Write(b []byte) (int, error) { - args := c.Called(b) - - return args.Int(0), args.Error(1) -} - -func (c *ConnMock) Close() error { - return c.Called().Error(0) -} - -func (c *ConnMock) LocalAddr() net.Addr { - return c.Called().Get(0).(net.Addr) -} - -func (c *ConnMock) RemoteAddr() net.Addr { - return c.Called().Get(0).(net.Addr) -} - -func (c *ConnMock) SetDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -func (c *ConnMock) SetReadDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -func (c *ConnMock) SetWriteDeadline(t time.Time) error { - return c.Called(t).Error(0) -} - -type DialerMock struct { - mock.Mock -} - -func (d *DialerMock) Dial(network, address string) (net.Conn, error) { - args := d.Called(network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - -func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - args := d.Called(ctx, network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} diff --git a/mtglib/network/circuit_breaker.go b/network/circuit_breaker.go similarity index 100% rename from mtglib/network/circuit_breaker.go rename to network/circuit_breaker.go diff --git a/mtglib/network/circuit_breaker_internal_test.go b/network/circuit_breaker_internal_test.go similarity index 96% rename from mtglib/network/circuit_breaker_internal_test.go rename to network/circuit_breaker_internal_test.go index cc26064..a29eec3 100644 --- a/mtglib/network/circuit_breaker_internal_test.go +++ b/network/circuit_breaker_internal_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -20,7 +21,7 @@ type CircuitBreakerTestSuite struct { mutex sync.Mutex ctx context.Context ctxCancel context.CancelFunc - connMock *ConnMock + connMock *testlib.NetConnMock baseDialerMock *DialerMock } @@ -28,7 +29,7 @@ func (suite *CircuitBreakerTestSuite) SetupTest() { suite.mutex = sync.Mutex{} suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) suite.baseDialerMock = &DialerMock{} - suite.connMock = &ConnMock{} + suite.connMock = &testlib.NetConnMock{} suite.d = newCircuitBreakerDialer(suite.baseDialerMock, 3, 100*time.Millisecond, 50*time.Millisecond) } diff --git a/mtglib/network/default.go b/network/default.go similarity index 100% rename from mtglib/network/default.go rename to network/default.go diff --git a/mtglib/network/default_test.go b/network/default_test.go similarity index 97% rename from mtglib/network/default_test.go rename to network/default_test.go index c954456..a6b24c7 100644 --- a/mtglib/network/default_test.go +++ b/network/default_test.go @@ -5,7 +5,7 @@ import ( "net/http" "testing" - "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/9seconds/mtg/v2/network" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/network/init.go b/network/init.go similarity index 71% rename from mtglib/network/init.go rename to network/init.go index 82416af..3bab936 100644 --- a/mtglib/network/init.go +++ b/network/init.go @@ -4,7 +4,6 @@ import ( "context" "errors" "net" - "net/http" "time" ) @@ -27,18 +26,7 @@ var ( ErrCannotDialWithAllProxies = errors.New("cannot dial with all proxies") ) -type DialFunc func(ctx context.Context, protocol, address string) (net.Conn, error) - type Dialer interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) } - -type Network interface { - Dialer - - DNSResolve(network, hostname string) (ips []string, err error) - MakeHTTPClient(DialFunc) *http.Client - IdleTimeout() time.Duration - HTTPTimeout() time.Duration -} diff --git a/network/init_internal_test.go b/network/init_internal_test.go new file mode 100644 index 0000000..9335818 --- /dev/null +++ b/network/init_internal_test.go @@ -0,0 +1,24 @@ +package network + +import ( + "context" + "net" + + "github.com/stretchr/testify/mock" +) + +type DialerMock struct { + mock.Mock +} + +func (d *DialerMock) Dial(network, address string) (net.Conn, error) { + args := d.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (d *DialerMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := d.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} diff --git a/mtglib/network/init_test.go b/network/init_test.go similarity index 97% rename from mtglib/network/init_test.go rename to network/init_test.go index 8f648d9..f42d1c3 100644 --- a/mtglib/network/init_test.go +++ b/network/init_test.go @@ -8,7 +8,7 @@ import ( "net/url" "strings" - "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/9seconds/mtg/v2/network" socks5 "github.com/armon/go-socks5" "github.com/mccutchen/go-httpbin/httpbin" "github.com/stretchr/testify/mock" diff --git a/mtglib/network/load_balanced_socks5.go b/network/load_balanced_socks5.go similarity index 100% rename from mtglib/network/load_balanced_socks5.go rename to network/load_balanced_socks5.go diff --git a/mtglib/network/load_balanced_socks5_test.go b/network/load_balanced_socks5_test.go similarity index 98% rename from mtglib/network/load_balanced_socks5_test.go rename to network/load_balanced_socks5_test.go index 03dd6f5..b6983c8 100644 --- a/mtglib/network/load_balanced_socks5_test.go +++ b/network/load_balanced_socks5_test.go @@ -8,7 +8,7 @@ import ( "net/url" "testing" - "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/9seconds/mtg/v2/network" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/network/network.go b/network/network.go similarity index 86% rename from mtglib/network/network.go rename to network/network.go index 1835eb0..a897899 100644 --- a/mtglib/network/network.go +++ b/network/network.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/9seconds/mtg/v2/mtglib" doh "github.com/babolivier/go-doh-client" ) @@ -38,7 +39,7 @@ func (n *network) Dial(protocol, address string) (net.Conn, error) { func (n *network) DialContext(ctx context.Context, protocol, address string) (net.Conn, error) { host, port, _ := net.SplitHostPort(address) - ips, err := n.DNSResolve(protocol, host) + ips, err := n.dnsResolve(protocol, host) if err != nil { return nil, fmt.Errorf("cannot resolve dns names: %w", err) } @@ -61,7 +62,20 @@ func (n *network) DialContext(ctx context.Context, protocol, address string) (ne return nil, fmt.Errorf("cannot dial to %s:%s: %w", protocol, address, err) } -func (n *network) DNSResolve(protocol, address string) ([]string, error) { +func (n *network) MakeHTTPClient(dialFunc func(ctx context.Context, + network, address string) (net.Conn, error)) *http.Client { + if dialFunc == nil { + dialFunc = n.DialContext + } + + return makeHTTPClient(n.userAgent, n.httpTimeout, dialFunc) +} + +func (n *network) IdleTimeout() time.Duration { + return n.idleTimeout +} + +func (n *network) dnsResolve(protocol, address string) ([]string, error) { if net.ParseIP(address) != nil { return []string{address}, nil } @@ -115,25 +129,9 @@ func (n *network) DNSResolve(protocol, address string) ([]string, error) { return ips, nil } -func (n *network) MakeHTTPClient(dialFunc DialFunc) *http.Client { - if dialFunc == nil { - dialFunc = n.DialContext - } - - return makeHTTPClient(n.userAgent, n.httpTimeout, dialFunc) -} - -func (n *network) IdleTimeout() time.Duration { - return n.idleTimeout -} - -func (n *network) HTTPTimeout() time.Duration { - return n.httpTimeout -} - func NewNetwork(dialer Dialer, userAgent, dohHostname string, - httpTimeout, idleTimeout time.Duration) (Network, error) { + httpTimeout, idleTimeout time.Duration) (mtglib.Network, error) { switch { case idleTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout) @@ -158,7 +156,9 @@ func NewNetwork(dialer Dialer, }, nil } -func makeHTTPClient(userAgent string, timeout time.Duration, dialFunc DialFunc) *http.Client { +func makeHTTPClient(userAgent string, + timeout time.Duration, + dialFunc func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client { return &http.Client{ Timeout: timeout, Transport: networkHTTPTransport{ diff --git a/mtglib/network/proxy_dialer.go b/network/proxy_dialer.go similarity index 100% rename from mtglib/network/proxy_dialer.go rename to network/proxy_dialer.go diff --git a/mtglib/network/proxy_dialer_internal_test.go b/network/proxy_dialer_internal_test.go similarity index 100% rename from mtglib/network/proxy_dialer_internal_test.go rename to network/proxy_dialer_internal_test.go diff --git a/mtglib/network/socks5.go b/network/socks5.go similarity index 100% rename from mtglib/network/socks5.go rename to network/socks5.go diff --git a/mtglib/network/socks5_test.go b/network/socks5_test.go similarity index 96% rename from mtglib/network/socks5_test.go rename to network/socks5_test.go index 82b471f..3460f4d 100644 --- a/mtglib/network/socks5_test.go +++ b/network/socks5_test.go @@ -4,7 +4,7 @@ import ( "net/http" "testing" - "github.com/9seconds/mtg/v2/mtglib/network" + "github.com/9seconds/mtg/v2/network" "github.com/stretchr/testify/suite" ) diff --git a/testlib/capture_output.go b/testlib/capture_output.go new file mode 100644 index 0000000..aa538c2 --- /dev/null +++ b/testlib/capture_output.go @@ -0,0 +1,42 @@ +package testlib + +import ( + "bytes" + "io" + "os" + "strings" +) + +func CaptureStdout(callback func()) string { + return captureOutput(&os.Stdout, callback) +} + +func CaptureStderr(callback func()) string { + return captureOutput(&os.Stderr, callback) +} + +func captureOutput(filefp **os.File, callback func()) string { + oldFp := *filefp + + defer func() { + *filefp = oldFp + }() + + reader, writer, _ := os.Pipe() + buf := &bytes.Buffer{} + closeChan := make(chan bool) + + go func() { + io.Copy(buf, reader) // nolint: errcheck + close(closeChan) + }() + + *filefp = writer + + callback() + + writer.Close() + <-closeChan + + return strings.TrimSpace(buf.String()) +} diff --git a/testlib/net_conn_mock.go b/testlib/net_conn_mock.go new file mode 100644 index 0000000..b2fc1a3 --- /dev/null +++ b/testlib/net_conn_mock.go @@ -0,0 +1,48 @@ +package testlib + +import ( + "net" + "time" + + "github.com/stretchr/testify/mock" +) + +type NetConnMock struct { + mock.Mock +} + +func (n *NetConnMock) Read(b []byte) (int, error) { + args := n.Called(b) + + return args.Int(0), args.Error(1) +} + +func (n *NetConnMock) Write(b []byte) (int, error) { + args := n.Called(b) + + return args.Int(0), args.Error(1) +} + +func (n *NetConnMock) Close() error { + return n.Called().Error(0) +} + +func (n *NetConnMock) LocalAddr() net.Addr { + return n.Called().Get(0).(net.Addr) +} + +func (n *NetConnMock) RemoteAddr() net.Addr { + return n.Called().Get(0).(net.Addr) +} + +func (n *NetConnMock) SetDeadline(t time.Time) error { + return n.Called(t).Error(0) +} + +func (n *NetConnMock) SetReadDeadline(t time.Time) error { + return n.Called(t).Error(0) +} + +func (n *NetConnMock) SetWriteDeadline(t time.Time) error { + return n.Called(t).Error(0) +} diff --git a/testlib/network_mock.go b/testlib/network_mock.go new file mode 100644 index 0000000..325dfd3 --- /dev/null +++ b/testlib/network_mock.go @@ -0,0 +1,35 @@ +package testlib + +import ( + "context" + "net" + "net/http" + "time" + + "github.com/stretchr/testify/mock" +) + +type NetworkMock struct { + mock.Mock +} + +func (n *NetworkMock) Dial(network, address string) (net.Conn, error) { + args := n.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (n *NetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := n.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (n *NetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, + network, address string) (net.Conn, error)) *http.Client { + return n.Called(dialFunc).Get(0).(*http.Client) +} + +func (n *NetworkMock) IdleTimeout() time.Duration { + return n.Called().Get(0).(time.Duration) +} From 91a1ee956ca527426cc86422fa64ff7b30fd79b3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 14 Mar 2021 21:51:14 +0300 Subject: [PATCH 064/206] Add noop logger --- logger/noop.go | 19 +++++++++++++++++++ logger/noop_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 logger/noop.go create mode 100644 logger/noop_test.go diff --git a/logger/noop.go b/logger/noop.go new file mode 100644 index 0000000..6d7289c --- /dev/null +++ b/logger/noop.go @@ -0,0 +1,19 @@ +package logger + +import "github.com/9seconds/mtg/v2/mtglib" + +type noopLogger struct{} + +func (n noopLogger) Named(_ string) mtglib.Logger { return n } +func (n noopLogger) BindInt(_ string, _ int) mtglib.Logger { return n } +func (n noopLogger) BindStr(_, _ string) mtglib.Logger { return n } +func (n noopLogger) Info(_ string) {} +func (n noopLogger) Warning(_ string) {} +func (n noopLogger) Debug(_ string) {} +func (n noopLogger) InfoError(_ string, _ error) {} +func (n noopLogger) WarningError(_ string, _ error) {} +func (n noopLogger) DebugError(_ string, _ error) {} + +func NewNoopLogger() mtglib.Logger { + return noopLogger{} +} diff --git a/logger/noop_test.go b/logger/noop_test.go new file mode 100644 index 0000000..add8464 --- /dev/null +++ b/logger/noop_test.go @@ -0,0 +1,34 @@ +package logger_test + +import ( + "io" + "testing" + + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/suite" +) + +type NoopLoggerTestSuite struct { + suite.Suite +} + +func (suite *NoopLoggerTestSuite) TestLog() { + suite.Empty(testlib.CaptureStdout(func() { + suite.Empty(testlib.CaptureStderr(func() { + log := logger.NewNoopLogger().Named("name") + + log.BindInt("int", 1).BindStr("str", "1").Info("info") + log.BindInt("int", 1).BindStr("str", "1").Warning("info") + log.BindInt("int", 1).BindStr("str", "1").Debug("info") + log.BindInt("int", 1).BindStr("str", "1").InfoError("info", io.EOF) + log.BindInt("int", 1).BindStr("str", "1").WarningError("info", io.EOF) + log.BindInt("int", 1).BindStr("str", "1").DebugError("info", io.EOF) + })) + })) +} + +func TestNoopLogger(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopLoggerTestSuite{}) +} From ab585fd6998fc8cf1b1cfb19376b33a08b0e4fc3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 14 Mar 2021 22:25:33 +0300 Subject: [PATCH 065/206] Add zerolog implementation --- logger/zerolog.go | 115 ++++++++++++++++++++++++++++++++++++++++ logger/zerolog_test.go | 116 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 logger/zerolog.go create mode 100644 logger/zerolog_test.go diff --git a/logger/zerolog.go b/logger/zerolog.go new file mode 100644 index 0000000..4142453 --- /dev/null +++ b/logger/zerolog.go @@ -0,0 +1,115 @@ +package logger + +import ( + "github.com/9seconds/mtg/v2/mtglib" + "github.com/rs/zerolog" +) + +const loggerFieldName = "logger" + +type zeroLogContextVarType uint8 + +const ( + zeroLogContextVarTypeUnknown zeroLogContextVarType = iota + zeroLogContextVarTypeStr + zeroLogContextVarTypeInt +) + +type zeroLogContext struct { + name string + log *zerolog.Logger + + ctxVarType zeroLogContextVarType + ctxVarName string + ctxVarStr string + ctxVarInt int + + parent *zeroLogContext +} + +func (z *zeroLogContext) Named(name string) mtglib.Logger { + loggerName := z.name + if loggerName == "" { + loggerName = name + } else { + loggerName += "." + name + } + + return &zeroLogContext{ + name: loggerName, + log: z.log, + parent: z, + } +} + +func (z *zeroLogContext) BindInt(name string, value int) mtglib.Logger { + return &zeroLogContext{ + name: z.name, + log: z.log, + ctxVarType: zeroLogContextVarTypeInt, + ctxVarInt: value, + ctxVarName: name, + parent: z, + } +} + +func (z *zeroLogContext) BindStr(name, value string) mtglib.Logger { + return &zeroLogContext{ + name: z.name, + log: z.log, + ctxVarType: zeroLogContextVarTypeStr, + ctxVarStr: value, + ctxVarName: name, + parent: z, + } +} + +func (z *zeroLogContext) Info(msg string) { + z.InfoError(msg, nil) +} + +func (z *zeroLogContext) Warning(msg string) { + z.WarningError(msg, nil) +} + +func (z *zeroLogContext) Debug(msg string) { + z.DebugError(msg, nil) +} + +func (z *zeroLogContext) InfoError(msg string, err error) { + z.emitLog(z.log.Info(), msg, err) +} + +func (z *zeroLogContext) WarningError(msg string, err error) { + z.emitLog(z.log.Warn(), msg, err) +} + +func (z *zeroLogContext) DebugError(msg string, err error) { + z.emitLog(z.log.Debug(), msg, err) +} + +func (z *zeroLogContext) emitLog(evt *zerolog.Event, msg string, err error) { + z.attachCtx(evt) + + for current := z.parent; current != nil; current = current.parent { + current.attachCtx(evt) + } + + evt.Str(loggerFieldName, z.name).Err(err).Msg(msg) +} + +func (z *zeroLogContext) attachCtx(evt *zerolog.Event) { + switch z.ctxVarType { + case zeroLogContextVarTypeStr: + evt.Str(z.ctxVarName, z.ctxVarStr) + case zeroLogContextVarTypeInt: + evt.Int(z.ctxVarName, z.ctxVarInt) + case zeroLogContextVarTypeUnknown: + } +} + +func NewZeroLogger(log zerolog.Logger) mtglib.Logger { + return &zeroLogContext{ + log: &log, + } +} diff --git a/logger/zerolog_test.go b/logger/zerolog_test.go new file mode 100644 index 0000000..4b2308f --- /dev/null +++ b/logger/zerolog_test.go @@ -0,0 +1,116 @@ +package logger_test + +import ( + "bytes" + "encoding/json" + "io" + "strings" + "testing" + "time" + + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type zeroLoggerLogMessage struct { + Timestamp int64 `json:"timestamp"` + Level string `json:"level"` + StrParam string `json:"strparam"` + IntParam int `json:"intparam"` + Logger string `json:"logger"` + Error string `json:"error"` + Message string `json:"message"` +} + +type ZeroLoggerTestSuite struct { + suite.Suite +} + +func (suite *ZeroLoggerTestSuite) SetupSuite() { + zerolog.SetGlobalLevel(zerolog.TraceLevel) + + zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs + zerolog.TimestampFieldName = "timestamp" + zerolog.LevelFieldName = "level" +} + +func (suite *ZeroLoggerTestSuite) TestLog() { + testData := map[string]func(mtglib.Logger){ + "info": func(l mtglib.Logger) { l.Info("hello") }, + "warn": func(l mtglib.Logger) { l.Warning("hello") }, + "debug": func(l mtglib.Logger) { l.Debug("hello") }, + "info-error": func(l mtglib.Logger) { l.InfoError("hello", io.EOF) }, + "warn-error": func(l mtglib.Logger) { l.WarningError("hello", io.EOF) }, + "debug-error": func(l mtglib.Logger) { l.DebugError("hello", io.EOF) }, + } + + for k, v := range testData { + name := k + callback := v + level := strings.TrimSuffix(name, "-error") + + suite.T().Run(name, func(t *testing.T) { + buf := &bytes.Buffer{} + log := logger.NewZeroLogger(zerolog.New(buf).With().Timestamp().Logger()) + + callback(log.Named("name").BindInt("intparam", 1).BindStr("strparam", name)) + + msg := &zeroLoggerLogMessage{} + assert.NoError(t, json.Unmarshal(buf.Bytes(), msg)) + + timestamp := time.Unix(msg.Timestamp/1000, (msg.Timestamp%1000)*1_000_000) + assert.WithinDuration(t, time.Now(), timestamp, 100*time.Millisecond) + + assert.Equal(t, level, msg.Level) + assert.Equal(t, name, msg.StrParam) + assert.EqualValues(t, 1, msg.IntParam) + assert.Equal(t, "name", msg.Logger) + assert.Equal(t, "hello", msg.Message) + + if level != name { + assert.Equal(t, io.EOF.Error(), msg.Error) + } else { + assert.Empty(t, msg.Error) + } + }) + } +} + +func (suite *ZeroLoggerTestSuite) TestIndependence() { + buf := &bytes.Buffer{} + log := logger.NewZeroLogger(zerolog.New(buf).With().Timestamp().Logger()) + + log1 := log.Named("1") + log2 := log.Named("2") + log12 := log1.Named("2") + + log1.BindInt("param", 1).Info("hello") + + log1Output := buf.String() + + buf.Reset() + + log2.BindInt("lalala", 2).Info("hello") + + log2Output := buf.String() + + buf.Reset() + + log12.BindStr("tttt", "qqq").Info("hello") + + log12Output := buf.String() + + suite.NotContains("lalala", log1Output) + suite.NotContains("tttt", log1Output) + suite.NotContains("param", log2Output) + suite.NotContains("tttt", log1Output) + suite.NotContains("param", log12Output) + suite.NotContains("lalala", log12Output) +} + +func TestZeroLogger(t *testing.T) { // nolint: paralleltest + suite.Run(t, &ZeroLoggerTestSuite{}) +} From ab1b3b486324422b4c48ed146c04d405a6d70143 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 10:43:32 +0300 Subject: [PATCH 066/206] Add antireplay cache --- antireplay/noop.go | 11 +++++++++++ antireplay/noop_test.go | 26 ++++++++++++++++++++++++++ antireplay/stable_bloom_filter.go | 24 ++++++++++++++++++++++++ antireplay/stable_bloom_filter_test.go | 26 ++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 4 ++++ mtglib/init.go | 4 ++++ 7 files changed, 97 insertions(+) create mode 100644 antireplay/noop.go create mode 100644 antireplay/noop_test.go create mode 100644 antireplay/stable_bloom_filter.go create mode 100644 antireplay/stable_bloom_filter_test.go diff --git a/antireplay/noop.go b/antireplay/noop.go new file mode 100644 index 0000000..8613e40 --- /dev/null +++ b/antireplay/noop.go @@ -0,0 +1,11 @@ +package antireplay + +import "github.com/9seconds/mtg/v2/mtglib" + +type noop struct{} + +func (n noop) SeenBefore(_ []byte) bool { return false } + +func NewNoop() mtglib.AntiReplayCache { + return noop{} +} diff --git a/antireplay/noop_test.go b/antireplay/noop_test.go new file mode 100644 index 0000000..b78c48d --- /dev/null +++ b/antireplay/noop_test.go @@ -0,0 +1,26 @@ +package antireplay_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite +} + +func (suite *NoopTestSuite) TestOp() { + filter := antireplay.NewNoop() + + suite.False(filter.SeenBefore([]byte{1, 2, 3})) + suite.False(filter.SeenBefore([]byte{4, 5, 6})) + suite.False(filter.SeenBefore([]byte{1, 2, 3})) + suite.False(filter.SeenBefore([]byte{4, 5, 6})) +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go new file mode 100644 index 0000000..5ab15f3 --- /dev/null +++ b/antireplay/stable_bloom_filter.go @@ -0,0 +1,24 @@ +package antireplay + +import ( + "github.com/9seconds/mtg/v2/mtglib" + "github.com/OneOfOne/xxhash" + boom "github.com/tylertreat/BoomFilters" +) + +type stableBloomFilter struct { + filter boom.StableBloomFilter +} + +func (s *stableBloomFilter) SeenBefore(digest []byte) bool { + return s.filter.TestAndAdd(digest) +} + +func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { + sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd + sf.SetHash(xxhash.New64()) + + return &stableBloomFilter{ + filter: *sf, + } +} diff --git a/antireplay/stable_bloom_filter_test.go b/antireplay/stable_bloom_filter_test.go new file mode 100644 index 0000000..accb590 --- /dev/null +++ b/antireplay/stable_bloom_filter_test.go @@ -0,0 +1,26 @@ +package antireplay_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/stretchr/testify/suite" +) + +type StableBloomFilterTestSuite struct { + suite.Suite +} + +func (suite *StableBloomFilterTestSuite) TestOp() { + filter := antireplay.NewStableBloomFilter(500, 0.001) + + suite.False(filter.SeenBefore([]byte{1, 2, 3})) + suite.False(filter.SeenBefore([]byte{4, 5, 6})) + suite.True(filter.SeenBefore([]byte{1, 2, 3})) + suite.True(filter.SeenBefore([]byte{4, 5, 6})) +} + +func TestStableBloomFilter(t *testing.T) { + t.Parallel() + suite.Run(t, &StableBloomFilterTestSuite{}) +} diff --git a/go.mod b/go.mod index 3f1b78b..1438615 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/9seconds/mtg/v2 go 1.16 require ( + github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/alecthomas/kong v0.2.16 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 @@ -15,6 +16,7 @@ require ( github.com/rs/zerolog v1.20.0 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 + github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c // indirect github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect diff --git a/go.sum b/go.sum index e6d0a7f..5f3cd5b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/alecthomas/kong v0.2.16 h1:F232CiYSn54Tnl1sJGTeHmx4vJDNLVP2b9yCVMOQwHQ= github.com/alecthomas/kong v0.2.16/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= @@ -40,6 +42,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c h1:pGEq55pv/5i+G/Dy+kDsUugzvg6R02jzegxtrwOhE7A= +github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= diff --git a/mtglib/init.go b/mtglib/init.go index 1c50d61..2bb6975 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -17,6 +17,10 @@ type Network interface { IdleTimeout() time.Duration } +type AntiReplayCache interface { + SeenBefore(data []byte) bool +} + type Logger interface { Named(name string) Logger From 8eb5fed92fc6af7ee1003c1d6b779b2bfedcd7af Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 10:45:01 +0300 Subject: [PATCH 067/206] Rename network_mock to mtglib_netwock_mock --- cli/init_test.go | 4 ++-- testlib/mtglib_network_mock.go | 35 ++++++++++++++++++++++++++++++++++ testlib/network_mock.go | 35 ---------------------------------- 3 files changed, 37 insertions(+), 37 deletions(-) create mode 100644 testlib/mtglib_network_mock.go delete mode 100644 testlib/network_mock.go diff --git a/cli/init_test.go b/cli/init_test.go index 197d6d6..77304e9 100644 --- a/cli/init_test.go +++ b/cli/init_test.go @@ -14,12 +14,12 @@ type CommonTestSuite struct { suite.Suite cli *cli.CLI - networkMock *testlib.NetworkMock + networkMock *testlib.MtglibNetworkMock httpClient *http.Client } func (suite *CommonTestSuite) SetupTest() { - suite.networkMock = &testlib.NetworkMock{} + suite.networkMock = &testlib.MtglibNetworkMock{} suite.httpClient = &http.Client{} suite.cli = &cli.CLI{} diff --git a/testlib/mtglib_network_mock.go b/testlib/mtglib_network_mock.go new file mode 100644 index 0000000..5a8192f --- /dev/null +++ b/testlib/mtglib_network_mock.go @@ -0,0 +1,35 @@ +package testlib + +import ( + "context" + "net" + "net/http" + "time" + + "github.com/stretchr/testify/mock" +) + +type MtglibNetworkMock struct { + mock.Mock +} + +func (n *MtglibNetworkMock) Dial(network, address string) (net.Conn, error) { + args := n.Called(network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (n *MtglibNetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := n.Called(ctx, network, address) + + return args.Get(0).(net.Conn), args.Error(1) +} + +func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, + network, address string) (net.Conn, error)) *http.Client { + return m.Called(dialFunc).Get(0).(*http.Client) +} + +func (m *MtglibNetworkMock) IdleTimeout() time.Duration { + return m.Called().Get(0).(time.Duration) +} diff --git a/testlib/network_mock.go b/testlib/network_mock.go deleted file mode 100644 index 325dfd3..0000000 --- a/testlib/network_mock.go +++ /dev/null @@ -1,35 +0,0 @@ -package testlib - -import ( - "context" - "net" - "net/http" - "time" - - "github.com/stretchr/testify/mock" -) - -type NetworkMock struct { - mock.Mock -} - -func (n *NetworkMock) Dial(network, address string) (net.Conn, error) { - args := n.Called(network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - -func (n *NetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - args := n.Called(ctx, network, address) - - return args.Get(0).(net.Conn), args.Error(1) -} - -func (n *NetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, - network, address string) (net.Conn, error)) *http.Client { - return n.Called(dialFunc).Get(0).(*http.Client) -} - -func (n *NetworkMock) IdleTimeout() time.Duration { - return n.Called().Get(0).(time.Duration) -} From 3dc263d6d7f540b56e549bee5def3db0322d8b85 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 10:52:14 +0300 Subject: [PATCH 068/206] Add antireplay cache mock --- antireplay/stable_bloom_filter.go | 2 +- testlib/mtglib_antireplay_cache_mock.go | 11 +++++++++++ testlib/mtglib_network_mock.go | 8 ++++---- 3 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 testlib/mtglib_antireplay_cache_mock.go diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index 5ab15f3..6eef3f5 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -16,7 +16,7 @@ func (s *stableBloomFilter) SeenBefore(digest []byte) bool { func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd - sf.SetHash(xxhash.New64()) + sf.SetHash(xxhash.New64()) return &stableBloomFilter{ filter: *sf, diff --git a/testlib/mtglib_antireplay_cache_mock.go b/testlib/mtglib_antireplay_cache_mock.go new file mode 100644 index 0000000..20ac993 --- /dev/null +++ b/testlib/mtglib_antireplay_cache_mock.go @@ -0,0 +1,11 @@ +package testlib + +import "github.com/stretchr/testify/mock" + +type MtglibAntiReplayCacheMock struct { + mock.Mock +} + +func (m *MtglibAntiReplayCacheMock) SeenBefore(data []byte) bool { + return m.Called(data).Bool(0) +} diff --git a/testlib/mtglib_network_mock.go b/testlib/mtglib_network_mock.go index 5a8192f..23fa8ba 100644 --- a/testlib/mtglib_network_mock.go +++ b/testlib/mtglib_network_mock.go @@ -13,14 +13,14 @@ type MtglibNetworkMock struct { mock.Mock } -func (n *MtglibNetworkMock) Dial(network, address string) (net.Conn, error) { - args := n.Called(network, address) +func (m *MtglibNetworkMock) Dial(network, address string) (net.Conn, error) { + args := m.Called(network, address) return args.Get(0).(net.Conn), args.Error(1) } -func (n *MtglibNetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - args := n.Called(ctx, network, address) +func (m *MtglibNetworkMock) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + args := m.Called(ctx, network, address) return args.Get(0).(net.Conn), args.Error(1) } From cae33a22e65cd023645107af45177577683210d3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 10:53:29 +0300 Subject: [PATCH 069/206] Make stable bloom filter threadsafe --- antireplay/stable_bloom_filter.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index 6eef3f5..b52631e 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -1,6 +1,8 @@ package antireplay import ( + "sync" + "github.com/9seconds/mtg/v2/mtglib" "github.com/OneOfOne/xxhash" boom "github.com/tylertreat/BoomFilters" @@ -8,9 +10,13 @@ import ( type stableBloomFilter struct { filter boom.StableBloomFilter + mutex sync.Mutex } func (s *stableBloomFilter) SeenBefore(digest []byte) bool { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.filter.TestAndAdd(digest) } From f52b3391d1cb5e1649890e3a70c4f50c583ecf84 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 16:21:07 +0300 Subject: [PATCH 070/206] Add firehol blocklist --- go.mod | 2 + go.sum | 5 + ipblocklist/firehol.go | 327 ++++++++++++++++++++++++ ipblocklist/firehol_test.go | 137 ++++++++++ ipblocklist/noop.go | 15 ++ ipblocklist/noop_test.go | 23 ++ ipblocklist/testdata/broken_ipset.ipset | 5 + ipblocklist/testdata/good_ipset.ipset | 7 + ipblocklist/testdata/remote_ipset.ipset | 1 + mtglib/init.go | 4 + network/network.go | 7 + 11 files changed, 533 insertions(+) create mode 100644 ipblocklist/firehol.go create mode 100644 ipblocklist/firehol_test.go create mode 100644 ipblocklist/noop.go create mode 100644 ipblocklist/noop_test.go create mode 100644 ipblocklist/testdata/broken_ipset.ipset create mode 100644 ipblocklist/testdata/good_ipset.ipset create mode 100644 ipblocklist/testdata/remote_ipset.ipset diff --git a/go.mod b/go.mod index 1438615..f9caefe 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,11 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/jarcoal/httpmock v1.0.8 + github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 + github.com/panjf2000/ants v1.3.0 // indirect github.com/pelletier/go-toml v1.8.1 github.com/rs/zerolog v1.20.0 // indirect github.com/stretchr/objx v0.3.0 // indirect diff --git a/go.sum b/go.sum index 5f3cd5b..bcb02ae 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 h1:D7qhJP3R49ZjUzpzKQ6B2H3lgejPs6DTO5gRomhhOpE= +github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -23,6 +25,8 @@ github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyC github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE= github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I= +github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= +github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -36,6 +40,7 @@ github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go new file mode 100644 index 0000000..e4c0370 --- /dev/null +++ b/ipblocklist/firehol.go @@ -0,0 +1,327 @@ +package ipblocklist + +import ( + "bufio" + "context" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "sync" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/kentik/patricia" + "github.com/kentik/patricia/bool_tree" + "github.com/panjf2000/ants" +) + +const ( + fireholIPv4DefaultCIDR = 32 + fireholIPv6DefaultCIDR = 128 +) + +var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) + +type Firehol struct { + logger mtglib.Logger + rwMutex sync.RWMutex + remoteURLs []string + localFiles []string + httpClient *http.Client + workerPool *ants.Pool + treeV4 *bool_tree.TreeV4 + treeV6 *bool_tree.TreeV6 +} + +func (f *Firehol) Contains(ip net.IP) bool { + if ip == nil { + return true + } + + ip4 := ip.To4() + + f.rwMutex.RLock() + defer f.rwMutex.RUnlock() + + if ip4 != nil { + return f.containsIPv4(ip4) + } + + return f.containsIPv6(ip.To16()) +} + +func (f *Firehol) containsIPv4(addr net.IP) bool { + ip := patricia.NewIPv4AddressFromBytes(addr, 32) + + if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) containsIPv6(addr net.IP) bool { + ip := patricia.NewIPv6Address(addr, 128) + + if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) Run(ctx context.Context, updateEach time.Duration) { + ticker := time.NewTicker(updateEach) + + defer func() { + ticker.Stop() + + select { + case <-ticker.C: + default: + } + }() + + if err := f.update(ctx); err != nil { + f.logger.WarningError("cannot update blocklist", err) + } + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := f.update(ctx); err != nil { + f.logger.WarningError("cannot update blocklist", err) + } + } + } +} + +func (f *Firehol) update(ctx context.Context) error { // nolint: funlen, cyclop + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + wg := &sync.WaitGroup{} + wg.Add(len(f.remoteURLs) + len(f.localFiles)) + + treeMutex := &sync.Mutex{} + v4tree := bool_tree.NewTreeV4() + v6tree := bool_tree.NewTreeV6() + + errorChan := make(chan error, 1) + defer close(errorChan) + + for _, v := range f.localFiles { + go func(filename string) { + defer wg.Done() + + if err := f.updateLocalFile(ctx, filename, treeMutex, v4tree, v6tree); err != nil { + cancel() + f.logger.BindStr("filename", filename).WarningError("cannot update", err) + + select { + case errorChan <- err: + default: + } + } + }(v) + } + + for _, v := range f.remoteURLs { + value := v + + f.workerPool.Submit(func() { // nolint: errcheck + defer wg.Done() + + if err := f.updateRemoteURL(ctx, value, treeMutex, v4tree, v6tree); err != nil { + cancel() + f.logger.BindStr("url", value).WarningError("cannot update", err) + + select { + case errorChan <- err: + default: + } + } + }) + } + + wg.Wait() + + select { + case err := <-errorChan: + return fmt.Errorf("cannot update trees: %w", err) + default: + } + + f.rwMutex.Lock() + defer f.rwMutex.Unlock() + + f.treeV4 = v4tree + f.treeV6 = v6tree + + return nil +} + +func (f *Firehol) updateLocalFile(ctx context.Context, filename string, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + filefp, err := os.Open(filename) + if err != nil { + return fmt.Errorf("cannot open file: %w", err) + } + + defer filefp.Close() + + return f.updateTrees(ctx, mutex, filefp, v4tree, v6tree) +} + +func (f *Firehol) updateRemoteURL(ctx context.Context, url string, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("cannot build a request: %w", err) + } + + resp, err := f.httpClient.Do(req) + if err != nil { + return fmt.Errorf("cannot request a remote URL %s: %w", url, err) + } + + defer func() { + io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck + resp.Body.Close() + }() + + return f.updateTrees(ctx, mutex, resp.Body, v4tree, v6tree) +} + +func (f *Firehol) updateTrees(ctx context.Context, + mutex sync.Locker, + reader io.Reader, + v4tree *bool_tree.TreeV4, + v6tree *bool_tree.TreeV6) error { + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + text := scanner.Text() + text = fireholRegexpComment.ReplaceAllLiteralString(text, "") + text = strings.TrimSpace(text) + + if text == "" { + continue + } + + ip, cidr, err := f.updateParseLine(text) + if err != nil { + return fmt.Errorf("cannot parse a line: %w", err) + } + + if err := f.updateAddToTrees(ip, cidr, mutex, v4tree, v6tree); err != nil { + return fmt.Errorf("cannot add a node to the tree: %w", err) + } + } + + if scanner.Err() != nil { + return fmt.Errorf("cannot parse a response: %w", scanner.Err()) + } + + return nil +} + +func (f *Firehol) updateParseLine(text string) (net.IP, uint, error) { + _, ipnet, err := net.ParseCIDR(text) + if err != nil { + ipaddr := net.ParseIP(text) + if ipaddr == nil { + return nil, 0, fmt.Errorf("incorrect ip address %s", text) + } + + ip4 := ipaddr.To4() + if ip4 != nil { + return ip4, fireholIPv4DefaultCIDR, nil + } + + return ipaddr.To16(), fireholIPv6DefaultCIDR, nil + } + + ones, _ := ipnet.Mask.Size() + + return ipnet.IP, uint(ones), nil +} + +func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, + mutex sync.Locker, + v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { + mutex.Lock() + defer mutex.Unlock() + + if ip.To4() != nil { + addr := patricia.NewIPv4AddressFromBytes(ip, cidr) + + if _, _, err := v4tree.Set(addr, true); err != nil { + return err // nolint: wrapcheck + } + } else { + addr := patricia.NewIPv6Address(ip, cidr) + + if _, _, err := v6tree.Set(addr, true); err != nil { + return err // nolint: wrapcheck + } + } + + return nil +} + +func NewFirehol(logger mtglib.Logger, network mtglib.Network, + downloadConcurrency uint, + remoteURLs []string, + localFiles []string) (*Firehol, error) { + for _, v := range remoteURLs { + parsed, err := url.Parse(v) + if err != nil { + return nil, fmt.Errorf("incorrect url %s: %w", v, err) + } + + switch parsed.Scheme { + case "http", "https": + default: + return nil, fmt.Errorf("unsupported url %s", v) + } + } + + for _, v := range localFiles { + if stat, err := os.Stat(v); os.IsNotExist(err) || stat.IsDir() || stat.Mode().Perm()&0o400 == 0 { + return nil, fmt.Errorf("%s is not a readable file", v) + } + } + + if downloadConcurrency == 0 { + downloadConcurrency = 1 + } + + workerPool, _ := ants.NewPool(int(downloadConcurrency)) + + return &Firehol{ + logger: logger.Named("firehol"), + httpClient: network.MakeHTTPClient(nil), + treeV4: bool_tree.NewTreeV4(), + treeV6: bool_tree.NewTreeV6(), + workerPool: workerPool, + remoteURLs: remoteURLs, + localFiles: localFiles, + }, nil +} diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go new file mode 100644 index 0000000..5b18820 --- /dev/null +++ b/ipblocklist/firehol_test.go @@ -0,0 +1,137 @@ +package ipblocklist_test + +import ( + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/network" + "github.com/9seconds/mtg/v2/testlib" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type FireholTestSuite struct { + suite.Suite + + networkMock *testlib.MtglibNetworkMock + httpServer *httptest.Server +} + +func (suite *FireholTestSuite) SetupSuite() { + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + filefp, err := os.Open(filepath.Join("testdata", "remote_ipset.ipset")) + if err != nil { + panic(err) + } + + defer filefp.Close() + + io.Copy(w, filefp) // nolint: errcheck + }) + + suite.httpServer = httptest.NewServer(mux) +} + +func (suite *FireholTestSuite) SetupTest() { + httpClient := &http.Client{} + suite.networkMock = &testlib.MtglibNetworkMock{} + + httpmock.ActivateNonDefault(httpClient) + + suite.networkMock. + On("MakeHTTPClient", mock.Anything). + Maybe(). + Return(httpClient) +} + +func (suite *FireholTestSuite) TearDownTest() { + suite.networkMock.AssertExpectations(suite.T()) + httpmock.DeactivateAndReset() +} + +func (suite *FireholTestSuite) TearDownSuite() { + suite.httpServer.Close() +} + +func (suite *FireholTestSuite) TestLocalFail() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + nil, []string{filepath.Join("testdata", "broken_ipset.ipset")}) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.False(blocklist.Contains(net.ParseIP("10.0.0.10"))) + suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) +} + +func (suite *FireholTestSuite) TestLocalOk() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + nil, []string{filepath.Join("testdata", "good_ipset.ipset")}) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.True(blocklist.Contains(net.ParseIP("10.0.0.10"))) + suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) +} + +func (suite *FireholTestSuite) TestRemoteFail() { + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + suite.networkMock, 2, + []string{"https://google.com"}, nil) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.False(blocklist.Contains(net.ParseIP("10.2.2.2"))) +} + +func (suite *FireholTestSuite) TestMixed() { + dialer, _ := network.NewDefaultDialer(0, 0) + ntw, _ := network.NewNetwork(dialer, "mtg", "1.1.1.1", 0, 0) + + blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), + ntw, 2, + []string{ + suite.httpServer.URL, + }, []string{ + filepath.Join("testdata", "good_ipset.ipset"), + }) + + suite.NoError(err) + + go blocklist.Run(context.Background(), time.Hour) + + time.Sleep(500 * time.Millisecond) + + suite.True(blocklist.Contains(net.ParseIP("10.2.2.2"))) + suite.True(blocklist.Contains(net.ParseIP("10.1.0.100"))) +} + +func TestFirehol(t *testing.T) { + t.Parallel() + suite.Run(t, &FireholTestSuite{}) +} diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go new file mode 100644 index 0000000..dcbed19 --- /dev/null +++ b/ipblocklist/noop.go @@ -0,0 +1,15 @@ +package ipblocklist + +import ( + "net" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type noop struct{} + +func (n noop) Contains(ip net.IP) bool { return false } + +func NewNoop() mtglib.IPBlocklist { + return noop{} +} diff --git a/ipblocklist/noop_test.go b/ipblocklist/noop_test.go new file mode 100644 index 0000000..f5bb346 --- /dev/null +++ b/ipblocklist/noop_test.go @@ -0,0 +1,23 @@ +package ipblocklist_test + +import ( + "net" + "testing" + + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite +} + +func (suite *NoopTestSuite) TestOp() { + suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) + suite.False(ipblocklist.NewNoop().Contains(net.ParseIP("10.0.0.10"))) +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/ipblocklist/testdata/broken_ipset.ipset b/ipblocklist/testdata/broken_ipset.ipset new file mode 100644 index 0000000..2758925 --- /dev/null +++ b/ipblocklist/testdata/broken_ipset.ipset @@ -0,0 +1,5 @@ +# +# This is an intentionally broken ipset. +# + +ajsdkfbd diff --git a/ipblocklist/testdata/good_ipset.ipset b/ipblocklist/testdata/good_ipset.ipset new file mode 100644 index 0000000..411ba01 --- /dev/null +++ b/ipblocklist/testdata/good_ipset.ipset @@ -0,0 +1,7 @@ +# +# This is very good ipset +# + +10.0.0.10 # just an example +10.1.0.0/24 +2001:0db8:85a3:0000:0000:8a2e:0370:7334 diff --git a/ipblocklist/testdata/remote_ipset.ipset b/ipblocklist/testdata/remote_ipset.ipset new file mode 100644 index 0000000..80d49e4 --- /dev/null +++ b/ipblocklist/testdata/remote_ipset.ipset @@ -0,0 +1 @@ +10.2.2.2 diff --git a/mtglib/init.go b/mtglib/init.go index 2bb6975..12e1393 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -21,6 +21,10 @@ type AntiReplayCache interface { SeenBefore(data []byte) bool } +type IPBlocklist interface { + Contains(net.IP) bool +} + type Logger interface { Named(name string) Logger diff --git a/network/network.go b/network/network.go index a897899..585ff3c 100644 --- a/network/network.go +++ b/network/network.go @@ -132,6 +132,13 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { func NewNetwork(dialer Dialer, userAgent, dohHostname string, httpTimeout, idleTimeout time.Duration) (mtglib.Network, error) { + switch { + case httpTimeout < 0: + return nil, fmt.Errorf("timeout should be positive number %s", httpTimeout) + case httpTimeout == 0: + httpTimeout = DefaultHTTPTimeout + } + switch { case idleTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout) From abff0cf211c39ad97670ede54bdfc8904f68d890 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 16:22:01 +0300 Subject: [PATCH 071/206] Run ci tests in parallel --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b11dccd..d5c15ff 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ test: .PHONY: citest citest: - @go test -coverprofile=coverage.txt -covermode=atomic -race -v ./... + @go test -coverprofile=coverage.txt -covermode=atomic -parallel 2 -race -v ./... .PHONY: crosscompile crosscompile: $(CC_BINARIES) From b08d945d7c62b6dc861d02fe490dc1cf5767ae6e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 21:31:07 +0300 Subject: [PATCH 072/206] Add event stream module --- antireplay/noop.go | 1 + antireplay/noop_test.go | 2 + antireplay/stable_bloom_filter.go | 2 + antireplay/stable_bloom_filter_test.go | 2 + events/event_stream.go | 64 ++++++++++++++++++++++++++ events/init.go | 12 +++++ events/multi_observer.go | 59 ++++++++++++++++++++++++ ipblocklist/firehol.go | 33 +++++++++---- ipblocklist/firehol_test.go | 21 +++++++-- ipblocklist/noop.go | 1 + mtglib/events.go | 25 ++++++++++ mtglib/init.go | 11 +++++ testlib/events_observer_mock.go | 7 +++ 13 files changed, 225 insertions(+), 15 deletions(-) create mode 100644 events/event_stream.go create mode 100644 events/init.go create mode 100644 events/multi_observer.go create mode 100644 mtglib/events.go create mode 100644 testlib/events_observer_mock.go diff --git a/antireplay/noop.go b/antireplay/noop.go index 8613e40..b07d34f 100644 --- a/antireplay/noop.go +++ b/antireplay/noop.go @@ -5,6 +5,7 @@ import "github.com/9seconds/mtg/v2/mtglib" type noop struct{} func (n noop) SeenBefore(_ []byte) bool { return false } +func (n noop) Shutdown() {} func NewNoop() mtglib.AntiReplayCache { return noop{} diff --git a/antireplay/noop_test.go b/antireplay/noop_test.go index b78c48d..42f39eb 100644 --- a/antireplay/noop_test.go +++ b/antireplay/noop_test.go @@ -18,6 +18,8 @@ func (suite *NoopTestSuite) TestOp() { suite.False(filter.SeenBefore([]byte{4, 5, 6})) suite.False(filter.SeenBefore([]byte{1, 2, 3})) suite.False(filter.SeenBefore([]byte{4, 5, 6})) + + filter.Shutdown() } func TestNoop(t *testing.T) { diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index b52631e..ea4f5a7 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -20,6 +20,8 @@ func (s *stableBloomFilter) SeenBefore(digest []byte) bool { return s.filter.TestAndAdd(digest) } +func (s *stableBloomFilter) Shutdown() {} + func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd sf.SetHash(xxhash.New64()) diff --git a/antireplay/stable_bloom_filter_test.go b/antireplay/stable_bloom_filter_test.go index accb590..9b2517e 100644 --- a/antireplay/stable_bloom_filter_test.go +++ b/antireplay/stable_bloom_filter_test.go @@ -18,6 +18,8 @@ func (suite *StableBloomFilterTestSuite) TestOp() { suite.False(filter.SeenBefore([]byte{4, 5, 6})) suite.True(filter.SeenBefore([]byte{1, 2, 3})) suite.True(filter.SeenBefore([]byte{4, 5, 6})) + + filter.Shutdown() } func TestStableBloomFilter(t *testing.T) { diff --git a/events/event_stream.go b/events/event_stream.go new file mode 100644 index 0000000..577e713 --- /dev/null +++ b/events/event_stream.go @@ -0,0 +1,64 @@ +package events + +import ( + "context" + "runtime" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/OneOfOne/xxhash" +) + +type eventStream struct { + ctx context.Context + ctxCancel context.CancelFunc + chans []chan mtglib.Event +} + +func (e eventStream) Send(ctx context.Context, evt mtglib.Event) { + chanNo := int(xxhash.ChecksumString32(evt.ConnectionID())) % len(e.chans) + + select { + case <-ctx.Done(): + case <-e.ctx.Done(): + case e.chans[chanNo] <- evt: + } +} + +func (e eventStream) Shutdown() { + e.ctxCancel() +} + +func NewEventStream(observerFactories []ObserverFactory) mtglib.EventStream { + ctx, cancel := context.WithCancel(context.Background()) + rv := eventStream{ + ctx: ctx, + ctxCancel: cancel, + chans: make([]chan mtglib.Event, runtime.NumCPU()), + } + + for i := 0; i < runtime.NumCPU(); i++ { + rv.chans[i] = make(chan mtglib.Event, 1) + + go eventStreamProcessor(ctx, rv.chans[i], newMultiObserver(observerFactories)) + } + + return rv +} + +func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, observer Observer) { + defer observer.Shutdown() + + for { + select { + case <-ctx.Done(): + return + case evt := <-eventChan: + switch typedEvt := evt.(type) { + case mtglib.EventStart: + observer.EventStart(typedEvt) + case mtglib.EventFinish: + observer.EventFinish(typedEvt) + } + } + } +} diff --git a/events/init.go b/events/init.go new file mode 100644 index 0000000..f84b03b --- /dev/null +++ b/events/init.go @@ -0,0 +1,12 @@ +package events + +import "github.com/9seconds/mtg/v2/mtglib" + +type Observer interface { + EventStart(mtglib.EventStart) + EventFinish(mtglib.EventFinish) + + Shutdown() +} + +type ObserverFactory func() Observer diff --git a/events/multi_observer.go b/events/multi_observer.go new file mode 100644 index 0000000..9370fb0 --- /dev/null +++ b/events/multi_observer.go @@ -0,0 +1,59 @@ +package events + +import ( + "sync" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type multiObserver struct { + observers []Observer +} + +func (m multiObserver) EventStart(evt mtglib.EventStart) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventStart(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventFinish(evt mtglib.EventFinish) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventFinish(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) Shutdown() { + for _, v := range m.observers { + v.Shutdown() + } +} + +func newMultiObserver(factories []ObserverFactory) Observer { + observers := make([]Observer, len(factories)) + + for i, v := range factories { + observers[i] = v() + } + + return multiObserver{ + observers: observers, + } +} diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index e4c0370..076ab6e 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -29,14 +29,20 @@ const ( var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) type Firehol struct { - logger mtglib.Logger - rwMutex sync.RWMutex + ctx context.Context + ctxCancel context.CancelFunc + logger mtglib.Logger + + rwMutex sync.RWMutex + remoteURLs []string localFiles []string + httpClient *http.Client workerPool *ants.Pool - treeV4 *bool_tree.TreeV4 - treeV6 *bool_tree.TreeV6 + + treeV4 *bool_tree.TreeV4 + treeV6 *bool_tree.TreeV6 } func (f *Firehol) Contains(ip net.IP) bool { @@ -76,7 +82,7 @@ func (f *Firehol) containsIPv6(addr net.IP) bool { return false } -func (f *Firehol) Run(ctx context.Context, updateEach time.Duration) { +func (f *Firehol) Run(updateEach time.Duration) { ticker := time.NewTicker(updateEach) defer func() { @@ -88,24 +94,28 @@ func (f *Firehol) Run(ctx context.Context, updateEach time.Duration) { } }() - if err := f.update(ctx); err != nil { + if err := f.update(); err != nil { f.logger.WarningError("cannot update blocklist", err) } for { select { - case <-ctx.Done(): + case <-f.ctx.Done(): return case <-ticker.C: - if err := f.update(ctx); err != nil { + if err := f.update(); err != nil { f.logger.WarningError("cannot update blocklist", err) } } } } -func (f *Firehol) update(ctx context.Context) error { // nolint: funlen, cyclop - ctx, cancel := context.WithCancel(ctx) +func (f *Firehol) Shutdown() { + f.ctxCancel() +} + +func (f *Firehol) update() error { // nolint: funlen, cyclop + ctx, cancel := context.WithCancel(f.ctx) defer cancel() wg := &sync.WaitGroup{} @@ -314,8 +324,11 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, } workerPool, _ := ants.NewPool(int(downloadConcurrency)) + ctx, cancel := context.WithCancel(context.Background()) return &Firehol{ + ctx: ctx, + ctxCancel: cancel, logger: logger.Named("firehol"), httpClient: network.MakeHTTPClient(nil), treeV4: bool_tree.NewTreeV4(), diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go index 5b18820..ad353fe 100644 --- a/ipblocklist/firehol_test.go +++ b/ipblocklist/firehol_test.go @@ -1,7 +1,6 @@ package ipblocklist_test import ( - "context" "io" "net" "net/http" @@ -72,12 +71,15 @@ func (suite *FireholTestSuite) TestLocalFail() { suite.NoError(err) - go blocklist.Run(context.Background(), time.Hour) + go blocklist.Run(time.Hour) time.Sleep(500 * time.Millisecond) suite.False(blocklist.Contains(net.ParseIP("10.0.0.10"))) suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) } func (suite *FireholTestSuite) TestLocalOk() { @@ -87,12 +89,15 @@ func (suite *FireholTestSuite) TestLocalOk() { suite.NoError(err) - go blocklist.Run(context.Background(), time.Hour) + go blocklist.Run(time.Hour) time.Sleep(500 * time.Millisecond) suite.True(blocklist.Contains(net.ParseIP("10.0.0.10"))) suite.False(blocklist.Contains(net.ParseIP("127.0.0.1"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) } func (suite *FireholTestSuite) TestRemoteFail() { @@ -102,11 +107,14 @@ func (suite *FireholTestSuite) TestRemoteFail() { suite.NoError(err) - go blocklist.Run(context.Background(), time.Hour) + go blocklist.Run(time.Hour) time.Sleep(500 * time.Millisecond) suite.False(blocklist.Contains(net.ParseIP("10.2.2.2"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) } func (suite *FireholTestSuite) TestMixed() { @@ -123,12 +131,15 @@ func (suite *FireholTestSuite) TestMixed() { suite.NoError(err) - go blocklist.Run(context.Background(), time.Hour) + go blocklist.Run(time.Hour) time.Sleep(500 * time.Millisecond) suite.True(blocklist.Contains(net.ParseIP("10.2.2.2"))) suite.True(blocklist.Contains(net.ParseIP("10.1.0.100"))) + + blocklist.Shutdown() + time.Sleep(500 * time.Millisecond) } func TestFirehol(t *testing.T) { diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go index dcbed19..4b29449 100644 --- a/ipblocklist/noop.go +++ b/ipblocklist/noop.go @@ -9,6 +9,7 @@ import ( type noop struct{} func (n noop) Contains(ip net.IP) bool { return false } +func (n noop) Shutdown() {} func NewNoop() mtglib.IPBlocklist { return noop{} diff --git a/mtglib/events.go b/mtglib/events.go new file mode 100644 index 0000000..ef361a2 --- /dev/null +++ b/mtglib/events.go @@ -0,0 +1,25 @@ +package mtglib + +import ( + "net" + "time" +) + +type eventBase struct { + CreatedAt time.Time + ConnID string +} + +func (e eventBase) ConnectionID() string { + return e.ConnID +} + +type EventStart struct { + eventBase + + RemoteIP net.IP +} + +type EventFinish struct { + eventBase +} diff --git a/mtglib/init.go b/mtglib/init.go index 12e1393..8a3b764 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -19,10 +19,21 @@ type Network interface { type AntiReplayCache interface { SeenBefore(data []byte) bool + Shutdown() } type IPBlocklist interface { Contains(net.IP) bool + Shutdown() +} + +type Event interface { + ConnectionID() string +} + +type EventStream interface { + Send(context.Context, Event) + Shutdown() } type Logger interface { diff --git a/testlib/events_observer_mock.go b/testlib/events_observer_mock.go new file mode 100644 index 0000000..e3d8979 --- /dev/null +++ b/testlib/events_observer_mock.go @@ -0,0 +1,7 @@ +package testlib + +import "github.com/stretchr/testify/mock" + +type EventsObserverMock struct { + mock.Mock +} From f0063ba0897a925795040f154f3d0814e815db15 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 15 Mar 2021 21:39:21 +0300 Subject: [PATCH 073/206] Add observer mock --- events/init_test.go | 22 ++++++++++++++++++++++ ipblocklist/firehol.go | 33 ++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 events/init_test.go diff --git a/events/init_test.go b/events/init_test.go new file mode 100644 index 0000000..829ca5c --- /dev/null +++ b/events/init_test.go @@ -0,0 +1,22 @@ +package events_test + +import ( + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/mock" +) + +type ObserverMock struct { + mock.Mock +} + +func (o *ObserverMock) EventStart(evt mtglib.EventStart) { + o.Called(evt) +} + +func (o *ObserverMock) EventFinish(evt mtglib.EventStart) { + o.Called(evt) +} + +func (o *ObserverMock) Shutdown() { + o.Called() +} diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 076ab6e..876ad82 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -187,9 +187,14 @@ func (f *Firehol) updateLocalFile(ctx context.Context, filename string, return fmt.Errorf("cannot open file: %w", err) } + go func(ctx context.Context, closer io.Closer) { + <-ctx.Done() + closer.Close() + }(ctx, filefp) + defer filefp.Close() - return f.updateTrees(ctx, mutex, filefp, v4tree, v6tree) + return f.updateTrees(mutex, filefp, v4tree, v6tree) } func (f *Firehol) updateRemoteURL(ctx context.Context, url string, @@ -200,33 +205,31 @@ func (f *Firehol) updateRemoteURL(ctx context.Context, url string, return fmt.Errorf("cannot build a request: %w", err) } - resp, err := f.httpClient.Do(req) + resp, err := f.httpClient.Do(req) // nolint: bodyclose if err != nil { return fmt.Errorf("cannot request a remote URL %s: %w", url, err) } - defer func() { - io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck - resp.Body.Close() - }() + go func(ctx context.Context, closer io.Closer) { + <-ctx.Done() + closer.Close() + }(ctx, resp.Body) - return f.updateTrees(ctx, mutex, resp.Body, v4tree, v6tree) + defer func(rc io.ReadCloser) { + io.Copy(ioutil.Discard, rc) // nolint: errcheck + rc.Close() + }(resp.Body) + + return f.updateTrees(mutex, resp.Body, v4tree, v6tree) } -func (f *Firehol) updateTrees(ctx context.Context, - mutex sync.Locker, +func (f *Firehol) updateTrees(mutex sync.Locker, reader io.Reader, v4tree *bool_tree.TreeV4, v6tree *bool_tree.TreeV6) error { scanner := bufio.NewScanner(reader) for scanner.Scan() { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - text := scanner.Text() text = fireholRegexpComment.ReplaceAllLiteralString(text, "") text = strings.TrimSpace(text) From e009d05a906f34c7d50cf9de7fd907b8afada33a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 16 Mar 2021 11:28:34 +0300 Subject: [PATCH 074/206] Add tests for noop event stream --- events/event_stream.go | 25 ++++++++++++-- events/init.go | 1 + events/init_test.go | 4 +++ events/multi_observer.go | 15 ++++++++ events/noop.go | 27 +++++++++++++++ events/noop_test.go | 75 ++++++++++++++++++++++++++++++++++++++++ mtglib/events.go | 24 ++++++++----- mtglib/init.go | 2 +- 8 files changed, 160 insertions(+), 13 deletions(-) create mode 100644 events/noop.go create mode 100644 events/noop_test.go diff --git a/events/event_stream.go b/events/event_stream.go index 577e713..0a053b4 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -2,6 +2,7 @@ package events import ( "context" + "math/rand" "runtime" "github.com/9seconds/mtg/v2/mtglib" @@ -15,12 +16,20 @@ type eventStream struct { } func (e eventStream) Send(ctx context.Context, evt mtglib.Event) { - chanNo := int(xxhash.ChecksumString32(evt.ConnectionID())) % len(e.chans) + var chanNo uint32 + + streamID := evt.StreamID() + + if streamID == "" { + chanNo = rand.Uint32() + } else { + chanNo = xxhash.ChecksumString32(streamID) + } select { case <-ctx.Done(): case <-e.ctx.Done(): - case e.chans[chanNo] <- evt: + case e.chans[int(chanNo)%len(e.chans)] <- evt: } } @@ -29,6 +38,10 @@ func (e eventStream) Shutdown() { } func NewEventStream(observerFactories []ObserverFactory) mtglib.EventStream { + if len(observerFactories) == 0 { + observerFactories = append(observerFactories, NewNoopObserver) + } + ctx, cancel := context.WithCancel(context.Background()) rv := eventStream{ ctx: ctx, @@ -39,7 +52,11 @@ func NewEventStream(observerFactories []ObserverFactory) mtglib.EventStream { for i := 0; i < runtime.NumCPU(); i++ { rv.chans[i] = make(chan mtglib.Event, 1) - go eventStreamProcessor(ctx, rv.chans[i], newMultiObserver(observerFactories)) + if len(observerFactories) == 1 { + go eventStreamProcessor(ctx, rv.chans[i], observerFactories[0]()) + } else { + go eventStreamProcessor(ctx, rv.chans[i], newMultiObserver(observerFactories)) + } } return rv @@ -58,6 +75,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob observer.EventStart(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) + case mtglib.EventConcurrencyLimited: + observer.EventConcurrencyLimited(typedEvt) } } } diff --git a/events/init.go b/events/init.go index f84b03b..9a632f4 100644 --- a/events/init.go +++ b/events/init.go @@ -5,6 +5,7 @@ import "github.com/9seconds/mtg/v2/mtglib" type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) + EventConcurrencyLimited(mtglib.EventConcurrencyLimited) Shutdown() } diff --git a/events/init_test.go b/events/init_test.go index 829ca5c..c67efa9 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -17,6 +17,10 @@ func (o *ObserverMock) EventFinish(evt mtglib.EventStart) { o.Called(evt) } +func (o *ObserverMock) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimited) { + o.Called(evt) +} + func (o *ObserverMock) Shutdown() { o.Called() } diff --git a/events/multi_observer.go b/events/multi_observer.go index 9370fb0..698d87c 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -40,6 +40,21 @@ func (m multiObserver) EventFinish(evt mtglib.EventFinish) { wg.Wait() } +func (m multiObserver) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimited) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventConcurrencyLimited(evt) + }(v) + } + + wg.Wait() +} + func (m multiObserver) Shutdown() { for _, v := range m.observers { v.Shutdown() diff --git a/events/noop.go b/events/noop.go new file mode 100644 index 0000000..7eb4041 --- /dev/null +++ b/events/noop.go @@ -0,0 +1,27 @@ +package events + +import ( + "context" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type noop struct{} + +func (n noop) Send(ctx context.Context, evt mtglib.Event) {} +func (n noop) Shutdown() {} + +func NewNoopStream() mtglib.EventStream { + return noop{} +} + +type noopObserver struct{} + +func (n noopObserver) EventStart(_ mtglib.EventStart) {} +func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} +func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} +func (n noopObserver) Shutdown() {} + +func NewNoopObserver() Observer { + return noopObserver{} +} diff --git a/events/noop_test.go b/events/noop_test.go new file mode 100644 index 0000000..b5120f3 --- /dev/null +++ b/events/noop_test.go @@ -0,0 +1,75 @@ +package events_test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite + + testData map[string]mtglib.Event + ctx context.Context +} + +func (suite *NoopTestSuite) SetupSuite() { + suite.testData = map[string]mtglib.Event{ + "start": mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("127.0.0.1"), + }, + "finish": mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "connID", + }, + "concurrency-limited": mtglib.EventConcurrencyLimited{}, + } + suite.ctx = context.Background() +} + +func (suite *NoopTestSuite) TestStream() { + stream := events.NewNoopStream() + + for name, v := range suite.testData { + value := v + + suite.T().Run(name, func(t *testing.T) { + stream.Send(suite.ctx, value) + }) + } + + stream.Shutdown() +} + +func (suite *NoopTestSuite) TestObserver() { + observer := events.NewNoopObserver() + + for name, v := range suite.testData { + value := v + + suite.T().Run(name, func(t *testing.T) { + switch typedEvt := value.(type) { + case mtglib.EventStart: + observer.EventStart(typedEvt) + case mtglib.EventFinish: + observer.EventFinish(typedEvt) + case mtglib.EventConcurrencyLimited: + observer.EventConcurrencyLimited(typedEvt) + } + }) + } + + observer.Shutdown() +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} diff --git a/mtglib/events.go b/mtglib/events.go index ef361a2..61f282a 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -5,21 +5,27 @@ import ( "time" ) -type eventBase struct { +type EventStart struct { + CreatedAt time.Time + ConnID string + RemoteIP net.IP +} + +func (e EventStart) StreamID() string { + return e.ConnID +} + +type EventFinish struct { CreatedAt time.Time ConnID string } -func (e eventBase) ConnectionID() string { +func (e EventFinish) StreamID() string { return e.ConnID } -type EventStart struct { - eventBase +type EventConcurrencyLimited struct{} - RemoteIP net.IP -} - -type EventFinish struct { - eventBase +func (e EventConcurrencyLimited) StreamID() string { + return "" } diff --git a/mtglib/init.go b/mtglib/init.go index 8a3b764..22c5472 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -28,7 +28,7 @@ type IPBlocklist interface { } type Event interface { - ConnectionID() string + StreamID() string } type EventStream interface { From 7617aeadda0fefd777c66ca22526310295e4fcc6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 16 Mar 2021 14:54:17 +0300 Subject: [PATCH 075/206] Test event stream --- events/event_stream.go | 8 +-- events/event_stream_test.go | 122 ++++++++++++++++++++++++++++++++++++ events/init.go | 2 +- events/init_test.go | 2 +- mtglib/events.go | 4 +- 5 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 events/event_stream_test.go diff --git a/events/event_stream.go b/events/event_stream.go index 0a053b4..be860f6 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -18,12 +18,10 @@ type eventStream struct { func (e eventStream) Send(ctx context.Context, evt mtglib.Event) { var chanNo uint32 - streamID := evt.StreamID() - - if streamID == "" { - chanNo = rand.Uint32() - } else { + if streamID := evt.StreamID(); streamID != "" { chanNo = xxhash.ChecksumString32(streamID) + } else { + chanNo = rand.Uint32() } select { diff --git a/events/event_stream_test.go b/events/event_stream_test.go new file mode 100644 index 0000000..b86b68b --- /dev/null +++ b/events/event_stream_test.go @@ -0,0 +1,122 @@ +package events_test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type EventStreamTestSuite struct { + suite.Suite + + ctx context.Context + ctxCancel context.CancelFunc + observerMock1 *ObserverMock + observerMock2 *ObserverMock + stream mtglib.EventStream +} + +func (suite *EventStreamTestSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) + + suite.observerMock1 = &ObserverMock{} + suite.observerMock2 = &ObserverMock{} + + suite.observerMock1.On("Shutdown") + suite.observerMock2.On("Shutdown") + + factories := make([]events.ObserverFactory, 2) + factories[0] = func() events.Observer { return suite.observerMock1 } + factories[1] = func() events.Observer { return suite.observerMock2 } + + suite.stream = events.NewEventStream(factories) +} + +func (suite *EventStreamTestSuite) TestEventStartOk() { + evt := mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.1"), + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventStart", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventStart) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.ConnID, caught.ConnID) + suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) + suite.Equal(evt.StreamID(), caught.StreamID()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventFinishOk() { + evt := mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "connID", + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventFinish", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventFinish) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.ConnID, caught.ConnID) + suite.Equal(evt.StreamID(), caught.StreamID()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventConcurrencyLimitedOk() { + evt := mtglib.EventConcurrencyLimited{ + CreatedAt: time.Now(), + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventConcurrencyLimited", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventConcurrencyLimited) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TearDownTest() { + suite.stream.Shutdown() + suite.ctxCancel() + + time.Sleep(100 * time.Millisecond) + + suite.observerMock1.AssertExpectations(suite.T()) + suite.observerMock2.AssertExpectations(suite.T()) +} + +func TestEventStream(t *testing.T) { + t.Parallel() + suite.Run(t, &EventStreamTestSuite{}) +} diff --git a/events/init.go b/events/init.go index 9a632f4..0e348f5 100644 --- a/events/init.go +++ b/events/init.go @@ -5,7 +5,7 @@ import "github.com/9seconds/mtg/v2/mtglib" type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) - EventConcurrencyLimited(mtglib.EventConcurrencyLimited) + EventConcurrencyLimited(mtglib.EventConcurrencyLimited) Shutdown() } diff --git a/events/init_test.go b/events/init_test.go index c67efa9..e75e5e9 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -13,7 +13,7 @@ func (o *ObserverMock) EventStart(evt mtglib.EventStart) { o.Called(evt) } -func (o *ObserverMock) EventFinish(evt mtglib.EventStart) { +func (o *ObserverMock) EventFinish(evt mtglib.EventFinish) { o.Called(evt) } diff --git a/mtglib/events.go b/mtglib/events.go index 61f282a..e7ea557 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -24,7 +24,9 @@ func (e EventFinish) StreamID() string { return e.ConnID } -type EventConcurrencyLimited struct{} +type EventConcurrencyLimited struct { + CreatedAt time.Time +} func (e EventConcurrencyLimited) StreamID() string { return "" From 46bd61758181092524af11c54aef3cea5c0f8eba Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 16 Mar 2021 17:28:03 +0300 Subject: [PATCH 076/206] Add tags for statsd in config --- config/config.go | 8 +- config/type_statsd_tag_format.go | 49 ++++++++++ config/type_statsd_tag_format_test.go | 130 ++++++++++++++++++++++++++ example.config.toml | 4 + 4 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 config/type_statsd_tag_format.go create mode 100644 config/type_statsd_tag_format_test.go diff --git a/config/config.go b/config/config.go index dd7f9c8..0bd079d 100644 --- a/config/config.go +++ b/config/config.go @@ -45,9 +45,10 @@ type Config struct { } `json:"network"` Stats struct { StatsD struct { - Enabled bool `json:"enabled"` - Address TypeHostPort `json:"address"` - MetricPrefix TypeMetricPrefix `json:"metric-prefix"` + Enabled bool `json:"enabled"` + Address TypeHostPort `json:"address"` + MetricPrefix TypeMetricPrefix `json:"metric-prefix"` + TagFormat TypeStatsdTagFormat `json:"tag-format"` } `json:"statsd"` Prometheus struct { Enabled bool `json:"enabled"` @@ -122,6 +123,7 @@ type configRaw struct { Enabled bool `toml:"enabled" json:"enabled,omitempty"` Address string `toml:"address" json:"address,omitempty"` MetricPrefix string `toml:"metric-prefix" json:"metric-prefix,omitempty"` + TagFormat string `toml:"tag-format" json:"tag-format,omitempty"` } `toml:"statsd" json:"statsd,omitempty"` Prometheus struct { Enabled bool `toml:"enabled" json:"enabled,omitempty"` diff --git a/config/type_statsd_tag_format.go b/config/type_statsd_tag_format.go new file mode 100644 index 0000000..7903b0c --- /dev/null +++ b/config/type_statsd_tag_format.go @@ -0,0 +1,49 @@ +package config + +import ( + "fmt" + "strings" +) + +const ( + TypeStatsdTagFormatInfluxdb = "influxdb" + TypeStatsdTagFormatDatadog = "datadog" + TypeStatsdTagFormatGraphite = "graphite" +) + +type TypeStatsdTagFormat struct { + value string +} + +func (c *TypeStatsdTagFormat) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := strings.ToLower(string(data)) + + switch text { + case TypeStatsdTagFormatInfluxdb, TypeStatsdTagFormatDatadog, TypeStatsdTagFormatGraphite: + c.value = text + default: + return fmt.Errorf("incorrect tag format value: %s", string(data)) + } + + return nil +} + +func (c TypeStatsdTagFormat) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + +func (c *TypeStatsdTagFormat) String() string { + return c.value +} + +func (c *TypeStatsdTagFormat) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/config/type_statsd_tag_format_test.go b/config/type_statsd_tag_format_test.go new file mode 100644 index 0000000..2c7074f --- /dev/null +++ b/config/type_statsd_tag_format_test.go @@ -0,0 +1,130 @@ +package config_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeStatsdTagFormatTestStruct struct { + Value config.TypeStatsdTagFormat `json:"value"` +} + +type TypeStatsdTagFormat struct { + suite.Suite +} + +func (suite *TypeStatsdTagFormat) TestUnmarshalFail() { + testData := []string{ + "p", + "ipv4", + "onlyipv4", + "ipv6prefer", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeStatsdTagFormatTestStruct{})) + }) + } +} + +func (suite *TypeStatsdTagFormat) TestUnmarshalOk() { + testData := []string{ + config.TypeStatsdTagFormatDatadog, + config.TypeStatsdTagFormatInfluxdb, + config.TypeStatsdTagFormatGraphite, + strings.ToUpper(config.TypeStatsdTagFormatDatadog), + strings.ToUpper(config.TypeStatsdTagFormatInfluxdb), + strings.ToUpper(config.TypeStatsdTagFormatGraphite), + strings.ToLower(config.TypeStatsdTagFormatDatadog), + strings.ToLower(config.TypeStatsdTagFormatInfluxdb), + strings.ToLower(config.TypeStatsdTagFormatGraphite), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeStatsdTagFormatTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, + strings.ToLower(value), + testStruct.Value.Value(config.TypeStatsdTagFormatDatadog)) + }) + } +} + +func (suite *TypeStatsdTagFormat) TestMarshalOk() { + testData := []string{ + config.TypeStatsdTagFormatDatadog, + config.TypeStatsdTagFormatInfluxdb, + config.TypeStatsdTagFormatGraphite, + strings.ToUpper(config.TypeStatsdTagFormatDatadog), + strings.ToUpper(config.TypeStatsdTagFormatInfluxdb), + strings.ToUpper(config.TypeStatsdTagFormatGraphite), + strings.ToLower(config.TypeStatsdTagFormatDatadog), + strings.ToLower(config.TypeStatsdTagFormatInfluxdb), + strings.ToLower(config.TypeStatsdTagFormatGraphite), + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeStatsdTagFormatTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, strings.ToLower(value), testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, strings.ToLower(value), string(marshalled)) + }) + } +} + +func (suite *TypeStatsdTagFormat) TestValue() { + testStruct := &typePreferIPTestStruct{} + + suite.EqualValues(config.TypePreferIPPreferIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv4)) + suite.EqualValues(config.TypePreferIPPreferIPv6, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) + + data, err := json.Marshal(map[string]string{ + "value": config.TypePreferOnlyIPv4, + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferOnlyIPv6)) + suite.EqualValues(config.TypePreferOnlyIPv4, + testStruct.Value.Value(config.TypePreferIPPreferIPv6)) +} + +func TestTypeStatsdTagFormat(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeStatsdTagFormat{}) +} diff --git a/example.config.toml b/example.config.toml index c3d3fbb..d573a16 100644 --- a/example.config.toml +++ b/example.config.toml @@ -174,6 +174,10 @@ enabled = false address = "127.0.0.1:8888" # prefix of metric for statsd metric-prefix = "mtg" +# tag format to use +# supported values are 'datadog', 'influxdb' and 'graphite' +# default format is graphite. +tag-format = "datadog" # prometheus metrics integration. [stats.prometheus] From c04f9e392a2c5e53ba6b24f49daf5fe9ce2ef649 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 16 Mar 2021 17:28:52 +0300 Subject: [PATCH 077/206] Small improvements for config parsing --- config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 0bd079d..179a2f1 100644 --- a/config/config.go +++ b/config/config.go @@ -148,11 +148,11 @@ func Parse(rawData []byte) (*Config, error) { } if err := jsonEncoder.Encode(rawConf); err != nil { - return nil, fmt.Errorf("cannot dump into interim format: %w", err) + panic(err) } if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { - return nil, fmt.Errorf("cannot parse final config: %w", err) + return nil, fmt.Errorf("cannot parse a config: %w", err) } if err := conf.Validate(); err != nil { From 8e7207d975829b6a8f7aa1f7cdb7415b0ebbfe0e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 11:07:16 +0300 Subject: [PATCH 078/206] Add statsd --- go.mod | 1 + go.sum | 2 + stats/init.go | 12 +++++ stats/statsd.go | 115 +++++++++++++++++++++++++++++++++++++++ stats/statsd_test.go | 126 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 stats/init.go create mode 100644 stats/statsd.go create mode 100644 stats/statsd_test.go diff --git a/go.mod b/go.mod index f9caefe..281af91 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/panjf2000/ants v1.3.0 // indirect github.com/pelletier/go-toml v1.8.1 github.com/rs/zerolog v1.20.0 // indirect + github.com/smira/go-statsd v1.3.2 // indirect github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c // indirect diff --git a/go.sum b/go.sum index bcb02ae..3aee538 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= +github.com/smira/go-statsd v1.3.2 h1:1EeuzxNZ/TD9apbTOFSM9nulqfcsQFmT4u1A2DREabI= +github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= diff --git a/stats/init.go b/stats/init.go new file mode 100644 index 0000000..7210888 --- /dev/null +++ b/stats/init.go @@ -0,0 +1,12 @@ +package stats + +const ( + MetricActiveConnection = "active_connections" + MetricSessionDuration = "session_duration" + MetricConcurrencyLimited = "concurrency_limited" + + TagIPType = "ip_type" + + TagIPTypeIPv4 = "ipv4" + TagIPTypeIPv6 = "ipv4" +) diff --git a/stats/statsd.go b/stats/statsd.go new file mode 100644 index 0000000..92e0979 --- /dev/null +++ b/stats/statsd.go @@ -0,0 +1,115 @@ +package stats + +import ( + "fmt" + "net" + "strings" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + statsd "github.com/smira/go-statsd" +) + +type statsdFakeLogger struct{} + +func (s statsdFakeLogger) Printf(msg string, args ...interface{}) {} + +type statsdStreamInfo struct { + createdAt time.Time + clientIP net.IP +} + +func (s *statsdStreamInfo) ClientIPTag() statsd.Tag { + if s.clientIP.To4() == nil { + return statsd.StringTag(TagIPType, TagIPTypeIPv6) + } else { + return statsd.StringTag(TagIPType, TagIPTypeIPv4) + } +} + +type statsdProcessor struct { + streams map[string]*statsdStreamInfo + client *statsd.Client +} + +func (s statsdProcessor) EventStart(evt mtglib.EventStart) { + clientInfo := &statsdStreamInfo{ + createdAt: evt.CreatedAt, + clientIP: evt.RemoteIP, + } + s.streams[evt.StreamID()] = clientInfo + + s.client.GaugeDelta(MetricActiveConnection, 1, clientInfo.ClientIPTag()) +} + +func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { + clientInfo, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + defer delete(s.streams, evt.StreamID()) + + duration := evt.CreatedAt.Sub(clientInfo.createdAt) + + s.client.GaugeDelta(MetricActiveConnection, -1, clientInfo.ClientIPTag()) + s.client.PrecisionTiming(MetricSessionDuration, duration) +} + +func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { + s.client.Incr(MetricConcurrencyLimited, 1) +} + +func (s statsdProcessor) Shutdown() { + now := time.Now() + events := make([]mtglib.EventFinish, 0, len(s.streams)) + + for k := range s.streams { + events = append(events, mtglib.EventFinish{ + CreatedAt: now, + ConnID: k, + }) + } + + for i := range events { + s.EventFinish(events[i]) + } +} + +type StatsdFactory struct { + client *statsd.Client +} + +func (s StatsdFactory) Close() error { + return s.client.Close() +} + +func (s StatsdFactory) Make() events.Observer { + return statsdProcessor{ + client: s.client, + streams: make(map[string]*statsdStreamInfo), + } +} + +func NewStatsd(address, metricPrefix, tagFormat string) (StatsdFactory, error) { + options := []statsd.Option{ + statsd.MetricPrefix(metricPrefix), + statsd.Logger(statsdFakeLogger{}), + } + + switch strings.ToLower(tagFormat) { + case "datadog": + options = append(options, statsd.TagStyle(statsd.TagFormatDatadog)) + case "influxdb": + options = append(options, statsd.TagStyle(statsd.TagFormatInfluxDB)) + case "graphite": + options = append(options, statsd.TagStyle(statsd.TagFormatGraphite)) + default: + return StatsdFactory{}, fmt.Errorf("unknown tag format %s", tagFormat) + } + + return StatsdFactory{ + client: statsd.NewClient(address, options...), + }, nil +} diff --git a/stats/statsd_test.go b/stats/statsd_test.go new file mode 100644 index 0000000..13b6f02 --- /dev/null +++ b/stats/statsd_test.go @@ -0,0 +1,126 @@ +package stats_test + +import ( + "bytes" + "net" + "strings" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/stats" + statsd "github.com/smira/go-statsd" + "github.com/stretchr/testify/suite" +) + +type statsdFakeServer struct { + conn *net.UDPConn + buf *bytes.Buffer +} + +func (s statsdFakeServer) Addr() string { + return s.conn.LocalAddr().String() +} + +func (s statsdFakeServer) Close() error { + if s.conn != nil { + return s.conn.Close() + } + + return nil +} + +func (s statsdFakeServer) String() string { + return strings.TrimSpace(s.buf.String()) +} + +func statsdNewFakeServer() statsdFakeServer { + conn, err := net.ListenUDP("udp", &net.UDPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }) + if err != nil { + panic(err) + } + + buf := &bytes.Buffer{} + + go func() { + currentBuffer := make([]byte, 4096) + + for { + n, _, err := conn.ReadFromUDP(currentBuffer) + if n > 0 { + buf.Write(currentBuffer[:n]) + } + + if err != nil { + return + } + } + }() + + return statsdFakeServer{ + conn: conn, + buf: buf, + } +} + +type StatsdTestSuite struct { + suite.Suite + + statsdServer statsdFakeServer + factory stats.StatsdFactory + statsd events.Observer +} + +func (suite *StatsdTestSuite) SetupTest() { + suite.statsdServer = statsdNewFakeServer() + + factory, err := stats.NewStatsd(suite.statsdServer.Addr(), "mtg.", "datadog") + if err != nil { + panic(err) + } + + suite.factory = factory + suite.statsd = suite.factory.Make() +} + +func (suite *StatsdTestSuite) TearDownTest() { + suite.statsd.Shutdown() + suite.factory.Close() + suite.statsdServer.Close() +} + +func (suite *StatsdTestSuite) TestEventStartFinish() { + suite.statsd.EventStart(mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + + time.Sleep(2 * statsd.DefaultFlushInterval) + suite.Equal("mtg.active_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String()) + + suite.statsd.EventFinish(mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + + time.Sleep(2 * statsd.DefaultFlushInterval) + suite.Contains(suite.statsdServer.String(), "mtg.session_duration") +} + +func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { + suite.statsd.EventConcurrencyLimited(mtglib.EventConcurrencyLimited{ + CreatedAt: time.Now(), + }) + + time.Sleep(2 * statsd.DefaultFlushInterval) + suite.Equal("mtg.concurrency_limited:1|c", suite.statsdServer.String()) +} + +func TestStatsd(t *testing.T) { + t.Parallel() + suite.Run(t, &StatsdTestSuite{}) +} From dbbfdaf918bc367dd56dd5cf8201cc1c65506dc3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 12:00:36 +0300 Subject: [PATCH 079/206] Add prometheus --- config/config.go | 2 +- go.mod | 1 + go.sum | 375 +++++++++++++++++++++++++++++++++++++++ stats/prometheus.go | 123 +++++++++++++ stats/prometheus_test.go | 97 ++++++++++ stats/statsd.go | 32 ++-- stats/statsd_test.go | 1 + stats/stream_info.go | 19 ++ 8 files changed, 627 insertions(+), 23 deletions(-) create mode 100644 stats/prometheus.go create mode 100644 stats/prometheus_test.go create mode 100644 stats/stream_info.go diff --git a/config/config.go b/config/config.go index 179a2f1..cbb2156 100644 --- a/config/config.go +++ b/config/config.go @@ -152,7 +152,7 @@ func Parse(rawData []byte) (*Config, error) { } if err := json.NewDecoder(jsonBuf).Decode(conf); err != nil { - return nil, fmt.Errorf("cannot parse a config: %w", err) + return nil, fmt.Errorf("cannot parse a config: %w", err) } if err := conf.Validate(); err != nil { diff --git a/go.mod b/go.mod index 281af91..05e2b83 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/mccutchen/go-httpbin v1.1.1 github.com/panjf2000/ants v1.3.0 // indirect github.com/pelletier/go-toml v1.8.1 + github.com/prometheus/client_golang v1.9.0 // indirect github.com/rs/zerolog v1.20.0 // indirect github.com/smira/go-statsd v1.3.2 // indirect github.com/stretchr/objx v0.3.0 // indirect diff --git a/go.sum b/go.sum index 3aee538..fd76767 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,160 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/kong v0.2.16 h1:F232CiYSn54Tnl1sJGTeHmx4vJDNLVP2b9yCVMOQwHQ= github.com/alecthomas/kong v0.2.16/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 h1:4NNbNM2Iq/k57qEu7WfL67UrbPq1uFWxW4qODCohi+0= github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6/go.mod h1:J29hk+f9lJrblVIfiJOtTFk+OblBawmib4uz/VdKzlg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 h1:D7qhJP3R49ZjUzpzKQ6B2H3lgejPs6DTO5gRomhhOpE= github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1/go.mod h1:2OfLA+0esiUJpwMjrH39pEk79cb8MvGTBS9YlZpejJ4= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -23,23 +162,125 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/libp2p/go-reuseport v0.0.2 h1:XSG94b1FJfGA01BUrT82imejHQyTxO4jEWqheyCXYvU= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mccutchen/go-httpbin v1.1.1 h1:aEws49HEJEyXHLDnshQVswfUlCVoS8g6h9YaDyaW7RE= github.com/mccutchen/go-httpbin v1.1.1/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smira/go-statsd v1.3.2 h1:1EeuzxNZ/TD9apbTOFSM9nulqfcsQFmT4u1A2DREabI= github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.1.5-0.20170809224252-890a5c3458b4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -49,33 +290,167 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c h1:pGEq55pv/5i+G/Dy+kDsUugzvg6R02jzegxtrwOhE7A= github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c/go.mod h1:OYRfF6eb5wY9VRFkXJH8FFBi3plw2v+giaIu7P054pM= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/stats/prometheus.go b/stats/prometheus.go new file mode 100644 index 0000000..69b23e3 --- /dev/null +++ b/stats/prometheus.go @@ -0,0 +1,123 @@ +package stats + +import ( + "context" + "net" + "net/http" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type prometheusProcessor struct { + streams map[string]*streamInfo + factory *PrometheusFactory +} + +func (p prometheusProcessor) EventStart(evt mtglib.EventStart) { + sInfo := &streamInfo{ + createdAt: evt.CreatedAt, + clientIP: evt.RemoteIP, + } + p.streams[evt.StreamID()] = sInfo + + p.factory.metricActiveConnections.WithLabelValues(sInfo.IPType()).Inc() +} + +func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { + sInfo, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + defer delete(p.streams, evt.StreamID()) + + duration := evt.CreatedAt.Sub(sInfo.createdAt) + + p.factory.metricActiveConnections.WithLabelValues(sInfo.IPType()).Dec() + p.factory.metricSessionDuration.Observe(float64(duration) / float64(time.Second)) +} + +func (p prometheusProcessor) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimited) { + p.factory.metricConcurrencyLimited.Inc() +} + +func (p prometheusProcessor) Shutdown() { + p.streams = make(map[string]*streamInfo) +} + +type PrometheusFactory struct { + httpServer *http.Server + + metricActiveConnections *prometheus.GaugeVec + metricConcurrencyLimited prometheus.Counter + metricSessionDuration prometheus.Histogram +} + +func (p *PrometheusFactory) Make() events.Observer { + return prometheusProcessor{ + streams: make(map[string]*streamInfo), + factory: p, + } +} + +func (p *PrometheusFactory) Serve(listener net.Listener) error { + return p.httpServer.Serve(listener) +} + +func (p *PrometheusFactory) Close() error { + return p.httpServer.Shutdown(context.Background()) +} + +func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { + registry := prometheus.NewPedanticRegistry() + httpHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ + EnableOpenMetrics: true, + }) + mux := http.NewServeMux() + + mux.Handle(httpPath, httpHandler) + + factory := &PrometheusFactory{ + httpServer: &http.Server{ + Handler: mux, + }, + + metricActiveConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricPrefix, + Name: MetricActiveConnection, + Help: "A number of connections under active processing.", + }, []string{TagIPType}), + metricSessionDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: metricPrefix, + Name: MetricSessionDuration, + Help: "Session duration.", + Buckets: []float64{ // per 30 seconds + 30, + 60, + 90, + 120, + 150, + 180, + 210, + 240, + 270, + 300, + }, + }), + metricConcurrencyLimited: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricConcurrencyLimited, + Help: "A number of sessions that were rejected by concurrency limiter.", + }), + } + + registry.MustRegister(factory.metricActiveConnections) + registry.MustRegister(factory.metricSessionDuration) + registry.MustRegister(factory.metricConcurrencyLimited) + + return factory +} diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go new file mode 100644 index 0000000..319a6d1 --- /dev/null +++ b/stats/prometheus_test.go @@ -0,0 +1,97 @@ +package stats_test + +import ( + "fmt" + "io/ioutil" + "net" + "net/http" + "testing" + "time" + + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/stats" + "github.com/stretchr/testify/suite" +) + +type PrometheusTestSuite struct { + suite.Suite + + httpListener net.Listener + factory *stats.PrometheusFactory + prometheus events.Observer +} + +func (suite *PrometheusTestSuite) Get() (string, error) { + addr := fmt.Sprintf("http://%s/", suite.httpListener.Addr().String()) + + resp, err := http.Get(addr) // nolint: noctx + if err != nil { + return "", err // nolint: wrapcheck + } + + defer resp.Body.Close() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err // nolint: wrapcheck + } + + return string(data), nil +} + +func (suite *PrometheusTestSuite) SetupTest() { + suite.httpListener, _ = net.Listen("tcp", "127.0.0.1:0") + suite.factory = stats.NewPrometheus("mtg", "/") + suite.prometheus = suite.factory.Make() + + go suite.factory.Serve(suite.httpListener) // nolint: errcheck +} + +func (suite *PrometheusTestSuite) TearDownTest() { + suite.prometheus.Shutdown() + suite.NoError(suite.factory.Close()) + suite.httpListener.Close() +} + +func (suite *PrometheusTestSuite) TestEventStartFinish() { + suite.prometheus.EventStart(mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.10"), + }) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_active_connections{ip_type="ipv4"} 1`) + + suite.prometheus.EventFinish(mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_active_connections{ip_type="ipv4"} 0`) +} + +func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { + suite.prometheus.EventConcurrencyLimited(mtglib.EventConcurrencyLimited{ + CreatedAt: time.Now(), + }) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_concurrency_limited 1`) +} + +func TestPrometheus(t *testing.T) { + t.Parallel() + suite.Run(t, &PrometheusTestSuite{}) +} diff --git a/stats/statsd.go b/stats/statsd.go index 92e0979..be04c51 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -2,7 +2,6 @@ package stats import ( "fmt" - "net" "strings" "time" @@ -15,45 +14,34 @@ type statsdFakeLogger struct{} func (s statsdFakeLogger) Printf(msg string, args ...interface{}) {} -type statsdStreamInfo struct { - createdAt time.Time - clientIP net.IP -} - -func (s *statsdStreamInfo) ClientIPTag() statsd.Tag { - if s.clientIP.To4() == nil { - return statsd.StringTag(TagIPType, TagIPTypeIPv6) - } else { - return statsd.StringTag(TagIPType, TagIPTypeIPv4) - } -} - type statsdProcessor struct { - streams map[string]*statsdStreamInfo + streams map[string]*streamInfo client *statsd.Client } func (s statsdProcessor) EventStart(evt mtglib.EventStart) { - clientInfo := &statsdStreamInfo{ + sInfo := &streamInfo{ createdAt: evt.CreatedAt, clientIP: evt.RemoteIP, } - s.streams[evt.StreamID()] = clientInfo + s.streams[evt.StreamID()] = sInfo + ipTypeTag := statsd.StringTag(TagIPType, sInfo.IPType()) - s.client.GaugeDelta(MetricActiveConnection, 1, clientInfo.ClientIPTag()) + s.client.GaugeDelta(MetricActiveConnection, 1, ipTypeTag) } func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { - clientInfo, ok := s.streams[evt.StreamID()] + sInfo, ok := s.streams[evt.StreamID()] if !ok { return } defer delete(s.streams, evt.StreamID()) - duration := evt.CreatedAt.Sub(clientInfo.createdAt) + duration := evt.CreatedAt.Sub(sInfo.createdAt) + ipTypeTag := statsd.StringTag(TagIPType, sInfo.IPType()) - s.client.GaugeDelta(MetricActiveConnection, -1, clientInfo.ClientIPTag()) + s.client.GaugeDelta(MetricActiveConnection, -1, ipTypeTag) s.client.PrecisionTiming(MetricSessionDuration, duration) } @@ -88,7 +76,7 @@ func (s StatsdFactory) Close() error { func (s StatsdFactory) Make() events.Observer { return statsdProcessor{ client: s.client, - streams: make(map[string]*statsdStreamInfo), + streams: make(map[string]*streamInfo), } } diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 13b6f02..8732463 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -97,6 +97,7 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { suite.statsd.EventStart(mtglib.EventStart{ CreatedAt: time.Now(), ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.10"), }) time.Sleep(2 * statsd.DefaultFlushInterval) diff --git a/stats/stream_info.go b/stats/stream_info.go new file mode 100644 index 0000000..fe51136 --- /dev/null +++ b/stats/stream_info.go @@ -0,0 +1,19 @@ +package stats + +import ( + "net" + "time" +) + +type streamInfo struct { + createdAt time.Time + clientIP net.IP +} + +func (s *streamInfo) IPType() string { + if s.clientIP.To4() == nil { + return TagIPTypeIPv6 + } + + return TagIPTypeIPv4 +} From 7955ac6a465418138dda0046916993fed0caca97 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 13:44:27 +0300 Subject: [PATCH 080/206] Add tests for events --- mtglib/events_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 mtglib/events_test.go diff --git a/mtglib/events_test.go b/mtglib/events_test.go new file mode 100644 index 0000000..ed08aa5 --- /dev/null +++ b/mtglib/events_test.go @@ -0,0 +1,42 @@ +package mtglib_test + +import ( + "net" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/stretchr/testify/suite" +) + +type EventsTestSuite struct { + suite.Suite +} + +func (suite *EventsTestSuite) TestEventStart() { + evt := mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "CONNID", + RemoteIP: net.ParseIP("10.0.0.10"), + } + + suite.Equal("CONNID", evt.StreamID()) +} + +func (suite *EventsTestSuite) TestEventFinish() { + evt := mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "CONNID", + } + + suite.Equal("CONNID", evt.StreamID()) +} + +func (suite *EventsTestSuite) TestEventConcurrencyLimited() { + suite.Empty(mtglib.EventConcurrencyLimited{}.StreamID()) +} + +func TestEvents(t *testing.T) { + t.Parallel() + suite.Run(t, &EventsTestSuite{}) +} From 23519913f2d3a93d657f2ef7cce171c76e661651 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 16:12:44 +0300 Subject: [PATCH 081/206] Add skeleton of the proxy --- go.mod | 17 +++--- go.sum | 9 ++- ipblocklist/firehol.go | 2 +- mtglib/init.go | 14 ++++- mtglib/proxy.go | 120 +++++++++++++++++++++++++++++++++++++++ mtglib/proxy_opts.go | 12 ++++ mtglib/stream_context.go | 63 ++++++++++++++++++++ 7 files changed, 224 insertions(+), 13 deletions(-) create mode 100644 mtglib/proxy.go create mode 100644 mtglib/proxy_opts.go create mode 100644 mtglib/stream_context.go diff --git a/go.mod b/go.mod index 05e2b83..a09b213 100644 --- a/go.mod +++ b/go.mod @@ -3,26 +3,25 @@ module github.com/9seconds/mtg/v2 go 1.16 require ( - github.com/OneOfOne/xxhash v1.2.8 // indirect + github.com/OneOfOne/xxhash v1.2.8 github.com/alecthomas/kong v0.2.16 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 + github.com/d4l3k/messagediff v1.2.1 // indirect github.com/jarcoal/httpmock v1.0.8 - github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 // indirect - github.com/kr/pretty v0.1.0 // indirect + github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 github.com/libp2p/go-reuseport v0.0.2 github.com/mccutchen/go-httpbin v1.1.1 - github.com/panjf2000/ants v1.3.0 // indirect + github.com/panjf2000/ants/v2 v2.4.3 github.com/pelletier/go-toml v1.8.1 - github.com/prometheus/client_golang v1.9.0 // indirect - github.com/rs/zerolog v1.20.0 // indirect - github.com/smira/go-statsd v1.3.2 // indirect + github.com/prometheus/client_golang v1.9.0 + github.com/rs/zerolog v1.20.0 + github.com/smira/go-statsd v1.3.2 github.com/stretchr/objx v0.3.0 // indirect github.com/stretchr/testify v1.7.0 - github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c // indirect + github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index fd76767..d9e0113 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= +github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -100,6 +102,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -210,8 +213,8 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/panjf2000/ants v1.3.0 h1:8pQ+8leaLc9lys2viEEr8md0U4RN6uOSUCE9bOYjQ9M= -github.com/panjf2000/ants v1.3.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= +github.com/panjf2000/ants/v2 v2.4.3 h1:wHghL17YKFanB62QjPQ9o+DuM4q7WrQ7zAhoX8+eBXU= +github.com/panjf2000/ants/v2 v2.4.3/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= @@ -402,6 +405,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -445,6 +449,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 876ad82..2d8a748 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -18,7 +18,7 @@ import ( "github.com/9seconds/mtg/v2/mtglib" "github.com/kentik/patricia" "github.com/kentik/patricia/bool_tree" - "github.com/panjf2000/ants" + "github.com/panjf2000/ants/v2" ) const ( diff --git a/mtglib/init.go b/mtglib/init.go index 22c5472..e663c62 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -8,7 +8,19 @@ import ( "time" ) -var ErrSecretEmpty = errors.New("secret is empty") +var ( + ErrSecretEmpty = errors.New("secret is empty") + ErrSecretInvalid = errors.New("secret is invalid") + ErrNetworkIsNotDefined = errors.New("network is not defined") + ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") + ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") + ErrEventStreamIsNotDefined = errors.New("event stream is not defined") + ErrLoggerIsNotDefined = errors.New("logger is not defined") +) + +const ( + DefaultConcurrency = 4096 +) type Network interface { Dial(network, address string) (net.Conn, error) diff --git a/mtglib/proxy.go b/mtglib/proxy.go new file mode 100644 index 0000000..e603db1 --- /dev/null +++ b/mtglib/proxy.go @@ -0,0 +1,120 @@ +package mtglib + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "time" + + "github.com/panjf2000/ants/v2" +) + +type Proxy struct { + ctx context.Context + ctxCancel context.CancelFunc + streamWaitGroup sync.WaitGroup + workerPool *ants.PoolWithFunc + + secret Secret + network Network + antiReplayCache AntiReplayCache + ipBlocklist IPBlocklist + eventStream EventStream + logger Logger +} + +func (p *Proxy) ServeConn(conn net.Conn) { + ctx := newStreamContext(p.ctx, p.logger, conn) + defer ctx.Close() + + p.eventStream.Send(ctx, EventStart{ + CreatedAt: time.Now(), + ConnID: ctx.connID, + RemoteIP: ctx.ClientIP(), + }) + ctx.logger.Info("Stream has been started") + + defer func() { + p.eventStream.Send(ctx, EventFinish{ + CreatedAt: time.Now(), + ConnID: ctx.connID, + }) + ctx.logger.Info("Stream has been finished") + }() +} + +func (p *Proxy) Serve(listener net.Listener) error { + for { + conn, err := listener.Accept() + if err != nil { + return fmt.Errorf("cannot accept a new connection: %w", err) + } + + err = p.workerPool.Invoke(conn) + + switch { + case err == nil: + case errors.Is(err, ants.ErrPoolClosed): + return nil + case errors.Is(err, ants.ErrPoolOverload): + p.eventStream.Send(p.ctx, EventConcurrencyLimited{}) + } + } +} + +func (p *Proxy) Shutdown() { + p.ctxCancel() + p.streamWaitGroup.Wait() + p.workerPool.Release() +} + +type antsLogger struct{} + +func (a antsLogger) Printf(msg string, args ...interface{}) {} + +func NewProxy(opts ProxyOpts) (*Proxy, error) { + switch { + case opts.Network == nil: + return nil, ErrNetworkIsNotDefined + case opts.AntiReplayCache == nil: + return nil, ErrAntiReplayCacheIsNotDefined + case opts.IPBlocklist == nil: + return nil, ErrIPBlocklistIsNotDefined + case opts.EventStream == nil: + return nil, ErrEventStreamIsNotDefined + case opts.Logger == nil: + return nil, ErrLoggerIsNotDefined + case !opts.Secret.Valid(): + return nil, ErrSecretInvalid + } + + concurrency := opts.Concurrency + if concurrency == 0 { + concurrency = DefaultConcurrency + } + + ctx, cancel := context.WithCancel(context.Background()) + proxy := &Proxy{ + ctx: ctx, + ctxCancel: cancel, + secret: opts.Secret, + network: opts.Network, + antiReplayCache: opts.AntiReplayCache, + ipBlocklist: opts.IPBlocklist, + eventStream: opts.EventStream, + logger: opts.Logger.Named("proxy"), + } + + pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { + proxy.ServeConn(arg.(net.Conn)) + }, ants.WithLogger(antsLogger{})) + if err != nil { + return nil, fmt.Errorf("cannot initialize a pool: %w", err) + } + + proxy.workerPool = pool + + return proxy, nil +} diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go new file mode 100644 index 0000000..88130ca --- /dev/null +++ b/mtglib/proxy_opts.go @@ -0,0 +1,12 @@ +package mtglib + +type ProxyOpts struct { + Secret Secret + Network Network + AntiReplayCache AntiReplayCache + IPBlocklist IPBlocklist + EventStream EventStream + Logger Logger + + Concurrency uint +} diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go new file mode 100644 index 0000000..cd5d1e8 --- /dev/null +++ b/mtglib/stream_context.go @@ -0,0 +1,63 @@ +package mtglib + +import ( + "context" + "crypto/rand" + "encoding/base64" + "net" + "time" +) + +type streamContext struct { + ctx context.Context + ctxCancel context.CancelFunc + clientConn net.Conn + connID string + logger Logger +} + +func (s *streamContext) Deadline() (time.Time, bool) { + return s.ctx.Deadline() +} + +func (s *streamContext) Done() <-chan struct{} { + return s.ctx.Done() +} + +func (s *streamContext) Err() error { + return s.ctx.Err() +} + +func (s *streamContext) Value(key interface{}) interface{} { + return s.ctx.Value(key) +} + +func (s *streamContext) Close() { + s.ctxCancel() + s.clientConn.Close() +} + +func (s *streamContext) ClientIP() net.IP { + return s.clientConn.RemoteAddr().(*net.TCPAddr).IP +} + +func newStreamContext(ctx context.Context, logger Logger, clientConn net.Conn) *streamContext { + connIDBytes := make([]byte, 16) + + if _, err := rand.Read(connIDBytes); err != nil { + panic(err) + } + + ctx, cancel := context.WithCancel(ctx) + streamCtx := &streamContext{ + ctx: ctx, + ctxCancel: cancel, + clientConn: clientConn, + connID: base64.RawURLEncoding.EncodeToString(connIDBytes), + } + streamCtx.logger = logger. + BindStr("stream-id", streamCtx.connID). + BindStr("client-ip", streamCtx.ClientIP().String()) + + return streamCtx +} From 2408f1530fc8e026796d929dbcad63adebe77079 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 21:24:24 +0300 Subject: [PATCH 082/206] Add EventIPBlocklisted --- events/event_stream.go | 2 ++ events/event_stream_test.go | 23 +++++++++++++++++++++++ events/init.go | 1 + events/init_test.go | 4 ++++ events/multi_observer.go | 15 +++++++++++++++ events/noop.go | 1 + events/noop_test.go | 6 ++++++ mtglib/events.go | 9 +++++++++ mtglib/events_test.go | 4 ++++ mtglib/proxy.go | 10 ++++++++++ stats/init.go | 3 ++- stats/prometheus.go | 17 ++++++++++++++++- stats/prometheus_test.go | 13 +++++++++++++ stats/statsd.go | 12 ++++++++++++ stats/statsd_test.go | 10 ++++++++++ 15 files changed, 128 insertions(+), 2 deletions(-) diff --git a/events/event_stream.go b/events/event_stream.go index be860f6..5037fb6 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -73,6 +73,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob observer.EventStart(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) + case mtglib.EventIPBlocklisted: + observer.EventIPBlocklisted(typedEvt) case mtglib.EventConcurrencyLimited: observer.EventConcurrencyLimited(typedEvt) } diff --git a/events/event_stream_test.go b/events/event_stream_test.go index b86b68b..ec708a0 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -106,6 +106,29 @@ func (suite *EventStreamTestSuite) TestEventConcurrencyLimitedOk() { time.Sleep(100 * time.Millisecond) } +func (suite *EventStreamTestSuite) TestEventIPBlocklistedOk() { + evt := mtglib.EventIPBlocklisted{ + CreatedAt: time.Now(), + RemoteIP: net.ParseIP("10.0.0.10"), + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventIPBlocklisted", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventIPBlocklisted) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + func (suite *EventStreamTestSuite) TearDownTest() { suite.stream.Shutdown() suite.ctxCancel() diff --git a/events/init.go b/events/init.go index 0e348f5..6924071 100644 --- a/events/init.go +++ b/events/init.go @@ -6,6 +6,7 @@ type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) EventConcurrencyLimited(mtglib.EventConcurrencyLimited) + EventIPBlocklisted(mtglib.EventIPBlocklisted) Shutdown() } diff --git a/events/init_test.go b/events/init_test.go index e75e5e9..faa4063 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -21,6 +21,10 @@ func (o *ObserverMock) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimite o.Called(evt) } +func (o *ObserverMock) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { + o.Called(evt) +} + func (o *ObserverMock) Shutdown() { o.Called() } diff --git a/events/multi_observer.go b/events/multi_observer.go index 698d87c..d97d802 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -55,6 +55,21 @@ func (m multiObserver) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimite wg.Wait() } +func (m multiObserver) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventIPBlocklisted(evt) + }(v) + } + + wg.Wait() +} + func (m multiObserver) Shutdown() { for _, v := range m.observers { v.Shutdown() diff --git a/events/noop.go b/events/noop.go index 7eb4041..7acbb88 100644 --- a/events/noop.go +++ b/events/noop.go @@ -20,6 +20,7 @@ type noopObserver struct{} func (n noopObserver) EventStart(_ mtglib.EventStart) {} func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} +func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} func (n noopObserver) Shutdown() {} func NewNoopObserver() Observer { diff --git a/events/noop_test.go b/events/noop_test.go index b5120f3..d3a3109 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -30,6 +30,10 @@ func (suite *NoopTestSuite) SetupSuite() { ConnID: "connID", }, "concurrency-limited": mtglib.EventConcurrencyLimited{}, + "ip-blacklisted": mtglib.EventIPBlocklisted{ + RemoteIP: net.ParseIP("10.0.0.10"), + CreatedAt: time.Now(), + }, } suite.ctx = context.Background() } @@ -62,6 +66,8 @@ func (suite *NoopTestSuite) TestObserver() { observer.EventFinish(typedEvt) case mtglib.EventConcurrencyLimited: observer.EventConcurrencyLimited(typedEvt) + case mtglib.EventIPBlocklisted: + observer.EventIPBlocklisted(typedEvt) } }) } diff --git a/mtglib/events.go b/mtglib/events.go index e7ea557..0e9a2d1 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -31,3 +31,12 @@ type EventConcurrencyLimited struct { func (e EventConcurrencyLimited) StreamID() string { return "" } + +type EventIPBlocklisted struct { + CreatedAt time.Time + RemoteIP net.IP +} + +func (e EventIPBlocklisted) StreamID() string { + return "" +} diff --git a/mtglib/events_test.go b/mtglib/events_test.go index ed08aa5..f0e3975 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -36,6 +36,10 @@ func (suite *EventsTestSuite) TestEventConcurrencyLimited() { suite.Empty(mtglib.EventConcurrencyLimited{}.StreamID()) } +func (suite *EventsTestSuite) TestEventIPBlocklisted() { + suite.Empty(mtglib.EventIPBlocklisted{}.StreamID()) +} + func TestEvents(t *testing.T) { t.Parallel() suite.Run(t, &EventsTestSuite{}) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index e603db1..6c68415 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -52,6 +52,16 @@ func (p *Proxy) Serve(listener net.Listener) error { return fmt.Errorf("cannot accept a new connection: %w", err) } + if addr := conn.RemoteAddr().(*net.TCPAddr).IP; p.ipBlocklist.Contains(addr) { + conn.Close() + p.eventStream.Send(p.ctx, EventIPBlocklisted{ + CreatedAt: time.Now(), + RemoteIP: addr, + }) + + continue + } + err = p.workerPool.Invoke(conn) switch { diff --git a/stats/init.go b/stats/init.go index 7210888..063eb32 100644 --- a/stats/init.go +++ b/stats/init.go @@ -4,9 +4,10 @@ const ( MetricActiveConnection = "active_connections" MetricSessionDuration = "session_duration" MetricConcurrencyLimited = "concurrency_limited" + MetricIPBlocklisted = "ip_blocklisted" TagIPType = "ip_type" TagIPTypeIPv4 = "ipv4" - TagIPTypeIPv6 = "ipv4" + TagIPTypeIPv6 = "ipv6" ) diff --git a/stats/prometheus.go b/stats/prometheus.go index 69b23e3..be4c676 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -41,10 +41,18 @@ func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { p.factory.metricSessionDuration.Observe(float64(duration) / float64(time.Second)) } -func (p prometheusProcessor) EventConcurrencyLimited(evt mtglib.EventConcurrencyLimited) { +func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { p.factory.metricConcurrencyLimited.Inc() } +func (p prometheusProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { + if evt.RemoteIP.To4() == nil { + p.factory.metricIPBlocklisted.WithLabelValues(TagIPTypeIPv6).Inc() + } else { + p.factory.metricIPBlocklisted.WithLabelValues(TagIPTypeIPv4).Inc() + } +} + func (p prometheusProcessor) Shutdown() { p.streams = make(map[string]*streamInfo) } @@ -53,6 +61,7 @@ type PrometheusFactory struct { httpServer *http.Server metricActiveConnections *prometheus.GaugeVec + metricIPBlocklisted *prometheus.CounterVec metricConcurrencyLimited prometheus.Counter metricSessionDuration prometheus.Histogram } @@ -113,11 +122,17 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { Name: MetricConcurrencyLimited, Help: "A number of sessions that were rejected by concurrency limiter.", }), + metricIPBlocklisted: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricIPBlocklisted, + Help: "A number of rejected sessions due to ip blocklisting", + }, []string{TagIPType}), } registry.MustRegister(factory.metricActiveConnections) registry.MustRegister(factory.metricSessionDuration) registry.MustRegister(factory.metricConcurrencyLimited) + registry.MustRegister(factory.metricIPBlocklisted) return factory } diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index 319a6d1..4b2175a 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -91,6 +91,19 @@ func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { suite.Contains(data, `mtg_concurrency_limited 1`) } +func (suite *PrometheusTestSuite) TestEventIPBlocklisted() { + suite.prometheus.EventIPBlocklisted(mtglib.EventIPBlocklisted{ + CreatedAt: time.Now(), + RemoteIP: net.ParseIP("2001:db8::68"), + }) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_ip_blocklisted{ip_type="ipv6"} 1`) +} + func TestPrometheus(t *testing.T) { t.Parallel() suite.Run(t, &PrometheusTestSuite{}) diff --git a/stats/statsd.go b/stats/statsd.go index be04c51..69c8b37 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -49,6 +49,18 @@ func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimite s.client.Incr(MetricConcurrencyLimited, 1) } +func (s statsdProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { + var tag statsd.Tag + + if evt.RemoteIP.To4() == nil { + tag = statsd.StringTag(TagIPType, TagIPTypeIPv6) + } else { + tag = statsd.StringTag(TagIPType, TagIPTypeIPv4) + } + + s.client.Incr(MetricIPBlocklisted, 1, tag) +} + func (s statsdProcessor) Shutdown() { now := time.Now() events := make([]mtglib.EventFinish, 0, len(s.streams)) diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 8732463..1babb5b 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -121,6 +121,16 @@ func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { suite.Equal("mtg.concurrency_limited:1|c", suite.statsdServer.String()) } +func (suite *StatsdTestSuite) TestEventIPBlocklisted() { + suite.statsd.EventIPBlocklisted(mtglib.EventIPBlocklisted{ + CreatedAt: time.Now(), + RemoteIP: net.ParseIP("10.0.0.10"), + }) + + time.Sleep(2 * statsd.DefaultFlushInterval) + suite.Equal("mtg.ip_blocklisted:1|c|#ip_type:ipv4", suite.statsdServer.String()) +} + func TestStatsd(t *testing.T) { t.Parallel() suite.Run(t, &StatsdTestSuite{}) From 80a1de60e61ddc8ef71ae89c3d4f51ce4918081c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 21:24:42 +0300 Subject: [PATCH 083/206] Always close connection on context finished --- mtglib/proxy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 6c68415..6cec3bc 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -29,6 +29,11 @@ func (p *Proxy) ServeConn(conn net.Conn) { ctx := newStreamContext(p.ctx, p.logger, conn) defer ctx.Close() + go func() { + <-ctx.Done() + ctx.Close() + }() + p.eventStream.Send(ctx, EventStart{ CreatedAt: time.Now(), ConnID: ctx.connID, From 5e31b95bb59256c2474ad4054c9b78a5d1fcf4f9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 21:24:55 +0300 Subject: [PATCH 084/206] Add timestamp to EventConcurrencyLimited --- mtglib/proxy.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 6cec3bc..f3b32bb 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -74,7 +74,9 @@ func (p *Proxy) Serve(listener net.Listener) error { case errors.Is(err, ants.ErrPoolClosed): return nil case errors.Is(err, ants.ErrPoolOverload): - p.eventStream.Send(p.ctx, EventConcurrencyLimited{}) + p.eventStream.Send(p.ctx, EventConcurrencyLimited{ + CreatedAt: time.Now(), + }) } } } From 57cb1b5aa0199301ec9b40330c088b15003c9fa9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 21:44:42 +0300 Subject: [PATCH 085/206] Add new TCPBufferSize parameter to network --- mtglib/init.go | 1 + network/default.go | 4 ++++ network/default_test.go | 4 ++++ network/init.go | 1 + network/init_internal_test.go | 4 ++++ network/init_test.go | 4 ++++ network/load_balanced_socks5.go | 10 ++++++++-- network/load_balanced_socks5_test.go | 1 + network/network.go | 4 ++++ network/socks5.go | 20 +++++++++++++++++++- testlib/mtglib_network_mock.go | 4 ++++ 11 files changed, 54 insertions(+), 3 deletions(-) diff --git a/mtglib/init.go b/mtglib/init.go index e663c62..5f68661 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -27,6 +27,7 @@ type Network interface { DialContext(ctx context.Context, network, address string) (net.Conn, error) MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client IdleTimeout() time.Duration + TCPBufferSize() int } type AntiReplayCache interface { diff --git a/network/default.go b/network/default.go index 72b3da5..db104f0 100644 --- a/network/default.go +++ b/network/default.go @@ -60,6 +60,10 @@ func (d *defaultDialer) DialContext(ctx context.Context, network, address string return tcpConn, nil } +func (d *defaultDialer) TCPBufferSize() int { + return d.bufferSize +} + func NewDefaultDialer(timeout time.Duration, bufferSize int) (Dialer, error) { switch { case timeout < 0: diff --git a/network/default_test.go b/network/default_test.go index a6b24c7..4136770 100644 --- a/network/default_test.go +++ b/network/default_test.go @@ -71,6 +71,10 @@ func (suite *DefaultDialerTestSuite) TestHTTPRequest() { suite.Equal(http.StatusOK, resp.StatusCode) } +func (suite *DefaultDialerTestSuite) TestTCPBufferSize() { + suite.Equal(network.DefaultBufferSize, suite.d.TCPBufferSize()) +} + func TestDefaultDialer(t *testing.T) { t.Parallel() suite.Run(t, &DefaultDialerTestSuite{}) diff --git a/network/init.go b/network/init.go index 3bab936..bdb4b26 100644 --- a/network/init.go +++ b/network/init.go @@ -29,4 +29,5 @@ var ( type Dialer interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) + TCPBufferSize() int } diff --git a/network/init_internal_test.go b/network/init_internal_test.go index 9335818..b908a01 100644 --- a/network/init_internal_test.go +++ b/network/init_internal_test.go @@ -22,3 +22,7 @@ func (d *DialerMock) DialContext(ctx context.Context, network, address string) ( return args.Get(0).(net.Conn), args.Error(1) } + +func (d *DialerMock) TCPBufferSize() int { + return d.Called().Int(0) +} diff --git a/network/init_test.go b/network/init_test.go index f42d1c3..b75ff11 100644 --- a/network/init_test.go +++ b/network/init_test.go @@ -30,6 +30,10 @@ func (d *DialerMock) DialContext(ctx context.Context, network, address string) ( return args.Get(0).(net.Conn), args.Error(1) } +func (d *DialerMock) TCPBufferSize() int { + return d.Called().Int(0) +} + type HTTPServerTestSuite struct { httpServer *httptest.Server } diff --git a/network/load_balanced_socks5.go b/network/load_balanced_socks5.go index 2a60d69..41331a5 100644 --- a/network/load_balanced_socks5.go +++ b/network/load_balanced_socks5.go @@ -9,13 +9,18 @@ import ( ) type loadBalancedSocks5Dialer struct { - dialers []Dialer + dialers []Dialer + bufferSize int } func (l loadBalancedSocks5Dialer) Dial(network, address string) (net.Conn, error) { return l.DialContext(context.Background(), network, address) } +func (l loadBalancedSocks5Dialer) TCPBufferSize() int { + return l.bufferSize +} + func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { length := len(l.dialers) start := rand.Intn(length) @@ -45,6 +50,7 @@ func NewLoadBalancedSocks5Dialer(baseDialer Dialer, proxyURLs []*url.URL) (Diale } return loadBalancedSocks5Dialer{ - dialers: dialers, + dialers: dialers, + bufferSize: baseDialer.TCPBufferSize(), }, nil } diff --git a/network/load_balanced_socks5_test.go b/network/load_balanced_socks5_test.go index b6983c8..25ca3b3 100644 --- a/network/load_balanced_socks5_test.go +++ b/network/load_balanced_socks5_test.go @@ -57,6 +57,7 @@ func (suite *LoadBalancedSocks5TestSuite) TestCannotDial() { baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.2:1080"). Times(network.ProxyDialerOpenThreshold). Return(&net.TCPConn{}, io.EOF) + baseDialer.On("TCPBufferSize").Return(network.DefaultBufferSize) lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ {Scheme: "socks5", User: url.UserPassword("user", "password"), Host: "127.0.0.1:1080"}, diff --git a/network/network.go b/network/network.go index 585ff3c..f2cb044 100644 --- a/network/network.go +++ b/network/network.go @@ -75,6 +75,10 @@ func (n *network) IdleTimeout() time.Duration { return n.idleTimeout } +func (n *network) TCPBufferSize() int { + return n.dialer.TCPBufferSize() +} + func (n *network) dnsResolve(protocol, address string) ([]string, error) { if net.ParseIP(address) != nil { return []string{address}, nil diff --git a/network/socks5.go b/network/socks5.go index 91c9c61..66c1f23 100644 --- a/network/socks5.go +++ b/network/socks5.go @@ -2,16 +2,34 @@ package network import ( "fmt" + "net" "net/url" "golang.org/x/net/proxy" ) +type socks5Dialer struct { + proxy.ContextDialer + + bufferSize int +} + +func (s socks5Dialer) Dial(protocol, address string) (net.Conn, error) { + return s.ContextDialer.(proxy.Dialer).Dial(protocol, address) +} + +func (s socks5Dialer) TCPBufferSize() int { + return s.bufferSize +} + func NewSocks5Dialer(baseDialer Dialer, proxyURL *url.URL) (Dialer, error) { rv, err := proxy.FromURL(proxyURL, baseDialer) if err != nil { return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) } - return rv.(Dialer), nil + return socks5Dialer{ + ContextDialer: rv.(proxy.ContextDialer), + bufferSize: baseDialer.TCPBufferSize(), + }, nil } diff --git a/testlib/mtglib_network_mock.go b/testlib/mtglib_network_mock.go index 23fa8ba..3e89fdd 100644 --- a/testlib/mtglib_network_mock.go +++ b/testlib/mtglib_network_mock.go @@ -33,3 +33,7 @@ func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, func (m *MtglibNetworkMock) IdleTimeout() time.Duration { return m.Called().Get(0).(time.Duration) } + +func (m *MtglibNetworkMock) TCPBufferSize() int { + return m.Called().Int(0) +} From 83eeedc0089ec4cf46a73468ae0e1547755f3c0e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 21:47:13 +0300 Subject: [PATCH 086/206] Add cloakport and prefer ip parameters to proxyopts --- mtglib/proxy_opts.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index 88130ca..6441c73 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -9,4 +9,6 @@ type ProxyOpts struct { Logger Logger Concurrency uint + CloakPort uint + PreferIP string } From adf4ab1a35682d5582510bb5be4974665f0918df Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 22:03:36 +0300 Subject: [PATCH 087/206] Revert "Add new TCPBufferSize parameter to network" This reverts commit 57cb1b5aa0199301ec9b40330c088b15003c9fa9. --- mtglib/init.go | 1 - network/default.go | 4 ---- network/default_test.go | 4 ---- network/init.go | 1 - network/init_internal_test.go | 4 ---- network/init_test.go | 4 ---- network/load_balanced_socks5.go | 10 ++-------- network/load_balanced_socks5_test.go | 1 - network/network.go | 4 ---- network/socks5.go | 20 +------------------- testlib/mtglib_network_mock.go | 4 ---- 11 files changed, 3 insertions(+), 54 deletions(-) diff --git a/mtglib/init.go b/mtglib/init.go index 5f68661..e663c62 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -27,7 +27,6 @@ type Network interface { DialContext(ctx context.Context, network, address string) (net.Conn, error) MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client IdleTimeout() time.Duration - TCPBufferSize() int } type AntiReplayCache interface { diff --git a/network/default.go b/network/default.go index db104f0..72b3da5 100644 --- a/network/default.go +++ b/network/default.go @@ -60,10 +60,6 @@ func (d *defaultDialer) DialContext(ctx context.Context, network, address string return tcpConn, nil } -func (d *defaultDialer) TCPBufferSize() int { - return d.bufferSize -} - func NewDefaultDialer(timeout time.Duration, bufferSize int) (Dialer, error) { switch { case timeout < 0: diff --git a/network/default_test.go b/network/default_test.go index 4136770..a6b24c7 100644 --- a/network/default_test.go +++ b/network/default_test.go @@ -71,10 +71,6 @@ func (suite *DefaultDialerTestSuite) TestHTTPRequest() { suite.Equal(http.StatusOK, resp.StatusCode) } -func (suite *DefaultDialerTestSuite) TestTCPBufferSize() { - suite.Equal(network.DefaultBufferSize, suite.d.TCPBufferSize()) -} - func TestDefaultDialer(t *testing.T) { t.Parallel() suite.Run(t, &DefaultDialerTestSuite{}) diff --git a/network/init.go b/network/init.go index bdb4b26..3bab936 100644 --- a/network/init.go +++ b/network/init.go @@ -29,5 +29,4 @@ var ( type Dialer interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) - TCPBufferSize() int } diff --git a/network/init_internal_test.go b/network/init_internal_test.go index b908a01..9335818 100644 --- a/network/init_internal_test.go +++ b/network/init_internal_test.go @@ -22,7 +22,3 @@ func (d *DialerMock) DialContext(ctx context.Context, network, address string) ( return args.Get(0).(net.Conn), args.Error(1) } - -func (d *DialerMock) TCPBufferSize() int { - return d.Called().Int(0) -} diff --git a/network/init_test.go b/network/init_test.go index b75ff11..f42d1c3 100644 --- a/network/init_test.go +++ b/network/init_test.go @@ -30,10 +30,6 @@ func (d *DialerMock) DialContext(ctx context.Context, network, address string) ( return args.Get(0).(net.Conn), args.Error(1) } -func (d *DialerMock) TCPBufferSize() int { - return d.Called().Int(0) -} - type HTTPServerTestSuite struct { httpServer *httptest.Server } diff --git a/network/load_balanced_socks5.go b/network/load_balanced_socks5.go index 41331a5..2a60d69 100644 --- a/network/load_balanced_socks5.go +++ b/network/load_balanced_socks5.go @@ -9,18 +9,13 @@ import ( ) type loadBalancedSocks5Dialer struct { - dialers []Dialer - bufferSize int + dialers []Dialer } func (l loadBalancedSocks5Dialer) Dial(network, address string) (net.Conn, error) { return l.DialContext(context.Background(), network, address) } -func (l loadBalancedSocks5Dialer) TCPBufferSize() int { - return l.bufferSize -} - func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { length := len(l.dialers) start := rand.Intn(length) @@ -50,7 +45,6 @@ func NewLoadBalancedSocks5Dialer(baseDialer Dialer, proxyURLs []*url.URL) (Diale } return loadBalancedSocks5Dialer{ - dialers: dialers, - bufferSize: baseDialer.TCPBufferSize(), + dialers: dialers, }, nil } diff --git a/network/load_balanced_socks5_test.go b/network/load_balanced_socks5_test.go index 25ca3b3..b6983c8 100644 --- a/network/load_balanced_socks5_test.go +++ b/network/load_balanced_socks5_test.go @@ -57,7 +57,6 @@ func (suite *LoadBalancedSocks5TestSuite) TestCannotDial() { baseDialer.On("DialContext", mock.Anything, "tcp", "127.0.0.2:1080"). Times(network.ProxyDialerOpenThreshold). Return(&net.TCPConn{}, io.EOF) - baseDialer.On("TCPBufferSize").Return(network.DefaultBufferSize) lbDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, []*url.URL{ {Scheme: "socks5", User: url.UserPassword("user", "password"), Host: "127.0.0.1:1080"}, diff --git a/network/network.go b/network/network.go index f2cb044..585ff3c 100644 --- a/network/network.go +++ b/network/network.go @@ -75,10 +75,6 @@ func (n *network) IdleTimeout() time.Duration { return n.idleTimeout } -func (n *network) TCPBufferSize() int { - return n.dialer.TCPBufferSize() -} - func (n *network) dnsResolve(protocol, address string) ([]string, error) { if net.ParseIP(address) != nil { return []string{address}, nil diff --git a/network/socks5.go b/network/socks5.go index 66c1f23..91c9c61 100644 --- a/network/socks5.go +++ b/network/socks5.go @@ -2,34 +2,16 @@ package network import ( "fmt" - "net" "net/url" "golang.org/x/net/proxy" ) -type socks5Dialer struct { - proxy.ContextDialer - - bufferSize int -} - -func (s socks5Dialer) Dial(protocol, address string) (net.Conn, error) { - return s.ContextDialer.(proxy.Dialer).Dial(protocol, address) -} - -func (s socks5Dialer) TCPBufferSize() int { - return s.bufferSize -} - func NewSocks5Dialer(baseDialer Dialer, proxyURL *url.URL) (Dialer, error) { rv, err := proxy.FromURL(proxyURL, baseDialer) if err != nil { return nil, fmt.Errorf("cannot initialize socks5 proxy dialer: %w", err) } - return socks5Dialer{ - ContextDialer: rv.(proxy.ContextDialer), - bufferSize: baseDialer.TCPBufferSize(), - }, nil + return rv.(Dialer), nil } diff --git a/testlib/mtglib_network_mock.go b/testlib/mtglib_network_mock.go index 3e89fdd..23fa8ba 100644 --- a/testlib/mtglib_network_mock.go +++ b/testlib/mtglib_network_mock.go @@ -33,7 +33,3 @@ func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, func (m *MtglibNetworkMock) IdleTimeout() time.Duration { return m.Called().Get(0).(time.Duration) } - -func (m *MtglibNetworkMock) TCPBufferSize() int { - return m.Called().Int(0) -} From f21ee40baf7c7a2fd494f39b44299225a3a12e80 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 17 Mar 2021 22:08:47 +0300 Subject: [PATCH 088/206] Remove idle timeout from network --- cli/base.go | 7 +++---- ipblocklist/firehol_test.go | 2 +- mtglib/init.go | 2 -- network/init.go | 1 - network/network.go | 15 +-------------- stats/statsd_test.go | 10 ++++++---- testlib/mtglib_network_mock.go | 5 ----- 7 files changed, 11 insertions(+), 31 deletions(-) diff --git a/cli/base.go b/cli/base.go index def25d8..f4b6635 100644 --- a/cli/base.go +++ b/cli/base.go @@ -40,7 +40,6 @@ func (b *base) ReadConfig(path, version string) error { func (b *base) makeNetwork(conf *config.Config, version string) (mtglib.Network, error) { tcpTimeout := conf.Network.Timeout.TCP.Value(network.DefaultTimeout) - idleTimeout := conf.Network.Timeout.Idle.Value(network.DefaultIdleTimeout) httpTimeout := conf.Network.Timeout.HTTP.Value(network.DefaultHTTPTimeout) dohIP := conf.Network.DOHIP.Value(net.ParseIP(network.DefaultDOHHostname)).String() bufferSize := conf.TCPBuffer.Value(network.DefaultBufferSize) @@ -61,14 +60,14 @@ func (b *base) makeNetwork(conf *config.Config, version string) (mtglib.Network, switch len(proxyURLs) { case 0: - return network.NewNetwork(baseDialer, userAgent, dohIP, httpTimeout, idleTimeout) + return network.NewNetwork(baseDialer, userAgent, dohIP, httpTimeout) case 1: socksDialer, err := network.NewSocks5Dialer(baseDialer, proxyURLs[0]) if err != nil { return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout, idleTimeout) + return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout) } socksDialer, err := network.NewLoadBalancedSocks5Dialer(baseDialer, proxyURLs) @@ -76,5 +75,5 @@ func (b *base) makeNetwork(conf *config.Config, version string) (mtglib.Network, return nil, fmt.Errorf("cannot build socks5 dialer: %w", err) } - return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout, idleTimeout) + return network.NewNetwork(socksDialer, userAgent, dohIP, httpTimeout) } diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go index ad353fe..9879afd 100644 --- a/ipblocklist/firehol_test.go +++ b/ipblocklist/firehol_test.go @@ -119,7 +119,7 @@ func (suite *FireholTestSuite) TestRemoteFail() { func (suite *FireholTestSuite) TestMixed() { dialer, _ := network.NewDefaultDialer(0, 0) - ntw, _ := network.NewNetwork(dialer, "mtg", "1.1.1.1", 0, 0) + ntw, _ := network.NewNetwork(dialer, "mtg", "1.1.1.1", 0) blocklist, err := ipblocklist.NewFirehol(logger.NewNoopLogger(), ntw, 2, diff --git a/mtglib/init.go b/mtglib/init.go index e663c62..b3d53a3 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -5,7 +5,6 @@ import ( "errors" "net" "net/http" - "time" ) var ( @@ -26,7 +25,6 @@ type Network interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client - IdleTimeout() time.Duration } type AntiReplayCache interface { diff --git a/network/init.go b/network/init.go index 3bab936..29c3056 100644 --- a/network/init.go +++ b/network/init.go @@ -9,7 +9,6 @@ import ( const ( DefaultTimeout = 10 * time.Second - DefaultIdleTimeout = time.Minute DefaultHTTPTimeout = 10 * time.Second DefaultBufferSize = 4096 diff --git a/network/network.go b/network/network.go index 585ff3c..9f85744 100644 --- a/network/network.go +++ b/network/network.go @@ -27,7 +27,6 @@ func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, erro type network struct { dialer Dialer dns doh.Resolver - idleTimeout time.Duration httpTimeout time.Duration userAgent string } @@ -71,10 +70,6 @@ func (n *network) MakeHTTPClient(dialFunc func(ctx context.Context, return makeHTTPClient(n.userAgent, n.httpTimeout, dialFunc) } -func (n *network) IdleTimeout() time.Duration { - return n.idleTimeout -} - func (n *network) dnsResolve(protocol, address string) ([]string, error) { if net.ParseIP(address) != nil { return []string{address}, nil @@ -131,7 +126,7 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { func NewNetwork(dialer Dialer, userAgent, dohHostname string, - httpTimeout, idleTimeout time.Duration) (mtglib.Network, error) { + httpTimeout time.Duration) (mtglib.Network, error) { switch { case httpTimeout < 0: return nil, fmt.Errorf("timeout should be positive number %s", httpTimeout) @@ -139,20 +134,12 @@ func NewNetwork(dialer Dialer, httpTimeout = DefaultHTTPTimeout } - switch { - case idleTimeout < 0: - return nil, fmt.Errorf("timeout should be positive number %s", idleTimeout) - case idleTimeout == 0: - idleTimeout = DefaultIdleTimeout - } - if net.ParseIP(dohHostname) == nil { return nil, fmt.Errorf("hostname %s should be IP address", dohHostname) } return &network{ dialer: dialer, - idleTimeout: idleTimeout, httpTimeout: httpTimeout, userAgent: userAgent, dns: doh.Resolver{ diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 1babb5b..0fad63a 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/suite" ) +const statsdSleepTime = 3 * statsd.DefaultFlushInterval + type statsdFakeServer struct { conn *net.UDPConn buf *bytes.Buffer @@ -100,7 +102,7 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { RemoteIP: net.ParseIP("10.0.0.10"), }) - time.Sleep(2 * statsd.DefaultFlushInterval) + time.Sleep(statsdSleepTime) suite.Equal("mtg.active_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String()) suite.statsd.EventFinish(mtglib.EventFinish{ @@ -108,7 +110,7 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { ConnID: "connID", }) - time.Sleep(2 * statsd.DefaultFlushInterval) + time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.session_duration") } @@ -117,7 +119,7 @@ func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { CreatedAt: time.Now(), }) - time.Sleep(2 * statsd.DefaultFlushInterval) + time.Sleep(statsdSleepTime) suite.Equal("mtg.concurrency_limited:1|c", suite.statsdServer.String()) } @@ -127,7 +129,7 @@ func (suite *StatsdTestSuite) TestEventIPBlocklisted() { RemoteIP: net.ParseIP("10.0.0.10"), }) - time.Sleep(2 * statsd.DefaultFlushInterval) + time.Sleep(statsdSleepTime) suite.Equal("mtg.ip_blocklisted:1|c|#ip_type:ipv4", suite.statsdServer.String()) } diff --git a/testlib/mtglib_network_mock.go b/testlib/mtglib_network_mock.go index 23fa8ba..092bd28 100644 --- a/testlib/mtglib_network_mock.go +++ b/testlib/mtglib_network_mock.go @@ -4,7 +4,6 @@ import ( "context" "net" "net/http" - "time" "github.com/stretchr/testify/mock" ) @@ -29,7 +28,3 @@ func (m *MtglibNetworkMock) MakeHTTPClient(dialFunc func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client { return m.Called(dialFunc).Get(0).(*http.Client) } - -func (m *MtglibNetworkMock) IdleTimeout() time.Duration { - return m.Called().Get(0).(time.Duration) -} From 60d5b3cadd99a6de963281ff698fb18155408948 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 10:03:13 +0300 Subject: [PATCH 089/206] Pass new options to proxy opts --- mtglib/proxy_opts.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index 6441c73..a18763f 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -1,5 +1,7 @@ package mtglib +import "time" + type ProxyOpts struct { Secret Secret Network Network @@ -8,7 +10,9 @@ type ProxyOpts struct { EventStream EventStream Logger Logger + BufferSize uint Concurrency uint CloakPort uint + IdleTimeout time.Duration PreferIP string } From b258581f471a963704690979669cd60ff7d9b567 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 10:15:22 +0300 Subject: [PATCH 090/206] Add Printf method to logger --- logger/noop.go | 1 + logger/noop_test.go | 1 + logger/zerolog.go | 6 ++++++ logger/zerolog_test.go | 7 ++++++- mtglib/init.go | 1 + 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/logger/noop.go b/logger/noop.go index 6d7289c..d416542 100644 --- a/logger/noop.go +++ b/logger/noop.go @@ -7,6 +7,7 @@ type noopLogger struct{} func (n noopLogger) Named(_ string) mtglib.Logger { return n } func (n noopLogger) BindInt(_ string, _ int) mtglib.Logger { return n } func (n noopLogger) BindStr(_, _ string) mtglib.Logger { return n } +func (n noopLogger) Printf(_ string, _ ...interface{}) {} func (n noopLogger) Info(_ string) {} func (n noopLogger) Warning(_ string) {} func (n noopLogger) Debug(_ string) {} diff --git a/logger/noop_test.go b/logger/noop_test.go index add8464..ca05901 100644 --- a/logger/noop_test.go +++ b/logger/noop_test.go @@ -18,6 +18,7 @@ func (suite *NoopLoggerTestSuite) TestLog() { suite.Empty(testlib.CaptureStderr(func() { log := logger.NewNoopLogger().Named("name") + log.BindInt("int", 1).BindStr("str", "1").Printf("info", 1, 2) log.BindInt("int", 1).BindStr("str", "1").Info("info") log.BindInt("int", 1).BindStr("str", "1").Warning("info") log.BindInt("int", 1).BindStr("str", "1").Debug("info") diff --git a/logger/zerolog.go b/logger/zerolog.go index 4142453..d504afa 100644 --- a/logger/zerolog.go +++ b/logger/zerolog.go @@ -1,6 +1,8 @@ package logger import ( + "fmt" + "github.com/9seconds/mtg/v2/mtglib" "github.com/rs/zerolog" ) @@ -64,6 +66,10 @@ func (z *zeroLogContext) BindStr(name, value string) mtglib.Logger { } } +func (z *zeroLogContext) Printf(format string, args ...interface{}) { + z.Debug(fmt.Sprintf(format, args...)) +} + func (z *zeroLogContext) Info(msg string) { z.InfoError(msg, nil) } diff --git a/logger/zerolog_test.go b/logger/zerolog_test.go index 4b2308f..a3e9ae2 100644 --- a/logger/zerolog_test.go +++ b/logger/zerolog_test.go @@ -41,6 +41,7 @@ func (suite *ZeroLoggerTestSuite) TestLog() { testData := map[string]func(mtglib.Logger){ "info": func(l mtglib.Logger) { l.Info("hello") }, "warn": func(l mtglib.Logger) { l.Warning("hello") }, + "printf": func(l mtglib.Logger) { l.Printf("hello") }, "debug": func(l mtglib.Logger) { l.Debug("hello") }, "info-error": func(l mtglib.Logger) { l.InfoError("hello", io.EOF) }, "warn-error": func(l mtglib.Logger) { l.WarningError("hello", io.EOF) }, @@ -64,13 +65,17 @@ func (suite *ZeroLoggerTestSuite) TestLog() { timestamp := time.Unix(msg.Timestamp/1000, (msg.Timestamp%1000)*1_000_000) assert.WithinDuration(t, time.Now(), timestamp, 100*time.Millisecond) + if level == "printf" { + level = "debug" + } + assert.Equal(t, level, msg.Level) assert.Equal(t, name, msg.StrParam) assert.EqualValues(t, 1, msg.IntParam) assert.Equal(t, "name", msg.Logger) assert.Equal(t, "hello", msg.Message) - if level != name { + if level != name && name != "printf" { assert.Equal(t, io.EOF.Error(), msg.Error) } else { assert.Empty(t, msg.Error) diff --git a/mtglib/init.go b/mtglib/init.go index b3d53a3..3ea5e30 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -52,6 +52,7 @@ type Logger interface { BindInt(name string, value int) Logger BindStr(name, value string) Logger + Printf(format string, args ...interface{}) Info(msg string) InfoError(msg string, err error) Warning(msg string) From 172b596cf1df9dc200ad82baac7ca65577b36516 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 10:37:58 +0300 Subject: [PATCH 091/206] Intergrate custom logger where applicable --- logger/init.go | 5 +++++ mtglib/proxy.go | 6 +----- stats/statsd.go | 10 ++++------ stats/statsd_test.go | 4 +++- 4 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 logger/init.go diff --git a/logger/init.go b/logger/init.go new file mode 100644 index 0000000..afcaf1d --- /dev/null +++ b/logger/init.go @@ -0,0 +1,5 @@ +package logger + +type StdLikeLogger interface { + Printf(format string, args ...interface{}) +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index f3b32bb..7ae99a8 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -87,10 +87,6 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } -type antsLogger struct{} - -func (a antsLogger) Printf(msg string, args ...interface{}) {} - func NewProxy(opts ProxyOpts) (*Proxy, error) { switch { case opts.Network == nil: @@ -126,7 +122,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { proxy.ServeConn(arg.(net.Conn)) - }, ants.WithLogger(antsLogger{})) + }, ants.WithLogger(opts.Logger.Named("ants"))) if err != nil { return nil, fmt.Errorf("cannot initialize a pool: %w", err) } diff --git a/stats/statsd.go b/stats/statsd.go index 69c8b37..f7c437c 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -6,14 +6,11 @@ import ( "time" "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/logger" "github.com/9seconds/mtg/v2/mtglib" statsd "github.com/smira/go-statsd" ) -type statsdFakeLogger struct{} - -func (s statsdFakeLogger) Printf(msg string, args ...interface{}) {} - type statsdProcessor struct { streams map[string]*streamInfo client *statsd.Client @@ -92,10 +89,11 @@ func (s StatsdFactory) Make() events.Observer { } } -func NewStatsd(address, metricPrefix, tagFormat string) (StatsdFactory, error) { +func NewStatsd(address string, log logger.StdLikeLogger, + metricPrefix, tagFormat string) (StatsdFactory, error) { options := []statsd.Option{ statsd.MetricPrefix(metricPrefix), - statsd.Logger(statsdFakeLogger{}), + statsd.Logger(log), } switch strings.ToLower(tagFormat) { diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 0fad63a..af9491f 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/logger" "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/stats" statsd "github.com/smira/go-statsd" @@ -80,7 +81,8 @@ type StatsdTestSuite struct { func (suite *StatsdTestSuite) SetupTest() { suite.statsdServer = statsdNewFakeServer() - factory, err := stats.NewStatsd(suite.statsdServer.Addr(), "mtg.", "datadog") + factory, err := stats.NewStatsd(suite.statsdServer.Addr(), + logger.NewNoopLogger(), "mtg.", "datadog") if err != nil { panic(err) } From f3112d4ba6fec97b148638f37f415592c92ab92e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 11:07:51 +0300 Subject: [PATCH 092/206] Add timeattack detector --- mtglib/init.go | 5 +++++ mtglib/proxy.go | 13 +++++++------ mtglib/proxy_opts.go | 13 +++++++------ timeattack/detector.go | 36 ++++++++++++++++++++++++++++++++++++ timeattack/detector_test.go | 28 ++++++++++++++++++++++++++++ timeattack/noop.go | 15 +++++++++++++++ timeattack/noop_test.go | 26 ++++++++++++++++++++++++++ 7 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 timeattack/detector.go create mode 100644 timeattack/detector_test.go create mode 100644 timeattack/noop.go create mode 100644 timeattack/noop_test.go diff --git a/mtglib/init.go b/mtglib/init.go index 3ea5e30..826f124 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -5,6 +5,7 @@ import ( "errors" "net" "net/http" + "time" ) var ( @@ -46,6 +47,10 @@ type EventStream interface { Shutdown() } +type TimeAttackDetector interface { + Valid(time.Time) error +} + type Logger interface { Named(name string) Logger diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 7ae99a8..90adaa2 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -17,12 +17,13 @@ type Proxy struct { streamWaitGroup sync.WaitGroup workerPool *ants.PoolWithFunc - secret Secret - network Network - antiReplayCache AntiReplayCache - ipBlocklist IPBlocklist - eventStream EventStream - logger Logger + secret Secret + network Network + timeAttackDetector TimeAttackDetector + antiReplayCache AntiReplayCache + ipBlocklist IPBlocklist + eventStream EventStream + logger Logger } func (p *Proxy) ServeConn(conn net.Conn) { diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index a18763f..9067fb3 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -3,12 +3,13 @@ package mtglib import "time" type ProxyOpts struct { - Secret Secret - Network Network - AntiReplayCache AntiReplayCache - IPBlocklist IPBlocklist - EventStream EventStream - Logger Logger + Secret Secret + Network Network + AntiReplayCache AntiReplayCache + TimeAttackDetector TimeAttackDetector + IPBlocklist IPBlocklist + EventStream EventStream + Logger Logger BufferSize uint Concurrency uint diff --git a/timeattack/detector.go b/timeattack/detector.go new file mode 100644 index 0000000..ba70d3b --- /dev/null +++ b/timeattack/detector.go @@ -0,0 +1,36 @@ +package timeattack + +import ( + "fmt" + "time" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type detector struct { + time.Duration +} + +func (d detector) Valid(then time.Time) error { + now := time.Now() + + diff := now.Sub(then) + if diff < 0 { + diff = -diff + } + + if diff > d.Duration { + return fmt.Errorf("time is invalid. now=%d, then=%d, diff=%v", + now.Unix(), + then.Unix(), + diff) + } + + return nil +} + +func NewDetector(duration time.Duration) mtglib.TimeAttackDetector { + return detector{ + Duration: duration, + } +} diff --git a/timeattack/detector_test.go b/timeattack/detector_test.go new file mode 100644 index 0000000..dd4bd9f --- /dev/null +++ b/timeattack/detector_test.go @@ -0,0 +1,28 @@ +package timeattack_test + +import ( + "testing" + "time" + + "github.com/9seconds/mtg/v2/timeattack" + "github.com/stretchr/testify/suite" +) + +type DetectorTestSuite struct { + suite.Suite +} + +func (suite *DetectorTestSuite) TestOp() { + d := timeattack.NewDetector(time.Second) + + suite.NoError(d.Valid(time.Now())) + suite.NoError(d.Valid(time.Now().Add(100 * time.Millisecond))) + suite.NoError(d.Valid(time.Now().Add(-100 * time.Millisecond))) + suite.Error(d.Valid(time.Now().Add(time.Hour))) + suite.Error(d.Valid(time.Now().Add(-time.Hour))) +} + +func TestDetector(t *testing.T) { + t.Parallel() + suite.Run(t, &DetectorTestSuite{}) +} diff --git a/timeattack/noop.go b/timeattack/noop.go new file mode 100644 index 0000000..43537c5 --- /dev/null +++ b/timeattack/noop.go @@ -0,0 +1,15 @@ +package timeattack + +import ( + "time" + + "github.com/9seconds/mtg/v2/mtglib" +) + +type noop struct{} + +func (n noop) Valid(_ time.Time) error { return nil } + +func NewNoop() mtglib.TimeAttackDetector { + return noop{} +} diff --git a/timeattack/noop_test.go b/timeattack/noop_test.go new file mode 100644 index 0000000..4b3da79 --- /dev/null +++ b/timeattack/noop_test.go @@ -0,0 +1,26 @@ +package timeattack_test + +import ( + "testing" + "time" + + "github.com/9seconds/mtg/v2/timeattack" + "github.com/stretchr/testify/suite" +) + +type NoopTestSuite struct { + suite.Suite +} + +func (suite *NoopTestSuite) TestOp() { + d := timeattack.NewNoop() + + suite.NoError(d.Valid(time.Now())) + suite.NoError(d.Valid(time.Now().Add(time.Hour))) + suite.NoError(d.Valid(time.Now().Add(-time.Hour))) +} + +func TestNoop(t *testing.T) { + t.Parallel() + suite.Run(t, &NoopTestSuite{}) +} From 657a74a5c28e555bb47ba79e862b80f09335307a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 11:16:08 +0300 Subject: [PATCH 093/206] Add defaults for proxy opts --- mtglib/init.go | 4 ++++ mtglib/internal/obfuscated2/frame.go | 1 + network/init.go | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 mtglib/internal/obfuscated2/frame.go diff --git a/mtglib/init.go b/mtglib/init.go index 826f124..a0e5f8f 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -20,6 +20,10 @@ var ( const ( DefaultConcurrency = 4096 + DefaultBufferSize = 16 * 1024 // 16 kib + DefaultCloakPort = 443 + DefaultIdleTimeout = time.Minute + DefaultPreferIP = "prefer-ipv6" ) type Network interface { diff --git a/mtglib/internal/obfuscated2/frame.go b/mtglib/internal/obfuscated2/frame.go new file mode 100644 index 0000000..07570d8 --- /dev/null +++ b/mtglib/internal/obfuscated2/frame.go @@ -0,0 +1 @@ +package obfuscated2 diff --git a/network/init.go b/network/init.go index 29c3056..57053af 100644 --- a/network/init.go +++ b/network/init.go @@ -10,7 +10,7 @@ import ( const ( DefaultTimeout = 10 * time.Second DefaultHTTPTimeout = 10 * time.Second - DefaultBufferSize = 4096 + DefaultBufferSize = 16 * 1024 // 16 kib ProxyDialerOpenThreshold = 5 ProxyDialerHalfOpenTimeout = time.Minute From 58335b3e5916eca9d52a2f763a0addcaef93659f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 13:42:14 +0300 Subject: [PATCH 094/206] Support obfuscated2 --- .../internal/obfuscated2/client_handshake.go | 37 +++++++++ mtglib/internal/obfuscated2/conn.go | 33 ++++++++ mtglib/internal/obfuscated2/frame.go | 78 +++++++++++++++++++ mtglib/internal/obfuscated2/pools.go | 22 ++++++ mtglib/internal/obfuscated2/utils.go | 15 ++++ mtglib/proxy.go | 46 +++++++++-- mtglib/stream_context.go | 1 + 7 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 mtglib/internal/obfuscated2/client_handshake.go create mode 100644 mtglib/internal/obfuscated2/conn.go create mode 100644 mtglib/internal/obfuscated2/pools.go create mode 100644 mtglib/internal/obfuscated2/utils.go diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go new file mode 100644 index 0000000..275afb0 --- /dev/null +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -0,0 +1,37 @@ +package obfuscated2 + +import ( + "crypto/cipher" + "crypto/subtle" + "encoding/hex" + "fmt" +) + +// Connection Type secure. We support only fake tls. +var clientHandshakeMagic = []byte{0xdd, 0xdd, 0xdd, 0xdd} + +func ClientHandshake(secret []byte, handshakeFrame *HandhakeFrame) (int16, cipher.Stream, cipher.Stream, error) { + decHasher := acquireSha256Hasher() + defer releaseSha256Hasher(decHasher) + + decHasher.Write(handshakeFrame.key()) // nolint: errcheck + decHasher.Write(secret) // nolint: errcheck + decryptor := makeAesCtr(decHasher.Sum(nil), handshakeFrame.iv()) + + encHasher := acquireSha256Hasher() + defer releaseSha256Hasher(encHasher) + + invertedFrame := handshakeFrame.invert() + encHasher.Write(invertedFrame.key()) // nolint: errcheck + encHasher.Write(secret) // nolint: errcheck + encryptor := makeAesCtr(encHasher.Sum(nil), invertedFrame.iv()) + + decryptedFrame := HandhakeFrame{} + decryptor.XORKeyStream(decryptedFrame.data[:], handshakeFrame.data[:]) + + if magic := decryptedFrame.magic(); subtle.ConstantTimeCompare(clientHandshakeMagic, magic) != 1 { + return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(magic)) + } + + return decryptedFrame.dc(), encryptor, decryptor, nil +} diff --git a/mtglib/internal/obfuscated2/conn.go b/mtglib/internal/obfuscated2/conn.go new file mode 100644 index 0000000..dbd69ed --- /dev/null +++ b/mtglib/internal/obfuscated2/conn.go @@ -0,0 +1,33 @@ +package obfuscated2 + +import ( + "crypto/cipher" + "net" +) + +type Conn struct { + net.Conn + + Encryptor cipher.Stream + Decryptor cipher.Stream + + writeBuf []byte +} + +func (c *Conn) Read(p []byte) (int, error) { + n, err := c.Conn.Read(p) + if err != nil { + return n, err // nolint: wrapcheck + } + + c.Decryptor.XORKeyStream(p, p[:n]) + + return n, nil +} + +func (c *Conn) Write(p []byte) (int, error) { + c.writeBuf = append(c.writeBuf[:0], p...) + c.Encryptor.XORKeyStream(c.writeBuf, c.writeBuf) + + return c.Conn.Write(c.writeBuf) +} diff --git a/mtglib/internal/obfuscated2/frame.go b/mtglib/internal/obfuscated2/frame.go index 07570d8..0526972 100644 --- a/mtglib/internal/obfuscated2/frame.go +++ b/mtglib/internal/obfuscated2/frame.go @@ -1 +1,79 @@ package obfuscated2 + +import ( + "encoding/binary" + "fmt" + "io" +) + +const ( + handshakeFrameLen = 64 + + handshakeFrameLenKey = 32 + handshakeFrameLenIV = 16 + handshakeFrameLenMagic = 4 + handshakeFrameLenDC = 2 + + handshakeFrameOffsetStart = 8 + handshakeFrameOffsetKey = handshakeFrameOffsetStart + handshakeFrameOffsetIV = handshakeFrameOffsetKey + handshakeFrameLenKey + handshakeFrameOffsetMagic = handshakeFrameOffsetIV + handshakeFrameLenIV + handshakeFrameOffsetDC = handshakeFrameOffsetMagic + handshakeFrameLenMagic + handshakeFrameOffsetEnd = handshakeFrameOffsetDC + handshakeFrameLenDC +) + +// A structure of obfuscated2 handshake frame is following: +// +// [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd]. +// +// - 8 bytes of noise +// - 32 bytes of AES Key +// - 16 bytes of AES IV +// - 4 bytes of 'magic' - this has some settings like a connection type +// - 2 bytes of 'DC'. DC is little endian int16 +// - 2 bytes of noise +type HandhakeFrame struct { + data [handshakeFrameLen]byte +} + +func (f *HandhakeFrame) Fingerprint() []byte { + return f.data[handshakeFrameOffsetStart:handshakeFrameOffsetEnd] +} + +func (f *HandhakeFrame) dc() int16 { + data := f.data[handshakeFrameOffsetDC:handshakeFrameOffsetEnd] + + return int16(binary.LittleEndian.Uint16(data)) +} + +func (f *HandhakeFrame) key() []byte { + return f.data[handshakeFrameLenKey:handshakeFrameOffsetIV] +} + +func (f *HandhakeFrame) iv() []byte { + return f.data[handshakeFrameOffsetIV:handshakeFrameOffsetMagic] +} + +func (f *HandhakeFrame) magic() []byte { + return f.data[handshakeFrameOffsetMagic:handshakeFrameOffsetDC] +} + +func (f *HandhakeFrame) invert() *HandhakeFrame { + newFrame := &HandhakeFrame{} + + for i, v := range f.data { + newFrame.data[handshakeFrameLen-1-i] = v + } + + return newFrame +} + +func ReadHandshakeFrame(reader io.Reader) (*HandhakeFrame, error) { + frame := &HandhakeFrame{} + + if _, err := io.ReadFull(reader, frame.data[:]); err != nil { + return nil, fmt.Errorf("cannot read frame data: %w", err) + } + + return frame, nil +} diff --git a/mtglib/internal/obfuscated2/pools.go b/mtglib/internal/obfuscated2/pools.go new file mode 100644 index 0000000..ce3204f --- /dev/null +++ b/mtglib/internal/obfuscated2/pools.go @@ -0,0 +1,22 @@ +package obfuscated2 + +import ( + "crypto/sha256" + "hash" + "sync" +) + +var sha256HasherPool = sync.Pool{ + New: func() interface{} { + return sha256.New() + }, +} + +func acquireSha256Hasher() hash.Hash { + return sha256HasherPool.Get().(hash.Hash) +} + +func releaseSha256Hasher(h hash.Hash) { + h.Reset() + sha256HasherPool.Put(h) +} diff --git a/mtglib/internal/obfuscated2/utils.go b/mtglib/internal/obfuscated2/utils.go new file mode 100644 index 0000000..a3fdcf9 --- /dev/null +++ b/mtglib/internal/obfuscated2/utils.go @@ -0,0 +1,15 @@ +package obfuscated2 + +import ( + "crypto/aes" + "crypto/cipher" +) + +func makeAesCtr(key, iv []byte) cipher.Stream { + block, err := aes.NewCipher(key) + if err != nil { + panic(err) + } + + return cipher.NewCTR(block, iv) +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 90adaa2..3c3bad2 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" "github.com/panjf2000/ants/v2" ) @@ -17,13 +18,12 @@ type Proxy struct { streamWaitGroup sync.WaitGroup workerPool *ants.PoolWithFunc - secret Secret - network Network - timeAttackDetector TimeAttackDetector - antiReplayCache AntiReplayCache - ipBlocklist IPBlocklist - eventStream EventStream - logger Logger + secret Secret + network Network + antiReplayCache AntiReplayCache + ipBlocklist IPBlocklist + eventStream EventStream + logger Logger } func (p *Proxy) ServeConn(conn net.Conn) { @@ -49,6 +49,12 @@ func (p *Proxy) ServeConn(conn net.Conn) { }) ctx.logger.Info("Stream has been finished") }() + + if err := p.doObfuscated2Handshake(ctx); err != nil { + p.logger.InfoError("obfuscated2 handshake is failed", err) + + return + } } func (p *Proxy) Serve(listener net.Listener) error { @@ -88,6 +94,32 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } +func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { + handshakeFrame, err := obfuscated2.ReadHandshakeFrame(ctx.clientConn) + if err != nil { + return fmt.Errorf("cannot read handshake frame: %w", err) + } + + dc, encryptor, decryptor, err := obfuscated2.ClientHandshake(p.secret.Key[:], handshakeFrame) + if err != nil { + return fmt.Errorf("cannot process client handshake: %w", err) + } + + if dc < 0 { + dc = -dc + } + + ctx.dc = int(dc) + ctx.logger = ctx.logger.BindInt("dc", ctx.dc) + ctx.clientConn = &obfuscated2.Conn{ + Conn: ctx.clientConn, + Encryptor: encryptor, + Decryptor: decryptor, + } + + return nil +} + func NewProxy(opts ProxyOpts) (*Proxy, error) { switch { case opts.Network == nil: diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go index cd5d1e8..1c75482 100644 --- a/mtglib/stream_context.go +++ b/mtglib/stream_context.go @@ -13,6 +13,7 @@ type streamContext struct { ctxCancel context.CancelFunc clientConn net.Conn connID string + dc int logger Logger } From 3a44bcb8545cda70bb1946a5774c4801c7f201ca Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 15:01:22 +0300 Subject: [PATCH 095/206] Introduce pools to obfuscated2 --- .../internal/obfuscated2/client_handshake.go | 27 ++++++++--- mtglib/internal/obfuscated2/frame.go | 48 ++++--------------- mtglib/internal/obfuscated2/pools.go | 25 ++++++++-- mtglib/proxy.go | 7 +-- 4 files changed, 51 insertions(+), 56 deletions(-) diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index 275afb0..f5e692f 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -5,12 +5,20 @@ import ( "crypto/subtle" "encoding/hex" "fmt" + "io" ) // Connection Type secure. We support only fake tls. var clientHandshakeMagic = []byte{0xdd, 0xdd, 0xdd, 0xdd} -func ClientHandshake(secret []byte, handshakeFrame *HandhakeFrame) (int16, cipher.Stream, cipher.Stream, error) { +func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cipher.Stream, error) { + handshakeFrame := acquireHandshakeFrame() + defer releaseHandshakeFrame(handshakeFrame) + + if _, err := io.ReadFull(reader, handshakeFrame.data[:]); err != nil { + return 0, nil, nil, fmt.Errorf("cannot read frame: %w", err) + } + decHasher := acquireSha256Hasher() defer releaseSha256Hasher(decHasher) @@ -21,17 +29,22 @@ func ClientHandshake(secret []byte, handshakeFrame *HandhakeFrame) (int16, ciphe encHasher := acquireSha256Hasher() defer releaseSha256Hasher(encHasher) - invertedFrame := handshakeFrame.invert() + invertedFrame := acquireHandshakeFrame() + defer releaseHandshakeFrame(invertedFrame) + + for i, v := range handshakeFrame.data { + invertedFrame.data[handshakeFrameLen-1-i] = v + } + encHasher.Write(invertedFrame.key()) // nolint: errcheck - encHasher.Write(secret) // nolint: errcheck + encHasher.Write(secret) // nolint: errcheck encryptor := makeAesCtr(encHasher.Sum(nil), invertedFrame.iv()) - decryptedFrame := HandhakeFrame{} - decryptor.XORKeyStream(decryptedFrame.data[:], handshakeFrame.data[:]) + decryptor.XORKeyStream(handshakeFrame.data[:], handshakeFrame.data[:]) - if magic := decryptedFrame.magic(); subtle.ConstantTimeCompare(clientHandshakeMagic, magic) != 1 { + if magic := handshakeFrame.magic(); subtle.ConstantTimeCompare(clientHandshakeMagic, magic) != 1 { return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(magic)) } - return decryptedFrame.dc(), encryptor, decryptor, nil + return handshakeFrame.dc(), encryptor, decryptor, nil } diff --git a/mtglib/internal/obfuscated2/frame.go b/mtglib/internal/obfuscated2/frame.go index 0526972..f84a07d 100644 --- a/mtglib/internal/obfuscated2/frame.go +++ b/mtglib/internal/obfuscated2/frame.go @@ -1,10 +1,6 @@ package obfuscated2 -import ( - "encoding/binary" - "fmt" - "io" -) +import "encoding/binary" const ( handshakeFrameLen = 64 @@ -32,48 +28,24 @@ const ( // - 4 bytes of 'magic' - this has some settings like a connection type // - 2 bytes of 'DC'. DC is little endian int16 // - 2 bytes of noise -type HandhakeFrame struct { +type handshakeFrame struct { data [handshakeFrameLen]byte } -func (f *HandhakeFrame) Fingerprint() []byte { - return f.data[handshakeFrameOffsetStart:handshakeFrameOffsetEnd] -} - -func (f *HandhakeFrame) dc() int16 { - data := f.data[handshakeFrameOffsetDC:handshakeFrameOffsetEnd] +func (h *handshakeFrame) dc() int16 { + data := h.data[handshakeFrameOffsetDC:handshakeFrameOffsetEnd] return int16(binary.LittleEndian.Uint16(data)) } -func (f *HandhakeFrame) key() []byte { - return f.data[handshakeFrameLenKey:handshakeFrameOffsetIV] +func (h *handshakeFrame) key() []byte { + return h.data[handshakeFrameLenKey:handshakeFrameOffsetIV] } -func (f *HandhakeFrame) iv() []byte { - return f.data[handshakeFrameOffsetIV:handshakeFrameOffsetMagic] +func (h *handshakeFrame) iv() []byte { + return h.data[handshakeFrameOffsetIV:handshakeFrameOffsetMagic] } -func (f *HandhakeFrame) magic() []byte { - return f.data[handshakeFrameOffsetMagic:handshakeFrameOffsetDC] -} - -func (f *HandhakeFrame) invert() *HandhakeFrame { - newFrame := &HandhakeFrame{} - - for i, v := range f.data { - newFrame.data[handshakeFrameLen-1-i] = v - } - - return newFrame -} - -func ReadHandshakeFrame(reader io.Reader) (*HandhakeFrame, error) { - frame := &HandhakeFrame{} - - if _, err := io.ReadFull(reader, frame.data[:]); err != nil { - return nil, fmt.Errorf("cannot read frame data: %w", err) - } - - return frame, nil +func (h *handshakeFrame) magic() []byte { + return h.data[handshakeFrameOffsetMagic:handshakeFrameOffsetDC] } diff --git a/mtglib/internal/obfuscated2/pools.go b/mtglib/internal/obfuscated2/pools.go index ce3204f..222514e 100644 --- a/mtglib/internal/obfuscated2/pools.go +++ b/mtglib/internal/obfuscated2/pools.go @@ -6,11 +6,18 @@ import ( "sync" ) -var sha256HasherPool = sync.Pool{ - New: func() interface{} { - return sha256.New() - }, -} +var ( + sha256HasherPool = sync.Pool{ + New: func() interface{} { + return sha256.New() + }, + } + handshakeFramePool = sync.Pool{ + New: func() interface{} { + return &handshakeFrame{} + }, + } +) func acquireSha256Hasher() hash.Hash { return sha256HasherPool.Get().(hash.Hash) @@ -20,3 +27,11 @@ func releaseSha256Hasher(h hash.Hash) { h.Reset() sha256HasherPool.Put(h) } + +func acquireHandshakeFrame() *handshakeFrame { + return handshakeFramePool.Get().(*handshakeFrame) +} + +func releaseHandshakeFrame(h *handshakeFrame) { + handshakeFramePool.Put(h) +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 3c3bad2..1dd76cd 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -95,12 +95,7 @@ func (p *Proxy) Shutdown() { } func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { - handshakeFrame, err := obfuscated2.ReadHandshakeFrame(ctx.clientConn) - if err != nil { - return fmt.Errorf("cannot read handshake frame: %w", err) - } - - dc, encryptor, decryptor, err := obfuscated2.ClientHandshake(p.secret.Key[:], handshakeFrame) + dc, encryptor, decryptor, err := obfuscated2.ClientHandshake(p.secret.Key[:], ctx.clientConn) if err != nil { return fmt.Errorf("cannot process client handshake: %w", err) } From f155b9f37ee54a3e81566e2bf6d2e7411d92d5ba Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 15:01:50 +0300 Subject: [PATCH 096/206] Rename frame to handshake frame --- mtglib/internal/obfuscated2/{frame.go => handshake_frame.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mtglib/internal/obfuscated2/{frame.go => handshake_frame.go} (100%) diff --git a/mtglib/internal/obfuscated2/frame.go b/mtglib/internal/obfuscated2/handshake_frame.go similarity index 100% rename from mtglib/internal/obfuscated2/frame.go rename to mtglib/internal/obfuscated2/handshake_frame.go From 0ad2d6174269f9c9b0ed100c782c31b250aeceb5 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 15:20:54 +0300 Subject: [PATCH 097/206] Move config path cli parameter to base --- cli/access.go | 6 ++---- cli/base.go | 10 ++++++---- cli/base_internal_test.go | 6 ++++-- cli/generate_secret.go | 2 -- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cli/access.go b/cli/access.go index d6212b6..b020372 100644 --- a/cli/access.go +++ b/cli/access.go @@ -34,18 +34,16 @@ type accessResponseURLs struct { } type Access struct { - base `kong:"-"` + base PublicIPv4 net.IP `kong:"help='Public IPv4 address for proxy. By default it is resolved via remote website',name='ipv4',short='i'"` // nolint: lll PublicIPv6 net.IP `kong:"help='Public IPv6 address for proxy. By default it is resolved via remote website',name='ipv6',short='I'"` // nolint: lll Port uint `kong:"help='Port number. Default port is taken from configuration file, bind-to parameter',type:'uint',short='p'"` // nolint: lll Hex bool `kong:"help='Print secret in hex encoding.',short='x'"` - - ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll } func (c *Access) Run(cli *CLI, version string) error { - if err := c.ReadConfig(cli.Access.ConfigPath, version); err != nil { + if err := c.ReadConfig(version); err != nil { return fmt.Errorf("cannot init config: %w", err) } diff --git a/cli/base.go b/cli/base.go index f4b6635..9c19ec1 100644 --- a/cli/base.go +++ b/cli/base.go @@ -12,12 +12,14 @@ import ( ) type base struct { - Network mtglib.Network - Config *config.Config + ConfigPath string `kong:"arg,required,type='existingfile',help='Path to the configuration file.',name='config-path'"` // nolint: lll + + Network mtglib.Network `kong:"-"` + Config *config.Config `kong:"-"` } -func (b *base) ReadConfig(path, version string) error { - content, err := ioutil.ReadFile(path) +func (b *base) ReadConfig(version string) error { + content, err := ioutil.ReadFile(b.ConfigPath) if err != nil { return fmt.Errorf("cannot read config file: %w", err) } diff --git a/cli/base_internal_test.go b/cli/base_internal_test.go index 1cd0b97..51909d4 100644 --- a/cli/base_internal_test.go +++ b/cli/base_internal_test.go @@ -18,11 +18,13 @@ func (suite *BaseTestSuite) SetupTest() { } func (suite *BaseTestSuite) TestReadConfigNok() { - suite.Error(suite.b.ReadConfig(filepath.Join("testdata", "unknown"), "dev")) + suite.b.ConfigPath = filepath.Join("testdata", "unknown") + suite.Error(suite.b.ReadConfig("dev")) } func (suite *BaseTestSuite) TestReadConfig() { - suite.NoError(suite.b.ReadConfig(filepath.Join("testdata", "minimal.toml"), "dev")) + suite.b.ConfigPath = filepath.Join("testdata", "minimal.toml") + suite.NoError(suite.b.ReadConfig("dev")) } func TestBase(t *testing.T) { diff --git a/cli/generate_secret.go b/cli/generate_secret.go index e618dfd..e3fb0e8 100644 --- a/cli/generate_secret.go +++ b/cli/generate_secret.go @@ -7,8 +7,6 @@ import ( ) type GenerateSecret struct { - base `kong:"-"` - HostName string `kong:"arg,required,help='Hostname to use for domain fronting.',name='hostname'"` Hex bool `kong:"help='Print secret in hex encoding.',short='x'"` } From 0330a0e5cd266560702d111ea1c45a13880f520f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 17:15:55 +0300 Subject: [PATCH 098/206] Fixes for obfuscated2 --- antireplay/init.go | 6 + cli/cli.go | 1 + cli/proxy.go | 205 ++++++++++++++++++ ipblocklist/firehol.go | 2 +- ipblocklist/init.go | 8 + main.go | 5 + .../internal/obfuscated2/client_handshake.go | 6 +- .../internal/obfuscated2/handshake_frame.go | 30 +-- stats/init.go | 5 + timeattack/init.go | 7 + utils/rlimit.go | 24 ++ utils/rlimit_windows.go | 7 + utils/root_context.go | 25 +++ utils/root_context_windows.go | 23 ++ 14 files changed, 335 insertions(+), 19 deletions(-) create mode 100644 antireplay/init.go create mode 100644 cli/proxy.go create mode 100644 ipblocklist/init.go create mode 100644 timeattack/init.go create mode 100644 utils/rlimit.go create mode 100644 utils/rlimit_windows.go create mode 100644 utils/root_context.go create mode 100644 utils/root_context_windows.go diff --git a/antireplay/init.go b/antireplay/init.go new file mode 100644 index 0000000..4585a65 --- /dev/null +++ b/antireplay/init.go @@ -0,0 +1,6 @@ +package antireplay + +const ( + DefaultMaxSize = 10 * 1024 * 1024 // 10mib + DefaultErrorRate = 0.0001 +) diff --git a/cli/cli.go b/cli/cli.go index f620e21..d2d813c 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -5,5 +5,6 @@ import "github.com/alecthomas/kong" type CLI struct { GenerateSecret GenerateSecret `kong:"cmd,help='Generate new proxy secret'"` Access Access `kong:"cmd,help='Print access information.'"` + Run Proxy `kong:"cmd,help='Run proxy.'"` Version kong.VersionFlag `kong:"help='Print version.',short='v'"` } diff --git a/cli/proxy.go b/cli/proxy.go new file mode 100644 index 0000000..eaf7035 --- /dev/null +++ b/cli/proxy.go @@ -0,0 +1,205 @@ +package cli + +import ( + "fmt" + "net" + "os" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/stats" + "github.com/9seconds/mtg/v2/timeattack" + "github.com/9seconds/mtg/v2/utils" + "github.com/rs/zerolog" +) + +type Proxy struct { + base + + prometheusListener net.Listener + prometheus *stats.PrometheusFactory + statsdFactory *stats.StatsdFactory +} + +func (c *Proxy) Run(cli *CLI, version string) error { + if err := c.ReadConfig(version); err != nil { + return fmt.Errorf("cannot init config: %w", err) + } + + return c.Execute() +} + +func (c *Proxy) Execute() error { // nolint: funlen + zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs + zerolog.TimestampFieldName = "timestamp" + zerolog.LevelFieldName = "level" + + ctx := utils.RootContext() + opts := mtglib.ProxyOpts{ + Logger: logger.NewZeroLogger(zerolog.New(os.Stdout).With().Timestamp().Logger()), + Network: c.Network, + AntiReplayCache: antireplay.NewNoop(), + IPBlocklist: ipblocklist.NewNoop(), + TimeAttackDetector: timeattack.NewNoop(), + EventStream: events.NewNoopStream(), + + Secret: c.Config.Secret, + BufferSize: c.Config.TCPBuffer.Value(mtglib.DefaultBufferSize), + CloakPort: c.Config.CloakPort.Value(mtglib.DefaultCloakPort), + IdleTimeout: c.Config.Network.Timeout.Idle.Value(mtglib.DefaultIdleTimeout), + PreferIP: c.Config.PreferIP.Value(mtglib.DefaultPreferIP), + } + + defer func() { + opts.AntiReplayCache.Shutdown() + opts.IPBlocklist.Shutdown() + opts.EventStream.Shutdown() + }() + + if opts.Concurrency == 0 { + opts.Concurrency = mtglib.DefaultConcurrency + } + + opts.Logger.BindStr("configuration", c.Config.String()).Debug("configuration") + + c.setupAntiReplayCache(&opts) + c.setupTimeAttackDetector(&opts) + + if err := c.setupIPBlocklist(&opts); err != nil { + return fmt.Errorf("cannot setup ipblocklist: %w", err) + } + + if err := c.setupEventStream(&opts); err != nil { + return fmt.Errorf("cannot setup event stream: %w", err) + } + + proxy, err := mtglib.NewProxy(opts) + if err != nil { + return fmt.Errorf("cannot create a proxy: %w", err) + } + + listener, err := net.Listen("tcp", c.Config.BindTo.String()) + if err != nil { + return fmt.Errorf("cannot start proxy: %w", err) + } + + go proxy.Serve(listener) // nolint: errcheck + + <-ctx.Done() + + listener.Close() + + if c.prometheusListener != nil { + c.prometheusListener.Close() + } + + if c.prometheus != nil { + c.prometheus.Close() + } + + if c.statsdFactory != nil { + c.statsdFactory.Close() + } + + return nil +} + +func (c *Proxy) setupAntiReplayCache(opts *mtglib.ProxyOpts) { + if !c.Config.Defense.AntiReplay.Enabled { + return + } + + opts.AntiReplayCache = antireplay.NewStableBloomFilter( + c.Config.Defense.AntiReplay.MaxSize.Value(antireplay.DefaultMaxSize), + c.Config.Defense.AntiReplay.ErrorRate.Value(antireplay.DefaultErrorRate), + ) +} + +func (c *Proxy) setupTimeAttackDetector(opts *mtglib.ProxyOpts) { + if !c.Config.Defense.Time.Enabled { + return + } + + opts.TimeAttackDetector = timeattack.NewDetector( + c.Config.Defense.Time.AllowSkewness.Value(timeattack.DefaultDuration), + ) +} + +func (c *Proxy) setupIPBlocklist(opts *mtglib.ProxyOpts) error { + if !c.Config.Defense.Blocklist.Enabled { + return nil + } + + remoteURLs := []string{} + localFiles := []string{} + + for _, v := range c.Config.Defense.Blocklist.URLs { + if v.IsRemote() { + remoteURLs = append(remoteURLs, v.String()) + } else { + localFiles = append(localFiles, v.String()) + } + } + + firehol, err := ipblocklist.NewFirehol(opts.Logger.Named("ipblockist"), + c.Network, + c.Config.Defense.Blocklist.DownloadConcurrency, + remoteURLs, + localFiles) + if err != nil { + return err // nolint: wrapcheck + } + + go firehol.Run(c.Config.Defense.Blocklist.UpdateEach.Value(ipblocklist.DefaultUpdateEach)) + + opts.IPBlocklist = firehol + + return nil +} + +func (c *Proxy) setupEventStream(opts *mtglib.ProxyOpts) error { + factories := make([]events.ObserverFactory, 0, 2) + + if c.Config.Stats.StatsD.Enabled { + statsdFactory, err := stats.NewStatsd( + c.Config.Stats.StatsD.Address.String(), + opts.Logger.Named("statsd"), + c.Config.Stats.StatsD.MetricPrefix.Value(stats.DefaultStatsdMetricPrefix), + c.Config.Stats.StatsD.TagFormat.Value(stats.DefaultStatsdTagFormat)) + if err != nil { + return fmt.Errorf("cannot build statsd observer: %w", err) + } + + c.statsdFactory = &statsdFactory + + factories = append(factories, statsdFactory.Make) + } + + if c.Config.Stats.Prometheus.Enabled { + prometheus := stats.NewPrometheus( + c.Config.Stats.Prometheus.MetricPrefix.Value(stats.DefaultMetricPrefix), + c.Config.Stats.Prometheus.HTTPPath.Value("/"), + ) + + listener, err := net.Listen("tcp", c.Config.Stats.Prometheus.BindTo.String()) + if err != nil { + return fmt.Errorf("cannot start a listener for prometheus: %w", err) + } + + go prometheus.Serve(listener) // nolint: errcheck + + c.prometheusListener = listener + c.prometheus = prometheus + + factories = append(factories, prometheus.Make) + } + + if len(factories) > 0 { + opts.EventStream = events.NewEventStream(factories) + } + + return nil +} diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 2d8a748..49f1be7 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -323,7 +323,7 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, } if downloadConcurrency == 0 { - downloadConcurrency = 1 + downloadConcurrency = DefaultDownloadConcurrency } workerPool, _ := ants.NewPool(int(downloadConcurrency)) diff --git a/ipblocklist/init.go b/ipblocklist/init.go new file mode 100644 index 0000000..cfa8df8 --- /dev/null +++ b/ipblocklist/init.go @@ -0,0 +1,8 @@ +package ipblocklist + +import "time" + +const ( + DefaultDownloadConcurrency = 1 + DefaultUpdateEach = 12 * time.Hour +) diff --git a/main.go b/main.go index e74a760..d554166 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "time" "github.com/9seconds/mtg/v2/cli" + "github.com/9seconds/mtg/v2/utils" "github.com/alecthomas/kong" ) @@ -13,6 +14,10 @@ var version = "dev" // has to be set by ldflags func main() { rand.Seed(time.Now().UTC().UnixNano()) + if err := utils.SetLimits(); err != nil { + panic(err) + } + cli := &cli.CLI{} ctx := kong.Parse(cli, kong.Vars{ "version": version, diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index f5e692f..eb6a5a8 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -9,7 +9,7 @@ import ( ) // Connection Type secure. We support only fake tls. -var clientHandshakeMagic = []byte{0xdd, 0xdd, 0xdd, 0xdd} +var clientHandshakeConnectionType = []byte{0xdd, 0xdd, 0xdd, 0xdd} func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cipher.Stream, error) { handshakeFrame := acquireHandshakeFrame() @@ -42,8 +42,8 @@ func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cip decryptor.XORKeyStream(handshakeFrame.data[:], handshakeFrame.data[:]) - if magic := handshakeFrame.magic(); subtle.ConstantTimeCompare(clientHandshakeMagic, magic) != 1 { - return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(magic)) + if val := handshakeFrame.connectionType(); subtle.ConstantTimeCompare(clientHandshakeConnectionType, val) != 1 { + return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(val)) } return handshakeFrame.dc(), encryptor, decryptor, nil diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go index f84a07d..71a993c 100644 --- a/mtglib/internal/obfuscated2/handshake_frame.go +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -5,17 +5,17 @@ import "encoding/binary" const ( handshakeFrameLen = 64 - handshakeFrameLenKey = 32 - handshakeFrameLenIV = 16 - handshakeFrameLenMagic = 4 - handshakeFrameLenDC = 2 + handshakeFrameLenKey = 32 + handshakeFrameLenIV = 16 + handshakeFrameLenConnectionType = 4 + handshakeFrameLenDC = 2 - handshakeFrameOffsetStart = 8 - handshakeFrameOffsetKey = handshakeFrameOffsetStart - handshakeFrameOffsetIV = handshakeFrameOffsetKey + handshakeFrameLenKey - handshakeFrameOffsetMagic = handshakeFrameOffsetIV + handshakeFrameLenIV - handshakeFrameOffsetDC = handshakeFrameOffsetMagic + handshakeFrameLenMagic - handshakeFrameOffsetEnd = handshakeFrameOffsetDC + handshakeFrameLenDC + handshakeFrameOffsetStart = 8 + handshakeFrameOffsetKey = handshakeFrameOffsetStart + handshakeFrameOffsetIV = handshakeFrameOffsetKey + handshakeFrameLenKey + handshakeFrameOffsetConnectionType = handshakeFrameOffsetIV + handshakeFrameLenIV + handshakeFrameOffsetDC = handshakeFrameOffsetConnectionType + handshakeFrameLenConnectionType + handshakeFrameOffsetEnd = handshakeFrameOffsetDC + handshakeFrameLenDC ) // A structure of obfuscated2 handshake frame is following: @@ -25,7 +25,7 @@ const ( // - 8 bytes of noise // - 32 bytes of AES Key // - 16 bytes of AES IV -// - 4 bytes of 'magic' - this has some settings like a connection type +// - 4 bytes of 'connection type' - this has some setting like a connection type // - 2 bytes of 'DC'. DC is little endian int16 // - 2 bytes of noise type handshakeFrame struct { @@ -39,13 +39,13 @@ func (h *handshakeFrame) dc() int16 { } func (h *handshakeFrame) key() []byte { - return h.data[handshakeFrameLenKey:handshakeFrameOffsetIV] + return h.data[handshakeFrameOffsetKey:handshakeFrameOffsetIV] } func (h *handshakeFrame) iv() []byte { - return h.data[handshakeFrameOffsetIV:handshakeFrameOffsetMagic] + return h.data[handshakeFrameOffsetIV:handshakeFrameOffsetConnectionType] } -func (h *handshakeFrame) magic() []byte { - return h.data[handshakeFrameOffsetMagic:handshakeFrameOffsetDC] +func (h *handshakeFrame) connectionType() []byte { + return h.data[handshakeFrameOffsetConnectionType:handshakeFrameOffsetDC] } diff --git a/stats/init.go b/stats/init.go index 063eb32..7d095c5 100644 --- a/stats/init.go +++ b/stats/init.go @@ -1,6 +1,11 @@ package stats const ( + DefaultMetricPrefix = "mtg" + + DefaultStatsdMetricPrefix = DefaultMetricPrefix + "." + DefaultStatsdTagFormat = "datadog" + MetricActiveConnection = "active_connections" MetricSessionDuration = "session_duration" MetricConcurrencyLimited = "concurrency_limited" diff --git a/timeattack/init.go b/timeattack/init.go new file mode 100644 index 0000000..385b9d7 --- /dev/null +++ b/timeattack/init.go @@ -0,0 +1,7 @@ +package timeattack + +import "time" + +const ( + DefaultDuration = time.Second +) diff --git a/utils/rlimit.go b/utils/rlimit.go new file mode 100644 index 0000000..b474076 --- /dev/null +++ b/utils/rlimit.go @@ -0,0 +1,24 @@ +// +build !windows + +package utils + +import ( + "fmt" + + "golang.org/x/sys/unix" +) + +func SetLimits() error { + rLimit := unix.Rlimit{} + if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("cannot get rlimit: %w", err) + } + + rLimit.Cur = rLimit.Max + + if err := unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit); err != nil { + return fmt.Errorf("cannot set rlimit: %w", err) + } + + return nil +} diff --git a/utils/rlimit_windows.go b/utils/rlimit_windows.go new file mode 100644 index 0000000..58bdf39 --- /dev/null +++ b/utils/rlimit_windows.go @@ -0,0 +1,7 @@ +// +build windows + +package utils + +func SetLimits() error { + return nil +} diff --git a/utils/root_context.go b/utils/root_context.go new file mode 100644 index 0000000..9fe4b49 --- /dev/null +++ b/utils/root_context.go @@ -0,0 +1,25 @@ +// +build !windows + +package utils + +import ( + "context" + "os" + "os/signal" + "syscall" +) + +func RootContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + sigChan := make(chan os.Signal, 1) + + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + go func() { + for range sigChan { + cancel() + } + }() + + return ctx +} diff --git a/utils/root_context_windows.go b/utils/root_context_windows.go new file mode 100644 index 0000000..9a5b6e0 --- /dev/null +++ b/utils/root_context_windows.go @@ -0,0 +1,23 @@ +// +build windows + +package utils + +import ( + "context" + "os" + "os/signal" +) + +func RootContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + sigChan := make(chan os.Signal, 1) + + signal.Notify(sigChan, os.Interrupt) + go func() { + for range sigChan { + cancel() + } + }() + + return ctx +} From 188fa6a04aa721c14650db0d13f1d154b8fb7da2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 17:50:54 +0300 Subject: [PATCH 099/206] Remove redundand utils file --- mtglib/internal/obfuscated2/client_handshake.go | 9 +++++++++ mtglib/internal/obfuscated2/utils.go | 15 --------------- 2 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 mtglib/internal/obfuscated2/utils.go diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index eb6a5a8..41e80b3 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -48,3 +48,12 @@ func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cip return handshakeFrame.dc(), encryptor, decryptor, nil } + +func makeAesCtr(key, iv []byte) cipher.Stream { + block, err := aes.NewCipher(key) + if err != nil { + panic(err) + } + + return cipher.NewCTR(block, iv) +} diff --git a/mtglib/internal/obfuscated2/utils.go b/mtglib/internal/obfuscated2/utils.go deleted file mode 100644 index a3fdcf9..0000000 --- a/mtglib/internal/obfuscated2/utils.go +++ /dev/null @@ -1,15 +0,0 @@ -package obfuscated2 - -import ( - "crypto/aes" - "crypto/cipher" -) - -func makeAesCtr(key, iv []byte) cipher.Stream { - block, err := aes.NewCipher(key) - if err != nil { - panic(err) - } - - return cipher.NewCTR(block, iv) -} From 8de727932d1c99ea43a39698e79fcdd1c5887467 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 18 Mar 2021 22:06:51 +0300 Subject: [PATCH 100/206] Unpool handshake Anyway, we are going to use stack here, no need for connection pooling. Even arrays are allocaed on a stack because we do not use slices here but real ones. --- .../internal/obfuscated2/client_handshake.go | 31 +++++++++---------- mtglib/internal/obfuscated2/pools.go | 25 +++------------ 2 files changed, 20 insertions(+), 36 deletions(-) diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index 41e80b3..035086e 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -1,6 +1,7 @@ package obfuscated2 import ( + "crypto/aes" "crypto/cipher" "crypto/subtle" "encoding/hex" @@ -12,41 +13,39 @@ import ( var clientHandshakeConnectionType = []byte{0xdd, 0xdd, 0xdd, 0xdd} func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cipher.Stream, error) { - handshakeFrame := acquireHandshakeFrame() - defer releaseHandshakeFrame(handshakeFrame) + handshake := handshakeFrame{} - if _, err := io.ReadFull(reader, handshakeFrame.data[:]); err != nil { + if _, err := io.ReadFull(reader, handshake.data[:]); err != nil { return 0, nil, nil, fmt.Errorf("cannot read frame: %w", err) } decHasher := acquireSha256Hasher() defer releaseSha256Hasher(decHasher) - decHasher.Write(handshakeFrame.key()) // nolint: errcheck - decHasher.Write(secret) // nolint: errcheck - decryptor := makeAesCtr(decHasher.Sum(nil), handshakeFrame.iv()) + decHasher.Write(handshake.key()) // nolint: errcheck + decHasher.Write(secret) // nolint: errcheck + decryptor := makeAesCtr(decHasher.Sum(nil), handshake.iv()) encHasher := acquireSha256Hasher() defer releaseSha256Hasher(encHasher) - invertedFrame := acquireHandshakeFrame() - defer releaseHandshakeFrame(invertedFrame) + invertedHandshake := handshakeFrame{} - for i, v := range handshakeFrame.data { - invertedFrame.data[handshakeFrameLen-1-i] = v + for i, v := range handshake.data { + invertedHandshake.data[handshakeFrameLen-1-i] = v } - encHasher.Write(invertedFrame.key()) // nolint: errcheck - encHasher.Write(secret) // nolint: errcheck - encryptor := makeAesCtr(encHasher.Sum(nil), invertedFrame.iv()) + encHasher.Write(invertedHandshake.key()) // nolint: errcheck + encHasher.Write(secret) // nolint: errcheck + encryptor := makeAesCtr(encHasher.Sum(nil), invertedHandshake.iv()) - decryptor.XORKeyStream(handshakeFrame.data[:], handshakeFrame.data[:]) + decryptor.XORKeyStream(handshake.data[:], handshake.data[:]) - if val := handshakeFrame.connectionType(); subtle.ConstantTimeCompare(clientHandshakeConnectionType, val) != 1 { + if val := handshake.connectionType(); subtle.ConstantTimeCompare(clientHandshakeConnectionType, val) != 1 { return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(val)) } - return handshakeFrame.dc(), encryptor, decryptor, nil + return handshake.dc(), encryptor, decryptor, nil } func makeAesCtr(key, iv []byte) cipher.Stream { diff --git a/mtglib/internal/obfuscated2/pools.go b/mtglib/internal/obfuscated2/pools.go index 222514e..ce3204f 100644 --- a/mtglib/internal/obfuscated2/pools.go +++ b/mtglib/internal/obfuscated2/pools.go @@ -6,18 +6,11 @@ import ( "sync" ) -var ( - sha256HasherPool = sync.Pool{ - New: func() interface{} { - return sha256.New() - }, - } - handshakeFramePool = sync.Pool{ - New: func() interface{} { - return &handshakeFrame{} - }, - } -) +var sha256HasherPool = sync.Pool{ + New: func() interface{} { + return sha256.New() + }, +} func acquireSha256Hasher() hash.Hash { return sha256HasherPool.Get().(hash.Hash) @@ -27,11 +20,3 @@ func releaseSha256Hasher(h hash.Hash) { h.Reset() sha256HasherPool.Put(h) } - -func acquireHandshakeFrame() *handshakeFrame { - return handshakeFramePool.Get().(*handshakeFrame) -} - -func releaseHandshakeFrame(h *handshakeFrame) { - handshakeFramePool.Put(h) -} From 4d2d21e1012ae0cdd2a999c2a4fab46132d6342a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 21 Mar 2021 21:19:26 +0300 Subject: [PATCH 101/206] Small refactorings --- .../internal/obfuscated2/client_handshake.go | 38 ++----------------- .../obfuscated2/client_handshake_frame.go | 30 +++++++++++++++ .../internal/obfuscated2/handshake_frame.go | 3 ++ mtglib/internal/obfuscated2/utils.go | 23 +++++++++++ 4 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 mtglib/internal/obfuscated2/client_handshake_frame.go create mode 100644 mtglib/internal/obfuscated2/utils.go diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index 035086e..08bef5d 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -1,7 +1,6 @@ package obfuscated2 import ( - "crypto/aes" "crypto/cipher" "crypto/subtle" "encoding/hex" @@ -9,50 +8,21 @@ import ( "io" ) -// Connection Type secure. We support only fake tls. -var clientHandshakeConnectionType = []byte{0xdd, 0xdd, 0xdd, 0xdd} - func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cipher.Stream, error) { - handshake := handshakeFrame{} + handshake := clientHandhakeFrame{} if _, err := io.ReadFull(reader, handshake.data[:]); err != nil { return 0, nil, nil, fmt.Errorf("cannot read frame: %w", err) } - decHasher := acquireSha256Hasher() - defer releaseSha256Hasher(decHasher) - - decHasher.Write(handshake.key()) // nolint: errcheck - decHasher.Write(secret) // nolint: errcheck - decryptor := makeAesCtr(decHasher.Sum(nil), handshake.iv()) - - encHasher := acquireSha256Hasher() - defer releaseSha256Hasher(encHasher) - - invertedHandshake := handshakeFrame{} - - for i, v := range handshake.data { - invertedHandshake.data[handshakeFrameLen-1-i] = v - } - - encHasher.Write(invertedHandshake.key()) // nolint: errcheck - encHasher.Write(secret) // nolint: errcheck - encryptor := makeAesCtr(encHasher.Sum(nil), invertedHandshake.iv()) + decryptor := handshake.decryptor(secret) + encryptor := handshake.encryptor(secret) decryptor.XORKeyStream(handshake.data[:], handshake.data[:]) - if val := handshake.connectionType(); subtle.ConstantTimeCompare(clientHandshakeConnectionType, val) != 1 { + if val := handshake.connectionType(); subtle.ConstantTimeCompare(handshakeConnectionType, val) != 1 { return 0, nil, nil, fmt.Errorf("unsupported connection type: %s", hex.EncodeToString(val)) } return handshake.dc(), encryptor, decryptor, nil } - -func makeAesCtr(key, iv []byte) cipher.Stream { - block, err := aes.NewCipher(key) - if err != nil { - panic(err) - } - - return cipher.NewCTR(block, iv) -} diff --git a/mtglib/internal/obfuscated2/client_handshake_frame.go b/mtglib/internal/obfuscated2/client_handshake_frame.go new file mode 100644 index 0000000..4d238f8 --- /dev/null +++ b/mtglib/internal/obfuscated2/client_handshake_frame.go @@ -0,0 +1,30 @@ +package obfuscated2 + +import "crypto/cipher" + +type clientHandhakeFrame struct { + handshakeFrame +} + +func (c *clientHandhakeFrame) decryptor(secret []byte) cipher.Stream { + hasher := acquireSha256Hasher() + defer releaseSha256Hasher(hasher) + + hasher.Write(c.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck + + return makeAesCtr(hasher.Sum(nil), c.iv()) +} + +func (c *clientHandhakeFrame) encryptor(secret []byte) cipher.Stream { + arr := clientHandhakeFrame{} + invertByteSlices(arr.data[:], c.data[:]) + + hasher := acquireSha256Hasher() + defer releaseSha256Hasher(hasher) + + hasher.Write(arr.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck + + return makeAesCtr(hasher.Sum(nil), arr.iv()) +} diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go index 71a993c..919d7bb 100644 --- a/mtglib/internal/obfuscated2/handshake_frame.go +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -18,6 +18,9 @@ const ( handshakeFrameOffsetEnd = handshakeFrameOffsetDC + handshakeFrameLenDC ) +// Connection-Type: Secure. We support only fake tls. +var handshakeConnectionType = []byte{0xdd, 0xdd, 0xdd, 0xdd} + // A structure of obfuscated2 handshake frame is following: // // [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd]. diff --git a/mtglib/internal/obfuscated2/utils.go b/mtglib/internal/obfuscated2/utils.go new file mode 100644 index 0000000..f32dde0 --- /dev/null +++ b/mtglib/internal/obfuscated2/utils.go @@ -0,0 +1,23 @@ +package obfuscated2 + +import ( + "crypto/aes" + "crypto/cipher" +) + +func makeAesCtr(key, iv []byte) cipher.Stream { + block, err := aes.NewCipher(key) + if err != nil { + panic(err) + } + + return cipher.NewCTR(block, iv) +} + +func invertByteSlices(dst, src []byte) { + lenDst := len(dst) - 1 + + for i, v := range src { + dst[lenDst-i] = v + } +} From 69203f3e23894232a033f74c84788bf96da437ff Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 22 Mar 2021 11:34:32 +0300 Subject: [PATCH 102/206] Add tests for obfuscated2 clientside --- .github/workflows/ci.yaml | 2 - cli/access.go | 5 +- cli/base.go | 4 +- config/config_test.go | 4 +- ipblocklist/firehol.go | 3 +- .../internal/obfuscated2/client_handshake.go | 2 +- .../obfuscated2/client_handshake_test.go | 85 +++++++++++++++++++ .../internal/obfuscated2/handshake_frame.go | 12 ++- mtglib/internal/obfuscated2/init_test.go | 83 ++++++++++++++++++ .../snapshot-ok-4529d55776e2d427 | 13 +++ .../snapshot-ok-585c944d672f60a2 | 13 +++ .../snapshot-ok-e3cedb3c8b05c39d | 13 +++ .../snapshot-ok-e7d2a1e5e479cf50 | 13 +++ .../snapshot-ok-eae4de34f1ab0bdf | 13 +++ .../snapshot-ok-ec9c53fa17f4b39b | 13 +++ .../snapshot-ok-ee4b5ad981dcce87 | 13 +++ .../snapshot-ok-f122ea426238564c | 13 +++ .../snapshot-ok-f3e117d7ac4e44f9 | 13 +++ .../snapshot-ok-f5e00132a81f7a61 | 13 +++ mtglib/proxy.go | 8 +- stats/prometheus_test.go | 4 +- 21 files changed, 320 insertions(+), 22 deletions(-) create mode 100644 mtglib/internal/obfuscated2/client_handshake_test.go create mode 100644 mtglib/internal/obfuscated2/init_test.go create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-4529d55776e2d427 create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-585c944d672f60a2 create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 create mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4bd3b39..c122a34 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -30,8 +30,6 @@ jobs: strategy: matrix: go_version: - - ~1.14 - - ~1.15 - ^1.16 steps: - name: Checkout diff --git a/cli/access.go b/cli/access.go index b020372..d443f87 100644 --- a/cli/access.go +++ b/cli/access.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -123,11 +122,11 @@ func (c *Access) getIP(protocol string) net.IP { } defer func() { - io.Copy(ioutil.Discard, resp.Body) // nolint: errcheck + io.Copy(io.Discard, resp.Body) // nolint: errcheck resp.Body.Close() }() - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil } diff --git a/cli/base.go b/cli/base.go index 9c19ec1..6b6f892 100644 --- a/cli/base.go +++ b/cli/base.go @@ -2,9 +2,9 @@ package cli import ( "fmt" - "io/ioutil" "net" "net/url" + "os" "github.com/9seconds/mtg/v2/config" "github.com/9seconds/mtg/v2/mtglib" @@ -19,7 +19,7 @@ type base struct { } func (b *base) ReadConfig(version string) error { - content, err := ioutil.ReadFile(b.ConfigPath) + content, err := os.ReadFile(b.ConfigPath) if err != nil { return fmt.Errorf("cannot read config file: %w", err) } diff --git a/config/config_test.go b/config/config_test.go index 3233c13..8ef99df 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,7 +1,7 @@ package config_test import ( - "io/ioutil" + "os" "path/filepath" "testing" @@ -14,7 +14,7 @@ type ConfigTestSuite struct { } func (suite *ConfigTestSuite) ReadConfig(filename string) []byte { - data, err := ioutil.ReadFile(filepath.Join("testdata", filename)) + data, err := os.ReadFile(filepath.Join("testdata", filename)) suite.NoError(err) return data diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 49f1be7..ee1d473 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -216,7 +215,7 @@ func (f *Firehol) updateRemoteURL(ctx context.Context, url string, }(ctx, resp.Body) defer func(rc io.ReadCloser) { - io.Copy(ioutil.Discard, rc) // nolint: errcheck + io.Copy(io.Discard, rc) // nolint: errcheck rc.Close() }(resp.Body) diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index 08bef5d..93a164e 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -8,7 +8,7 @@ import ( "io" ) -func ClientHandshake(secret []byte, reader io.Reader) (int16, cipher.Stream, cipher.Stream, error) { +func ClientHandshake(secret []byte, reader io.Reader) (int, cipher.Stream, cipher.Stream, error) { handshake := clientHandhakeFrame{} if _, err := io.ReadFull(reader, handshake.data[:]); err != nil { diff --git a/mtglib/internal/obfuscated2/client_handshake_test.go b/mtglib/internal/obfuscated2/client_handshake_test.go new file mode 100644 index 0000000..d112f44 --- /dev/null +++ b/mtglib/internal/obfuscated2/client_handshake_test.go @@ -0,0 +1,85 @@ +package obfuscated2_test + +import ( + "bytes" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ClientHandshakeTestSuite struct { + suite.Suite + SnapshotTestSuite +} + +func (suite *ClientHandshakeTestSuite) SetupSuite() { + suite.NoError(suite.IngestSnapshots("client_snapshots", "snapshot-ok-")) +} + +func (suite *ClientHandshakeTestSuite) TestCannotRead() { + buf := bytes.NewBuffer([]byte{1, 2, 3}) + _, _, _, err := obfuscated2.ClientHandshake([]byte{1, 2, 3}, buf) // nolint: dogsled + + suite.Error(err) +} + +func (suite *ClientHandshakeTestSuite) TestOk() { + for nameV, snapshotV := range suite.snapshots { + snapshot := snapshotV + + suite.T().Run(nameV, func(t *testing.T) { + buf := bytes.NewBuffer(snapshot.Frame.data) + + dc, encryptor, decryptor, err := obfuscated2.ClientHandshake( + snapshot.Secret.data, buf) + assert.NoError(t, err) + assert.EqualValues(t, snapshot.DC, dc) + + writeData := make([]byte, len(snapshot.Encrypted.Text.data)) + readData := make([]byte, len(snapshot.Decrypted.Text.data)) + + connMock := &testlib.NetConnMock{} + connMock.On("Read", mock.Anything). + Once(). + Return(len(snapshot.Decrypted.Text.data), nil). + Run(func(args mock.Arguments) { + arr := args.Get(0).([]byte) + copy(arr, snapshot.Decrypted.Cipher.data) + }) + connMock.On("Write", mock.Anything). + Once(). + Return(len(snapshot.Encrypted.Text.data), nil). + Run(func(args mock.Arguments) { + arr := args.Get(0).([]byte) + copy(writeData, arr) + }) + + conn := &obfuscated2.Conn{ + Conn: connMock, + Encryptor: encryptor, + Decryptor: decryptor, + } + + n, err := conn.Read(readData) + assert.Equal(t, len(readData), n) + assert.NoError(t, err) + assert.Equal(t, snapshot.Decrypted.Text.data, readData) + + n, err = conn.Write(snapshot.Encrypted.Text.data) + assert.Equal(t, len(writeData), n) + assert.NoError(t, err) + assert.Equal(t, snapshot.Encrypted.Cipher.data, writeData) + + connMock.AssertExpectations(t) + }) + } +} + +func TestClientHandshake(t *testing.T) { + t.Parallel() + suite.Run(t, &ClientHandshakeTestSuite{}) +} diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go index 919d7bb..1309981 100644 --- a/mtglib/internal/obfuscated2/handshake_frame.go +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -35,10 +35,18 @@ type handshakeFrame struct { data [handshakeFrameLen]byte } -func (h *handshakeFrame) dc() int16 { +func (h *handshakeFrame) dc() int { data := h.data[handshakeFrameOffsetDC:handshakeFrameOffsetEnd] + idx := int16(binary.LittleEndian.Uint16(data)) - return int16(binary.LittleEndian.Uint16(data)) + switch { + case idx > 0: + return int(idx) - 1 + case idx < 0: + return -int(idx + 1) + default: + return 0 + } } func (h *handshakeFrame) key() []byte { diff --git a/mtglib/internal/obfuscated2/init_test.go b/mtglib/internal/obfuscated2/init_test.go new file mode 100644 index 0000000..001c84b --- /dev/null +++ b/mtglib/internal/obfuscated2/init_test.go @@ -0,0 +1,83 @@ +package obfuscated2_test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +type snapshotBytes struct { + data []byte +} + +func (s snapshotBytes) MarshalText() ([]byte, error) { + if len(s.data) == 0 { + return nil, nil + } + + return []byte(base64.RawStdEncoding.EncodeToString(s.data)), nil +} + +func (s *snapshotBytes) UnmarshalText(data []byte) error { + val, err := base64.RawStdEncoding.DecodeString(string(data)) + if err != nil { + return fmt.Errorf("cannot unmarshal %v: %w", len(val), err) + } + + s.data = val + + return nil +} + +type Obfuscated2Snapshot struct { + Secret snapshotBytes `json:"secret"` + Frame snapshotBytes `json:"frame"` + DC int16 `json:"dc"` + Encrypted struct { + Text snapshotBytes `json:"text"` + Cipher snapshotBytes `json:"cipher"` + } `json:"encrypted"` + Decrypted struct { + Text snapshotBytes `json:"text"` + Cipher snapshotBytes `json:"cipher"` + } `json:"decrypted"` +} + +type SnapshotTestSuite struct { + snapshots map[string]*Obfuscated2Snapshot +} + +func (suite *SnapshotTestSuite) IngestSnapshots(dirname, namePrefix string) error { + suite.snapshots = map[string]*Obfuscated2Snapshot{} + + files, err := os.ReadDir(filepath.Join("testdata", dirname)) + if err != nil { + return fmt.Errorf("cannot ingest snapshots: %w", err) + } + + for _, v := range files { + if !strings.HasPrefix(v.Name(), namePrefix) { + continue + } + + filename := filepath.Join("testdata", dirname, v.Name()) + + contents, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("cannot read %s: %w", filename, err) + } + + value := &Obfuscated2Snapshot{} + + if err := json.Unmarshal(contents, value); err != nil { + return fmt.Errorf("cannot unmarshal %s: %w", filename, err) + } + + suite.snapshots[v.Name()] = value + } + + return nil +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-4529d55776e2d427 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-4529d55776e2d427 new file mode 100644 index 0000000..00701aa --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-4529d55776e2d427 @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "gDcXwaMY4RwlR+nJw+ILDr123UJHHjjE/U5pF4m/Y04AmH7lEpEL6UYRnIYDbDlOHSDxc1ToziPvNlJJh8RMow", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "wZV3TR39l9nRoQ" + }, + "decrypted": { + "text": "4wZj6mUUew", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-585c944d672f60a2 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-585c944d672f60a2 new file mode 100644 index 0000000..c098a6e --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-585c944d672f60a2 @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "M2WyxeiwIQB+ZOFxNzSNHtu9OdESkfxv3JkKFimCxUoYA3BD/Ql9nXB/OIonCKLUKCcS0VzZ2P6/+5oQ9GI8YA", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "tzAwrCz00odERg" + }, + "decrypted": { + "text": "QkIvwGQDgA", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d new file mode 100644 index 0000000..7e58092 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "ZEZY1K3SZZgHX2MgMeYMMVoIPYR6eP+bgKxjI7IHl6sPLhfH2jRitS7/VA6Kz8E2L+uLqVom7x4zO+D5Q5iARA", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "WptF2u4ZSDMZxQ" + }, + "decrypted": { + "text": "DT+Ob9yGWA", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 new file mode 100644 index 0000000..7e7cfa9 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "/rbO1fCVLR7o28nrc9inrdDU+4Z4uOqbC2kMnNnzItv0fJmn4hcUXK6YBJQZVI01i7rFlgiCtTgrHtyfQp9p9w", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "jKZzALBemf72Cw" + }, + "decrypted": { + "text": "fMOlRiN20A", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf new file mode 100644 index 0000000..145cf0f --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "SfqaPWt1F8WTfrTkTl06s7F1nUcBR6AUp5uniCcYpVCJvgYpOO3TQHUZbLaFbI5qYRjJBQDe9OpXDJjR/5z4lg", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "0DtHeQPN2bv7Qw" + }, + "decrypted": { + "text": "BWPiYKe4bA", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b new file mode 100644 index 0000000..01b7ee0 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "MF6BfhvxyI61vFHZN/ecrDF1sZux/JdgVjKX1Yzy7SmBhYu+8bS25ta8iFsj/4y2moBpeNFp7rqekCE3FRb29A", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "MujITanGx9xf9g" + }, + "decrypted": { + "text": "0xVTx0JKug", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 new file mode 100644 index 0000000..7adb26f --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "YY3y0amcDqggxeNbnyyacKl++b3Q7X0XL9coxctXxJOYZQ/uXIaWFm1KD3s0VIKQ6C7NqJ8hnnJOcpp2Bdau6A", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "mni7eHb24tufLQ" + }, + "decrypted": { + "text": "Aqi/2cgHXA", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c new file mode 100644 index 0000000..421d633 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "k0f9Sv7V7svvxmHlIP4Ajg0sTGb0NXHfJDk1h6VgAnV0my1F+NU8KPG35kqXC9IDQZK/6fOABb7npO8/TCHEzQ", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "efUzYxRvqEYPlw" + }, + "decrypted": { + "text": "FFPwzZkL2A", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 new file mode 100644 index 0000000..e89bc3c --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "NiK5AnDAJw+fsytYBnxOmcDl6jx3uQECznBS4WIaHXGWP0tcioPikE1mVtkp33aXT7bCfFlst+b0PvcldGSARw", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "56aF+oauHVDbPQ" + }, + "decrypted": { + "text": "KycK+00LvQ", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 new file mode 100644 index 0000000..6f84d72 --- /dev/null +++ b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 @@ -0,0 +1,13 @@ +{ + "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", + "frame": "nYvAavvf0TywZ4hkfGEgf6ZmnIUsDWjRKzLktqF65D+3ha/LqViPdpvsexguXBd5HClDaY6YNXjb1TbjWWfYGQ", + "dc": 1, + "encrypted": { + "text": "AQIDBAUGBwgJCg", + "cipher": "t4GyXOdaa/phBw" + }, + "decrypted": { + "text": "lWNj5kAaug", + "cipher": "YWJjZGVmZw" + } +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 1dd76cd..312a1bf 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -100,12 +100,8 @@ func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { return fmt.Errorf("cannot process client handshake: %w", err) } - if dc < 0 { - dc = -dc - } - - ctx.dc = int(dc) - ctx.logger = ctx.logger.BindInt("dc", ctx.dc) + ctx.dc = dc + ctx.logger = ctx.logger.BindInt("dc", dc) ctx.clientConn = &obfuscated2.Conn{ Conn: ctx.clientConn, Encryptor: encryptor, diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index 4b2175a..00623cd 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -2,7 +2,7 @@ package stats_test import ( "fmt" - "io/ioutil" + "io" "net" "net/http" "testing" @@ -32,7 +32,7 @@ func (suite *PrometheusTestSuite) Get() (string, error) { defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return "", err // nolint: wrapcheck } From 7108fe99ec106bacaa48d7cf476ad4b5a948466b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 22 Mar 2021 12:02:21 +0300 Subject: [PATCH 103/206] Add implementation of telegram dialer --- mtglib/internal/telegram/init.go | 53 ++++++++++++++++++++ mtglib/internal/telegram/telegram.go | 75 ++++++++++++++++++++++++++++ network/network.go | 8 ++- 3 files changed, 131 insertions(+), 5 deletions(-) create mode 100644 mtglib/internal/telegram/init.go create mode 100644 mtglib/internal/telegram/telegram.go diff --git a/mtglib/internal/telegram/init.go b/mtglib/internal/telegram/init.go new file mode 100644 index 0000000..3e00353 --- /dev/null +++ b/mtglib/internal/telegram/init.go @@ -0,0 +1,53 @@ +package telegram + +import ( + "context" + "net" +) + +type preferIP uint8 + +const ( + preferIPOnlyIPv4 preferIP = iota + preferIPOnlyIPv6 + preferIPPreferIPv4 + preferIPPreferIPv6 +) + +type tgAddr struct { + network string + address string +} + +// https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/mtproto_dc_options.cpp#L30 +var ( + v4Addresses = [5][]tgAddr{ + { + {network: "tcp4", address: "149.154.175.50:443"}, + }, + { + {network: "tcp4", address: "149.154.167.51:443"}, + {network: "tcp4", address: "95.161.76.100:443"}, + }, + { + {network: "tcp4", address: "149.154.175.100:443"}, + }, + { + {network: "tcp4", address: "149.154.167.91:443"}, + }, + { + {network: "tcp4", address: "149.154.171.5"}, + }, + } + v6Addresses = [5]tgAddr{ + {network: "tcp6", address: "[2001:b28:f23d:f001::a]:443"}, + {network: "tcp6", address: "[2001:67c:04e8:f002::a]:443"}, + {network: "tcp6", address: "[2001:b28:f23d:f003::a]:443"}, + {network: "tcp6", address: "[2001:67c:04e8:f004::a]:443"}, + {network: "tcp6", address: "[2001:b28:f23f:f005::a]:443"}, + } +) + +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} diff --git a/mtglib/internal/telegram/telegram.go b/mtglib/internal/telegram/telegram.go new file mode 100644 index 0000000..c8e3a27 --- /dev/null +++ b/mtglib/internal/telegram/telegram.go @@ -0,0 +1,75 @@ +package telegram + +import ( + "context" + "fmt" + "math/rand" + "net" + "strings" +) + +type Telegram struct { + dialer Dialer + preferIP preferIP +} + +func (t Telegram) Dial(ctx context.Context, dc int) (net.Conn, error) { + if dc < 0 || dc > 4 { + return nil, fmt.Errorf("do not know how to dial to %d", dc) + } + + var addresses []tgAddr + + if t.preferIP == preferIPOnlyIPv6 { + addresses = []tgAddr{v6Addresses[dc]} + } else { + addresses = append(addresses, v4Addresses[dc]...) + rand.Shuffle(len(addresses), func(i, j int) { + addresses[i], addresses[j] = addresses[j], addresses[i] + }) + } + + switch t.preferIP { + case preferIPPreferIPv4: + addresses = append(addresses, v6Addresses[dc]) + case preferIPPreferIPv6: + addresses = append([]tgAddr{v6Addresses[dc]}, addresses...) + case preferIPOnlyIPv4, preferIPOnlyIPv6: + } + + var ( + conn net.Conn + err error + ) + + for _, v := range addresses { + conn, err = t.dialer.DialContext(ctx, v.network, v.address) + if err == nil { + return conn, nil + } + } + + return nil, fmt.Errorf("cannot dial to %d dc: %w", dc, err) +} + +func NewTelegramDialer(dialer Dialer, ipPreference string) (*Telegram, error) { + var pref preferIP + + switch strings.ToLower(ipPreference) { + case "prefer-ipv4": + pref = preferIPPreferIPv4 + case "prefer-ipv6": + pref = preferIPPreferIPv6 + case "only-ipv4": + pref = preferIPOnlyIPv4 + case "only-ipv6": + pref = preferIPOnlyIPv6 + default: + return nil, fmt.Errorf("unknown ip preference %s", ipPreference) + } + + return &Telegram{ + dialer: dialer, + preferIP: pref, + }, nil +} diff --git a/network/network.go b/network/network.go index 9f85744..9df70a0 100644 --- a/network/network.go +++ b/network/network.go @@ -43,11 +43,9 @@ func (n *network) DialContext(ctx context.Context, protocol, address string) (ne return nil, fmt.Errorf("cannot resolve dns names: %w", err) } - if len(ips) > 1 { - rand.Shuffle(len(ips), func(i, j int) { - ips[i], ips[j] = ips[j], ips[i] - }) - } + rand.Shuffle(len(ips), func(i, j int) { + ips[i], ips[j] = ips[j], ips[i] + }) var conn net.Conn for _, v := range ips { From 925a02dac38f794193360336afde039434a66ea6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 22 Mar 2021 14:51:18 +0300 Subject: [PATCH 104/206] Test telegram dc dialer --- mtglib/internal/telegram/telegram.go | 2 +- .../telegram/telegram_internal_test.go | 140 ++++++++++++++++++ 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 mtglib/internal/telegram/telegram_internal_test.go diff --git a/mtglib/internal/telegram/telegram.go b/mtglib/internal/telegram/telegram.go index c8e3a27..90f21ca 100644 --- a/mtglib/internal/telegram/telegram.go +++ b/mtglib/internal/telegram/telegram.go @@ -52,7 +52,7 @@ func (t Telegram) Dial(ctx context.Context, dc int) (net.Conn, error) { return nil, fmt.Errorf("cannot dial to %d dc: %w", dc, err) } -func NewTelegramDialer(dialer Dialer, ipPreference string) (*Telegram, error) { +func New(dialer Dialer, ipPreference string) (*Telegram, error) { var pref preferIP switch strings.ToLower(ipPreference) { diff --git a/mtglib/internal/telegram/telegram_internal_test.go b/mtglib/internal/telegram/telegram_internal_test.go new file mode 100644 index 0000000..891eff3 --- /dev/null +++ b/mtglib/internal/telegram/telegram_internal_test.go @@ -0,0 +1,140 @@ +package telegram + +import ( + "context" + "errors" + "io" + "net" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type TelegramTestSuite struct { + suite.Suite + + dialerMock *testlib.MtglibNetworkMock + t *Telegram +} + +func (suite *TelegramTestSuite) SetupTest() { + suite.dialerMock = &testlib.MtglibNetworkMock{} + suite.t, _ = New(suite.dialerMock, "prefer-ipv4") +} + +func (suite *TelegramTestSuite) TearDownTest() { + suite.dialerMock.AssertExpectations(suite.T()) +} + +func (suite *TelegramTestSuite) TestUnknownDC() { + testData := []int{ + -1, + 5, + 100, + } + + for _, v := range testData { + value := v + + suite.T().Run(strconv.Itoa(value), func(t *testing.T) { + _, err := suite.t.Dial(context.Background(), value) + assert.Error(t, err) + }) + } +} + +func (suite *TelegramTestSuite) TestDialToCorrectIPs() { + testData := map[int][]tgAddr{} + + for i := 0; i < 5; i++ { + testData[i] = []tgAddr{} + testData[i] = append(testData[i], v4Addresses[i]...) + testData[i] = append(testData[i], v6Addresses[i]) + } + + for i, v := range testData { + idx := i + addresses := v + + suite.T().Run(strconv.Itoa(idx), func(t *testing.T) { + for _, addr := range addresses { + suite.dialerMock. + On("DialContext", mock.Anything, addr.network, addr.address). + Once(). + Return((*net.TCPConn)(nil), io.EOF) + } + + _, err := suite.t.Dial(context.Background(), idx) + assert.True(t, errors.Is(err, io.EOF)) + }) + } +} + +func (suite *TelegramTestSuite) TestDialPreferIPRange() { + testData := map[string][]tgAddr{ + "prefer-ipv4": {v4Addresses[0][0], v6Addresses[0]}, + "prefer-ipv6": {v6Addresses[0], v4Addresses[0][0]}, + "only-ipv4": {v4Addresses[0][0]}, + "only-ipv6": {v6Addresses[0]}, + } + + for k, v := range testData { + name := k + addresses := v + + suite.T().Run(name, func(t *testing.T) { + for _, addr := range addresses { + suite.dialerMock. + On("DialContext", mock.Anything, addr.network, addr.address). + Once(). + Return((*net.TCPConn)(nil), io.EOF) + } + + tg, _ := New(suite.dialerMock, name) + _, err := tg.Dial(context.Background(), 0) + + assert.True(t, errors.Is(err, io.EOF)) + }) + } +} + +func (suite *TelegramTestSuite) TestDialPreferIPPriority() { + testData := map[string]tgAddr{ + "prefer-ipv4": v4Addresses[0][0], + "prefer-ipv6": v6Addresses[0], + } + + for k, v := range testData { + name := k + addr := v + + suite.T().Run(name, func(t *testing.T) { + conn := &net.TCPConn{} + + suite.dialerMock. + On("DialContext", mock.Anything, addr.network, addr.address). + Once(). + Return(conn, nil) + + tg, _ := New(suite.dialerMock, name) + + res, err := tg.Dial(context.Background(), 0) + assert.NoError(t, err) + assert.Equal(t, conn, res) + }) + } +} + +func (suite *TelegramTestSuite) TestUnknownPreferIP() { + _, err := New(suite.dialerMock, "xxx") + suite.Error(err) +} + +func TestTelegram(t *testing.T) { + t.Parallel() + suite.Run(t, &TelegramTestSuite{}) +} From 42160a08fe9c7f151b742f9bb95ccc8785e4aa69 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 22 Mar 2021 17:54:46 +0300 Subject: [PATCH 105/206] Add EventTraffic --- events/event_stream.go | 4 + events/event_stream_test.go | 62 ++++++++++++++- events/init.go | 2 + events/init_test.go | 8 ++ events/multi_observer.go | 30 ++++++++ events/noop.go | 2 + events/noop_test.go | 14 ++++ mtglib/conns.go | 91 ++++++++++++++++++++++ mtglib/events.go | 22 ++++++ mtglib/internal/obfuscated2/conn.go | 4 +- mtglib/proxy.go | 65 ++++++++++++++-- mtglib/stream_context.go | 22 ++++-- stats/init.go | 22 ++++-- stats/prometheus.go | 115 +++++++++++++++++++++++++--- stats/prometheus_test.go | 45 ++++++++++- stats/statsd.go | 58 ++++++++++++-- stats/statsd_test.go | 38 ++++++++- stats/stream_info.go | 20 ++++- 18 files changed, 569 insertions(+), 55 deletions(-) create mode 100644 mtglib/conns.go diff --git a/events/event_stream.go b/events/event_stream.go index 5037fb6..1726c81 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -71,6 +71,10 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob switch typedEvt := evt.(type) { case mtglib.EventStart: observer.EventStart(typedEvt) + case mtglib.EventConnectedToDC: + observer.EventConnectedToDC(typedEvt) + case mtglib.EventTraffic: + observer.EventTraffic(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) case mtglib.EventIPBlocklisted: diff --git a/events/event_stream_test.go b/events/event_stream_test.go index ec708a0..90dee03 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -38,7 +38,7 @@ func (suite *EventStreamTestSuite) SetupTest() { suite.stream = events.NewEventStream(factories) } -func (suite *EventStreamTestSuite) TestEventStartOk() { +func (suite *EventStreamTestSuite) TestEventStart() { evt := mtglib.EventStart{ CreatedAt: time.Now(), ConnID: "connID", @@ -63,7 +63,61 @@ func (suite *EventStreamTestSuite) TestEventStartOk() { time.Sleep(100 * time.Millisecond) } -func (suite *EventStreamTestSuite) TestEventFinishOk() { +func (suite *EventStreamTestSuite) TestEventConnectedToDC() { + evt := mtglib.EventConnectedToDC{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.1"), + DC: 3, + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventConnectedToDC", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventConnectedToDC) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.ConnID, caught.ConnID) + suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.DC, caught.DC) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventTraffic() { + evt := mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 1024, + IsRead: true, + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventTraffic", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventTraffic) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.ConnID, caught.ConnID) + suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Traffic, caught.Traffic) + suite.Equal(evt.IsRead, caught.IsRead) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + +func (suite *EventStreamTestSuite) TestEventFinish() { evt := mtglib.EventFinish{ CreatedAt: time.Now(), ConnID: "connID", @@ -86,7 +140,7 @@ func (suite *EventStreamTestSuite) TestEventFinishOk() { time.Sleep(100 * time.Millisecond) } -func (suite *EventStreamTestSuite) TestEventConcurrencyLimitedOk() { +func (suite *EventStreamTestSuite) TestEventConcurrencyLimited() { evt := mtglib.EventConcurrencyLimited{ CreatedAt: time.Now(), } @@ -106,7 +160,7 @@ func (suite *EventStreamTestSuite) TestEventConcurrencyLimitedOk() { time.Sleep(100 * time.Millisecond) } -func (suite *EventStreamTestSuite) TestEventIPBlocklistedOk() { +func (suite *EventStreamTestSuite) TestEventIPBlocklisted() { evt := mtglib.EventIPBlocklisted{ CreatedAt: time.Now(), RemoteIP: net.ParseIP("10.0.0.10"), diff --git a/events/init.go b/events/init.go index 6924071..4720a3b 100644 --- a/events/init.go +++ b/events/init.go @@ -5,6 +5,8 @@ import "github.com/9seconds/mtg/v2/mtglib" type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) + EventConnectedToDC(mtglib.EventConnectedToDC) + EventTraffic(mtglib.EventTraffic) EventConcurrencyLimited(mtglib.EventConcurrencyLimited) EventIPBlocklisted(mtglib.EventIPBlocklisted) diff --git a/events/init_test.go b/events/init_test.go index faa4063..58e3635 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -13,6 +13,14 @@ func (o *ObserverMock) EventStart(evt mtglib.EventStart) { o.Called(evt) } +func (o *ObserverMock) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + o.Called(evt) +} + +func (o *ObserverMock) EventTraffic(evt mtglib.EventTraffic) { + o.Called(evt) +} + func (o *ObserverMock) EventFinish(evt mtglib.EventFinish) { o.Called(evt) } diff --git a/events/multi_observer.go b/events/multi_observer.go index d97d802..1dec489 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -25,6 +25,36 @@ func (m multiObserver) EventStart(evt mtglib.EventStart) { wg.Wait() } +func (m multiObserver) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventConnectedToDC(evt) + }(v) + } + + wg.Wait() +} + +func (m multiObserver) EventTraffic(evt mtglib.EventTraffic) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventTraffic(evt) + }(v) + } + + wg.Wait() +} + func (m multiObserver) EventFinish(evt mtglib.EventFinish) { wg := &sync.WaitGroup{} wg.Add(len(m.observers)) diff --git a/events/noop.go b/events/noop.go index 7acbb88..8dcbf57 100644 --- a/events/noop.go +++ b/events/noop.go @@ -18,6 +18,8 @@ func NewNoopStream() mtglib.EventStream { type noopObserver struct{} func (n noopObserver) EventStart(_ mtglib.EventStart) {} +func (n noopObserver) EventConnectedToDC(_ mtglib.EventConnectedToDC) {} +func (n noopObserver) EventTraffic(_ mtglib.EventTraffic) {} func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} diff --git a/events/noop_test.go b/events/noop_test.go index d3a3109..2621e38 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -25,6 +25,18 @@ func (suite *NoopTestSuite) SetupSuite() { ConnID: "connID", RemoteIP: net.ParseIP("127.0.0.1"), }, + "connected-to-dc": mtglib.EventConnectedToDC{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("127.1.0.1"), + DC: 2, + }, + "traffic": mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 1000, + IsRead: true, + }, "finish": mtglib.EventFinish{ CreatedAt: time.Now(), ConnID: "connID", @@ -62,6 +74,8 @@ func (suite *NoopTestSuite) TestObserver() { switch typedEvt := value.(type) { case mtglib.EventStart: observer.EventStart(typedEvt) + case mtglib.EventConnectedToDC: + observer.EventConnectedToDC(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) case mtglib.EventConcurrencyLimited: diff --git a/mtglib/conns.go b/mtglib/conns.go new file mode 100644 index 0000000..685c7db --- /dev/null +++ b/mtglib/conns.go @@ -0,0 +1,91 @@ +package mtglib + +import ( + "context" + "fmt" + "net" + "time" +) + +type connStandard struct { + conn net.Conn + idleTimeout time.Duration +} + +func (c connStandard) Read(b []byte) (int, error) { + if err := c.conn.SetReadDeadline(time.Now().Add(c.idleTimeout)); err != nil { + return 0, fmt.Errorf("cannot set read deadline: %w", err) + } + + return c.conn.Read(b) +} + +func (c connStandard) Write(b []byte) (int, error) { + if err := c.conn.SetWriteDeadline(time.Now().Add(c.idleTimeout)); err != nil { + return 0, fmt.Errorf("cannot set write deadline: %w", err) + } + + return c.conn.Write(b) +} + +func (c connStandard) Close() error { + return c.conn.Close() +} + +func (c connStandard) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +func (c connStandard) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +func (c connStandard) SetDeadline(t time.Time) error { + return c.conn.SetDeadline(t) +} + +func (c connStandard) SetReadDeadline(t time.Time) error { + return c.conn.SetReadDeadline(t) +} + +func (c connStandard) SetWriteDeadline(t time.Time) error { + return c.conn.SetWriteDeadline(t) +} + +type connEventTraffic struct { + net.Conn + + connID string + stream EventStream + ctx context.Context +} + +func (c connEventTraffic) Read(b []byte) (int, error) { + n, err := c.Conn.Read(b) + + if n > 0 { + c.stream.Send(c.ctx, EventTraffic{ + CreatedAt: time.Now(), + ConnID: c.connID, + Traffic: uint(n), + IsRead: true, + }) + } + + return n, err // nolint: wrapcheck +} + +func (c connEventTraffic) Write(b []byte) (int, error) { + n, err := c.Conn.Write(b) + + if n > 0 { + c.stream.Send(c.ctx, EventTraffic{ + CreatedAt: time.Now(), + ConnID: c.connID, + Traffic: uint(n), + IsRead: false, + }) + } + + return n, err // nolint: wrapcheck +} diff --git a/mtglib/events.go b/mtglib/events.go index 0e9a2d1..0e49e48 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -15,6 +15,28 @@ func (e EventStart) StreamID() string { return e.ConnID } +type EventConnectedToDC struct { + CreatedAt time.Time + ConnID string + RemoteIP net.IP + DC int +} + +func (e EventConnectedToDC) StreamID() string { + return e.ConnID +} + +type EventTraffic struct { + CreatedAt time.Time + ConnID string + Traffic uint + IsRead bool +} + +func (e EventTraffic) StreamID() string { + return e.ConnID +} + type EventFinish struct { CreatedAt time.Time ConnID string diff --git a/mtglib/internal/obfuscated2/conn.go b/mtglib/internal/obfuscated2/conn.go index dbd69ed..7e25d47 100644 --- a/mtglib/internal/obfuscated2/conn.go +++ b/mtglib/internal/obfuscated2/conn.go @@ -14,7 +14,7 @@ type Conn struct { writeBuf []byte } -func (c *Conn) Read(p []byte) (int, error) { +func (c Conn) Read(p []byte) (int, error) { n, err := c.Conn.Read(p) if err != nil { return n, err // nolint: wrapcheck @@ -25,7 +25,7 @@ func (c *Conn) Read(p []byte) (int, error) { return n, nil } -func (c *Conn) Write(p []byte) (int, error) { +func (c Conn) Write(p []byte) (int, error) { c.writeBuf = append(c.writeBuf[:0], p...) c.Encryptor.XORKeyStream(c.writeBuf, c.writeBuf) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 312a1bf..9cedf31 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -9,6 +9,7 @@ import ( "time" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/9seconds/mtg/v2/mtglib/internal/telegram" "github.com/panjf2000/ants/v2" ) @@ -16,10 +17,12 @@ type Proxy struct { ctx context.Context ctxCancel context.CancelFunc streamWaitGroup sync.WaitGroup - workerPool *ants.PoolWithFunc + + idleTimeout time.Duration + workerPool *ants.PoolWithFunc + telegram *telegram.Telegram secret Secret - network Network antiReplayCache AntiReplayCache ipBlocklist IPBlocklist eventStream EventStream @@ -55,6 +58,12 @@ func (p *Proxy) ServeConn(conn net.Conn) { return } + + if err := p.doTelegramCall(ctx); err != nil { + p.logger.WarningError("cannot dial to telegram", err) + + return + } } func (p *Proxy) Serve(listener net.Listener) error { @@ -102,16 +111,45 @@ func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { ctx.dc = dc ctx.logger = ctx.logger.BindInt("dc", dc) - ctx.clientConn = &obfuscated2.Conn{ - Conn: ctx.clientConn, - Encryptor: encryptor, - Decryptor: decryptor, + ctx.clientConn = connStandard{ + conn: obfuscated2.Conn{ + Conn: ctx.clientConn, + Encryptor: encryptor, + Decryptor: decryptor, + }, + idleTimeout: p.idleTimeout, } return nil } -func NewProxy(opts ProxyOpts) (*Proxy, error) { +func (p *Proxy) doTelegramCall(ctx *streamContext) error { + conn, err := p.telegram.Dial(ctx, ctx.dc) + if err != nil { + return fmt.Errorf("cannot dial to Telegram: %w", err) + } + + ctx.telegramConn = connEventTraffic{ + Conn: connStandard{ + conn: conn, + idleTimeout: p.idleTimeout, + }, + connID: ctx.connID, + stream: p.eventStream, + ctx: ctx, + } + + p.eventStream.Send(ctx, EventConnectedToDC{ + CreatedAt: time.Now(), + ConnID: ctx.connID, + RemoteIP: conn.RemoteAddr().(*net.TCPAddr).IP, + DC: ctx.dc, + }) + + return nil +} + +func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop switch { case opts.Network == nil: return nil, ErrNetworkIsNotDefined @@ -127,21 +165,32 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { return nil, ErrSecretInvalid } + tg, err := telegram.New(opts.Network, opts.PreferIP) + if err != nil { + return nil, fmt.Errorf("cannot build telegram dialer: %w", err) + } + concurrency := opts.Concurrency if concurrency == 0 { concurrency = DefaultConcurrency } + idleTimeout := opts.IdleTimeout + if idleTimeout < 1 { + idleTimeout = DefaultIdleTimeout + } + ctx, cancel := context.WithCancel(context.Background()) proxy := &Proxy{ ctx: ctx, ctxCancel: cancel, secret: opts.Secret, - network: opts.Network, antiReplayCache: opts.AntiReplayCache, ipBlocklist: opts.IPBlocklist, eventStream: opts.EventStream, logger: opts.Logger.Named("proxy"), + idleTimeout: idleTimeout, + telegram: tg, } pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go index 1c75482..2ab91a0 100644 --- a/mtglib/stream_context.go +++ b/mtglib/stream_context.go @@ -9,12 +9,13 @@ import ( ) type streamContext struct { - ctx context.Context - ctxCancel context.CancelFunc - clientConn net.Conn - connID string - dc int - logger Logger + ctx context.Context + ctxCancel context.CancelFunc + clientConn net.Conn + telegramConn net.Conn + connID string + dc int + logger Logger } func (s *streamContext) Deadline() (time.Time, bool) { @@ -35,7 +36,14 @@ func (s *streamContext) Value(key interface{}) interface{} { func (s *streamContext) Close() { s.ctxCancel() - s.clientConn.Close() + + if s.clientConn != nil { + s.clientConn.Close() + } + + if s.telegramConn != nil { + s.telegramConn.Close() + } } func (s *streamContext) ClientIP() net.IP { diff --git a/stats/init.go b/stats/init.go index 7d095c5..4bf1aa8 100644 --- a/stats/init.go +++ b/stats/init.go @@ -6,13 +6,21 @@ const ( DefaultStatsdMetricPrefix = DefaultMetricPrefix + "." DefaultStatsdTagFormat = "datadog" - MetricActiveConnection = "active_connections" - MetricSessionDuration = "session_duration" - MetricConcurrencyLimited = "concurrency_limited" - MetricIPBlocklisted = "ip_blocklisted" + MetricClientConnections = "client_connections" + MetricTelegramConnections = "telegram_connections" + MetricTraffic = "traffic" + MetricSessionDuration = "session_duration" + MetricSessionTraffic = "session_traffic" + MetricConcurrencyLimited = "concurrency_limited" + MetricIPBlocklisted = "ip_blocklisted" - TagIPType = "ip_type" + TagIPType = "ip_type" + TagTelegramIP = "ip" + TagDC = "dc" + TagDirection = "direction" - TagIPTypeIPv4 = "ipv4" - TagIPTypeIPv6 = "ipv6" + TagIPTypeIPv4 = "ipv4" + TagIPTypeIPv6 = "ipv6" + TagDirectionTelegram = "telegram" + TagDirectionClient = "client" ) diff --git a/stats/prometheus.go b/stats/prometheus.go index be4c676..4c9b0a3 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -4,6 +4,7 @@ import ( "context" "net" "net/http" + "strconv" "time" "github.com/9seconds/mtg/v2/events" @@ -24,7 +25,47 @@ func (p prometheusProcessor) EventStart(evt mtglib.EventStart) { } p.streams[evt.StreamID()] = sInfo - p.factory.metricActiveConnections.WithLabelValues(sInfo.IPType()).Inc() + p.factory.metricClientConnections.WithLabelValues(sInfo.GetClientIPType()).Inc() +} + +func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + sInfo, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + sInfo.remoteIP = evt.RemoteIP + sInfo.dc = evt.DC + + p.factory.metricTelegramConnections.WithLabelValues( + sInfo.GetRemoteIPType(), + sInfo.remoteIP.String(), + strconv.Itoa(sInfo.dc)).Inc() +} + +func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) { + sInfo, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + labels := []string{ + sInfo.GetRemoteIPType(), + sInfo.remoteIP.String(), + strconv.Itoa(sInfo.dc), + } + + if evt.IsRead { + sInfo.bytesRecvFromTelegram += evt.Traffic + + labels = append(labels, TagDirectionClient) + } else { + sInfo.bytesSentToTelegram += evt.Traffic + + labels = append(labels, TagDirectionTelegram) + } + + p.factory.metricTraffic.WithLabelValues(labels...).Add(float64(evt.Traffic)) } func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { @@ -37,8 +78,30 @@ func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { duration := evt.CreatedAt.Sub(sInfo.createdAt) - p.factory.metricActiveConnections.WithLabelValues(sInfo.IPType()).Dec() + p.factory.metricClientConnections.WithLabelValues(sInfo.GetRemoteIPType()).Dec() p.factory.metricSessionDuration.Observe(float64(duration) / float64(time.Second)) + + if sInfo.remoteIP == nil { + return + } + + labels := []string{ + sInfo.GetRemoteIPType(), + sInfo.remoteIP.String(), + strconv.Itoa(sInfo.dc), + } + + p.factory.metricTelegramConnections.WithLabelValues(labels...).Dec() + + labels = append(labels, TagDirectionClient) + p.factory.metricSessionTraffic. + WithLabelValues(labels...). + Observe(float64(sInfo.bytesRecvFromTelegram)) + + labels[3] = TagDirectionTelegram + p.factory.metricSessionTraffic. + WithLabelValues(labels...). + Observe(float64(sInfo.bytesSentToTelegram)) } func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { @@ -60,10 +123,13 @@ func (p prometheusProcessor) Shutdown() { type PrometheusFactory struct { httpServer *http.Server - metricActiveConnections *prometheus.GaugeVec - metricIPBlocklisted *prometheus.CounterVec - metricConcurrencyLimited prometheus.Counter - metricSessionDuration prometheus.Histogram + metricClientConnections *prometheus.GaugeVec + metricTelegramConnections *prometheus.GaugeVec + metricTraffic *prometheus.CounterVec + metricIPBlocklisted *prometheus.CounterVec + metricSessionTraffic *prometheus.HistogramVec + metricConcurrencyLimited prometheus.Counter + metricSessionDuration prometheus.Histogram } func (p *PrometheusFactory) Make() events.Observer { @@ -81,7 +147,7 @@ func (p *PrometheusFactory) Close() error { return p.httpServer.Shutdown(context.Background()) } -func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { +func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint: funlen registry := prometheus.NewPedanticRegistry() httpHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ EnableOpenMetrics: true, @@ -95,11 +161,16 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { Handler: mux, }, - metricActiveConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + metricClientConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: metricPrefix, - Name: MetricActiveConnection, + Name: MetricClientConnections, Help: "A number of connections under active processing.", }, []string{TagIPType}), + metricTelegramConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: metricPrefix, + Name: MetricTelegramConnections, + Help: "A number of connections to Telegram servers.", + }, []string{TagIPType, TagTelegramIP, TagDC}), metricSessionDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ Namespace: metricPrefix, Name: MetricSessionDuration, @@ -117,6 +188,27 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { 300, }, }), + metricSessionTraffic: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: metricPrefix, + Name: MetricSessionTraffic, + Help: "A traffic size which flew via proxy within a single session.", + Buckets: []float64{ // per 1mb + 1 * 1024 * 1024, + 2 * 1024 * 1024, + 3 * 1024 * 1024, + 4 * 1024 * 1024, + 5 * 1024 * 1024, + 6 * 1024 * 1024, + 7 * 1024 * 1024, + 8 * 1024 * 1024, + 9 * 1024 * 1024, + }, + }, []string{TagIPType, TagTelegramIP, TagDC, TagDirection}), + metricTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricTraffic, + Help: "Traffic which is sent through this proxy.", + }, []string{TagIPType, TagTelegramIP, TagDC, TagDirection}), metricConcurrencyLimited: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metricPrefix, Name: MetricConcurrencyLimited, @@ -129,7 +221,10 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { }, []string{TagIPType}), } - registry.MustRegister(factory.metricActiveConnections) + registry.MustRegister(factory.metricClientConnections) + registry.MustRegister(factory.metricTelegramConnections) + registry.MustRegister(factory.metricTraffic) + registry.MustRegister(factory.metricSessionTraffic) registry.MustRegister(factory.metricSessionDuration) registry.MustRegister(factory.metricConcurrencyLimited) registry.MustRegister(factory.metricIPBlocklisted) diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index 00623cd..c1425b5 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -60,23 +60,60 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { ConnID: "connID", RemoteIP: net.ParseIP("10.0.0.10"), }) - time.Sleep(100 * time.Millisecond) data, err := suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_active_connections{ip_type="ipv4"} 1`) + suite.Contains(data, `mtg_client_connections{ip_type="ipv4"} 1`) + + suite.prometheus.EventConnectedToDC(mtglib.EventConnectedToDC{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.1"), + DC: 4, + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_telegram_connections{dc="4",ip="10.0.0.1",ip_type="ipv4"} 1`) + + suite.prometheus.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 200, + IsRead: true, + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_traffic{dc="4",direction="client",ip="10.0.0.1",ip_type="ipv4"} 200`) + + suite.prometheus.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 100, + IsRead: false, + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_traffic{dc="4",direction="telegram",ip="10.0.0.1",ip_type="ipv4"} 100`) suite.prometheus.EventFinish(mtglib.EventFinish{ CreatedAt: time.Now(), ConnID: "connID", }) - time.Sleep(100 * time.Millisecond) data, err = suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_active_connections{ip_type="ipv4"} 0`) + suite.Contains(data, `mtg_client_connections{ip_type="ipv4"} 0`) + suite.Contains(data, `mtg_telegram_connections{dc="4",ip="10.0.0.1",ip_type="ipv4"} 0`) + suite.Contains(data, `mtg_traffic{dc="4",direction="client",ip="10.0.0.1",ip_type="ipv4"} 200`) + suite.Contains(data, `mtg_traffic{dc="4",direction="telegram",ip="10.0.0.1",ip_type="ipv4"} 100`) } func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { diff --git a/stats/statsd.go b/stats/statsd.go index f7c437c..beea0f8 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -22,9 +22,47 @@ func (s statsdProcessor) EventStart(evt mtglib.EventStart) { clientIP: evt.RemoteIP, } s.streams[evt.StreamID()] = sInfo - ipTypeTag := statsd.StringTag(TagIPType, sInfo.IPType()) - s.client.GaugeDelta(MetricActiveConnection, 1, ipTypeTag) + s.client.GaugeDelta(MetricClientConnections, + 1, + statsd.StringTag(TagIPType, sInfo.GetClientIPType())) +} + +func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { + sInfo, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + sInfo.remoteIP = evt.RemoteIP + sInfo.dc = evt.DC + + s.client.GaugeDelta(MetricTelegramConnections, + 1, + statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()), + statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()), + statsd.IntTag(TagDC, sInfo.dc)) +} + +func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { + sInfo, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + tags := []statsd.Tag{ + statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()), + statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()), + statsd.IntTag(TagDC, sInfo.dc), + } + + if evt.IsRead { + tags = append(tags, statsd.StringTag(TagDirection, TagDirectionClient)) + s.client.Incr(MetricTraffic, int64(evt.Traffic), tags...) + } else { + tags = append(tags, statsd.StringTag(TagDirection, TagDirectionTelegram)) + s.client.Incr(MetricTraffic, int64(evt.Traffic), tags...) + } } func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { @@ -35,11 +73,19 @@ func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { defer delete(s.streams, evt.StreamID()) - duration := evt.CreatedAt.Sub(sInfo.createdAt) - ipTypeTag := statsd.StringTag(TagIPType, sInfo.IPType()) + s.client.GaugeDelta(MetricClientConnections, + -1, + statsd.StringTag(TagIPType, sInfo.GetClientIPType())) + s.client.PrecisionTiming(MetricSessionDuration, + evt.CreatedAt.Sub(sInfo.createdAt)) - s.client.GaugeDelta(MetricActiveConnection, -1, ipTypeTag) - s.client.PrecisionTiming(MetricSessionDuration, duration) + if sInfo.remoteIP != nil { + s.client.GaugeDelta(MetricTelegramConnections, + -1, + statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()), + statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()), + statsd.IntTag(TagDC, sInfo.dc)) + } } func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { diff --git a/stats/statsd_test.go b/stats/statsd_test.go index af9491f..9ac40bd 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -103,17 +103,49 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { ConnID: "connID", RemoteIP: net.ParseIP("10.0.0.10"), }) - time.Sleep(statsdSleepTime) - suite.Equal("mtg.active_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String()) + suite.Equal("mtg.client_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String()) + + suite.statsd.EventConnectedToDC(mtglib.EventConnectedToDC{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.1.0.10"), + DC: 2, + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.telegram_connections:+1|g|#ip_type:ipv4,ip:10.1.0.10,dc:2") + + suite.statsd.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 30, + IsRead: true, + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.traffic:30|c|#ip_type:ipv4,ip:10.1.0.10,dc:2,direction:client") + + suite.statsd.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 90, + IsRead: false, + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.traffic:90|c|#ip_type:ipv4,ip:10.1.0.10,dc:2,direction:telegram") suite.statsd.EventFinish(mtglib.EventFinish{ CreatedAt: time.Now(), ConnID: "connID", }) - time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.session_duration") + suite.Contains(suite.statsdServer.String(), + "mtg.telegram_connections:-1|g|#ip_type:ipv4,ip:10.1.0.10,dc:2") + suite.Contains(suite.statsdServer.String(), + "mtg.client_connections:-1|g|#ip_type:ipv4") } func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { diff --git a/stats/stream_info.go b/stats/stream_info.go index fe51136..2827792 100644 --- a/stats/stream_info.go +++ b/stats/stream_info.go @@ -6,12 +6,24 @@ import ( ) type streamInfo struct { - createdAt time.Time - clientIP net.IP + createdAt time.Time + clientIP net.IP + remoteIP net.IP + dc int + bytesSentToTelegram uint + bytesRecvFromTelegram uint } -func (s *streamInfo) IPType() string { - if s.clientIP.To4() == nil { +func (s *streamInfo) GetClientIPType() string { + return s.getIPType(s.clientIP) +} + +func (s *streamInfo) GetRemoteIPType() string { + return s.getIPType(s.remoteIP) +} + +func (s *streamInfo) getIPType(ip net.IP) string { + if ip.To4() == nil { return TagIPTypeIPv6 } From f7c33ee333353c940bdadbe731a7607b78bf760a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 22 Mar 2021 18:41:54 +0300 Subject: [PATCH 106/206] Actually embedd connStandard into connEventTraffic --- mtglib/conns.go | 10 +++++----- mtglib/proxy.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mtglib/conns.go b/mtglib/conns.go index 685c7db..6793daf 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -53,7 +53,7 @@ func (c connStandard) SetWriteDeadline(t time.Time) error { } type connEventTraffic struct { - net.Conn + connStandard connID string stream EventStream @@ -61,7 +61,7 @@ type connEventTraffic struct { } func (c connEventTraffic) Read(b []byte) (int, error) { - n, err := c.Conn.Read(b) + n, err := c.connStandard.Read(b) if n > 0 { c.stream.Send(c.ctx, EventTraffic{ @@ -72,11 +72,11 @@ func (c connEventTraffic) Read(b []byte) (int, error) { }) } - return n, err // nolint: wrapcheck + return n, err } func (c connEventTraffic) Write(b []byte) (int, error) { - n, err := c.Conn.Write(b) + n, err := c.connStandard.Write(b) if n > 0 { c.stream.Send(c.ctx, EventTraffic{ @@ -87,5 +87,5 @@ func (c connEventTraffic) Write(b []byte) (int, error) { }) } - return n, err // nolint: wrapcheck + return n, err } diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 9cedf31..d25f3c9 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -130,7 +130,7 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { } ctx.telegramConn = connEventTraffic{ - Conn: connStandard{ + connStandard: connStandard{ conn: conn, idleTimeout: p.idleTimeout, }, From 66c45dc83bc52005b4e31dfa8238105d55c3a47e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 23 Mar 2021 10:27:22 +0300 Subject: [PATCH 107/206] Add obfuscated2 server handshake --- mtglib/conns.go | 58 ++-------------- .../internal/obfuscated2/client_handshake.go | 27 ++++++++ .../obfuscated2/client_handshake_frame.go | 30 -------- mtglib/internal/obfuscated2/conn.go | 6 +- .../internal/obfuscated2/server_handshake.go | 68 +++++++++++++++++++ mtglib/proxy.go | 33 +++++---- 6 files changed, 123 insertions(+), 99 deletions(-) delete mode 100644 mtglib/internal/obfuscated2/client_handshake_frame.go create mode 100644 mtglib/internal/obfuscated2/server_handshake.go diff --git a/mtglib/conns.go b/mtglib/conns.go index 6793daf..985cad0 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -2,66 +2,20 @@ package mtglib import ( "context" - "fmt" "net" "time" ) -type connStandard struct { - conn net.Conn - idleTimeout time.Duration -} - -func (c connStandard) Read(b []byte) (int, error) { - if err := c.conn.SetReadDeadline(time.Now().Add(c.idleTimeout)); err != nil { - return 0, fmt.Errorf("cannot set read deadline: %w", err) - } - - return c.conn.Read(b) -} - -func (c connStandard) Write(b []byte) (int, error) { - if err := c.conn.SetWriteDeadline(time.Now().Add(c.idleTimeout)); err != nil { - return 0, fmt.Errorf("cannot set write deadline: %w", err) - } - - return c.conn.Write(b) -} - -func (c connStandard) Close() error { - return c.conn.Close() -} - -func (c connStandard) LocalAddr() net.Addr { - return c.conn.LocalAddr() -} - -func (c connStandard) RemoteAddr() net.Addr { - return c.conn.RemoteAddr() -} - -func (c connStandard) SetDeadline(t time.Time) error { - return c.conn.SetDeadline(t) -} - -func (c connStandard) SetReadDeadline(t time.Time) error { - return c.conn.SetReadDeadline(t) -} - -func (c connStandard) SetWriteDeadline(t time.Time) error { - return c.conn.SetWriteDeadline(t) -} - -type connEventTraffic struct { - connStandard +type connTelegramTraffic struct { + net.Conn connID string stream EventStream ctx context.Context } -func (c connEventTraffic) Read(b []byte) (int, error) { - n, err := c.connStandard.Read(b) +func (c connTelegramTraffic) Read(b []byte) (int, error) { + n, err := c.Conn.Read(b) if n > 0 { c.stream.Send(c.ctx, EventTraffic{ @@ -75,8 +29,8 @@ func (c connEventTraffic) Read(b []byte) (int, error) { return n, err } -func (c connEventTraffic) Write(b []byte) (int, error) { - n, err := c.connStandard.Write(b) +func (c connTelegramTraffic) Write(b []byte) (int, error) { + n, err := c.Conn.Write(b) if n > 0 { c.stream.Send(c.ctx, EventTraffic{ diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index 93a164e..8601766 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -8,6 +8,33 @@ import ( "io" ) +type clientHandhakeFrame struct { + handshakeFrame +} + +func (c *clientHandhakeFrame) decryptor(secret []byte) cipher.Stream { + hasher := acquireSha256Hasher() + defer releaseSha256Hasher(hasher) + + hasher.Write(c.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck + + return makeAesCtr(hasher.Sum(nil), c.iv()) +} + +func (c *clientHandhakeFrame) encryptor(secret []byte) cipher.Stream { + arr := clientHandhakeFrame{} + invertByteSlices(arr.data[:], c.data[:]) + + hasher := acquireSha256Hasher() + defer releaseSha256Hasher(hasher) + + hasher.Write(arr.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck + + return makeAesCtr(hasher.Sum(nil), arr.iv()) +} + func ClientHandshake(secret []byte, reader io.Reader) (int, cipher.Stream, cipher.Stream, error) { handshake := clientHandhakeFrame{} diff --git a/mtglib/internal/obfuscated2/client_handshake_frame.go b/mtglib/internal/obfuscated2/client_handshake_frame.go deleted file mode 100644 index 4d238f8..0000000 --- a/mtglib/internal/obfuscated2/client_handshake_frame.go +++ /dev/null @@ -1,30 +0,0 @@ -package obfuscated2 - -import "crypto/cipher" - -type clientHandhakeFrame struct { - handshakeFrame -} - -func (c *clientHandhakeFrame) decryptor(secret []byte) cipher.Stream { - hasher := acquireSha256Hasher() - defer releaseSha256Hasher(hasher) - - hasher.Write(c.key()) // nolint: errcheck - hasher.Write(secret) // nolint: errcheck - - return makeAesCtr(hasher.Sum(nil), c.iv()) -} - -func (c *clientHandhakeFrame) encryptor(secret []byte) cipher.Stream { - arr := clientHandhakeFrame{} - invertByteSlices(arr.data[:], c.data[:]) - - hasher := acquireSha256Hasher() - defer releaseSha256Hasher(hasher) - - hasher.Write(arr.key()) // nolint: errcheck - hasher.Write(secret) // nolint: errcheck - - return makeAesCtr(hasher.Sum(nil), arr.iv()) -} diff --git a/mtglib/internal/obfuscated2/conn.go b/mtglib/internal/obfuscated2/conn.go index 7e25d47..24b5a81 100644 --- a/mtglib/internal/obfuscated2/conn.go +++ b/mtglib/internal/obfuscated2/conn.go @@ -14,7 +14,7 @@ type Conn struct { writeBuf []byte } -func (c Conn) Read(p []byte) (int, error) { +func (c *Conn) Read(p []byte) (int, error) { n, err := c.Conn.Read(p) if err != nil { return n, err // nolint: wrapcheck @@ -25,9 +25,9 @@ func (c Conn) Read(p []byte) (int, error) { return n, nil } -func (c Conn) Write(p []byte) (int, error) { +func (c *Conn) Write(p []byte) (int, error) { c.writeBuf = append(c.writeBuf[:0], p...) - c.Encryptor.XORKeyStream(c.writeBuf, c.writeBuf) + c.Encryptor.XORKeyStream(c.writeBuf, p) return c.Conn.Write(c.writeBuf) } diff --git a/mtglib/internal/obfuscated2/server_handshake.go b/mtglib/internal/obfuscated2/server_handshake.go new file mode 100644 index 0000000..3b69d6a --- /dev/null +++ b/mtglib/internal/obfuscated2/server_handshake.go @@ -0,0 +1,68 @@ +package obfuscated2 + +import ( + "crypto/cipher" + "crypto/rand" + "encoding/binary" + "fmt" + "net" +) + +type serverHandshakeFrame struct { + handshakeFrame +} + +func (s *serverHandshakeFrame) decryptor() cipher.Stream { + return makeAesCtr(s.key(), s.iv()) +} + +func (s *serverHandshakeFrame) encryptor() cipher.Stream { + arr := serverHandshakeFrame{} + invertByteSlices(arr.data[:], s.data[:]) + + return makeAesCtr(arr.key(), arr.iv()) +} + +func ServerHandshake(conn net.Conn) (cipher.Stream, cipher.Stream, error) { + handshake := generateServerHanshakeFrame() + copyHandshake := handshake + encryptor := handshake.encryptor() + decryptor := handshake.decryptor() + + encryptor.XORKeyStream(handshake.data[:], handshake.data[:]) + copy(handshake.key(), copyHandshake.key()) + copy(handshake.iv(), copyHandshake.iv()) + + if _, err := conn.Write(handshake.data[:]); err != nil { + return nil, nil, fmt.Errorf("cannot send a handshake frame to telegram: %w", err) + } + + return encryptor, decryptor, nil +} + +func generateServerHanshakeFrame() serverHandshakeFrame { + frame := serverHandshakeFrame{} + + for { + if _, err := rand.Read(frame.data[:]); err != nil { + panic(err) + } + + if frame.data[0] == 0xef { + continue + } + + switch binary.LittleEndian.Uint32(frame.data[:4]) { + case 0x44414548, 0x54534f50, 0x20544547, 0x4954504f, 0xeeeeeeee: + continue + } + + if (frame.data[4] | frame.data[5] | frame.data[6] | frame.data[7]) == 0 { + continue + } + + copy(frame.connectionType(), handshakeConnectionType) + + return frame + } +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index d25f3c9..6678513 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -111,13 +111,10 @@ func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { ctx.dc = dc ctx.logger = ctx.logger.BindInt("dc", dc) - ctx.clientConn = connStandard{ - conn: obfuscated2.Conn{ - Conn: ctx.clientConn, - Encryptor: encryptor, - Decryptor: decryptor, - }, - idleTimeout: p.idleTimeout, + ctx.clientConn = &obfuscated2.Conn{ + Conn: ctx.clientConn, + Encryptor: encryptor, + Decryptor: decryptor, } return nil @@ -129,14 +126,22 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { return fmt.Errorf("cannot dial to Telegram: %w", err) } - ctx.telegramConn = connEventTraffic{ - connStandard: connStandard{ - conn: conn, - idleTimeout: p.idleTimeout, + encryptor, decryptor, err := obfuscated2.ServerHandshake(conn) + if err != nil { + conn.Close() + + return fmt.Errorf("cannot perform obfuscated2 handshake: %w", err) + } + + ctx.telegramConn = &obfuscated2.Conn{ + Conn: connTelegramTraffic{ + Conn: conn, + connID: ctx.connID, + stream: p.eventStream, + ctx: ctx, }, - connID: ctx.connID, - stream: p.eventStream, - ctx: ctx, + Encryptor: encryptor, + Decryptor: decryptor, } p.eventStream.Send(ctx, EventConnectedToDC{ From a3362c7ea4c7b8eec6b37dfb542c81c310ddae18 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 23 Mar 2021 15:14:36 +0300 Subject: [PATCH 108/206] Proxy is working in a simple mode now --- mtglib/conns.go | 4 +- .../internal/obfuscated2/client_handshake.go | 9 +- .../internal/obfuscated2/handshake_frame.go | 10 ++ .../internal/obfuscated2/server_handshake.go | 21 ++-- mtglib/internal/obfuscated2/utils.go | 8 -- mtglib/internal/relay/conn.go | 33 ++++++ mtglib/internal/relay/pools.go | 46 ++++++++ mtglib/internal/relay/relay.go | 100 ++++++++++++++++++ mtglib/proxy.go | 15 +++ 9 files changed, 220 insertions(+), 26 deletions(-) create mode 100644 mtglib/internal/relay/conn.go create mode 100644 mtglib/internal/relay/pools.go create mode 100644 mtglib/internal/relay/relay.go diff --git a/mtglib/conns.go b/mtglib/conns.go index 985cad0..e30cf0d 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -26,7 +26,7 @@ func (c connTelegramTraffic) Read(b []byte) (int, error) { }) } - return n, err + return n, err // nolint: wrapcheck } func (c connTelegramTraffic) Write(b []byte) (int, error) { @@ -41,5 +41,5 @@ func (c connTelegramTraffic) Write(b []byte) (int, error) { }) } - return n, err + return n, err // nolint: wrapcheck } diff --git a/mtglib/internal/obfuscated2/client_handshake.go b/mtglib/internal/obfuscated2/client_handshake.go index 8601766..89cc396 100644 --- a/mtglib/internal/obfuscated2/client_handshake.go +++ b/mtglib/internal/obfuscated2/client_handshake.go @@ -23,16 +23,15 @@ func (c *clientHandhakeFrame) decryptor(secret []byte) cipher.Stream { } func (c *clientHandhakeFrame) encryptor(secret []byte) cipher.Stream { - arr := clientHandhakeFrame{} - invertByteSlices(arr.data[:], c.data[:]) + invertedHandshake := c.invert() hasher := acquireSha256Hasher() defer releaseSha256Hasher(hasher) - hasher.Write(arr.key()) // nolint: errcheck - hasher.Write(secret) // nolint: errcheck + hasher.Write(invertedHandshake.key()) // nolint: errcheck + hasher.Write(secret) // nolint: errcheck - return makeAesCtr(hasher.Sum(nil), arr.iv()) + return makeAesCtr(hasher.Sum(nil), invertedHandshake.iv()) } func ClientHandshake(secret []byte, reader io.Reader) (int, cipher.Stream, cipher.Stream, error) { diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go index 1309981..82327f1 100644 --- a/mtglib/internal/obfuscated2/handshake_frame.go +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -60,3 +60,13 @@ func (h *handshakeFrame) iv() []byte { func (h *handshakeFrame) connectionType() []byte { return h.data[handshakeFrameOffsetConnectionType:handshakeFrameOffsetDC] } + +func (h *handshakeFrame) invert() handshakeFrame { + copyFrame := *h + + for i := 0; i < handshakeFrameLenKey+handshakeFrameLenIV; i++ { + copyFrame.data[handshakeFrameOffsetKey+i] = h.data[handshakeFrameOffsetConnectionType-1-i] + } + + return copyFrame +} diff --git a/mtglib/internal/obfuscated2/server_handshake.go b/mtglib/internal/obfuscated2/server_handshake.go index 3b69d6a..9712fde 100644 --- a/mtglib/internal/obfuscated2/server_handshake.go +++ b/mtglib/internal/obfuscated2/server_handshake.go @@ -5,7 +5,7 @@ import ( "crypto/rand" "encoding/binary" "fmt" - "net" + "io" ) type serverHandshakeFrame struct { @@ -13,17 +13,16 @@ type serverHandshakeFrame struct { } func (s *serverHandshakeFrame) decryptor() cipher.Stream { - return makeAesCtr(s.key(), s.iv()) + invertedHandshake := s.invert() + + return makeAesCtr(invertedHandshake.key(), invertedHandshake.iv()) } func (s *serverHandshakeFrame) encryptor() cipher.Stream { - arr := serverHandshakeFrame{} - invertByteSlices(arr.data[:], s.data[:]) - - return makeAesCtr(arr.key(), arr.iv()) + return makeAesCtr(s.key(), s.iv()) } -func ServerHandshake(conn net.Conn) (cipher.Stream, cipher.Stream, error) { +func ServerHandshake(writer io.Writer) (cipher.Stream, cipher.Stream, error) { handshake := generateServerHanshakeFrame() copyHandshake := handshake encryptor := handshake.encryptor() @@ -33,7 +32,7 @@ func ServerHandshake(conn net.Conn) (cipher.Stream, cipher.Stream, error) { copy(handshake.key(), copyHandshake.key()) copy(handshake.iv(), copyHandshake.iv()) - if _, err := conn.Write(handshake.data[:]); err != nil { + if _, err := writer.Write(handshake.data[:]); err != nil { return nil, nil, fmt.Errorf("cannot send a handshake frame to telegram: %w", err) } @@ -48,16 +47,16 @@ func generateServerHanshakeFrame() serverHandshakeFrame { panic(err) } - if frame.data[0] == 0xef { + if frame.data[0] == 0xef { // nolint: gomnd // taken from tg sources continue } switch binary.LittleEndian.Uint32(frame.data[:4]) { - case 0x44414548, 0x54534f50, 0x20544547, 0x4954504f, 0xeeeeeeee: + case 0x44414548, 0x54534f50, 0x20544547, 0x4954504f, 0xeeeeeeee: // nolint: gomnd // taken from tg sources continue } - if (frame.data[4] | frame.data[5] | frame.data[6] | frame.data[7]) == 0 { + if frame.data[4]|frame.data[5]|frame.data[6]|frame.data[7] == 0 { continue } diff --git a/mtglib/internal/obfuscated2/utils.go b/mtglib/internal/obfuscated2/utils.go index f32dde0..a3fdcf9 100644 --- a/mtglib/internal/obfuscated2/utils.go +++ b/mtglib/internal/obfuscated2/utils.go @@ -13,11 +13,3 @@ func makeAesCtr(key, iv []byte) cipher.Stream { return cipher.NewCTR(block, iv) } - -func invertByteSlices(dst, src []byte) { - lenDst := len(dst) - 1 - - for i, v := range src { - dst[lenDst-i] = v - } -} diff --git a/mtglib/internal/relay/conn.go b/mtglib/internal/relay/conn.go new file mode 100644 index 0000000..211a32a --- /dev/null +++ b/mtglib/internal/relay/conn.go @@ -0,0 +1,33 @@ +package relay + +import "io" + +type conn struct { + io.ReadWriteCloser + + relay *Relay +} + +func (c conn) Read(p []byte) (int, error) { + ctx := c.relay.ctx + n, err := c.ReadWriteCloser.Read(p) + + select { + case <-ctx.Done(): + case c.relay.tickChannel <- struct{}{}: + } + + return n, err // nolint: wrapcheck +} + +func (c conn) Write(p []byte) (int, error) { + ctx := c.relay.ctx + n, err := c.ReadWriteCloser.Write(p) + + select { + case <-ctx.Done(): + case c.relay.tickChannel <- struct{}{}: + } + + return n, err // nolint: wrapcheck +} diff --git a/mtglib/internal/relay/pools.go b/mtglib/internal/relay/pools.go new file mode 100644 index 0000000..dc51c0f --- /dev/null +++ b/mtglib/internal/relay/pools.go @@ -0,0 +1,46 @@ +package relay + +import ( + "context" + "sync" + "time" +) + +var relayPool = sync.Pool{ + New: func() interface{} { + return &Relay{ + tickChannel: make(chan struct{}), + errorChannel: make(chan error, 1), + } + }, +} + +func AcquireRelay(ctx context.Context, logger Logger, bufferSize int, idleTimeout time.Duration) *Relay { + ctx, cancel := context.WithCancel(ctx) + + r := relayPool.Get().(*Relay) + r.ctx = ctx + r.ctxCancel = cancel + r.logger = logger + r.tickTimeout = idleTimeout + + if len(r.eastBuffer) != bufferSize { + r.eastBuffer = make([]byte, bufferSize) + } + + if len(r.westBuffer) != bufferSize { + r.westBuffer = make([]byte, bufferSize) + } + + return r +} + +func ReleaseRelay(r *Relay) { + r.ctxCancel() + + r.ctx = nil + r.ctxCancel = nil + r.logger = nil + + relayPool.Put(r) +} diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go new file mode 100644 index 0000000..2a6e1b5 --- /dev/null +++ b/mtglib/internal/relay/relay.go @@ -0,0 +1,100 @@ +package relay + +import ( + "context" + "io" + "sync" + "time" +) + +type Relay struct { + ctx context.Context + ctxCancel context.CancelFunc + logger Logger + eastBuffer []byte + westBuffer []byte + tickChannel chan struct{} + errorChannel chan error + tickTimeout time.Duration +} + +func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { + eastConn = conn{ + ReadWriteCloser: eastConn, + relay: r, + } + westConn = conn{ + ReadWriteCloser: westConn, + relay: r, + } + + defer func() { + r.ctxCancel() + eastConn.Close() + westConn.Close() + }() + + go r.runObserver() + + wg := &sync.WaitGroup{} + wg.Add(2) // nolint: gomnd + + go r.transmit(eastConn, westConn, r.westBuffer, "west", wg) + + r.transmit(westConn, eastConn, r.eastBuffer, "east", wg) + + wg.Wait() + + return <-r.errorChannel +} + +func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, + buffer []byte, direction string, wg *sync.WaitGroup) { + defer func() { + wg.Done() + src.Close() + dst.Close() + }() + + if _, err := io.CopyBuffer(dst, src, buffer); err != nil { + r.logger.Printf("error '%v' happened on direction %s", err, direction) + + select { + case <-r.ctx.Done(): + case r.errorChannel <- err: + default: + } + } +} + +func (r *Relay) runObserver() { + ticker := time.NewTicker(time.Second) + + defer func() { + ticker.Stop() + + select { + case <-ticker.C: + default: + } + }() + + lastTickAt := time.Now() + ctx := r.ctx + + for { + select { + case <-ctx.Done(): + return + case <-r.tickChannel: + lastTickAt = time.Now() + case <-ticker.C: + if time.Since(lastTickAt) > r.tickTimeout { + r.logger.Printf("exit due to a timeout") + r.ctxCancel() + + return + } + } + } +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 6678513..43b1bbe 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -9,6 +9,7 @@ import ( "time" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/9seconds/mtg/v2/mtglib/internal/relay" "github.com/9seconds/mtg/v2/mtglib/internal/telegram" "github.com/panjf2000/ants/v2" ) @@ -19,6 +20,7 @@ type Proxy struct { streamWaitGroup sync.WaitGroup idleTimeout time.Duration + bufferSize int workerPool *ants.PoolWithFunc telegram *telegram.Telegram @@ -64,6 +66,13 @@ func (p *Proxy) ServeConn(conn net.Conn) { return } + + rel := relay.AcquireRelay(ctx, p.logger.Named("relay"), p.bufferSize, p.idleTimeout) + defer relay.ReleaseRelay(rel) + + if err := rel.Process(ctx.clientConn, ctx.telegramConn); err != nil { + p.logger.DebugError("relay has been finished", err) + } } func (p *Proxy) Serve(listener net.Listener) error { @@ -185,6 +194,11 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop idleTimeout = DefaultIdleTimeout } + bufferSize := opts.BufferSize + if bufferSize < 1 { + bufferSize = DefaultBufferSize + } + ctx, cancel := context.WithCancel(context.Background()) proxy := &Proxy{ ctx: ctx, @@ -195,6 +209,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop eventStream: opts.EventStream, logger: opts.Logger.Named("proxy"), idleTimeout: idleTimeout, + bufferSize: int(bufferSize), telegram: tg, } From e3b7472239e5e9eadc447e66db5b02c8d944c3fc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 23 Mar 2021 21:24:16 +0300 Subject: [PATCH 109/206] Add test for handshake frame --- .../handshake_frame_internal_test.go | 46 +++++++++++++++++++ mtglib/internal/relay/init.go | 5 ++ 2 files changed, 51 insertions(+) create mode 100644 mtglib/internal/obfuscated2/handshake_frame_internal_test.go create mode 100644 mtglib/internal/relay/init.go diff --git a/mtglib/internal/obfuscated2/handshake_frame_internal_test.go b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go new file mode 100644 index 0000000..c5c8eee --- /dev/null +++ b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go @@ -0,0 +1,46 @@ +package obfuscated2 + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/suite" +) + +type HandshakeFrameTestSuite struct { + suite.Suite +} + +func (suite *HandshakeFrameTestSuite) Decode(value string) []byte { + v, err := base64.RawStdEncoding.DecodeString(value) + suite.NoError(err) + + return v +} + +func (suite *HandshakeFrameTestSuite) Encode(value []byte) string { + return base64.RawStdEncoding.EncodeToString(value) +} + +func (suite *HandshakeFrameTestSuite) TestOk() { + hf := handshakeFrame{} + testFrame := suite.Decode( + "L9TmCzzxl9bPKODBpZeVM/qqNUxQ/axxBup1S2ymbIfUd6f7YSyzzM9EmTFv2/XzGqJGEHuj2zofmUGBLghu5g") + copy(hf.data[:], testFrame) + + suite.Equal("zyjgwaWXlTP6qjVMUP2scQbqdUtspmyH1Hen+2Ess8w", suite.Encode(hf.key())) + suite.Equal("z0SZMW/b9fMaokYQe6PbOg", suite.Encode(hf.iv())) + suite.Equal("H5lBgQ", suite.Encode(hf.connectionType())) + suite.EqualValues(2093, hf.dc()) + + inverted := hf.invert() + suite.Equal("OtujexBGohrz9dtvMZlEz8yzLGH7p3fUh2ymbEt16gY", suite.Encode(inverted.key())) + suite.Equal("caz9UEw1qvozlZelweAozw", suite.Encode(inverted.iv())) + suite.Equal("H5lBgQ", suite.Encode(inverted.connectionType())) + suite.EqualValues(2093, inverted.dc()) +} + +func TestHandshakeFrame(t *testing.T) { + t.Parallel() + suite.Run(t, &HandshakeFrameTestSuite{}) +} diff --git a/mtglib/internal/relay/init.go b/mtglib/internal/relay/init.go new file mode 100644 index 0000000..c381a4a --- /dev/null +++ b/mtglib/internal/relay/init.go @@ -0,0 +1,5 @@ +package relay + +type Logger interface { + Printf(msg string, args ...interface{}) +} From aefa0033550a0de4ba37a5df537c6b26b64828eb Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 23 Mar 2021 21:26:57 +0300 Subject: [PATCH 110/206] Add tests for events --- mtglib/events_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mtglib/events_test.go b/mtglib/events_test.go index f0e3975..133fce3 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -32,6 +32,28 @@ func (suite *EventsTestSuite) TestEventFinish() { suite.Equal("CONNID", evt.StreamID()) } +func (suite *EventsTestSuite) TestEventConnectedToDC() { + evt := mtglib.EventConnectedToDC{ + CreatedAt: time.Now(), + ConnID: "CONNID", + RemoteIP: net.ParseIP("10.0.0.10"), + DC: 3, + } + + suite.Equal("CONNID", evt.StreamID()) +} + +func (suite *EventsTestSuite) TestEventTraffic() { + evt := mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "CONNID", + Traffic: 3, + IsRead: true, + } + + suite.Equal("CONNID", evt.StreamID()) +} + func (suite *EventsTestSuite) TestEventConcurrencyLimited() { suite.Empty(mtglib.EventConcurrencyLimited{}.StreamID()) } From ea698259d5f50eda7e58705cc69bf8c74e85536f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 23 Mar 2021 22:06:39 +0300 Subject: [PATCH 111/206] Add raw version of server handshake test --- .../obfuscated2/server_handshake_test.go | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 mtglib/internal/obfuscated2/server_handshake_test.go diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go new file mode 100644 index 0000000..d0b40c4 --- /dev/null +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -0,0 +1,95 @@ +package obfuscated2_test + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ServerHandshakeTestSuite struct { + suite.Suite +} + +func (suite *ServerHandshakeTestSuite) TestOk() { + buf := &bytes.Buffer{} + connMock := &testlib.NetConnMock{} + + encryptor, decryptor, err := obfuscated2.ServerHandshake(buf) + suite.NotNil(encryptor) + suite.NotNil(decryptor) + suite.NoError(err) + + proxyConn := &obfuscated2.Conn{ + Conn: connMock, + Encryptor: encryptor, + Decryptor: decryptor, + } + + serverEncrypted := buf.Bytes() + + decBlock, _ := aes.NewCipher(serverEncrypted[8 : 8+32]) + serverDecryptor := cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16]) + serverDecrypted := make([]byte, len(serverEncrypted)) + serverDecryptor.XORKeyStream(serverDecrypted, serverEncrypted) + + suite.Equal("3d3d3Q", + base64.RawStdEncoding.EncodeToString(serverDecrypted[8+32+16:8+32+16+4])) + + serverEncryptedReverted := make([]byte, len(serverEncrypted)) + + for i := 0; i < 32+16; i++ { + serverEncryptedReverted[8+i] = serverEncrypted[8+32+16-1-i] + } + + encBlock, _ := aes.NewCipher(serverEncryptedReverted[8 : 8+32]) + serverEncryptor := cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16]) + + messageFromTelegram := []byte{1, 2, 3, 4, 5} + // messageToTelegram := []byte{10, 11, 13, 14} + bufferToRead := make([]byte, 5) + + connMock. + On("Read", mock.Anything). + Return(5, nil). + Once(). + Run(func(args mock.Arguments) { + messageToRead := make([]byte, len(messageFromTelegram)) + serverEncryptor.XORKeyStream(messageToRead, messageFromTelegram) + copy(args.Get(0).([]byte), messageToRead) + }) + + n, err := proxyConn.Read(bufferToRead) + suite.EqualValues(5, n) + suite.NoError(err) + suite.Equal(messageFromTelegram, bufferToRead) + + messageToTelegram := []byte{10, 11, 12, 13, 14} + + connMock. + On("Write", mock.Anything). + Return(5, nil). + Once(). + Run(func(args mock.Arguments) { + message := make([]byte, len(messageToTelegram)) + serverDecryptor.XORKeyStream(message, args.Get(0).([]byte)) + suite.Equal(messageToTelegram, message) + }) + + n, err = proxyConn.Write(messageToTelegram) + suite.EqualValues(5, n) + suite.NoError(err) + + connMock.AssertExpectations(suite.T()) +} + +func TestServerHandshake(t *testing.T) { + t.Parallel() + suite.Run(t, &ServerHandshakeTestSuite{}) +} From 933855a233f5bfbc26c5f45c8df6f34cfaf1bc7c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 09:18:44 +0300 Subject: [PATCH 112/206] Refactor server handshake test --- .../obfuscated2/server_handshake_test.go | 77 +++++++++++-------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go index d0b40c4..3638369 100644 --- a/mtglib/internal/obfuscated2/server_handshake_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -15,19 +15,22 @@ import ( type ServerHandshakeTestSuite struct { suite.Suite + + connMock *testlib.NetConnMock + proxyConn *obfuscated2.Conn + encryptor cipher.Stream + decryptor cipher.Stream } -func (suite *ServerHandshakeTestSuite) TestOk() { +func (suite *ServerHandshakeTestSuite) SetupTest() { buf := &bytes.Buffer{} - connMock := &testlib.NetConnMock{} + suite.connMock = &testlib.NetConnMock{} encryptor, decryptor, err := obfuscated2.ServerHandshake(buf) - suite.NotNil(encryptor) - suite.NotNil(decryptor) suite.NoError(err) - proxyConn := &obfuscated2.Conn{ - Conn: connMock, + suite.proxyConn = &obfuscated2.Conn{ + Conn: suite.connMock, Encryptor: encryptor, Decryptor: decryptor, } @@ -35,9 +38,10 @@ func (suite *ServerHandshakeTestSuite) TestOk() { serverEncrypted := buf.Bytes() decBlock, _ := aes.NewCipher(serverEncrypted[8 : 8+32]) - serverDecryptor := cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16]) + suite.decryptor = cipher.NewCTR(decBlock, serverEncrypted[8+32:8+32+16]) + serverDecrypted := make([]byte, len(serverEncrypted)) - serverDecryptor.XORKeyStream(serverDecrypted, serverEncrypted) + suite.decryptor.XORKeyStream(serverDecrypted, serverEncrypted) suite.Equal("3d3d3Q", base64.RawStdEncoding.EncodeToString(serverDecrypted[8+32+16:8+32+16+4])) @@ -49,44 +53,49 @@ func (suite *ServerHandshakeTestSuite) TestOk() { } encBlock, _ := aes.NewCipher(serverEncryptedReverted[8 : 8+32]) - serverEncryptor := cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16]) + suite.encryptor = cipher.NewCTR(encBlock, serverEncryptedReverted[8+32:8+32+16]) +} - messageFromTelegram := []byte{1, 2, 3, 4, 5} - // messageToTelegram := []byte{10, 11, 13, 14} - bufferToRead := make([]byte, 5) +func (suite *ServerHandshakeTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} - connMock. - On("Read", mock.Anything). - Return(5, nil). - Once(). - Run(func(args mock.Arguments) { - messageToRead := make([]byte, len(messageFromTelegram)) - serverEncryptor.XORKeyStream(messageToRead, messageFromTelegram) - copy(args.Get(0).([]byte), messageToRead) - }) +func (suite *ServerHandshakeTestSuite) TestSendToTelegram() { + messageToTelegram := []byte{10, 11, 12, 13, 14, 'a'} - n, err := proxyConn.Read(bufferToRead) - suite.EqualValues(5, n) - suite.NoError(err) - suite.Equal(messageFromTelegram, bufferToRead) - - messageToTelegram := []byte{10, 11, 12, 13, 14} - - connMock. + suite.connMock. On("Write", mock.Anything). - Return(5, nil). + Return(len(messageToTelegram), nil). Once(). Run(func(args mock.Arguments) { message := make([]byte, len(messageToTelegram)) - serverDecryptor.XORKeyStream(message, args.Get(0).([]byte)) + suite.decryptor.XORKeyStream(message, args.Get(0).([]byte)) suite.Equal(messageToTelegram, message) }) - n, err = proxyConn.Write(messageToTelegram) - suite.EqualValues(5, n) + n, err := suite.proxyConn.Write(messageToTelegram) + suite.EqualValues(len(messageToTelegram), n) suite.NoError(err) +} - connMock.AssertExpectations(suite.T()) +func (suite *ServerHandshakeTestSuite) TestRecieveFromTelegram() { + messageFromTelegram := []byte{10, 11, 12, 13, 14, 'a'} + buffer := make([]byte, len(messageFromTelegram)) + + suite.connMock. + On("Read", mock.Anything). + Return(len(messageFromTelegram), nil). + Once(). + Run(func(args mock.Arguments) { + message := make([]byte, len(messageFromTelegram)) + suite.encryptor.XORKeyStream(message, messageFromTelegram) + copy(args.Get(0).([]byte), message) + }) + + n, err := suite.proxyConn.Read(buffer) + suite.EqualValues(len(messageFromTelegram), n) + suite.NoError(err) + suite.Equal(messageFromTelegram, buffer) } func TestServerHandshake(t *testing.T) { From 3076c52adba5ea52a0586f89c5f5ac124b817f7f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 09:22:28 +0300 Subject: [PATCH 113/206] Refactor snapshot files --- .../internal/obfuscated2/client_handshake_test.go | 2 +- ...client-handshake-snapshot-4529d55776e2d427.json} | 0 ...client-handshake-snapshot-585c944d672f60a2.json} | 0 .../client_snapshots/snapshot-ok-e3cedb3c8b05c39d | 13 ------------- .../client_snapshots/snapshot-ok-e7d2a1e5e479cf50 | 13 ------------- .../client_snapshots/snapshot-ok-eae4de34f1ab0bdf | 13 ------------- .../client_snapshots/snapshot-ok-ec9c53fa17f4b39b | 13 ------------- .../client_snapshots/snapshot-ok-ee4b5ad981dcce87 | 13 ------------- .../client_snapshots/snapshot-ok-f122ea426238564c | 13 ------------- .../client_snapshots/snapshot-ok-f3e117d7ac4e44f9 | 13 ------------- .../client_snapshots/snapshot-ok-f5e00132a81f7a61 | 13 ------------- 11 files changed, 1 insertion(+), 105 deletions(-) rename mtglib/internal/obfuscated2/testdata/{client_snapshots/snapshot-ok-4529d55776e2d427 => client-handshake-snapshot-4529d55776e2d427.json} (100%) rename mtglib/internal/obfuscated2/testdata/{client_snapshots/snapshot-ok-585c944d672f60a2 => client-handshake-snapshot-585c944d672f60a2.json} (100%) delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 delete mode 100644 mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 diff --git a/mtglib/internal/obfuscated2/client_handshake_test.go b/mtglib/internal/obfuscated2/client_handshake_test.go index d112f44..3edbdba 100644 --- a/mtglib/internal/obfuscated2/client_handshake_test.go +++ b/mtglib/internal/obfuscated2/client_handshake_test.go @@ -17,7 +17,7 @@ type ClientHandshakeTestSuite struct { } func (suite *ClientHandshakeTestSuite) SetupSuite() { - suite.NoError(suite.IngestSnapshots("client_snapshots", "snapshot-ok-")) + suite.NoError(suite.IngestSnapshots(".", "client-handshake-snapshot-")) } func (suite *ClientHandshakeTestSuite) TestCannotRead() { diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-4529d55776e2d427 b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json similarity index 100% rename from mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-4529d55776e2d427 rename to mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-585c944d672f60a2 b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json similarity index 100% rename from mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-585c944d672f60a2 rename to mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d deleted file mode 100644 index 7e58092..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e3cedb3c8b05c39d +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "ZEZY1K3SZZgHX2MgMeYMMVoIPYR6eP+bgKxjI7IHl6sPLhfH2jRitS7/VA6Kz8E2L+uLqVom7x4zO+D5Q5iARA", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "WptF2u4ZSDMZxQ" - }, - "decrypted": { - "text": "DT+Ob9yGWA", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 deleted file mode 100644 index 7e7cfa9..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-e7d2a1e5e479cf50 +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "/rbO1fCVLR7o28nrc9inrdDU+4Z4uOqbC2kMnNnzItv0fJmn4hcUXK6YBJQZVI01i7rFlgiCtTgrHtyfQp9p9w", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "jKZzALBemf72Cw" - }, - "decrypted": { - "text": "fMOlRiN20A", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf deleted file mode 100644 index 145cf0f..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-eae4de34f1ab0bdf +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "SfqaPWt1F8WTfrTkTl06s7F1nUcBR6AUp5uniCcYpVCJvgYpOO3TQHUZbLaFbI5qYRjJBQDe9OpXDJjR/5z4lg", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "0DtHeQPN2bv7Qw" - }, - "decrypted": { - "text": "BWPiYKe4bA", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b deleted file mode 100644 index 01b7ee0..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ec9c53fa17f4b39b +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "MF6BfhvxyI61vFHZN/ecrDF1sZux/JdgVjKX1Yzy7SmBhYu+8bS25ta8iFsj/4y2moBpeNFp7rqekCE3FRb29A", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "MujITanGx9xf9g" - }, - "decrypted": { - "text": "0xVTx0JKug", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 deleted file mode 100644 index 7adb26f..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-ee4b5ad981dcce87 +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "YY3y0amcDqggxeNbnyyacKl++b3Q7X0XL9coxctXxJOYZQ/uXIaWFm1KD3s0VIKQ6C7NqJ8hnnJOcpp2Bdau6A", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "mni7eHb24tufLQ" - }, - "decrypted": { - "text": "Aqi/2cgHXA", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c deleted file mode 100644 index 421d633..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f122ea426238564c +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "k0f9Sv7V7svvxmHlIP4Ajg0sTGb0NXHfJDk1h6VgAnV0my1F+NU8KPG35kqXC9IDQZK/6fOABb7npO8/TCHEzQ", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "efUzYxRvqEYPlw" - }, - "decrypted": { - "text": "FFPwzZkL2A", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 deleted file mode 100644 index e89bc3c..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f3e117d7ac4e44f9 +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "NiK5AnDAJw+fsytYBnxOmcDl6jx3uQECznBS4WIaHXGWP0tcioPikE1mVtkp33aXT7bCfFlst+b0PvcldGSARw", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "56aF+oauHVDbPQ" - }, - "decrypted": { - "text": "KycK+00LvQ", - "cipher": "YWJjZGVmZw" - } -} diff --git a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 b/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 deleted file mode 100644 index 6f84d72..0000000 --- a/mtglib/internal/obfuscated2/testdata/client_snapshots/snapshot-ok-f5e00132a81f7a61 +++ /dev/null @@ -1,13 +0,0 @@ -{ - "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", - "frame": "nYvAavvf0TywZ4hkfGEgf6ZmnIUsDWjRKzLktqF65D+3ha/LqViPdpvsexguXBd5HClDaY6YNXjb1TbjWWfYGQ", - "dc": 1, - "encrypted": { - "text": "AQIDBAUGBwgJCg", - "cipher": "t4GyXOdaa/phBw" - }, - "decrypted": { - "text": "lWNj5kAaug", - "cipher": "YWJjZGVmZw" - } -} From 6219f4bd90a4b9a4fa53f95463dedb864dcf68dd Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 09:44:33 +0300 Subject: [PATCH 114/206] Fix detected race --- stats/statsd_test.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 9ac40bd..9129755 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -4,6 +4,7 @@ import ( "bytes" "net" "strings" + "sync" "testing" "time" @@ -18,15 +19,16 @@ import ( const statsdSleepTime = 3 * statsd.DefaultFlushInterval type statsdFakeServer struct { - conn *net.UDPConn - buf *bytes.Buffer + conn *net.UDPConn + buf *bytes.Buffer + mutex sync.Mutex } -func (s statsdFakeServer) Addr() string { +func (s *statsdFakeServer) Addr() string { return s.conn.LocalAddr().String() } -func (s statsdFakeServer) Close() error { +func (s *statsdFakeServer) Close() error { if s.conn != nil { return s.conn.Close() } @@ -34,11 +36,14 @@ func (s statsdFakeServer) Close() error { return nil } -func (s statsdFakeServer) String() string { +func (s *statsdFakeServer) String() string { + s.mutex.Lock() + defer s.mutex.Unlock() + return strings.TrimSpace(s.buf.String()) } -func statsdNewFakeServer() statsdFakeServer { +func statsdNewFakeServer() *statsdFakeServer { conn, err := net.ListenUDP("udp", &net.UDPAddr{ IP: net.ParseIP("127.0.0.1"), Port: 0, @@ -47,7 +52,10 @@ func statsdNewFakeServer() statsdFakeServer { panic(err) } - buf := &bytes.Buffer{} + rv := &statsdFakeServer{ + conn: conn, + buf: &bytes.Buffer{}, + } go func() { currentBuffer := make([]byte, 4096) @@ -55,7 +63,9 @@ func statsdNewFakeServer() statsdFakeServer { for { n, _, err := conn.ReadFromUDP(currentBuffer) if n > 0 { - buf.Write(currentBuffer[:n]) + rv.mutex.Lock() + rv.buf.Write(currentBuffer[:n]) + rv.mutex.Unlock() } if err != nil { @@ -64,16 +74,13 @@ func statsdNewFakeServer() statsdFakeServer { } }() - return statsdFakeServer{ - conn: conn, - buf: buf, - } + return rv } type StatsdTestSuite struct { suite.Suite - statsdServer statsdFakeServer + statsdServer *statsdFakeServer factory stats.StatsdFactory statsd events.Observer } From 6b1bfe7b171a6044cc4c42763ebdade7b5e51177 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 10:08:04 +0300 Subject: [PATCH 115/206] Add primitive relay tests --- mtglib/internal/relay/init_test.go | 49 +++++++++++++++++++++++++ mtglib/internal/relay/relay.go | 12 ++++--- mtglib/internal/relay/relay_test.go | 56 +++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 mtglib/internal/relay/init_test.go create mode 100644 mtglib/internal/relay/relay_test.go diff --git a/mtglib/internal/relay/init_test.go b/mtglib/internal/relay/init_test.go new file mode 100644 index 0000000..e19ecf4 --- /dev/null +++ b/mtglib/internal/relay/init_test.go @@ -0,0 +1,49 @@ +package relay_test + +import ( + "bytes" + "io" + "sync" +) + +type loggerMock struct{} + +func (l loggerMock) Printf(format string, args ...interface{}) {} + +type rwcMock struct { + bytes.Buffer + + closed bool + mutex sync.Mutex +} + +func (r *rwcMock) Read(p []byte) (int, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + + if r.closed { + return 0, io.EOF + } + + return r.Buffer.Read(p) +} + +func (r *rwcMock) Write(p []byte) (int, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + + if r.closed { + return 0, io.EOF + } + + return r.Buffer.Write(p) +} + +func (r *rwcMock) Close() error { + r.mutex.Lock() + defer r.mutex.Unlock() + + r.closed = true + + return nil +} diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go index 2a6e1b5..2785b03 100644 --- a/mtglib/internal/relay/relay.go +++ b/mtglib/internal/relay/relay.go @@ -34,7 +34,7 @@ func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { westConn.Close() }() - go r.runObserver() + go r.runObserver(r.ctx) wg := &sync.WaitGroup{} wg.Add(2) // nolint: gomnd @@ -45,7 +45,12 @@ func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { wg.Wait() - return <-r.errorChannel + select { + case err := <-r.errorChannel: + return err + default: + return nil + } } func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, @@ -67,7 +72,7 @@ func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, } } -func (r *Relay) runObserver() { +func (r *Relay) runObserver(ctx context.Context) { ticker := time.NewTicker(time.Second) defer func() { @@ -80,7 +85,6 @@ func (r *Relay) runObserver() { }() lastTickAt := time.Now() - ctx := r.ctx for { select { diff --git a/mtglib/internal/relay/relay_test.go b/mtglib/internal/relay/relay_test.go new file mode 100644 index 0000000..b9d7987 --- /dev/null +++ b/mtglib/internal/relay/relay_test.go @@ -0,0 +1,56 @@ +package relay_test + +import ( + "context" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib/internal/relay" + "github.com/stretchr/testify/suite" +) + +type RelayTestSuite struct { + suite.Suite + + ctx context.Context + ctxCancel context.CancelFunc + r *relay.Relay +} + +func (suite *RelayTestSuite) SetupTest() { + suite.ctx, suite.ctxCancel = context.WithCancel(context.Background()) + suite.r = relay.AcquireRelay(suite.ctx, loggerMock{}, 4096, time.Second) +} + +func (suite *RelayTestSuite) TearDownTest() { + suite.ctxCancel() + relay.ReleaseRelay(suite.r) + suite.r = nil +} + +func (suite *RelayTestSuite) TestCancelled() { + suite.ctxCancel() + + eastConn := &rwcMock{} + eastConn.Write([]byte{1, 2, 3, 4, 5}) + + westConn := &rwcMock{} + westConn.Write([]byte{100, 101, 102}) + + suite.Nil(suite.r.Process(eastConn, westConn)) +} + +func (suite *RelayTestSuite) TestCopyFine() { + eastConn := &rwcMock{} + eastConn.Write([]byte{1, 2, 3, 4, 5}) + + westConn := &rwcMock{} + westConn.Write([]byte{100, 101, 102}) + + suite.NotNil(suite.r.Process(eastConn, westConn)) +} + +func TestRelay(t *testing.T) { + t.Parallel() + suite.Run(t, &RelayTestSuite{}) +} From b42e51e300071b09c1a4c7d8b4700f1bdd1f8c28 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 10:12:22 +0300 Subject: [PATCH 116/206] Small improvements --- mtglib/internal/relay/relay_test.go | 10 +++++----- mtglib/stream_context.go | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mtglib/internal/relay/relay_test.go b/mtglib/internal/relay/relay_test.go index b9d7987..b771b76 100644 --- a/mtglib/internal/relay/relay_test.go +++ b/mtglib/internal/relay/relay_test.go @@ -32,22 +32,22 @@ func (suite *RelayTestSuite) TestCancelled() { suite.ctxCancel() eastConn := &rwcMock{} - eastConn.Write([]byte{1, 2, 3, 4, 5}) + eastConn.Write([]byte{1, 2, 3, 4, 5}) // nolint: errcheck westConn := &rwcMock{} - westConn.Write([]byte{100, 101, 102}) + westConn.Write([]byte{100, 101, 102}) // nolint: errcheck suite.Nil(suite.r.Process(eastConn, westConn)) } func (suite *RelayTestSuite) TestCopyFine() { eastConn := &rwcMock{} - eastConn.Write([]byte{1, 2, 3, 4, 5}) + eastConn.Write([]byte{1, 2, 3, 4, 5}) // nolint: errcheck westConn := &rwcMock{} - westConn.Write([]byte{100, 101, 102}) + westConn.Write([]byte{100, 101, 102}) // nolint: errcheck - suite.NotNil(suite.r.Process(eastConn, westConn)) + suite.r.Process(eastConn, westConn) } func TestRelay(t *testing.T) { diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go index 2ab91a0..9b1027d 100644 --- a/mtglib/stream_context.go +++ b/mtglib/stream_context.go @@ -8,6 +8,8 @@ import ( "time" ) +const ConnectionIDBytesLength = 16 + type streamContext struct { ctx context.Context ctxCancel context.CancelFunc @@ -51,7 +53,7 @@ func (s *streamContext) ClientIP() net.IP { } func newStreamContext(ctx context.Context, logger Logger, clientConn net.Conn) *streamContext { - connIDBytes := make([]byte, 16) + connIDBytes := make([]byte, ConnectionIDBytesLength) if _, err := rand.Read(connIDBytes); err != nil { panic(err) From 336e825e3a74161682617c578bb8a15d09668ba9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 10:16:50 +0300 Subject: [PATCH 117/206] Add wg to observer This is to ensure a case when we still can have observer hanging around --- mtglib/internal/relay/relay.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go index 2785b03..2a27708 100644 --- a/mtglib/internal/relay/relay.go +++ b/mtglib/internal/relay/relay.go @@ -34,10 +34,10 @@ func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { westConn.Close() }() - go r.runObserver(r.ctx) - wg := &sync.WaitGroup{} - wg.Add(2) // nolint: gomnd + wg.Add(3) // nolint: gomnd + + go r.runObserver(r.ctx, wg) go r.transmit(eastConn, westConn, r.westBuffer, "west", wg) @@ -72,7 +72,7 @@ func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, } } -func (r *Relay) runObserver(ctx context.Context) { +func (r *Relay) runObserver(ctx context.Context, wg *sync.WaitGroup) { ticker := time.NewTicker(time.Second) defer func() { @@ -82,6 +82,8 @@ func (r *Relay) runObserver(ctx context.Context) { case <-ticker.C: default: } + + wg.Done() }() lastTickAt := time.Now() From 4da2a4f10f080472d11dd0bf180be9f21d4d1d00 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 10:18:25 +0300 Subject: [PATCH 118/206] Add comment for relay test --- mtglib/internal/relay/relay_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mtglib/internal/relay/relay_test.go b/mtglib/internal/relay/relay_test.go index b771b76..54b257d 100644 --- a/mtglib/internal/relay/relay_test.go +++ b/mtglib/internal/relay/relay_test.go @@ -47,7 +47,9 @@ func (suite *RelayTestSuite) TestCopyFine() { westConn := &rwcMock{} westConn.Write([]byte{100, 101, 102}) // nolint: errcheck - suite.r.Process(eastConn, westConn) + // yes, this test is not good enough. but apparently, if it hangs, + // we can debug most of possible issues. + _ = suite.r.Process(eastConn, westConn) } func TestRelay(t *testing.T) { From 02aa969d6c30395fde8d81099cafd63b268015a1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 10:30:09 +0300 Subject: [PATCH 119/206] Use dc indexes accordingly --- mtglib/internal/obfuscated2/handshake_frame.go | 8 +++++--- .../obfuscated2/handshake_frame_internal_test.go | 4 ++-- .../client-handshake-snapshot-4529d55776e2d427.json | 2 +- .../client-handshake-snapshot-585c944d672f60a2.json | 2 +- mtglib/internal/telegram/telegram.go | 10 +++++----- mtglib/internal/telegram/telegram_internal_test.go | 13 +++++++------ 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go index 82327f1..fde59ae 100644 --- a/mtglib/internal/obfuscated2/handshake_frame.go +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -3,6 +3,8 @@ package obfuscated2 import "encoding/binary" const ( + DefaultDC = 2 + handshakeFrameLen = 64 handshakeFrameLenKey = 32 @@ -41,11 +43,11 @@ func (h *handshakeFrame) dc() int { switch { case idx > 0: - return int(idx) - 1 + return int(idx) case idx < 0: - return -int(idx + 1) + return -int(idx) default: - return 0 + return DefaultDC } } diff --git a/mtglib/internal/obfuscated2/handshake_frame_internal_test.go b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go index c5c8eee..4fedb3f 100644 --- a/mtglib/internal/obfuscated2/handshake_frame_internal_test.go +++ b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go @@ -31,13 +31,13 @@ func (suite *HandshakeFrameTestSuite) TestOk() { suite.Equal("zyjgwaWXlTP6qjVMUP2scQbqdUtspmyH1Hen+2Ess8w", suite.Encode(hf.key())) suite.Equal("z0SZMW/b9fMaokYQe6PbOg", suite.Encode(hf.iv())) suite.Equal("H5lBgQ", suite.Encode(hf.connectionType())) - suite.EqualValues(2093, hf.dc()) + suite.EqualValues(2094, hf.dc()) inverted := hf.invert() suite.Equal("OtujexBGohrz9dtvMZlEz8yzLGH7p3fUh2ymbEt16gY", suite.Encode(inverted.key())) suite.Equal("caz9UEw1qvozlZelweAozw", suite.Encode(inverted.iv())) suite.Equal("H5lBgQ", suite.Encode(inverted.connectionType())) - suite.EqualValues(2093, inverted.dc()) + suite.EqualValues(2094, inverted.dc()) } func TestHandshakeFrame(t *testing.T) { diff --git a/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json index 00701aa..f34dc15 100644 --- a/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json +++ b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-4529d55776e2d427.json @@ -1,7 +1,7 @@ { "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", "frame": "gDcXwaMY4RwlR+nJw+ILDr123UJHHjjE/U5pF4m/Y04AmH7lEpEL6UYRnIYDbDlOHSDxc1ToziPvNlJJh8RMow", - "dc": 1, + "dc": 2, "encrypted": { "text": "AQIDBAUGBwgJCg", "cipher": "wZV3TR39l9nRoQ" diff --git a/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json index c098a6e..a59adb4 100644 --- a/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json +++ b/mtglib/internal/obfuscated2/testdata/client-handshake-snapshot-585c944d672f60a2.json @@ -1,7 +1,7 @@ { "secret": "NnoYmu4Y+jHBkAVO/UqOlQ", "frame": "M2WyxeiwIQB+ZOFxNzSNHtu9OdESkfxv3JkKFimCxUoYA3BD/Ql9nXB/OIonCKLUKCcS0VzZ2P6/+5oQ9GI8YA", - "dc": 1, + "dc": 2, "encrypted": { "text": "AQIDBAUGBwgJCg", "cipher": "tzAwrCz00odERg" diff --git a/mtglib/internal/telegram/telegram.go b/mtglib/internal/telegram/telegram.go index 90f21ca..2ee86e2 100644 --- a/mtglib/internal/telegram/telegram.go +++ b/mtglib/internal/telegram/telegram.go @@ -14,16 +14,16 @@ type Telegram struct { } func (t Telegram) Dial(ctx context.Context, dc int) (net.Conn, error) { - if dc < 0 || dc > 4 { + if dc <= 0 || dc > 5 { return nil, fmt.Errorf("do not know how to dial to %d", dc) } var addresses []tgAddr if t.preferIP == preferIPOnlyIPv6 { - addresses = []tgAddr{v6Addresses[dc]} + addresses = []tgAddr{v6Addresses[dc-1]} } else { - addresses = append(addresses, v4Addresses[dc]...) + addresses = append(addresses, v4Addresses[dc-1]...) rand.Shuffle(len(addresses), func(i, j int) { addresses[i], addresses[j] = addresses[j], addresses[i] }) @@ -31,9 +31,9 @@ func (t Telegram) Dial(ctx context.Context, dc int) (net.Conn, error) { switch t.preferIP { case preferIPPreferIPv4: - addresses = append(addresses, v6Addresses[dc]) + addresses = append(addresses, v6Addresses[dc-1]) case preferIPPreferIPv6: - addresses = append([]tgAddr{v6Addresses[dc]}, addresses...) + addresses = append([]tgAddr{v6Addresses[dc-1]}, addresses...) case preferIPOnlyIPv4, preferIPOnlyIPv6: } diff --git a/mtglib/internal/telegram/telegram_internal_test.go b/mtglib/internal/telegram/telegram_internal_test.go index 891eff3..d4821ae 100644 --- a/mtglib/internal/telegram/telegram_internal_test.go +++ b/mtglib/internal/telegram/telegram_internal_test.go @@ -33,7 +33,8 @@ func (suite *TelegramTestSuite) TearDownTest() { func (suite *TelegramTestSuite) TestUnknownDC() { testData := []int{ -1, - 5, + 0, + 6, 100, } @@ -50,10 +51,10 @@ func (suite *TelegramTestSuite) TestUnknownDC() { func (suite *TelegramTestSuite) TestDialToCorrectIPs() { testData := map[int][]tgAddr{} - for i := 0; i < 5; i++ { + for i := 1; i <= 5; i++ { testData[i] = []tgAddr{} - testData[i] = append(testData[i], v4Addresses[i]...) - testData[i] = append(testData[i], v6Addresses[i]) + testData[i] = append(testData[i], v4Addresses[i-1]...) + testData[i] = append(testData[i], v6Addresses[i-1]) } for i, v := range testData { @@ -95,7 +96,7 @@ func (suite *TelegramTestSuite) TestDialPreferIPRange() { } tg, _ := New(suite.dialerMock, name) - _, err := tg.Dial(context.Background(), 0) + _, err := tg.Dial(context.Background(), 1) assert.True(t, errors.Is(err, io.EOF)) }) @@ -122,7 +123,7 @@ func (suite *TelegramTestSuite) TestDialPreferIPPriority() { tg, _ := New(suite.dialerMock, name) - res, err := tg.Dial(context.Background(), 0) + res, err := tg.Dial(context.Background(), 1) assert.NoError(t, err) assert.Equal(t, conn, res) }) From bbe8f34e72eae006203ab3443c392d6e0632c287 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 24 Mar 2021 21:08:32 +0300 Subject: [PATCH 120/206] Correctly decrease a value for prometheus metrics --- stats/prometheus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stats/prometheus.go b/stats/prometheus.go index 4c9b0a3..9069bbf 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -78,7 +78,7 @@ func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { duration := evt.CreatedAt.Sub(sInfo.createdAt) - p.factory.metricClientConnections.WithLabelValues(sInfo.GetRemoteIPType()).Dec() + p.factory.metricClientConnections.WithLabelValues(sInfo.GetClientIPType()).Dec() p.factory.metricSessionDuration.Observe(float64(duration) / float64(time.Second)) if sInfo.remoteIP == nil { From a3c64c1d1eed69b0dff020b5a0519c91e60b45a3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 25 Mar 2021 09:35:23 +0300 Subject: [PATCH 121/206] Fix lint issue --- mtglib/internal/telegram/telegram_internal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtglib/internal/telegram/telegram_internal_test.go b/mtglib/internal/telegram/telegram_internal_test.go index d4821ae..64f1d6b 100644 --- a/mtglib/internal/telegram/telegram_internal_test.go +++ b/mtglib/internal/telegram/telegram_internal_test.go @@ -33,7 +33,7 @@ func (suite *TelegramTestSuite) TearDownTest() { func (suite *TelegramTestSuite) TestUnknownDC() { testData := []int{ -1, - 0, + 0, 6, 100, } From 4a2d1df384a19090a3f559aecd8a475383bd55e6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 25 Mar 2021 14:27:54 +0300 Subject: [PATCH 122/206] Can correctly accept faketls messages --- mtglib/init.go | 15 ++-- .../faketls/clienthello/clienthello.go | 67 ++++++++++++++ mtglib/internal/faketls/clienthello/init.go | 17 ++++ mtglib/internal/faketls/record/init.go | 66 ++++++++++++++ mtglib/internal/faketls/record/pools.go | 18 ++++ mtglib/internal/faketls/record/record.go | 87 +++++++++++++++++++ mtglib/proxy.go | 57 ++++++++---- 7 files changed, 305 insertions(+), 22 deletions(-) create mode 100644 mtglib/internal/faketls/clienthello/clienthello.go create mode 100644 mtglib/internal/faketls/clienthello/init.go create mode 100644 mtglib/internal/faketls/record/init.go create mode 100644 mtglib/internal/faketls/record/pools.go create mode 100644 mtglib/internal/faketls/record/record.go diff --git a/mtglib/init.go b/mtglib/init.go index a0e5f8f..cb2893d 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -9,13 +9,14 @@ import ( ) var ( - ErrSecretEmpty = errors.New("secret is empty") - ErrSecretInvalid = errors.New("secret is invalid") - ErrNetworkIsNotDefined = errors.New("network is not defined") - ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") - ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") - ErrEventStreamIsNotDefined = errors.New("event stream is not defined") - ErrLoggerIsNotDefined = errors.New("logger is not defined") + ErrSecretEmpty = errors.New("secret is empty") + ErrSecretInvalid = errors.New("secret is invalid") + ErrNetworkIsNotDefined = errors.New("network is not defined") + ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") + ErrTimeAttackDetectorIsNotDefined = errors.New("time attack detector is not defined") + ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") + ErrEventStreamIsNotDefined = errors.New("event stream is not defined") + ErrLoggerIsNotDefined = errors.New("logger is not defined") ) const ( diff --git a/mtglib/internal/faketls/clienthello/clienthello.go b/mtglib/internal/faketls/clienthello/clienthello.go new file mode 100644 index 0000000..4789029 --- /dev/null +++ b/mtglib/internal/faketls/clienthello/clienthello.go @@ -0,0 +1,67 @@ +package clienthello + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/binary" + "fmt" + "time" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" +) + +type ClientHello struct { + Time time.Time + Digest [RandomLen]byte + SessionID []byte +} + +func ParseHandshake(secret, handshake []byte) (ClientHello, error) { + hello := ClientHello{} + + if len(handshake) < MinLen { + return hello, fmt.Errorf("lengh of handshake is too small: %d", len(handshake)) + } + + if handshake[0] != HandshakeTypeClient { + return hello, fmt.Errorf("unknown handshake type %#x", handshake[0]) + } + + copy(hello.Digest[:], handshake[RandomOffset:]) + + for i := RandomOffset; i < RandomOffset+RandomLen; i++ { + handshake[i] = 0 + } + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeHandshake + rec.Version = record.Version10 + rec.Payload.Write(handshake) + + // mac is calculated for the whole record, not only + // for the payload part + mac := hmac.New(sha256.New, secret) + rec.Dump(mac) + + computedDigest := mac.Sum(nil) + + for i := 0; i < RandomLen; i++ { + computedDigest[i] ^= hello.Digest[i] + } + + for i := 0; i < RandomLen-4; i++ { + if computedDigest[i] != 0 { + return hello, ErrBadDigest + } + } + + timestamp := int64(binary.LittleEndian.Uint32(computedDigest[RandomLen-4:])) + hello.Time = time.Unix(timestamp, 0) + + hello.SessionID = make([]byte, handshake[SessionIDOffset]) + copy(hello.SessionID, handshake[SessionIDOffset+1:]) + + return hello, nil +} diff --git a/mtglib/internal/faketls/clienthello/init.go b/mtglib/internal/faketls/clienthello/init.go new file mode 100644 index 0000000..c597b84 --- /dev/null +++ b/mtglib/internal/faketls/clienthello/init.go @@ -0,0 +1,17 @@ +package clienthello + +import "errors" + +const ( + RandomLen = 32 + RandomOffset = 6 + SessionIDOffset = RandomOffset + RandomLen + MinLen = SessionIDOffset + 1 + + HandshakeTypeClient = 0x01 +) + +var ( + ErrBadDigest = errors.New("bad digest") + ErrAntiReplayAttack = errors.New("antireplay attack was detected") +) diff --git a/mtglib/internal/faketls/record/init.go b/mtglib/internal/faketls/record/init.go new file mode 100644 index 0000000..1e543aa --- /dev/null +++ b/mtglib/internal/faketls/record/init.go @@ -0,0 +1,66 @@ +package record + +import "fmt" + +type Type uint8 + +const ( + TypeChangeCipherSpec Type = 0x14 + TypeHandshake Type = 0x16 + TypeApplicationData Type = 0x17 +) + +func (t Type) String() string { + switch t { + case TypeChangeCipherSpec: + return "changeCipher(0x14)" + case TypeHandshake: + return "handshake(0x16)" + case TypeApplicationData: + return "applicationData(0x17)" + } + + return fmt.Sprintf("unknown(%#x)", byte(t)) +} + +func (t Type) Valid() error { + switch t { + case TypeChangeCipherSpec, TypeHandshake, TypeApplicationData: + return nil + } + + return fmt.Errorf("unknown type %#x", byte(t)) +} + +type Version uint16 + +const ( + Version10 Version = 769 // 0x03 0x01 + Version11 Version = 770 // 0x03 0x02 + Version12 Version = 771 // 0x03 0x03 + Version13 Version = 772 // 0x03 0x04 +) + +func (v Version) String() string { + switch v { + case Version10: + return "tls1.0" + case Version11: + return "tls1.1" + case Version12: + return "tls1.2" + case Version13: + return "tls1.3" + } + + return fmt.Sprintf("tls(%d)", uint16(v)) +} + +func (v Version) Valid() error { + switch v { + case Version10, Version11, Version12, Version13: + return nil + } + + return fmt.Errorf("unknown version %d", uint16(v)) +} diff --git a/mtglib/internal/faketls/record/pools.go b/mtglib/internal/faketls/record/pools.go new file mode 100644 index 0000000..62f03e9 --- /dev/null +++ b/mtglib/internal/faketls/record/pools.go @@ -0,0 +1,18 @@ +package record + +import "sync" + +var recordPool = sync.Pool{ + New: func() interface{} { + return &Record{} + }, +} + +func AcquireRecord() *Record { + return recordPool.Get().(*Record) +} + +func ReleaseRecord(r *Record) { + r.Reset() + recordPool.Put(r) +} diff --git a/mtglib/internal/faketls/record/record.go b/mtglib/internal/faketls/record/record.go new file mode 100644 index 0000000..31a1d9e --- /dev/null +++ b/mtglib/internal/faketls/record/record.go @@ -0,0 +1,87 @@ +package record + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "fmt" + "io" +) + +type Record struct { + Type Type + Version Version + Payload bytes.Buffer +} + +func (r *Record) String() string { + return fmt.Sprintf("", + r.Type, + r.Version, + base64.StdEncoding.EncodeToString(r.Payload.Bytes())) +} + +func (r *Record) Reset() { + r.Payload.Reset() +} + +func (r *Record) Read(reader io.Reader) error { + r.Reset() + + buf := [2]byte{} + + if _, err := io.ReadFull(reader, buf[:1]); err != nil { + return fmt.Errorf("cannot read type: %w", err) + } + + r.Type = Type(buf[0]) + if err := r.Type.Valid(); err != nil { + return fmt.Errorf("invalid type: %w", err) + } + + if _, err := io.ReadFull(reader, buf[:]); err != nil { + return fmt.Errorf("cannot read version: %w", err) + } + + r.Version = Version(binary.BigEndian.Uint16(buf[:])) + if err := r.Version.Valid(); err != nil { + return fmt.Errorf("invalid version: %w", err) + } + + if _, err := io.ReadFull(reader, buf[:]); err != nil { + return fmt.Errorf("cannot read payload length: %w", err) + } + + length := int64(binary.BigEndian.Uint16(buf[:])) + if _, err := io.CopyN(&r.Payload, reader, length); err != nil { + return fmt.Errorf("cannot read payload: %w", err) + } + + return nil +} + +func (r *Record) Dump(writer io.Writer) error { + buf := [2]byte{byte(r.Type), 0} + + if _, err := writer.Write(buf[:1]); err != nil { + return fmt.Errorf("cannot dump type: %w", err) + } + + binary.BigEndian.PutUint16(buf[:], uint16(r.Version)) + + if _, err := writer.Write(buf[:]); err != nil { + return fmt.Errorf("cannot dump version: %w", err) + } + + binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len())) + + if _, err := writer.Write(buf[:]); err != nil { + return fmt.Errorf("cannot dump payload length: %w", err) + } + + if _, err := writer.Write(r.Payload.Bytes()); err != nil { + return fmt.Errorf("cannot dump payload: %w", err) + } + + return nil +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 43b1bbe..7273375 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/clienthello" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" "github.com/9seconds/mtg/v2/mtglib/internal/relay" "github.com/9seconds/mtg/v2/mtglib/internal/telegram" @@ -24,11 +26,12 @@ type Proxy struct { workerPool *ants.PoolWithFunc telegram *telegram.Telegram - secret Secret - antiReplayCache AntiReplayCache - ipBlocklist IPBlocklist - eventStream EventStream - logger Logger + secret Secret + antiReplayCache AntiReplayCache + timeAttackDetector TimeAttackDetector + ipBlocklist IPBlocklist + eventStream EventStream + logger Logger } func (p *Proxy) ServeConn(conn net.Conn) { @@ -55,6 +58,12 @@ func (p *Proxy) ServeConn(conn net.Conn) { ctx.logger.Info("Stream has been finished") }() + if err := p.doFakeTLSHandshake(ctx); err != nil { + p.logger.InfoError("faketls handshake is failed", err) + + return + } + if err := p.doObfuscated2Handshake(ctx); err != nil { p.logger.InfoError("obfuscated2 handshake is failed", err) @@ -112,6 +121,21 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) error { + clientHelloRecord := record.AcquireRecord() + defer record.ReleaseRecord(clientHelloRecord) + + if err := clientHelloRecord.Read(ctx.clientConn); err != nil { + return fmt.Errorf("cannot read client hello: %w", err) + } + + hello, _ := clienthello.ParseHandshake(p.secret.Key[:], + clientHelloRecord.Payload.Bytes()) + fmt.Println(hello) + + return fmt.Errorf("SUCCESS") +} + func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { dc, encryptor, decryptor, err := obfuscated2.ClientHandshake(p.secret.Key[:], ctx.clientConn) if err != nil { @@ -173,6 +197,8 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop return nil, ErrIPBlocklistIsNotDefined case opts.EventStream == nil: return nil, ErrEventStreamIsNotDefined + case opts.TimeAttackDetector == nil: + return nil, ErrTimeAttackDetectorIsNotDefined case opts.Logger == nil: return nil, ErrLoggerIsNotDefined case !opts.Secret.Valid(): @@ -201,16 +227,17 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop ctx, cancel := context.WithCancel(context.Background()) proxy := &Proxy{ - ctx: ctx, - ctxCancel: cancel, - secret: opts.Secret, - antiReplayCache: opts.AntiReplayCache, - ipBlocklist: opts.IPBlocklist, - eventStream: opts.EventStream, - logger: opts.Logger.Named("proxy"), - idleTimeout: idleTimeout, - bufferSize: int(bufferSize), - telegram: tg, + ctx: ctx, + ctxCancel: cancel, + secret: opts.Secret, + antiReplayCache: opts.AntiReplayCache, + timeAttackDetector: opts.TimeAttackDetector, + ipBlocklist: opts.IPBlocklist, + eventStream: opts.EventStream, + logger: opts.Logger.Named("proxy"), + idleTimeout: idleTimeout, + bufferSize: int(bufferSize), + telegram: tg, } pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { From d3551aa9cc34cbfb5e7be744750bd9401df41dc7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 25 Mar 2021 16:44:06 +0300 Subject: [PATCH 123/206] Move clienhello to faketls --- .../faketls/{clienthello => }/clienthello.go | 14 +++++------ mtglib/internal/faketls/clienthello/init.go | 17 -------------- mtglib/internal/faketls/init.go | 18 +++++++++++++++ mtglib/proxy.go | 23 ++++++++++++++----- 4 files changed, 42 insertions(+), 30 deletions(-) rename mtglib/internal/faketls/{clienthello => }/clienthello.go (73%) delete mode 100644 mtglib/internal/faketls/clienthello/init.go create mode 100644 mtglib/internal/faketls/init.go diff --git a/mtglib/internal/faketls/clienthello/clienthello.go b/mtglib/internal/faketls/clienthello.go similarity index 73% rename from mtglib/internal/faketls/clienthello/clienthello.go rename to mtglib/internal/faketls/clienthello.go index 4789029..3ee9aa0 100644 --- a/mtglib/internal/faketls/clienthello/clienthello.go +++ b/mtglib/internal/faketls/clienthello.go @@ -1,4 +1,4 @@ -package clienthello +package faketls import ( "crypto/hmac" @@ -16,10 +16,10 @@ type ClientHello struct { SessionID []byte } -func ParseHandshake(secret, handshake []byte) (ClientHello, error) { +func ParseClientHello(secret, handshake []byte) (ClientHello, error) { hello := ClientHello{} - if len(handshake) < MinLen { + if len(handshake) < ClientHelloMinLen { return hello, fmt.Errorf("lengh of handshake is too small: %d", len(handshake)) } @@ -27,9 +27,9 @@ func ParseHandshake(secret, handshake []byte) (ClientHello, error) { return hello, fmt.Errorf("unknown handshake type %#x", handshake[0]) } - copy(hello.Digest[:], handshake[RandomOffset:]) + copy(hello.Digest[:], handshake[ClientHelloRandomOffset:]) - for i := RandomOffset; i < RandomOffset+RandomLen; i++ { + for i := ClientHelloRandomOffset; i < ClientHelloRandomOffset+RandomLen; i++ { handshake[i] = 0 } @@ -60,8 +60,8 @@ func ParseHandshake(secret, handshake []byte) (ClientHello, error) { timestamp := int64(binary.LittleEndian.Uint32(computedDigest[RandomLen-4:])) hello.Time = time.Unix(timestamp, 0) - hello.SessionID = make([]byte, handshake[SessionIDOffset]) - copy(hello.SessionID, handshake[SessionIDOffset+1:]) + hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) + copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) return hello, nil } diff --git a/mtglib/internal/faketls/clienthello/init.go b/mtglib/internal/faketls/clienthello/init.go deleted file mode 100644 index c597b84..0000000 --- a/mtglib/internal/faketls/clienthello/init.go +++ /dev/null @@ -1,17 +0,0 @@ -package clienthello - -import "errors" - -const ( - RandomLen = 32 - RandomOffset = 6 - SessionIDOffset = RandomOffset + RandomLen - MinLen = SessionIDOffset + 1 - - HandshakeTypeClient = 0x01 -) - -var ( - ErrBadDigest = errors.New("bad digest") - ErrAntiReplayAttack = errors.New("antireplay attack was detected") -) diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go new file mode 100644 index 0000000..081a9ba --- /dev/null +++ b/mtglib/internal/faketls/init.go @@ -0,0 +1,18 @@ +package faketls + +import "errors" + +const ( + RandomLen = 32 + + ClientHelloRandomOffset = 6 + ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen + ClientHelloMinLen = ClientHelloSessionIDOffset + 1 + + HandshakeTypeClient = 0x01 +) + +var ( + ErrBadDigest = errors.New("bad digest") + ErrAntiReplayAttack = errors.New("antireplay attack was detected") +) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 7273375..8ce024d 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "github.com/9seconds/mtg/v2/mtglib/internal/faketls/clienthello" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" "github.com/9seconds/mtg/v2/mtglib/internal/relay" @@ -58,7 +58,7 @@ func (p *Proxy) ServeConn(conn net.Conn) { ctx.logger.Info("Stream has been finished") }() - if err := p.doFakeTLSHandshake(ctx); err != nil { + if err := p.doFakeTLSHandshake(ctx, ctx.clientConn); err != nil { p.logger.InfoError("faketls handshake is failed", err) return @@ -121,17 +121,28 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } -func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) error { +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn net.Conn) error { clientHelloRecord := record.AcquireRecord() defer record.ReleaseRecord(clientHelloRecord) - if err := clientHelloRecord.Read(ctx.clientConn); err != nil { + if err := clientHelloRecord.Read(conn); err != nil { return fmt.Errorf("cannot read client hello: %w", err) } - hello, _ := clienthello.ParseHandshake(p.secret.Key[:], + hello, err := faketls.ParseClientHello(p.secret.Key[:], clientHelloRecord.Payload.Bytes()) - fmt.Println(hello) + if err != nil { + return fmt.Errorf("cannot parse client hello: %w", err) + } + + if err := p.timeAttackDetector.Valid(hello.Time); err != nil { + return fmt.Errorf("invalid time: %w", err) + } + if p.antiReplayCache.SeenBefore(hello.SessionID) { + p.logger.Warning("anti replay attack was detected") + + return fmt.Errorf("anti replay attack from %s", ctx.ClientIP().String()) + } return fmt.Errorf("SUCCESS") } From a444a10bd01e305b7307977c76493a44e3bb302f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 25 Mar 2021 16:44:36 +0300 Subject: [PATCH 124/206] Rename client hello file --- mtglib/internal/faketls/{clienthello.go => client_hello.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mtglib/internal/faketls/{clienthello.go => client_hello.go} (100%) diff --git a/mtglib/internal/faketls/clienthello.go b/mtglib/internal/faketls/client_hello.go similarity index 100% rename from mtglib/internal/faketls/clienthello.go rename to mtglib/internal/faketls/client_hello.go From c3e8e8b1feb5bbb12bd5a4c2fb009721efeb6ec6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 25 Mar 2021 17:33:22 +0300 Subject: [PATCH 125/206] Add base faketls processing --- go.mod | 1 + go.sum | 2 + mtglib/internal/faketls/client_hello.go | 20 +++--- mtglib/internal/faketls/init.go | 17 ++++- mtglib/internal/faketls/pools.go | 21 ++++++ mtglib/internal/faketls/welcome.go | 88 +++++++++++++++++++++++++ mtglib/proxy.go | 16 +++-- 7 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 mtglib/internal/faketls/pools.go create mode 100644 mtglib/internal/faketls/welcome.go diff --git a/go.mod b/go.mod index a09b213..ad324f1 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c github.com/xeipuuv/gojsonschema v1.2.0 + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect ) diff --git a/go.sum b/go.sum index d9e0113..0cf083c 100644 --- a/go.sum +++ b/go.sum @@ -324,6 +324,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go index 3ee9aa0..6f31ee6 100644 --- a/mtglib/internal/faketls/client_hello.go +++ b/mtglib/internal/faketls/client_hello.go @@ -11,9 +11,10 @@ import ( ) type ClientHello struct { - Time time.Time - Digest [RandomLen]byte - SessionID []byte + Time time.Time + Random [RandomLen]byte + SessionID []byte + CipherSuite uint16 } func ParseClientHello(secret, handshake []byte) (ClientHello, error) { @@ -27,7 +28,7 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { return hello, fmt.Errorf("unknown handshake type %#x", handshake[0]) } - copy(hello.Digest[:], handshake[ClientHelloRandomOffset:]) + copy(hello.Random[:], handshake[ClientHelloRandomOffset:]) for i := ClientHelloRandomOffset; i < ClientHelloRandomOffset+RandomLen; i++ { handshake[i] = 0 @@ -45,23 +46,26 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { mac := hmac.New(sha256.New, secret) rec.Dump(mac) - computedDigest := mac.Sum(nil) + computedRandom := mac.Sum(nil) for i := 0; i < RandomLen; i++ { - computedDigest[i] ^= hello.Digest[i] + computedRandom[i] ^= hello.Random[i] } for i := 0; i < RandomLen-4; i++ { - if computedDigest[i] != 0 { + if computedRandom[i] != 0 { return hello, ErrBadDigest } } - timestamp := int64(binary.LittleEndian.Uint32(computedDigest[RandomLen-4:])) + timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:])) hello.Time = time.Unix(timestamp, 0) hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) + cipherSuiteOffset := ClientHelloSessionIDOffset + 1 + len(hello.SessionID) + 2 + hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2]) + return hello, nil } diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go index 081a9ba..5166cbf 100644 --- a/mtglib/internal/faketls/init.go +++ b/mtglib/internal/faketls/init.go @@ -7,12 +7,27 @@ const ( ClientHelloRandomOffset = 6 ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen - ClientHelloMinLen = ClientHelloSessionIDOffset + 1 + ClientHelloMinLen = ClientHelloSessionIDOffset + 1 + + WelcomePacketRandomOffset = 11 HandshakeTypeClient = 0x01 + HandshakeTypeServer = 0x02 ) var ( ErrBadDigest = errors.New("bad digest") ErrAntiReplayAttack = errors.New("antireplay attack was detected") + + serverHelloSuffix = []byte{ + 0x00, // no compression + 0x00, 0x2e, // 46 bytes of data + 0x00, 0x2b, // Extension - Supported Versions + 0x00, 0x02, // 2 bytes are following + 0x03, 0x04, // TLS 1.3 + 0x00, 0x33, // Extension - Key Share + 0x00, 0x24, // 36 bytes + 0x00, 0x1d, // x25519 curve + 0x00, 0x20, // 32 bytes of key + } ) diff --git a/mtglib/internal/faketls/pools.go b/mtglib/internal/faketls/pools.go new file mode 100644 index 0000000..35a7aea --- /dev/null +++ b/mtglib/internal/faketls/pools.go @@ -0,0 +1,21 @@ +package faketls + +import ( + "bytes" + "sync" +) + +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, +} + +func acquireBytesBuffer() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func releaseBytesBuffer(b *bytes.Buffer) { + b.Reset() + bytesBufferPool.Put(b) +} diff --git a/mtglib/internal/faketls/welcome.go b/mtglib/internal/faketls/welcome.go new file mode 100644 index 0000000..d943900 --- /dev/null +++ b/mtglib/internal/faketls/welcome.go @@ -0,0 +1,88 @@ +package faketls + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "io" + mrand "math/rand" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "golang.org/x/crypto/curve25519" +) + +func SendWelcomePacket(writer io.Writer, secret []byte, clientHello ClientHello) error { + buf := acquireBytesBuffer() + defer releaseBytesBuffer(buf) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeHandshake + rec.Version = record.Version12 + + generateServerHello(&rec.Payload, clientHello) + rec.Dump(buf) + rec.Reset() + + rec.Type = record.TypeChangeCipherSpec + rec.Version = record.Version12 + rec.Payload.WriteByte(0x01) + + rec.Dump(buf) + rec.Reset() + + rec.Type = record.TypeApplicationData + rec.Version = record.Version12 + + if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.Intn(3092))); err != nil { + panic(err) + } + + rec.Dump(buf) + + packet := buf.Bytes() + mac := hmac.New(sha256.New, secret) + + mac.Write(clientHello.Random[:]) + mac.Write(packet) + + copy(packet[WelcomePacketRandomOffset:], mac.Sum(nil)) + + if _, err := writer.Write(packet); err != nil { + return err // nolint: wrapcheck + } + + return nil +} + +func generateServerHello(writer io.Writer, clientHello ClientHello) { + bodyBuf := acquireBytesBuffer() + defer releaseBytesBuffer(bodyBuf) + + sliceBuf := [2]byte{} + digest := [RandomLen]byte{} + + binary.BigEndian.PutUint16(sliceBuf[:], uint16(record.Version12)) + bodyBuf.Write(sliceBuf[:]) + bodyBuf.Write(digest[:]) + bodyBuf.WriteByte(byte(len(clientHello.SessionID))) + bodyBuf.Write(clientHello.SessionID) + + binary.BigEndian.PutUint16(sliceBuf[:], clientHello.CipherSuite) + bodyBuf.Write(sliceBuf[:]) + bodyBuf.Write(serverHelloSuffix) + + scalar := [32]byte{} + rand.Read(scalar[:]) + curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) + bodyBuf.Write(curve) + + header := [4]byte{0, 0, 0, 0} + binary.BigEndian.PutUint32(header[:], uint32(bodyBuf.Len())) + header[0] = HandshakeTypeServer + + writer.Write(header[:]) + bodyBuf.WriteTo(writer) +} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 8ce024d..b9032c0 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -122,15 +122,14 @@ func (p *Proxy) Shutdown() { } func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn net.Conn) error { - clientHelloRecord := record.AcquireRecord() - defer record.ReleaseRecord(clientHelloRecord) + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) - if err := clientHelloRecord.Read(conn); err != nil { + if err := rec.Read(conn); err != nil { return fmt.Errorf("cannot read client hello: %w", err) } - hello, err := faketls.ParseClientHello(p.secret.Key[:], - clientHelloRecord.Payload.Bytes()) + hello, err := faketls.ParseClientHello(p.secret.Key[:], rec.Payload.Bytes()) if err != nil { return fmt.Errorf("cannot parse client hello: %w", err) } @@ -138,13 +137,18 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn net.Conn) error { if err := p.timeAttackDetector.Valid(hello.Time); err != nil { return fmt.Errorf("invalid time: %w", err) } + if p.antiReplayCache.SeenBefore(hello.SessionID) { p.logger.Warning("anti replay attack was detected") return fmt.Errorf("anti replay attack from %s", ctx.ClientIP().String()) } - return fmt.Errorf("SUCCESS") + if err := faketls.SendWelcomePacket(conn, p.secret.Key[:], hello); err != nil { + return fmt.Errorf("cannot send a welcome packet: %w", err) + } + + return nil } func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { From 6d92d5fe210dce757ac270181f58851f02f0f253 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 25 Mar 2021 17:40:38 +0300 Subject: [PATCH 126/206] Fix lint issues --- mtglib/internal/faketls/client_hello.go | 4 ++-- mtglib/internal/faketls/init.go | 2 ++ mtglib/internal/faketls/pools.go | 6 +++--- mtglib/internal/faketls/welcome.go | 24 ++++++++++++++---------- mtglib/proxy.go | 5 +++-- 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go index 6f31ee6..b32eea0 100644 --- a/mtglib/internal/faketls/client_hello.go +++ b/mtglib/internal/faketls/client_hello.go @@ -44,7 +44,7 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { // mac is calculated for the whole record, not only // for the payload part mac := hmac.New(sha256.New, secret) - rec.Dump(mac) + rec.Dump(mac) // nolint: errcheck computedRandom := mac.Sum(nil) @@ -64,7 +64,7 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) - cipherSuiteOffset := ClientHelloSessionIDOffset + 1 + len(hello.SessionID) + 2 + cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3 // nolint: gomnd hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2]) return hello, nil diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go index 5166cbf..54d1186 100644 --- a/mtglib/internal/faketls/init.go +++ b/mtglib/internal/faketls/init.go @@ -13,6 +13,8 @@ const ( HandshakeTypeClient = 0x01 HandshakeTypeServer = 0x02 + + ChangeCipherValue = 0x01 ) var ( diff --git a/mtglib/internal/faketls/pools.go b/mtglib/internal/faketls/pools.go index 35a7aea..52288b1 100644 --- a/mtglib/internal/faketls/pools.go +++ b/mtglib/internal/faketls/pools.go @@ -12,10 +12,10 @@ var bytesBufferPool = sync.Pool{ } func acquireBytesBuffer() *bytes.Buffer { - return bytesBufferPool.Get().(*bytes.Buffer) + return bytesBufferPool.Get().(*bytes.Buffer) } func releaseBytesBuffer(b *bytes.Buffer) { - b.Reset() - bytesBufferPool.Put(b) + b.Reset() + bytesBufferPool.Put(b) } diff --git a/mtglib/internal/faketls/welcome.go b/mtglib/internal/faketls/welcome.go index d943900..d0b48d2 100644 --- a/mtglib/internal/faketls/welcome.go +++ b/mtglib/internal/faketls/welcome.go @@ -23,30 +23,30 @@ func SendWelcomePacket(writer io.Writer, secret []byte, clientHello ClientHello) rec.Version = record.Version12 generateServerHello(&rec.Payload, clientHello) - rec.Dump(buf) + rec.Dump(buf) // nolint: errcheck rec.Reset() rec.Type = record.TypeChangeCipherSpec rec.Version = record.Version12 - rec.Payload.WriteByte(0x01) + rec.Payload.WriteByte(ChangeCipherValue) - rec.Dump(buf) + rec.Dump(buf) // nolint: errcheck rec.Reset() rec.Type = record.TypeApplicationData rec.Version = record.Version12 - if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.Intn(3092))); err != nil { + if _, err := io.CopyN(&rec.Payload, rand.Reader, int64(1024+mrand.Intn(3092))); err != nil { // nolint: gomnd panic(err) } - rec.Dump(buf) + rec.Dump(buf) // nolint: errcheck packet := buf.Bytes() mac := hmac.New(sha256.New, secret) - mac.Write(clientHello.Random[:]) - mac.Write(packet) + mac.Write(clientHello.Random[:]) // nolint: errcheck + mac.Write(packet) // nolint: errcheck copy(packet[WelcomePacketRandomOffset:], mac.Sum(nil)) @@ -75,7 +75,11 @@ func generateServerHello(writer io.Writer, clientHello ClientHello) { bodyBuf.Write(serverHelloSuffix) scalar := [32]byte{} - rand.Read(scalar[:]) + + if _, err := rand.Read(scalar[:]); err != nil { + panic(err) + } + curve, _ := curve25519.X25519(scalar[:], curve25519.Basepoint) bodyBuf.Write(curve) @@ -83,6 +87,6 @@ func generateServerHello(writer io.Writer, clientHello ClientHello) { binary.BigEndian.PutUint32(header[:], uint32(bodyBuf.Len())) header[0] = HandshakeTypeServer - writer.Write(header[:]) - bodyBuf.WriteTo(writer) + writer.Write(header[:]) // nolint: errcheck + bodyBuf.WriteTo(writer) // nolint: errcheck } diff --git a/mtglib/proxy.go b/mtglib/proxy.go index b9032c0..7bf9f7c 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "net" "sync" "time" @@ -121,7 +122,7 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } -func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn net.Conn) error { +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn io.ReadWriter) error { rec := record.AcquireRecord() defer record.ReleaseRecord(rec) @@ -202,7 +203,7 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { return nil } -func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop +func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen switch { case opts.Network == nil: return nil, ErrNetworkIsNotDefined From f0efa4697ee92466a871eaa985fb4b6c98fa8e8a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 12:15:26 +0300 Subject: [PATCH 127/206] Faketls works now --- mtglib/internal/faketls/conn.go | 70 ++++++++++++++++++++++++++ mtglib/internal/faketls/record/init.go | 2 + mtglib/internal/relay/conn.go | 6 +-- mtglib/proxy.go | 13 +++-- 4 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 mtglib/internal/faketls/conn.go diff --git a/mtglib/internal/faketls/conn.go b/mtglib/internal/faketls/conn.go new file mode 100644 index 0000000..9962cbb --- /dev/null +++ b/mtglib/internal/faketls/conn.go @@ -0,0 +1,70 @@ +package faketls + +import ( + "bytes" + "fmt" + "net" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" +) + +type Conn struct { + net.Conn + + readBuffer bytes.Buffer +} + +func (c *Conn) Read(p []byte) (int, error) { + if n, _ := c.readBuffer.Read(p); n > 0 { + return n, nil + } + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + for { + if err := rec.Read(c.Conn); err != nil { + return 0, err // nolint: wrapcheck + } + + switch rec.Type { // nolint: exhaustive + case record.TypeChangeCipherSpec: + case record.TypeApplicationData: + rec.Payload.WriteTo(&c.readBuffer) + + n, err := c.readBuffer.Read(p) + + return n, err + default: + return 0, fmt.Errorf("unsupported record type %v", rec.Type) + } + } +} + +func (c *Conn) Write(p []byte) (int, error) { + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeApplicationData + rec.Version = record.Version12 + written := 0 + + for len(p) > 0 { + chunkSize := record.TLSMaxRecordSize + if chunkSize > len(p) { + chunkSize = len(p) + } + + rec.Payload.Reset() + rec.Payload.Write(p[:chunkSize]) + + if err := rec.Dump(c.Conn); err != nil { + return 0, err + } + + written += chunkSize + p = p[chunkSize:] + } + + return written, nil +} diff --git a/mtglib/internal/faketls/record/init.go b/mtglib/internal/faketls/record/init.go index 1e543aa..764c29e 100644 --- a/mtglib/internal/faketls/record/init.go +++ b/mtglib/internal/faketls/record/init.go @@ -2,6 +2,8 @@ package record import "fmt" +const TLSMaxRecordSize = 65535 // max uint16 + type Type uint8 const ( diff --git a/mtglib/internal/relay/conn.go b/mtglib/internal/relay/conn.go index 211a32a..8f4e89d 100644 --- a/mtglib/internal/relay/conn.go +++ b/mtglib/internal/relay/conn.go @@ -9,11 +9,10 @@ type conn struct { } func (c conn) Read(p []byte) (int, error) { - ctx := c.relay.ctx n, err := c.ReadWriteCloser.Read(p) select { - case <-ctx.Done(): + case <-c.relay.ctx.Done(): case c.relay.tickChannel <- struct{}{}: } @@ -21,11 +20,10 @@ func (c conn) Read(p []byte) (int, error) { } func (c conn) Write(p []byte) (int, error) { - ctx := c.relay.ctx n, err := c.ReadWriteCloser.Write(p) select { - case <-ctx.Done(): + case <-c.relay.ctx.Done(): case c.relay.tickChannel <- struct{}{}: } diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 7bf9f7c..6766c00 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "net" "sync" "time" @@ -59,7 +58,7 @@ func (p *Proxy) ServeConn(conn net.Conn) { ctx.logger.Info("Stream has been finished") }() - if err := p.doFakeTLSHandshake(ctx, ctx.clientConn); err != nil { + if err := p.doFakeTLSHandshake(ctx); err != nil { p.logger.InfoError("faketls handshake is failed", err) return @@ -122,11 +121,11 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } -func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn io.ReadWriter) error { +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) error { rec := record.AcquireRecord() defer record.ReleaseRecord(rec) - if err := rec.Read(conn); err != nil { + if err := rec.Read(ctx.clientConn); err != nil { return fmt.Errorf("cannot read client hello: %w", err) } @@ -145,10 +144,14 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext, conn io.ReadWriter) error return fmt.Errorf("anti replay attack from %s", ctx.ClientIP().String()) } - if err := faketls.SendWelcomePacket(conn, p.secret.Key[:], hello); err != nil { + if err := faketls.SendWelcomePacket(ctx.clientConn, p.secret.Key[:], hello); err != nil { return fmt.Errorf("cannot send a welcome packet: %w", err) } + ctx.clientConn = &faketls.Conn{ + Conn: ctx.clientConn, + } + return nil } From 40b483357f189eee18267a370175522410500e59 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 14:11:56 +0300 Subject: [PATCH 128/206] Propagate network to proxy object --- mtglib/init.go | 3 +++ mtglib/proxy.go | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/mtglib/init.go b/mtglib/init.go index cb2893d..331bc38 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -17,6 +17,9 @@ var ( ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") ErrEventStreamIsNotDefined = errors.New("event stream is not defined") ErrLoggerIsNotDefined = errors.New("logger is not defined") + + errCannotSendWelcomePacket = errors.New("cannot send welcome packet") + errReplayAttackDetected = errors.New("replay attack detected") ) const ( diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 6766c00..b48d50f 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -27,6 +27,7 @@ type Proxy struct { telegram *telegram.Telegram secret Secret + network Network antiReplayCache AntiReplayCache timeAttackDetector TimeAttackDetector ipBlocklist IPBlocklist @@ -139,13 +140,13 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) error { } if p.antiReplayCache.SeenBefore(hello.SessionID) { - p.logger.Warning("anti replay attack was detected") - - return fmt.Errorf("anti replay attack from %s", ctx.ClientIP().String()) + return errReplayAttackDetected } if err := faketls.SendWelcomePacket(ctx.clientConn, p.secret.Key[:], hello); err != nil { - return fmt.Errorf("cannot send a welcome packet: %w", err) + p.logger.InfoError("cannot send welcome packet", err) + + return errCannotSendWelcomePacket } ctx.clientConn = &faketls.Conn{ @@ -249,6 +250,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen ctx: ctx, ctxCancel: cancel, secret: opts.Secret, + network: opts.Network, antiReplayCache: opts.AntiReplayCache, timeAttackDetector: opts.TimeAttackDetector, ipBlocklist: opts.IPBlocklist, From e2e464d32c46f3f4edb6d2b6b970ab75d67ceafe Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 14:12:05 +0300 Subject: [PATCH 129/206] Set random length of the tls packets --- mtglib/internal/faketls/conn.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mtglib/internal/faketls/conn.go b/mtglib/internal/faketls/conn.go index 9962cbb..3ad90d8 100644 --- a/mtglib/internal/faketls/conn.go +++ b/mtglib/internal/faketls/conn.go @@ -3,6 +3,7 @@ package faketls import ( "bytes" "fmt" + "math/rand" "net" "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" @@ -50,8 +51,8 @@ func (c *Conn) Write(p []byte) (int, error) { written := 0 for len(p) > 0 { - chunkSize := record.TLSMaxRecordSize - if chunkSize > len(p) { + chunkSize := rand.Intn(record.TLSMaxRecordSize) + if chunkSize > len(p) || chunkSize == 0 { chunkSize = len(p) } From 9b78f766e889685d2c7153e01842396262cef9ba Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 14:17:33 +0300 Subject: [PATCH 130/206] fix lint issues --- mtglib/internal/faketls/conn.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mtglib/internal/faketls/conn.go b/mtglib/internal/faketls/conn.go index 3ad90d8..ee560c3 100644 --- a/mtglib/internal/faketls/conn.go +++ b/mtglib/internal/faketls/conn.go @@ -31,11 +31,9 @@ func (c *Conn) Read(p []byte) (int, error) { switch rec.Type { // nolint: exhaustive case record.TypeChangeCipherSpec: case record.TypeApplicationData: - rec.Payload.WriteTo(&c.readBuffer) + rec.Payload.WriteTo(&c.readBuffer) // nolint: errcheck - n, err := c.readBuffer.Read(p) - - return n, err + return c.readBuffer.Read(p) default: return 0, fmt.Errorf("unsupported record type %v", rec.Type) } @@ -60,7 +58,7 @@ func (c *Conn) Write(p []byte) (int, error) { rec.Payload.Write(p[:chunkSize]) if err := rec.Dump(c.Conn); err != nil { - return 0, err + return written, err // nolint: wrapcheck } written += chunkSize From bddf180575d92d13c4b1cdcd1a212ad50af6c116 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 14:45:00 +0300 Subject: [PATCH 131/206] Implement domain fronting --- cli/proxy.go | 10 +++--- config/config.go | 32 ++++++++--------- example.config.toml | 2 +- mtglib/conns.go | 34 +++++++++++++++++++ mtglib/init.go | 13 +++---- mtglib/proxy.go | 81 +++++++++++++++++++++++++++++++++----------- mtglib/proxy_opts.go | 10 +++--- 7 files changed, 127 insertions(+), 55 deletions(-) diff --git a/cli/proxy.go b/cli/proxy.go index eaf7035..50d608e 100644 --- a/cli/proxy.go +++ b/cli/proxy.go @@ -46,11 +46,11 @@ func (c *Proxy) Execute() error { // nolint: funlen TimeAttackDetector: timeattack.NewNoop(), EventStream: events.NewNoopStream(), - Secret: c.Config.Secret, - BufferSize: c.Config.TCPBuffer.Value(mtglib.DefaultBufferSize), - CloakPort: c.Config.CloakPort.Value(mtglib.DefaultCloakPort), - IdleTimeout: c.Config.Network.Timeout.Idle.Value(mtglib.DefaultIdleTimeout), - PreferIP: c.Config.PreferIP.Value(mtglib.DefaultPreferIP), + Secret: c.Config.Secret, + BufferSize: c.Config.TCPBuffer.Value(mtglib.DefaultBufferSize), + DomainFrontingPort: c.Config.DomainFrontingPort.Value(mtglib.DefaultDomainFrontingPort), + IdleTimeout: c.Config.Network.Timeout.Idle.Value(mtglib.DefaultIdleTimeout), + PreferIP: c.Config.PreferIP.Value(mtglib.DefaultPreferIP), } defer func() { diff --git a/config/config.go b/config/config.go index cbb2156..8414b9a 100644 --- a/config/config.go +++ b/config/config.go @@ -10,14 +10,14 @@ import ( ) type Config struct { - Debug bool `json:"debug"` - Secret mtglib.Secret `json:"secret"` - BindTo TypeHostPort `json:"bind-to"` - TCPBuffer TypeBytes `json:"tcp-buffer"` - PreferIP TypePreferIP `json:"prefer-ip"` - CloakPort TypePort `json:"cloak-port"` - Concurrency uint `json:"concurrency"` - Defense struct { + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` + BindTo TypeHostPort `json:"bind-to"` + TCPBuffer TypeBytes `json:"tcp-buffer"` + PreferIP TypePreferIP `json:"prefer-ip"` + DomainFrontingPort TypePort `json:"domain-fronting-port"` + Concurrency uint `json:"concurrency"` + Defense struct { Time struct { Enabled bool `json:"enabled"` AllowSkewness TypeDuration `json:"allow-skewness"` @@ -85,14 +85,14 @@ func (c *Config) String() string { } type configRaw struct { - Debug bool `toml:"debug" json:"debug,omitempty"` - Secret string `toml:"secret" json:"secret"` - BindTo string `toml:"bind-to" json:"bind-to"` - TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` - PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` - CloakPort uint `toml:"cloak-port" json:"cloak-port,omitempty"` - Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"` - Defense struct { + Debug bool `toml:"debug" json:"debug,omitempty"` + Secret string `toml:"secret" json:"secret"` + BindTo string `toml:"bind-to" json:"bind-to"` + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` + DomainFrontingPort uint `toml:"domain-fronting-port" json:"domain-fronting-port,omitempty"` + Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"` + Defense struct { Time struct { Enabled bool `toml:"enabled" json:"enabled,omitempty"` AllowSkewness string `toml:"allow-skewness" json:"allow-skewness,omitempty"` diff --git a/example.config.toml b/example.config.toml index d573a16..618ec19 100644 --- a/example.config.toml +++ b/example.config.toml @@ -46,7 +46,7 @@ prefer-ip = "prefer-ipv6" # FakeTLS uses domain fronting protection. So it needs to know a port to # access. -cloak-port = 443 +domain-fronting-port = 443 # network defines different network-related settings [network] diff --git a/mtglib/conns.go b/mtglib/conns.go index e30cf0d..07b7086 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -1,8 +1,11 @@ package mtglib import ( + "bytes" "context" + "io" "net" + "sync" "time" ) @@ -43,3 +46,34 @@ func (c connTelegramTraffic) Write(b []byte) (int, error) { return n, err // nolint: wrapcheck } + +type connRewind struct { + net.Conn + + active io.Reader + buf bytes.Buffer + mutex sync.RWMutex +} + +func (c *connRewind) Read(p []byte) (int, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.active.Read(p) +} + +func (c *connRewind) Rewind() { + c.mutex.Lock() + defer c.mutex.Unlock() + + c.active = io.MultiReader(&c.buf, c.Conn) +} + +func newConnRewind(conn net.Conn) *connRewind { + rv := &connRewind{ + Conn: conn, + } + rv.active = io.TeeReader(conn, &rv.buf) + + return rv +} diff --git a/mtglib/init.go b/mtglib/init.go index 331bc38..44da825 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -17,17 +17,14 @@ var ( ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") ErrEventStreamIsNotDefined = errors.New("event stream is not defined") ErrLoggerIsNotDefined = errors.New("logger is not defined") - - errCannotSendWelcomePacket = errors.New("cannot send welcome packet") - errReplayAttackDetected = errors.New("replay attack detected") ) const ( - DefaultConcurrency = 4096 - DefaultBufferSize = 16 * 1024 // 16 kib - DefaultCloakPort = 443 - DefaultIdleTimeout = time.Minute - DefaultPreferIP = "prefer-ipv6" + DefaultConcurrency = 4096 + DefaultBufferSize = 16 * 1024 // 16 kib + DefaultDomainFrontingPort = 443 + DefaultIdleTimeout = time.Minute + DefaultPreferIP = "prefer-ipv6" ) type Network interface { diff --git a/mtglib/proxy.go b/mtglib/proxy.go index b48d50f..4096438 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "strconv" "sync" "time" @@ -21,10 +22,11 @@ type Proxy struct { ctxCancel context.CancelFunc streamWaitGroup sync.WaitGroup - idleTimeout time.Duration - bufferSize int - workerPool *ants.PoolWithFunc - telegram *telegram.Telegram + idleTimeout time.Duration + bufferSize int + domainFrontAddress string + workerPool *ants.PoolWithFunc + telegram *telegram.Telegram secret Secret network Network @@ -59,9 +61,7 @@ func (p *Proxy) ServeConn(conn net.Conn) { ctx.logger.Info("Stream has been finished") }() - if err := p.doFakeTLSHandshake(ctx); err != nil { - p.logger.InfoError("faketls handshake is failed", err) - + if !p.doFakeTLSHandshake(ctx) { return } @@ -77,7 +77,8 @@ func (p *Proxy) ServeConn(conn net.Conn) { return } - rel := relay.AcquireRelay(ctx, p.logger.Named("relay"), p.bufferSize, p.idleTimeout) + rel := relay.AcquireRelay(ctx, + p.logger.Named("relay"), p.bufferSize, p.idleTimeout) defer relay.ReleaseRelay(rel) if err := rel.Process(ctx.clientConn, ctx.telegramConn); err != nil { @@ -122,38 +123,52 @@ func (p *Proxy) Shutdown() { p.workerPool.Release() } -func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) error { +func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { rec := record.AcquireRecord() defer record.ReleaseRecord(rec) - if err := rec.Read(ctx.clientConn); err != nil { - return fmt.Errorf("cannot read client hello: %w", err) + rewind := newConnRewind(ctx.clientConn) + + if err := rec.Read(rewind); err != nil { + p.logger.InfoError("cannot read client hello", err) + p.doDomainFronting(ctx, rewind) + + return false } hello, err := faketls.ParseClientHello(p.secret.Key[:], rec.Payload.Bytes()) if err != nil { - return fmt.Errorf("cannot parse client hello: %w", err) + p.logger.InfoError("cannot parse client hello", err) + p.doDomainFronting(ctx, rewind) + + return false } if err := p.timeAttackDetector.Valid(hello.Time); err != nil { - return fmt.Errorf("invalid time: %w", err) + p.logger.InfoError("invalid faketls time", err) + p.doDomainFronting(ctx, rewind) + + return false } if p.antiReplayCache.SeenBefore(hello.SessionID) { - return errReplayAttackDetected + p.logger.Warning("replay attack has been detected!") + p.doDomainFronting(ctx, rewind) + + return false } - if err := faketls.SendWelcomePacket(ctx.clientConn, p.secret.Key[:], hello); err != nil { + if err := faketls.SendWelcomePacket(rewind, p.secret.Key[:], hello); err != nil { p.logger.InfoError("cannot send welcome packet", err) - return errCannotSendWelcomePacket + return false } ctx.clientConn = &faketls.Conn{ Conn: ctx.clientConn, } - return nil + return true } func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { @@ -207,6 +222,25 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { return nil } +func (p *Proxy) doDomainFronting(ctx context.Context, conn *connRewind) { + conn.Rewind() + + frontConn, err := p.network.DialContext(ctx, "tcp", p.domainFrontAddress) + if err != nil { + p.logger.WarningError("cannot dial to the fronting domain", err) + + return + } + + rel := relay.AcquireRelay(ctx, + p.logger.Named("domain-fronting"), p.bufferSize, p.idleTimeout) + defer relay.ReleaseRelay(rel) + + if err := rel.Process(conn, frontConn); err != nil { + p.logger.DebugError("domain fronting relay has been finished", err) + } +} + func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen switch { case opts.Network == nil: @@ -245,6 +279,11 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen bufferSize = DefaultBufferSize } + domainFrontingPort := int(opts.DomainFrontingPort) + if domainFrontingPort == 0 { + domainFrontingPort = DefaultDomainFrontingPort + } + ctx, cancel := context.WithCancel(context.Background()) proxy := &Proxy{ ctx: ctx, @@ -256,9 +295,11 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen ipBlocklist: opts.IPBlocklist, eventStream: opts.EventStream, logger: opts.Logger.Named("proxy"), - idleTimeout: idleTimeout, - bufferSize: int(bufferSize), - telegram: tg, + domainFrontAddress: net.JoinHostPort(opts.Secret.Host, + strconv.Itoa(domainFrontingPort)), + idleTimeout: idleTimeout, + bufferSize: int(bufferSize), + telegram: tg, } pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index 9067fb3..a20caaa 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -11,9 +11,9 @@ type ProxyOpts struct { EventStream EventStream Logger Logger - BufferSize uint - Concurrency uint - CloakPort uint - IdleTimeout time.Duration - PreferIP string + BufferSize uint + Concurrency uint + DomainFrontingPort uint + IdleTimeout time.Duration + PreferIP string } From d8d954bbe8e4e9d48284f3dda2834b7d8f3e47de Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 15:35:31 +0300 Subject: [PATCH 132/206] Set logging level --- cli/proxy.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/proxy.go b/cli/proxy.go index 50d608e..22ebc38 100644 --- a/cli/proxy.go +++ b/cli/proxy.go @@ -37,6 +37,12 @@ func (c *Proxy) Execute() error { // nolint: funlen zerolog.TimestampFieldName = "timestamp" zerolog.LevelFieldName = "level" + if c.Config.Debug { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + } else { + zerolog.SetGlobalLevel(zerolog.WarnLevel) + } + ctx := utils.RootContext() opts := mtglib.ProxyOpts{ Logger: logger.NewZeroLogger(zerolog.New(os.Stdout).With().Timestamp().Logger()), From dedd67780b8d21d8b5741d2b4940d7019e23f10b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 16:31:42 +0300 Subject: [PATCH 133/206] Notify that blocklist were updated --- ipblocklist/firehol.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index ee1d473..20dd0d8 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -95,6 +95,8 @@ func (f *Firehol) Run(updateEach time.Duration) { if err := f.update(); err != nil { f.logger.WarningError("cannot update blocklist", err) + } else { + f.logger.Info("blocklist was updated") } for { @@ -104,6 +106,8 @@ func (f *Firehol) Run(updateEach time.Duration) { case <-ticker.C: if err := f.update(); err != nil { f.logger.WarningError("cannot update blocklist", err) + } else { + f.logger.Info("blocklist was updated") } } } From ce30e128e57971fe30d7ba903f41af7651e07339 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 16:37:59 +0300 Subject: [PATCH 134/206] fixup address for dc 5 --- mtglib/internal/telegram/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtglib/internal/telegram/init.go b/mtglib/internal/telegram/init.go index 3e00353..dc8b45b 100644 --- a/mtglib/internal/telegram/init.go +++ b/mtglib/internal/telegram/init.go @@ -36,7 +36,7 @@ var ( {network: "tcp4", address: "149.154.167.91:443"}, }, { - {network: "tcp4", address: "149.154.171.5"}, + {network: "tcp4", address: "149.154.171.5:443"}, }, } v6Addresses = [5]tgAddr{ From 5eca6ecb053bdd326fef34e127e10823b5769df3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 26 Mar 2021 17:22:53 +0300 Subject: [PATCH 135/206] Validate hostname if it was found in SNI --- example.config.toml | 2 +- mtglib/internal/faketls/client_hello.go | 57 ++++++++++++++++++++++--- mtglib/internal/faketls/init.go | 4 +- mtglib/internal/telegram/init.go | 2 +- mtglib/proxy.go | 7 +++ 5 files changed, 64 insertions(+), 8 deletions(-) diff --git a/example.config.toml b/example.config.toml index 618ec19..7402a57 100644 --- a/example.config.toml +++ b/example.config.toml @@ -13,7 +13,7 @@ # Debug starts application in debug mode. It starts to be quite verbose # in output. Actually, the idea is that you run it in debug mode only if # you have any issue. -debug = false +debug = true # A secret. Please remember that mtg supports only FakeTLS mode, legacy # simple and secured mode are prohibited. For you it means that secret diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go index b32eea0..62d3278 100644 --- a/mtglib/internal/faketls/client_hello.go +++ b/mtglib/internal/faketls/client_hello.go @@ -14,6 +14,7 @@ type ClientHello struct { Time time.Time Random [RandomLen]byte SessionID []byte + Host string CipherSuite uint16 } @@ -28,6 +29,15 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { return hello, fmt.Errorf("unknown handshake type %#x", handshake[0]) } + handshakeSizeBytes := [4]byte{0, handshake[1], handshake[2], handshake[3]} + handshakeLength := binary.BigEndian.Uint32(handshakeSizeBytes[:]) + + if len(handshake)-4 != int(handshakeLength) { + return hello, + fmt.Errorf("incorrect handshake size. manifested=%d, real=%d", + handshakeLength, len(handshake)-4) // nolint: gomnd + } + copy(hello.Random[:], handshake[ClientHelloRandomOffset:]) for i := ClientHelloRandomOffset; i < ClientHelloRandomOffset+RandomLen; i++ { @@ -61,11 +71,48 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:])) hello.Time = time.Unix(timestamp, 0) - hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) - copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) - - cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3 // nolint: gomnd - hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2]) + parseSessionID(&hello, handshake) + parseCipherSuite(&hello, handshake) + parseSNI(&hello, handshake) return hello, nil } + +func parseSessionID(hello *ClientHello, handshake []byte) { + hello.SessionID = make([]byte, handshake[ClientHelloSessionIDOffset]) + copy(hello.SessionID, handshake[ClientHelloSessionIDOffset+1:]) +} + +func parseCipherSuite(hello *ClientHello, handshake []byte) { + cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 3 // nolint: gomnd + hello.CipherSuite = binary.BigEndian.Uint16(handshake[cipherSuiteOffset : cipherSuiteOffset+2]) +} + +func parseSNI(hello *ClientHello, handshake []byte) { + cipherSuiteOffset := ClientHelloSessionIDOffset + len(hello.SessionID) + 1 + handshake = handshake[cipherSuiteOffset:] + + cipherSuiteLength := binary.BigEndian.Uint16(handshake[:2]) + handshake = handshake[2+cipherSuiteLength:] + + compressionMethodsLength := int(handshake[0]) + handshake = handshake[1+compressionMethodsLength:] + + extensionsLength := binary.BigEndian.Uint16(handshake[:2]) + handshake = handshake[2 : 2+extensionsLength] + + for len(handshake) > 0 { + if binary.BigEndian.Uint16(handshake[:2]) != ExtensionSNI { + extensionsLength := binary.BigEndian.Uint16(handshake[2:4]) + handshake = handshake[4+extensionsLength:] + + continue + } + + hostnameLength := binary.BigEndian.Uint16(handshake[7:9]) + handshake = handshake[9:] + hello.Host = string(handshake[:int(hostnameLength)]) + + return + } +} diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go index 54d1186..dfa269b 100644 --- a/mtglib/internal/faketls/init.go +++ b/mtglib/internal/faketls/init.go @@ -7,7 +7,7 @@ const ( ClientHelloRandomOffset = 6 ClientHelloSessionIDOffset = ClientHelloRandomOffset + RandomLen - ClientHelloMinLen = ClientHelloSessionIDOffset + 1 + ClientHelloMinLen = 4 WelcomePacketRandomOffset = 11 @@ -15,6 +15,8 @@ const ( HandshakeTypeServer = 0x02 ChangeCipherValue = 0x01 + + ExtensionSNI = 0x00 ) var ( diff --git a/mtglib/internal/telegram/init.go b/mtglib/internal/telegram/init.go index dc8b45b..af707d3 100644 --- a/mtglib/internal/telegram/init.go +++ b/mtglib/internal/telegram/init.go @@ -36,7 +36,7 @@ var ( {network: "tcp4", address: "149.154.167.91:443"}, }, { - {network: "tcp4", address: "149.154.171.5:443"}, + {network: "tcp4", address: "149.154.171.5:443"}, }, } v6Addresses = [5]tgAddr{ diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 4096438..b6a8f12 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -144,6 +144,13 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { return false } + if hello.Host != "" && hello.Host != p.secret.Host { + p.logger.BindStr("hostname", hello.Host).Info("incorrect domain was found in SNI") + p.doDomainFronting(ctx, rewind) + + return false + } + if err := p.timeAttackDetector.Valid(hello.Time); err != nil { p.logger.InfoError("invalid faketls time", err) p.doDomainFronting(ctx, rewind) From bc2bd4510a954557a53d4310538619adc115bb81 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 27 Mar 2021 22:03:43 +0300 Subject: [PATCH 136/206] Rework stats --- events/event_stream.go | 4 +- events/event_stream_test.go | 8 +- events/init.go | 2 +- events/init_test.go | 2 +- events/multi_observer.go | 4 +- events/noop.go | 2 +- events/noop_test.go | 2 +- mtglib/conns.go | 4 +- mtglib/events.go | 28 ++++- mtglib/events_test.go | 23 ++++- mtglib/init.go | 1 + mtglib/proxy.go | 6 ++ stats/init.go | 36 ++++--- stats/pools.go | 20 ++++ stats/prometheus.go | 197 +++++++++++++++--------------------- stats/prometheus_test.go | 20 ++-- stats/statsd.go | 72 +++++-------- stats/statsd_test.go | 19 ++-- stats/stream_info.go | 55 +++++++--- 19 files changed, 273 insertions(+), 232 deletions(-) create mode 100644 stats/pools.go diff --git a/events/event_stream.go b/events/event_stream.go index 1726c81..c8fe758 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -73,8 +73,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob observer.EventStart(typedEvt) case mtglib.EventConnectedToDC: observer.EventConnectedToDC(typedEvt) - case mtglib.EventTraffic: - observer.EventTraffic(typedEvt) + case mtglib.EventTelegramTraffic: + observer.EventTelegramTraffic(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) case mtglib.EventIPBlocklisted: diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 90dee03..8daa939 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -90,8 +90,8 @@ func (suite *EventStreamTestSuite) TestEventConnectedToDC() { time.Sleep(100 * time.Millisecond) } -func (suite *EventStreamTestSuite) TestEventTraffic() { - evt := mtglib.EventTraffic{ +func (suite *EventStreamTestSuite) TestEventTelegramTraffic() { + evt := mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 1024, @@ -100,10 +100,10 @@ func (suite *EventStreamTestSuite) TestEventTraffic() { for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. - On("EventTraffic", mock.Anything). + On("EventTelegramTraffic", mock.Anything). Once(). Run(func(args mock.Arguments) { - caught := args.Get(0).(mtglib.EventTraffic) + caught := args.Get(0).(mtglib.EventTelegramTraffic) suite.Equal(evt.CreatedAt, caught.CreatedAt) suite.Equal(evt.ConnID, caught.ConnID) diff --git a/events/init.go b/events/init.go index 4720a3b..fbfe330 100644 --- a/events/init.go +++ b/events/init.go @@ -6,7 +6,7 @@ type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) EventConnectedToDC(mtglib.EventConnectedToDC) - EventTraffic(mtglib.EventTraffic) + EventTelegramTraffic(mtglib.EventTelegramTraffic) EventConcurrencyLimited(mtglib.EventConcurrencyLimited) EventIPBlocklisted(mtglib.EventIPBlocklisted) diff --git a/events/init_test.go b/events/init_test.go index 58e3635..96089c0 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -17,7 +17,7 @@ func (o *ObserverMock) EventConnectedToDC(evt mtglib.EventConnectedToDC) { o.Called(evt) } -func (o *ObserverMock) EventTraffic(evt mtglib.EventTraffic) { +func (o *ObserverMock) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { o.Called(evt) } diff --git a/events/multi_observer.go b/events/multi_observer.go index 1dec489..292ab2c 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -40,7 +40,7 @@ func (m multiObserver) EventConnectedToDC(evt mtglib.EventConnectedToDC) { wg.Wait() } -func (m multiObserver) EventTraffic(evt mtglib.EventTraffic) { +func (m multiObserver) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { wg := &sync.WaitGroup{} wg.Add(len(m.observers)) @@ -48,7 +48,7 @@ func (m multiObserver) EventTraffic(evt mtglib.EventTraffic) { go func(obs Observer) { defer wg.Done() - obs.EventTraffic(evt) + obs.EventTelegramTraffic(evt) }(v) } diff --git a/events/noop.go b/events/noop.go index 8dcbf57..9f37062 100644 --- a/events/noop.go +++ b/events/noop.go @@ -19,7 +19,7 @@ type noopObserver struct{} func (n noopObserver) EventStart(_ mtglib.EventStart) {} func (n noopObserver) EventConnectedToDC(_ mtglib.EventConnectedToDC) {} -func (n noopObserver) EventTraffic(_ mtglib.EventTraffic) {} +func (n noopObserver) EventTelegramTraffic(_ mtglib.EventTelegramTraffic) {} func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} diff --git a/events/noop_test.go b/events/noop_test.go index 2621e38..6c9b768 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -31,7 +31,7 @@ func (suite *NoopTestSuite) SetupSuite() { RemoteIP: net.ParseIP("127.1.0.1"), DC: 2, }, - "traffic": mtglib.EventTraffic{ + "telegram-traffic": mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 1000, diff --git a/mtglib/conns.go b/mtglib/conns.go index 07b7086..3575957 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -21,7 +21,7 @@ func (c connTelegramTraffic) Read(b []byte) (int, error) { n, err := c.Conn.Read(b) if n > 0 { - c.stream.Send(c.ctx, EventTraffic{ + c.stream.Send(c.ctx, EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: c.connID, Traffic: uint(n), @@ -36,7 +36,7 @@ func (c connTelegramTraffic) Write(b []byte) (int, error) { n, err := c.Conn.Write(b) if n > 0 { - c.stream.Send(c.ctx, EventTraffic{ + c.stream.Send(c.ctx, EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: c.connID, Traffic: uint(n), diff --git a/mtglib/events.go b/mtglib/events.go index 0e49e48..959ffea 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -15,6 +15,10 @@ func (e EventStart) StreamID() string { return e.ConnID } +func (e EventStart) Timestamp() time.Time { + return e.CreatedAt +} + type EventConnectedToDC struct { CreatedAt time.Time ConnID string @@ -26,17 +30,25 @@ func (e EventConnectedToDC) StreamID() string { return e.ConnID } -type EventTraffic struct { +func (e EventConnectedToDC) Timestamp() time.Time { + return e.CreatedAt +} + +type EventTelegramTraffic struct { CreatedAt time.Time ConnID string Traffic uint IsRead bool } -func (e EventTraffic) StreamID() string { +func (e EventTelegramTraffic) StreamID() string { return e.ConnID } +func (e EventTelegramTraffic) Timestamp() time.Time { + return e.CreatedAt +} + type EventFinish struct { CreatedAt time.Time ConnID string @@ -46,6 +58,10 @@ func (e EventFinish) StreamID() string { return e.ConnID } +func (e EventFinish) Timestamp() time.Time { + return e.CreatedAt +} + type EventConcurrencyLimited struct { CreatedAt time.Time } @@ -54,6 +70,10 @@ func (e EventConcurrencyLimited) StreamID() string { return "" } +func (e EventConcurrencyLimited) Timestamp() time.Time { + return e.CreatedAt +} + type EventIPBlocklisted struct { CreatedAt time.Time RemoteIP net.IP @@ -62,3 +82,7 @@ type EventIPBlocklisted struct { func (e EventIPBlocklisted) StreamID() string { return "" } + +func (e EventIPBlocklisted) Timestamp() time.Time { + return e.CreatedAt +} diff --git a/mtglib/events_test.go b/mtglib/events_test.go index 133fce3..d0745aa 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -21,6 +21,7 @@ func (suite *EventsTestSuite) TestEventStart() { } suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventFinish() { @@ -30,6 +31,7 @@ func (suite *EventsTestSuite) TestEventFinish() { } suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventConnectedToDC() { @@ -41,10 +43,11 @@ func (suite *EventsTestSuite) TestEventConnectedToDC() { } suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } -func (suite *EventsTestSuite) TestEventTraffic() { - evt := mtglib.EventTraffic{ +func (suite *EventsTestSuite) TestEventTelegramTraffic() { + evt := mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "CONNID", Traffic: 3, @@ -52,14 +55,26 @@ func (suite *EventsTestSuite) TestEventTraffic() { } suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventConcurrencyLimited() { - suite.Empty(mtglib.EventConcurrencyLimited{}.StreamID()) + evt := mtglib.EventConcurrencyLimited{ + CreatedAt: time.Now(), + } + + suite.Empty(evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventIPBlocklisted() { - suite.Empty(mtglib.EventIPBlocklisted{}.StreamID()) + evt := mtglib.EventIPBlocklisted{ + CreatedAt: time.Now(), + RemoteIP: net.ParseIP("10.0.0.10"), + } + + suite.Empty(evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func TestEvents(t *testing.T) { diff --git a/mtglib/init.go b/mtglib/init.go index 44da825..0660e9e 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -45,6 +45,7 @@ type IPBlocklist interface { type Event interface { StreamID() string + Timestamp() time.Time } type EventStream interface { diff --git a/mtglib/proxy.go b/mtglib/proxy.go index b6a8f12..50b038c 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -95,6 +95,9 @@ func (p *Proxy) Serve(listener net.Listener) error { if addr := conn.RemoteAddr().(*net.TCPAddr).IP; p.ipBlocklist.Contains(addr) { conn.Close() + p.logger. + BindStr("ip", conn.RemoteAddr().(*net.TCPAddr).IP.String()). + Info("ip was blacklisted") p.eventStream.Send(p.ctx, EventIPBlocklisted{ CreatedAt: time.Now(), RemoteIP: addr, @@ -110,6 +113,9 @@ func (p *Proxy) Serve(listener net.Listener) error { case errors.Is(err, ants.ErrPoolClosed): return nil case errors.Is(err, ants.ErrPoolOverload): + p.logger. + BindStr("ip", conn.RemoteAddr().(*net.TCPAddr).IP.String()). + Info("connection was concurrency limited") p.eventStream.Send(p.ctx, EventConcurrencyLimited{ CreatedAt: time.Now(), }) diff --git a/stats/init.go b/stats/init.go index 4bf1aa8..ddeea9d 100644 --- a/stats/init.go +++ b/stats/init.go @@ -6,21 +6,27 @@ const ( DefaultStatsdMetricPrefix = DefaultMetricPrefix + "." DefaultStatsdTagFormat = "datadog" - MetricClientConnections = "client_connections" - MetricTelegramConnections = "telegram_connections" - MetricTraffic = "traffic" - MetricSessionDuration = "session_duration" - MetricSessionTraffic = "session_traffic" - MetricConcurrencyLimited = "concurrency_limited" - MetricIPBlocklisted = "ip_blocklisted" + MetricClientConnections = "client_connections" + MetricTelegramConnections = "telegram_connections" + MetricDomainDisguisingConnections = "domain_disguising_connections" - TagIPType = "ip_type" - TagTelegramIP = "ip" - TagDC = "dc" - TagDirection = "direction" + MetricTelegramTraffic = "telegram_traffic" + MetricDomainDisguisingTraffic = "domain_disguising_traffic" - TagIPTypeIPv4 = "ipv4" - TagIPTypeIPv6 = "ipv6" - TagDirectionTelegram = "telegram" - TagDirectionClient = "client" + MetricDomainDisguising = "domain_disguising" + MetricConcurrencyLimited = "concurrency_limited" + MetricIPBlocklisted = "ip_blocklisted" + MetricReplayAttacks = "replay_attacks" + + TagIPFamily = "ip_family" + TagIPFamilyIPv4 = "ipv4" + TagIPFamilyIPv6 = "ipv6" + + TagTelegramIP = "telegram_ip" + + TagDC = "dc" + + TagDirection = "direction" + TagDirectionToClient = "to_client" + TagDirectionFromClient = "from_client" ) diff --git a/stats/pools.go b/stats/pools.go new file mode 100644 index 0000000..aaa6c95 --- /dev/null +++ b/stats/pools.go @@ -0,0 +1,20 @@ +package stats + +import "sync" + +var streamInfoPool = sync.Pool{ + New: func() interface{} { + return &streamInfo{ + tags: map[string]string{}, + } + }, +} + +func acquireStreamInfo() *streamInfo { + return streamInfoPool.Get().(*streamInfo) +} + +func releaseStreamInfo(info *streamInfo) { + info.Reset() + streamInfoPool.Put(info) +} diff --git a/stats/prometheus.go b/stats/prometheus.go index 9069bbf..e002934 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -4,8 +4,6 @@ import ( "context" "net" "net/http" - "strconv" - "time" "github.com/9seconds/mtg/v2/events" "github.com/9seconds/mtg/v2/mtglib" @@ -19,89 +17,61 @@ type prometheusProcessor struct { } func (p prometheusProcessor) EventStart(evt mtglib.EventStart) { - sInfo := &streamInfo{ - createdAt: evt.CreatedAt, - clientIP: evt.RemoteIP, - } - p.streams[evt.StreamID()] = sInfo + info := acquireStreamInfo() + info.SetStartTime(evt.CreatedAt) + info.SetClientIP(evt.RemoteIP) + p.streams[evt.StreamID()] = info - p.factory.metricClientConnections.WithLabelValues(sInfo.GetClientIPType()).Inc() + p.factory.metricClientConnections. + WithLabelValues(info.V(TagIPFamily)). + Inc() } func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { - sInfo, ok := p.streams[evt.StreamID()] + info, ok := p.streams[evt.StreamID()] if !ok { return } - sInfo.remoteIP = evt.RemoteIP - sInfo.dc = evt.DC + info.SetTelegramIP(evt.RemoteIP) + info.SetDC(evt.DC) - p.factory.metricTelegramConnections.WithLabelValues( - sInfo.GetRemoteIPType(), - sInfo.remoteIP.String(), - strconv.Itoa(sInfo.dc)).Inc() + p.factory.metricTelegramConnections. + WithLabelValues(info.V(TagTelegramIP), info.V(TagDC)). + Inc() } -func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) { - sInfo, ok := p.streams[evt.StreamID()] +func (p prometheusProcessor) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { + info, ok := p.streams[evt.StreamID()] if !ok { return } - labels := []string{ - sInfo.GetRemoteIPType(), - sInfo.remoteIP.String(), - strconv.Itoa(sInfo.dc), - } - - if evt.IsRead { - sInfo.bytesRecvFromTelegram += evt.Traffic - - labels = append(labels, TagDirectionClient) - } else { - sInfo.bytesSentToTelegram += evt.Traffic - - labels = append(labels, TagDirectionTelegram) - } - - p.factory.metricTraffic.WithLabelValues(labels...).Add(float64(evt.Traffic)) + p.factory.metricTelegramTraffic. + WithLabelValues(info.V(TagTelegramIP), info.V(TagDC), getDirection(evt.IsRead)). + Add(float64(evt.Traffic)) } func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { - sInfo, ok := p.streams[evt.StreamID()] + info, ok := p.streams[evt.StreamID()] if !ok { return } - defer delete(p.streams, evt.StreamID()) + defer func() { + delete(p.streams, evt.StreamID()) + releaseStreamInfo(info) + }() - duration := evt.CreatedAt.Sub(sInfo.createdAt) + p.factory.metricClientConnections. + WithLabelValues(info.V(TagIPFamily)). + Dec() - p.factory.metricClientConnections.WithLabelValues(sInfo.GetClientIPType()).Dec() - p.factory.metricSessionDuration.Observe(float64(duration) / float64(time.Second)) - - if sInfo.remoteIP == nil { - return + if info.V(TagTelegramIP) != "" { + p.factory.metricTelegramConnections. + WithLabelValues(info.V(TagTelegramIP), info.V(TagDC)). + Dec() } - - labels := []string{ - sInfo.GetRemoteIPType(), - sInfo.remoteIP.String(), - strconv.Itoa(sInfo.dc), - } - - p.factory.metricTelegramConnections.WithLabelValues(labels...).Dec() - - labels = append(labels, TagDirectionClient) - p.factory.metricSessionTraffic. - WithLabelValues(labels...). - Observe(float64(sInfo.bytesRecvFromTelegram)) - - labels[3] = TagDirectionTelegram - p.factory.metricSessionTraffic. - WithLabelValues(labels...). - Observe(float64(sInfo.bytesSentToTelegram)) } func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) { @@ -109,11 +79,7 @@ func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLi } func (p prometheusProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { - if evt.RemoteIP.To4() == nil { - p.factory.metricIPBlocklisted.WithLabelValues(TagIPTypeIPv6).Inc() - } else { - p.factory.metricIPBlocklisted.WithLabelValues(TagIPTypeIPv4).Inc() - } + p.factory.metricIPBlocklisted.Inc() } func (p prometheusProcessor) Shutdown() { @@ -123,13 +89,17 @@ func (p prometheusProcessor) Shutdown() { type PrometheusFactory struct { httpServer *http.Server - metricClientConnections *prometheus.GaugeVec - metricTelegramConnections *prometheus.GaugeVec - metricTraffic *prometheus.CounterVec - metricIPBlocklisted *prometheus.CounterVec - metricSessionTraffic *prometheus.HistogramVec - metricConcurrencyLimited prometheus.Counter - metricSessionDuration prometheus.Histogram + metricClientConnections *prometheus.GaugeVec + metricTelegramConnections *prometheus.GaugeVec + metricDomainDisguisingConnections *prometheus.GaugeVec + + metricTelegramTraffic *prometheus.CounterVec + metricDomainDisguisingTraffic *prometheus.CounterVec + + metricDomainDisguising prometheus.Counter + metricConcurrencyLimited prometheus.Counter + metricIPBlocklisted prometheus.Counter + metricReplayAttacks prometheus.Counter } func (p *PrometheusFactory) Make() events.Observer { @@ -164,70 +134,63 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint metricClientConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: metricPrefix, Name: MetricClientConnections, - Help: "A number of connections under active processing.", - }, []string{TagIPType}), + Help: "A number of actively processing client connections.", + }, []string{TagIPFamily}), metricTelegramConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: metricPrefix, Name: MetricTelegramConnections, Help: "A number of connections to Telegram servers.", - }, []string{TagIPType, TagTelegramIP, TagDC}), - metricSessionDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + }, []string{TagTelegramIP, TagDC}), + metricDomainDisguisingConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: metricPrefix, - Name: MetricSessionDuration, - Help: "Session duration.", - Buckets: []float64{ // per 30 seconds - 30, - 60, - 90, - 120, - 150, - 180, - 210, - 240, - 270, - 300, - }, + Name: MetricDomainDisguisingConnections, + Help: "A number of connections which talk with disguising domain.", + }, []string{TagIPFamily}), + + metricTelegramTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricTelegramTraffic, + Help: "Traffic which is generated talking with Telegram servers.", + }, []string{TagTelegramIP, TagDC, TagDirection}), + metricDomainDisguisingTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricDomainDisguisingTraffic, + Help: "Traffic which is generated talking with disguising domain.", + }, []string{TagDirection}), + + metricDomainDisguising: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricDomainDisguising, + Help: "A number of routings to disguising domain.", }), - metricSessionTraffic: prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: metricPrefix, - Name: MetricSessionTraffic, - Help: "A traffic size which flew via proxy within a single session.", - Buckets: []float64{ // per 1mb - 1 * 1024 * 1024, - 2 * 1024 * 1024, - 3 * 1024 * 1024, - 4 * 1024 * 1024, - 5 * 1024 * 1024, - 6 * 1024 * 1024, - 7 * 1024 * 1024, - 8 * 1024 * 1024, - 9 * 1024 * 1024, - }, - }, []string{TagIPType, TagTelegramIP, TagDC, TagDirection}), - metricTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: metricPrefix, - Name: MetricTraffic, - Help: "Traffic which is sent through this proxy.", - }, []string{TagIPType, TagTelegramIP, TagDC, TagDirection}), metricConcurrencyLimited: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metricPrefix, Name: MetricConcurrencyLimited, Help: "A number of sessions that were rejected by concurrency limiter.", }), - metricIPBlocklisted: prometheus.NewCounterVec(prometheus.CounterOpts{ + metricIPBlocklisted: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metricPrefix, Name: MetricIPBlocklisted, - Help: "A number of rejected sessions due to ip blocklisting", - }, []string{TagIPType}), + Help: "A number of rejected sessions due to ip blocklisting.", + }), + metricReplayAttacks: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: metricPrefix, + Name: MetricReplayAttacks, + Help: "A number of detected replay attacks.", + }), } registry.MustRegister(factory.metricClientConnections) registry.MustRegister(factory.metricTelegramConnections) - registry.MustRegister(factory.metricTraffic) - registry.MustRegister(factory.metricSessionTraffic) - registry.MustRegister(factory.metricSessionDuration) + registry.MustRegister(factory.metricDomainDisguisingConnections) + + registry.MustRegister(factory.metricTelegramTraffic) + registry.MustRegister(factory.metricDomainDisguisingTraffic) + + registry.MustRegister(factory.metricDomainDisguising) registry.MustRegister(factory.metricConcurrencyLimited) registry.MustRegister(factory.metricIPBlocklisted) + registry.MustRegister(factory.metricReplayAttacks) return factory } diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index c1425b5..d3db5a0 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -64,7 +64,7 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { data, err := suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_client_connections{ip_type="ipv4"} 1`) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 1`) suite.prometheus.EventConnectedToDC(mtglib.EventConnectedToDC{ CreatedAt: time.Now(), @@ -76,9 +76,9 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { data, err = suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_telegram_connections{dc="4",ip="10.0.0.1",ip_type="ipv4"} 1`) + suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 1`) - suite.prometheus.EventTraffic(mtglib.EventTraffic{ + suite.prometheus.EventTelegramTraffic(mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 200, @@ -88,9 +88,9 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { data, err = suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_traffic{dc="4",direction="client",ip="10.0.0.1",ip_type="ipv4"} 200`) + suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="to_client",telegram_ip="10.0.0.1"} 200`) - suite.prometheus.EventTraffic(mtglib.EventTraffic{ + suite.prometheus.EventTelegramTraffic(mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 100, @@ -100,7 +100,7 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { data, err = suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_traffic{dc="4",direction="telegram",ip="10.0.0.1",ip_type="ipv4"} 100`) + suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="from_client",telegram_ip="10.0.0.1"} 100`) suite.prometheus.EventFinish(mtglib.EventFinish{ CreatedAt: time.Now(), @@ -110,10 +110,8 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { data, err = suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_client_connections{ip_type="ipv4"} 0`) - suite.Contains(data, `mtg_telegram_connections{dc="4",ip="10.0.0.1",ip_type="ipv4"} 0`) - suite.Contains(data, `mtg_traffic{dc="4",direction="client",ip="10.0.0.1",ip_type="ipv4"} 200`) - suite.Contains(data, `mtg_traffic{dc="4",direction="telegram",ip="10.0.0.1",ip_type="ipv4"} 100`) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 0`) + suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 0`) } func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { @@ -138,7 +136,7 @@ func (suite *PrometheusTestSuite) TestEventIPBlocklisted() { data, err := suite.Get() suite.NoError(err) - suite.Contains(data, `mtg_ip_blocklisted{ip_type="ipv6"} 1`) + suite.Contains(data, `mtg_ip_blocklisted 1`) } func TestPrometheus(t *testing.T) { diff --git a/stats/statsd.go b/stats/statsd.go index beea0f8..61e3ced 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -17,74 +17,64 @@ type statsdProcessor struct { } func (s statsdProcessor) EventStart(evt mtglib.EventStart) { - sInfo := &streamInfo{ - createdAt: evt.CreatedAt, - clientIP: evt.RemoteIP, - } - s.streams[evt.StreamID()] = sInfo + info := acquireStreamInfo() + info.SetStartTime(evt.CreatedAt) + info.SetClientIP(evt.RemoteIP) + s.streams[evt.StreamID()] = info s.client.GaugeDelta(MetricClientConnections, 1, - statsd.StringTag(TagIPType, sInfo.GetClientIPType())) + info.TV(TagIPFamily)) } func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { - sInfo, ok := s.streams[evt.StreamID()] + info, ok := s.streams[evt.StreamID()] if !ok { return } - sInfo.remoteIP = evt.RemoteIP - sInfo.dc = evt.DC + info.SetTelegramIP(evt.RemoteIP) + info.SetDC(evt.DC) s.client.GaugeDelta(MetricTelegramConnections, 1, - statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()), - statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()), - statsd.IntTag(TagDC, sInfo.dc)) + info.TV(TagTelegramIP), + info.TV(TagDC)) } -func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { - sInfo, ok := s.streams[evt.StreamID()] +func (s statsdProcessor) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { + info, ok := s.streams[evt.StreamID()] if !ok { return } - tags := []statsd.Tag{ - statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()), - statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()), - statsd.IntTag(TagDC, sInfo.dc), - } - - if evt.IsRead { - tags = append(tags, statsd.StringTag(TagDirection, TagDirectionClient)) - s.client.Incr(MetricTraffic, int64(evt.Traffic), tags...) - } else { - tags = append(tags, statsd.StringTag(TagDirection, TagDirectionTelegram)) - s.client.Incr(MetricTraffic, int64(evt.Traffic), tags...) - } + s.client.Incr(MetricTelegramTraffic, + int64(evt.Traffic), + info.TV(TagTelegramIP), + info.TV(TagDC), + statsd.StringTag(TagDirection, getDirection(evt.IsRead))) } func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { - sInfo, ok := s.streams[evt.StreamID()] + info, ok := s.streams[evt.StreamID()] if !ok { return } - defer delete(s.streams, evt.StreamID()) + defer func() { + delete(s.streams, evt.StreamID()) + releaseStreamInfo(info) + }() s.client.GaugeDelta(MetricClientConnections, -1, - statsd.StringTag(TagIPType, sInfo.GetClientIPType())) - s.client.PrecisionTiming(MetricSessionDuration, - evt.CreatedAt.Sub(sInfo.createdAt)) + info.TV(TagIPFamily)) - if sInfo.remoteIP != nil { + if info.V(TagTelegramIP) != "" { s.client.GaugeDelta(MetricTelegramConnections, -1, - statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()), - statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()), - statsd.IntTag(TagDC, sInfo.dc)) + info.TV(TagTelegramIP), + info.TV(TagDC)) } } @@ -93,15 +83,7 @@ func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimite } func (s statsdProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { - var tag statsd.Tag - - if evt.RemoteIP.To4() == nil { - tag = statsd.StringTag(TagIPType, TagIPTypeIPv6) - } else { - tag = statsd.StringTag(TagIPType, TagIPTypeIPv4) - } - - s.client.Incr(MetricIPBlocklisted, 1, tag) + s.client.Incr(MetricIPBlocklisted, 1) } func (s statsdProcessor) Shutdown() { diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 9129755..217ca5d 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -111,7 +111,7 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { RemoteIP: net.ParseIP("10.0.0.10"), }) time.Sleep(statsdSleepTime) - suite.Equal("mtg.client_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String()) + suite.Equal("mtg.client_connections:+1|g|#ip_family:ipv4", suite.statsdServer.String()) suite.statsd.EventConnectedToDC(mtglib.EventConnectedToDC{ CreatedAt: time.Now(), @@ -121,9 +121,9 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { }) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), - "mtg.telegram_connections:+1|g|#ip_type:ipv4,ip:10.1.0.10,dc:2") + "mtg.telegram_connections:+1|g|#telegram_ip:10.1.0.10,dc:2") - suite.statsd.EventTraffic(mtglib.EventTraffic{ + suite.statsd.EventTelegramTraffic(mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 30, @@ -131,9 +131,9 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { }) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), - "mtg.traffic:30|c|#ip_type:ipv4,ip:10.1.0.10,dc:2,direction:client") + "mtg.telegram_traffic:30|c|#telegram_ip:10.1.0.10,dc:2,direction:to_client") - suite.statsd.EventTraffic(mtglib.EventTraffic{ + suite.statsd.EventTelegramTraffic(mtglib.EventTelegramTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 90, @@ -141,18 +141,17 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { }) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), - "mtg.traffic:90|c|#ip_type:ipv4,ip:10.1.0.10,dc:2,direction:telegram") + "mtg.telegram_traffic:90|c|#telegram_ip:10.1.0.10,dc:2,direction:from_client") suite.statsd.EventFinish(mtglib.EventFinish{ CreatedAt: time.Now(), ConnID: "connID", }) time.Sleep(statsdSleepTime) - suite.Contains(suite.statsdServer.String(), "mtg.session_duration") suite.Contains(suite.statsdServer.String(), - "mtg.telegram_connections:-1|g|#ip_type:ipv4,ip:10.1.0.10,dc:2") + "mtg.telegram_connections:-1|g|#telegram_ip:10.1.0.10,dc:2") suite.Contains(suite.statsdServer.String(), - "mtg.client_connections:-1|g|#ip_type:ipv4") + "mtg.client_connections:-1|g|#ip_family:ipv4") } func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { @@ -171,7 +170,7 @@ func (suite *StatsdTestSuite) TestEventIPBlocklisted() { }) time.Sleep(statsdSleepTime) - suite.Equal("mtg.ip_blocklisted:1|c|#ip_type:ipv4", suite.statsdServer.String()) + suite.Equal("mtg.ip_blocklisted:1|c", suite.statsdServer.String()) } func TestStatsd(t *testing.T) { diff --git a/stats/stream_info.go b/stats/stream_info.go index 2827792..0aeb904 100644 --- a/stats/stream_info.go +++ b/stats/stream_info.go @@ -2,30 +2,57 @@ package stats import ( "net" + "strconv" "time" + + statsd "github.com/smira/go-statsd" ) type streamInfo struct { - createdAt time.Time - clientIP net.IP - remoteIP net.IP - dc int - bytesSentToTelegram uint - bytesRecvFromTelegram uint + startTime time.Time + tags map[string]string } -func (s *streamInfo) GetClientIPType() string { - return s.getIPType(s.clientIP) +func (s *streamInfo) SetStartTime(tme time.Time) { + s.startTime = tme } -func (s *streamInfo) GetRemoteIPType() string { - return s.getIPType(s.remoteIP) +func (s *streamInfo) SetClientIP(ip net.IP) { + if ip.To4() != nil { + s.tags[TagIPFamily] = TagIPFamilyIPv4 + } else { + s.tags[TagIPFamily] = TagIPFamilyIPv6 + } } -func (s *streamInfo) getIPType(ip net.IP) string { - if ip.To4() == nil { - return TagIPTypeIPv6 +func (s *streamInfo) SetTelegramIP(ip net.IP) { + s.tags[TagTelegramIP] = ip.String() +} + +func (s *streamInfo) SetDC(dc int) { + s.tags[TagDC] = strconv.Itoa(dc) +} + +func (s *streamInfo) V(key string) string { + return s.tags[key] +} + +func (s *streamInfo) TV(key string) statsd.Tag { + return statsd.StringTag(key, s.tags[key]) +} + +func (s *streamInfo) Reset() { + s.startTime = time.Time{} + + for k := range s.tags { + delete(s.tags, k) + } +} + +func getDirection(isRead bool) string { + if isRead { // for telegram + return TagDirectionToClient } - return TagIPTypeIPv4 + return TagDirectionFromClient } From db8614999a1f4026721807581916d0817e1db2da Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 28 Mar 2021 21:43:17 +0300 Subject: [PATCH 137/206] Rename EventTelegramTraffic back to EventTraffic --- events/event_stream.go | 4 ++-- events/event_stream_test.go | 8 ++++---- events/init.go | 2 +- events/init_test.go | 2 +- events/multi_observer.go | 4 ++-- events/noop.go | 2 +- events/noop_test.go | 2 +- mtglib/conns.go | 4 ++-- mtglib/events.go | 6 +++--- mtglib/events_test.go | 4 ++-- stats/prometheus.go | 2 +- stats/prometheus_test.go | 4 ++-- stats/statsd.go | 2 +- stats/statsd_test.go | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/events/event_stream.go b/events/event_stream.go index c8fe758..1726c81 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -73,8 +73,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob observer.EventStart(typedEvt) case mtglib.EventConnectedToDC: observer.EventConnectedToDC(typedEvt) - case mtglib.EventTelegramTraffic: - observer.EventTelegramTraffic(typedEvt) + case mtglib.EventTraffic: + observer.EventTraffic(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) case mtglib.EventIPBlocklisted: diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 8daa939..90dee03 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -90,8 +90,8 @@ func (suite *EventStreamTestSuite) TestEventConnectedToDC() { time.Sleep(100 * time.Millisecond) } -func (suite *EventStreamTestSuite) TestEventTelegramTraffic() { - evt := mtglib.EventTelegramTraffic{ +func (suite *EventStreamTestSuite) TestEventTraffic() { + evt := mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 1024, @@ -100,10 +100,10 @@ func (suite *EventStreamTestSuite) TestEventTelegramTraffic() { for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. - On("EventTelegramTraffic", mock.Anything). + On("EventTraffic", mock.Anything). Once(). Run(func(args mock.Arguments) { - caught := args.Get(0).(mtglib.EventTelegramTraffic) + caught := args.Get(0).(mtglib.EventTraffic) suite.Equal(evt.CreatedAt, caught.CreatedAt) suite.Equal(evt.ConnID, caught.ConnID) diff --git a/events/init.go b/events/init.go index fbfe330..4720a3b 100644 --- a/events/init.go +++ b/events/init.go @@ -6,7 +6,7 @@ type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) EventConnectedToDC(mtglib.EventConnectedToDC) - EventTelegramTraffic(mtglib.EventTelegramTraffic) + EventTraffic(mtglib.EventTraffic) EventConcurrencyLimited(mtglib.EventConcurrencyLimited) EventIPBlocklisted(mtglib.EventIPBlocklisted) diff --git a/events/init_test.go b/events/init_test.go index 96089c0..58e3635 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -17,7 +17,7 @@ func (o *ObserverMock) EventConnectedToDC(evt mtglib.EventConnectedToDC) { o.Called(evt) } -func (o *ObserverMock) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { +func (o *ObserverMock) EventTraffic(evt mtglib.EventTraffic) { o.Called(evt) } diff --git a/events/multi_observer.go b/events/multi_observer.go index 292ab2c..1dec489 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -40,7 +40,7 @@ func (m multiObserver) EventConnectedToDC(evt mtglib.EventConnectedToDC) { wg.Wait() } -func (m multiObserver) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { +func (m multiObserver) EventTraffic(evt mtglib.EventTraffic) { wg := &sync.WaitGroup{} wg.Add(len(m.observers)) @@ -48,7 +48,7 @@ func (m multiObserver) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { go func(obs Observer) { defer wg.Done() - obs.EventTelegramTraffic(evt) + obs.EventTraffic(evt) }(v) } diff --git a/events/noop.go b/events/noop.go index 9f37062..8dcbf57 100644 --- a/events/noop.go +++ b/events/noop.go @@ -19,7 +19,7 @@ type noopObserver struct{} func (n noopObserver) EventStart(_ mtglib.EventStart) {} func (n noopObserver) EventConnectedToDC(_ mtglib.EventConnectedToDC) {} -func (n noopObserver) EventTelegramTraffic(_ mtglib.EventTelegramTraffic) {} +func (n noopObserver) EventTraffic(_ mtglib.EventTraffic) {} func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} diff --git a/events/noop_test.go b/events/noop_test.go index 6c9b768..2621e38 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -31,7 +31,7 @@ func (suite *NoopTestSuite) SetupSuite() { RemoteIP: net.ParseIP("127.1.0.1"), DC: 2, }, - "telegram-traffic": mtglib.EventTelegramTraffic{ + "traffic": mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 1000, diff --git a/mtglib/conns.go b/mtglib/conns.go index 3575957..07b7086 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -21,7 +21,7 @@ func (c connTelegramTraffic) Read(b []byte) (int, error) { n, err := c.Conn.Read(b) if n > 0 { - c.stream.Send(c.ctx, EventTelegramTraffic{ + c.stream.Send(c.ctx, EventTraffic{ CreatedAt: time.Now(), ConnID: c.connID, Traffic: uint(n), @@ -36,7 +36,7 @@ func (c connTelegramTraffic) Write(b []byte) (int, error) { n, err := c.Conn.Write(b) if n > 0 { - c.stream.Send(c.ctx, EventTelegramTraffic{ + c.stream.Send(c.ctx, EventTraffic{ CreatedAt: time.Now(), ConnID: c.connID, Traffic: uint(n), diff --git a/mtglib/events.go b/mtglib/events.go index 959ffea..3a19b5f 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -34,18 +34,18 @@ func (e EventConnectedToDC) Timestamp() time.Time { return e.CreatedAt } -type EventTelegramTraffic struct { +type EventTraffic struct { CreatedAt time.Time ConnID string Traffic uint IsRead bool } -func (e EventTelegramTraffic) StreamID() string { +func (e EventTraffic) StreamID() string { return e.ConnID } -func (e EventTelegramTraffic) Timestamp() time.Time { +func (e EventTraffic) Timestamp() time.Time { return e.CreatedAt } diff --git a/mtglib/events_test.go b/mtglib/events_test.go index d0745aa..5597fae 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -46,8 +46,8 @@ func (suite *EventsTestSuite) TestEventConnectedToDC() { suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } -func (suite *EventsTestSuite) TestEventTelegramTraffic() { - evt := mtglib.EventTelegramTraffic{ +func (suite *EventsTestSuite) TestEventTraffic() { + evt := mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "CONNID", Traffic: 3, diff --git a/stats/prometheus.go b/stats/prometheus.go index e002934..89b12ad 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -41,7 +41,7 @@ func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { Inc() } -func (p prometheusProcessor) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { +func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) { info, ok := p.streams[evt.StreamID()] if !ok { return diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index d3db5a0..b4a2335 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -78,7 +78,7 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { suite.NoError(err) suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 1`) - suite.prometheus.EventTelegramTraffic(mtglib.EventTelegramTraffic{ + suite.prometheus.EventTraffic(mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 200, @@ -90,7 +90,7 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { suite.NoError(err) suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="to_client",telegram_ip="10.0.0.1"} 200`) - suite.prometheus.EventTelegramTraffic(mtglib.EventTelegramTraffic{ + suite.prometheus.EventTraffic(mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 100, diff --git a/stats/statsd.go b/stats/statsd.go index 61e3ced..b621c73 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -42,7 +42,7 @@ func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { info.TV(TagDC)) } -func (s statsdProcessor) EventTelegramTraffic(evt mtglib.EventTelegramTraffic) { +func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { info, ok := s.streams[evt.StreamID()] if !ok { return diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 217ca5d..5141eca 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -123,7 +123,7 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { suite.Contains(suite.statsdServer.String(), "mtg.telegram_connections:+1|g|#telegram_ip:10.1.0.10,dc:2") - suite.statsd.EventTelegramTraffic(mtglib.EventTelegramTraffic{ + suite.statsd.EventTraffic(mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 30, @@ -133,7 +133,7 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { suite.Contains(suite.statsdServer.String(), "mtg.telegram_traffic:30|c|#telegram_ip:10.1.0.10,dc:2,direction:to_client") - suite.statsd.EventTelegramTraffic(mtglib.EventTelegramTraffic{ + suite.statsd.EventTraffic(mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", Traffic: 90, From 39920545600c5807a4fc02104a3bdb2a4084f8de Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 10:26:52 +0300 Subject: [PATCH 138/206] Correctly manage partial writes --- mtglib/internal/faketls/record/pools.go | 31 +++++++++++++++---- mtglib/internal/faketls/record/record.go | 26 +++++++--------- .../obfuscated2/client_handshake_test.go | 2 +- mtglib/internal/obfuscated2/conn.go | 19 +++++++----- mtglib/internal/obfuscated2/pools.go | 27 +++++++++++++--- .../obfuscated2/server_handshake_test.go | 4 +-- mtglib/proxy.go | 4 +-- 7 files changed, 75 insertions(+), 38 deletions(-) diff --git a/mtglib/internal/faketls/record/pools.go b/mtglib/internal/faketls/record/pools.go index 62f03e9..50b0fec 100644 --- a/mtglib/internal/faketls/record/pools.go +++ b/mtglib/internal/faketls/record/pools.go @@ -1,12 +1,22 @@ package record -import "sync" +import ( + "bytes" + "sync" +) -var recordPool = sync.Pool{ - New: func() interface{} { - return &Record{} - }, -} +var ( + recordPool = sync.Pool{ + New: func() interface{} { + return &Record{} + }, + } + bytesBufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, + } +) func AcquireRecord() *Record { return recordPool.Get().(*Record) @@ -16,3 +26,12 @@ func ReleaseRecord(r *Record) { r.Reset() recordPool.Put(r) } + +func acquireBytesBuffer() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func releaseBytesBuffer(buf *bytes.Buffer) { + buf.Reset() + bytesBufferPool.Put(buf) +} diff --git a/mtglib/internal/faketls/record/record.go b/mtglib/internal/faketls/record/record.go index 31a1d9e..bce03b0 100644 --- a/mtglib/internal/faketls/record/record.go +++ b/mtglib/internal/faketls/record/record.go @@ -61,26 +61,22 @@ func (r *Record) Read(reader io.Reader) error { } func (r *Record) Dump(writer io.Writer) error { - buf := [2]byte{byte(r.Type), 0} + buf := acquireBytesBuffer() + defer releaseBytesBuffer(buf) - if _, err := writer.Write(buf[:1]); err != nil { - return fmt.Errorf("cannot dump type: %w", err) - } + bufSlice := [2]byte{byte(r.Type), 0} + buf.Write(bufSlice[:1]) - binary.BigEndian.PutUint16(buf[:], uint16(r.Version)) + binary.BigEndian.PutUint16(bufSlice[:], uint16(r.Version)) + buf.Write(bufSlice[:]) - if _, err := writer.Write(buf[:]); err != nil { - return fmt.Errorf("cannot dump version: %w", err) - } + binary.BigEndian.PutUint16(bufSlice[:], uint16(r.Payload.Len())) + buf.Write(bufSlice[:]) - binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len())) + buf.Write(r.Payload.Bytes()) - if _, err := writer.Write(buf[:]); err != nil { - return fmt.Errorf("cannot dump payload length: %w", err) - } - - if _, err := writer.Write(r.Payload.Bytes()); err != nil { - return fmt.Errorf("cannot dump payload: %w", err) + if _, err := buf.WriteTo(writer); err != nil { + return fmt.Errorf("cannot dump record: %w", err) } return nil diff --git a/mtglib/internal/obfuscated2/client_handshake_test.go b/mtglib/internal/obfuscated2/client_handshake_test.go index 3edbdba..ef7402a 100644 --- a/mtglib/internal/obfuscated2/client_handshake_test.go +++ b/mtglib/internal/obfuscated2/client_handshake_test.go @@ -58,7 +58,7 @@ func (suite *ClientHandshakeTestSuite) TestOk() { copy(writeData, arr) }) - conn := &obfuscated2.Conn{ + conn := obfuscated2.Conn{ Conn: connMock, Encryptor: encryptor, Decryptor: decryptor, diff --git a/mtglib/internal/obfuscated2/conn.go b/mtglib/internal/obfuscated2/conn.go index 24b5a81..c3a4c05 100644 --- a/mtglib/internal/obfuscated2/conn.go +++ b/mtglib/internal/obfuscated2/conn.go @@ -10,11 +10,9 @@ type Conn struct { Encryptor cipher.Stream Decryptor cipher.Stream - - writeBuf []byte } -func (c *Conn) Read(p []byte) (int, error) { +func (c Conn) Read(p []byte) (int, error) { n, err := c.Conn.Read(p) if err != nil { return n, err // nolint: wrapcheck @@ -25,9 +23,16 @@ func (c *Conn) Read(p []byte) (int, error) { return n, nil } -func (c *Conn) Write(p []byte) (int, error) { - c.writeBuf = append(c.writeBuf[:0], p...) - c.Encryptor.XORKeyStream(c.writeBuf, p) +func (c Conn) Write(p []byte) (int, error) { + buf := acquireBytesBuffer() + defer releaseBytesBuffer(buf) - return c.Conn.Write(c.writeBuf) + buf.Write(p) + + payload := buf.Bytes() + c.Encryptor.XORKeyStream(payload, payload) + + n, err := buf.WriteTo(c.Conn) + + return int(n), err // nolint: wrapcheck } diff --git a/mtglib/internal/obfuscated2/pools.go b/mtglib/internal/obfuscated2/pools.go index ce3204f..811c540 100644 --- a/mtglib/internal/obfuscated2/pools.go +++ b/mtglib/internal/obfuscated2/pools.go @@ -1,16 +1,24 @@ package obfuscated2 import ( + "bytes" "crypto/sha256" "hash" "sync" ) -var sha256HasherPool = sync.Pool{ - New: func() interface{} { - return sha256.New() - }, -} +var ( + sha256HasherPool = sync.Pool{ + New: func() interface{} { + return sha256.New() + }, + } + bytesBufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, + } +) func acquireSha256Hasher() hash.Hash { return sha256HasherPool.Get().(hash.Hash) @@ -20,3 +28,12 @@ func releaseSha256Hasher(h hash.Hash) { h.Reset() sha256HasherPool.Put(h) } + +func acquireBytesBuffer() *bytes.Buffer { + return bytesBufferPool.Get().(*bytes.Buffer) +} + +func releaseBytesBuffer(buf *bytes.Buffer) { + buf.Reset() + bytesBufferPool.Put(buf) +} diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go index 3638369..9279295 100644 --- a/mtglib/internal/obfuscated2/server_handshake_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -17,7 +17,7 @@ type ServerHandshakeTestSuite struct { suite.Suite connMock *testlib.NetConnMock - proxyConn *obfuscated2.Conn + proxyConn obfuscated2.Conn encryptor cipher.Stream decryptor cipher.Stream } @@ -29,7 +29,7 @@ func (suite *ServerHandshakeTestSuite) SetupTest() { encryptor, decryptor, err := obfuscated2.ServerHandshake(buf) suite.NoError(err) - suite.proxyConn = &obfuscated2.Conn{ + suite.proxyConn = obfuscated2.Conn{ Conn: suite.connMock, Encryptor: encryptor, Decryptor: decryptor, diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 50b038c..6baf4a2 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -192,7 +192,7 @@ func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { ctx.dc = dc ctx.logger = ctx.logger.BindInt("dc", dc) - ctx.clientConn = &obfuscated2.Conn{ + ctx.clientConn = obfuscated2.Conn{ Conn: ctx.clientConn, Encryptor: encryptor, Decryptor: decryptor, @@ -214,7 +214,7 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { return fmt.Errorf("cannot perform obfuscated2 handshake: %w", err) } - ctx.telegramConn = &obfuscated2.Conn{ + ctx.telegramConn = obfuscated2.Conn{ Conn: connTelegramTraffic{ Conn: conn, connID: ctx.connID, From 4c3f42e264662b28fd5f9d68338465db5819fff9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 10:37:09 +0300 Subject: [PATCH 139/206] Simplify relay --- mtglib/internal/faketls/conn.go | 2 +- mtglib/internal/relay/conn.go | 16 ++++++++++------ mtglib/internal/relay/relay.go | 17 ++++++++++++----- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/mtglib/internal/faketls/conn.go b/mtglib/internal/faketls/conn.go index ee560c3..06592ff 100644 --- a/mtglib/internal/faketls/conn.go +++ b/mtglib/internal/faketls/conn.go @@ -29,11 +29,11 @@ func (c *Conn) Read(p []byte) (int, error) { } switch rec.Type { // nolint: exhaustive - case record.TypeChangeCipherSpec: case record.TypeApplicationData: rec.Payload.WriteTo(&c.readBuffer) // nolint: errcheck return c.readBuffer.Read(p) + case record.TypeChangeCipherSpec: default: return 0, fmt.Errorf("unsupported record type %v", rec.Type) } diff --git a/mtglib/internal/relay/conn.go b/mtglib/internal/relay/conn.go index 8f4e89d..31f8bb2 100644 --- a/mtglib/internal/relay/conn.go +++ b/mtglib/internal/relay/conn.go @@ -1,19 +1,23 @@ package relay -import "io" +import ( + "context" + "io" +) type conn struct { io.ReadWriteCloser - relay *Relay + ctx context.Context + tickChannel chan struct{} } func (c conn) Read(p []byte) (int, error) { n, err := c.ReadWriteCloser.Read(p) select { - case <-c.relay.ctx.Done(): - case c.relay.tickChannel <- struct{}{}: + case <-c.ctx.Done(): + case c.tickChannel <- struct{}{}: } return n, err // nolint: wrapcheck @@ -23,8 +27,8 @@ func (c conn) Write(p []byte) (int, error) { n, err := c.ReadWriteCloser.Write(p) select { - case <-c.relay.ctx.Done(): - case c.relay.tickChannel <- struct{}{}: + case <-c.ctx.Done(): + case c.tickChannel <- struct{}{}: } return n, err // nolint: wrapcheck diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go index 2a27708..bbd92df 100644 --- a/mtglib/internal/relay/relay.go +++ b/mtglib/internal/relay/relay.go @@ -21,11 +21,13 @@ type Relay struct { func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { eastConn = conn{ ReadWriteCloser: eastConn, - relay: r, + ctx: r.ctx, + tickChannel: r.tickChannel, } westConn = conn{ ReadWriteCloser: westConn, - relay: r, + ctx: r.ctx, + tickChannel: r.tickChannel, } defer func() { @@ -37,7 +39,7 @@ func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { wg := &sync.WaitGroup{} wg.Add(3) // nolint: gomnd - go r.runObserver(r.ctx, wg) + go r.runObserver(wg) go r.transmit(eastConn, westConn, r.westBuffer, "west", wg) @@ -66,13 +68,18 @@ func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, select { case <-r.ctx.Done(): + err = r.ctx.Err() + default: + } + + select { case r.errorChannel <- err: default: } } } -func (r *Relay) runObserver(ctx context.Context, wg *sync.WaitGroup) { +func (r *Relay) runObserver(wg *sync.WaitGroup) { ticker := time.NewTicker(time.Second) defer func() { @@ -90,7 +97,7 @@ func (r *Relay) runObserver(ctx context.Context, wg *sync.WaitGroup) { for { select { - case <-ctx.Done(): + case <-r.ctx.Done(): return case <-r.tickChannel: lastTickAt = time.Now() From 36d695118ef2db840e936ac74bb9ad42fd23419e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 11:03:54 +0300 Subject: [PATCH 140/206] Rename metrics --- stats/init.go | 12 ++++---- stats/pools.go | 10 +++---- stats/prometheus.go | 68 ++++++++++++++++++++++++-------------------- stats/statsd.go | 36 +++++++++++++---------- stats/stream_info.go | 49 +++++-------------------------- 5 files changed, 75 insertions(+), 100 deletions(-) diff --git a/stats/init.go b/stats/init.go index ddeea9d..d9d64b9 100644 --- a/stats/init.go +++ b/stats/init.go @@ -6,14 +6,14 @@ const ( DefaultStatsdMetricPrefix = DefaultMetricPrefix + "." DefaultStatsdTagFormat = "datadog" - MetricClientConnections = "client_connections" - MetricTelegramConnections = "telegram_connections" - MetricDomainDisguisingConnections = "domain_disguising_connections" + MetricClientConnections = "client_connections" + MetricTelegramConnections = "telegram_connections" + MetricDomainFrontingConnections = "domain_fronting_connections" - MetricTelegramTraffic = "telegram_traffic" - MetricDomainDisguisingTraffic = "domain_disguising_traffic" + MetricTelegramTraffic = "telegram_traffic" + MetricDomainFrontingTraffic = "domain_fronting_traffic" - MetricDomainDisguising = "domain_disguising" + MetricDomainFronting = "domain_fronting" MetricConcurrencyLimited = "concurrency_limited" MetricIPBlocklisted = "ip_blocklisted" MetricReplayAttacks = "replay_attacks" diff --git a/stats/pools.go b/stats/pools.go index aaa6c95..ec0505c 100644 --- a/stats/pools.go +++ b/stats/pools.go @@ -4,17 +4,15 @@ import "sync" var streamInfoPool = sync.Pool{ New: func() interface{} { - return &streamInfo{ - tags: map[string]string{}, - } + return streamInfo{} }, } -func acquireStreamInfo() *streamInfo { - return streamInfoPool.Get().(*streamInfo) +func acquireStreamInfo() streamInfo { + return streamInfoPool.Get().(streamInfo) } -func releaseStreamInfo(info *streamInfo) { +func releaseStreamInfo(info streamInfo) { info.Reset() streamInfoPool.Put(info) } diff --git a/stats/prometheus.go b/stats/prometheus.go index 89b12ad..51c4377 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -4,6 +4,7 @@ import ( "context" "net" "net/http" + "strconv" "github.com/9seconds/mtg/v2/events" "github.com/9seconds/mtg/v2/mtglib" @@ -12,18 +13,23 @@ import ( ) type prometheusProcessor struct { - streams map[string]*streamInfo + streams map[string]streamInfo factory *PrometheusFactory } func (p prometheusProcessor) EventStart(evt mtglib.EventStart) { info := acquireStreamInfo() - info.SetStartTime(evt.CreatedAt) - info.SetClientIP(evt.RemoteIP) + + if evt.RemoteIP.To4() != nil { + info[TagIPFamily] = TagIPFamilyIPv4 + } else { + info[TagIPFamily] = TagIPFamilyIPv6 + } + p.streams[evt.StreamID()] = info p.factory.metricClientConnections. - WithLabelValues(info.V(TagIPFamily)). + WithLabelValues(info[TagIPFamily]). Inc() } @@ -33,11 +39,11 @@ func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { return } - info.SetTelegramIP(evt.RemoteIP) - info.SetDC(evt.DC) + info[TagTelegramIP] = evt.RemoteIP.String() + info[TagDC] = strconv.Itoa(evt.DC) p.factory.metricTelegramConnections. - WithLabelValues(info.V(TagTelegramIP), info.V(TagDC)). + WithLabelValues(info[TagTelegramIP], info[TagDC]). Inc() } @@ -48,7 +54,7 @@ func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) { } p.factory.metricTelegramTraffic. - WithLabelValues(info.V(TagTelegramIP), info.V(TagDC), getDirection(evt.IsRead)). + WithLabelValues(info[TagTelegramIP], info[TagDC], getDirection(evt.IsRead)). Add(float64(evt.Traffic)) } @@ -64,12 +70,12 @@ func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { }() p.factory.metricClientConnections. - WithLabelValues(info.V(TagIPFamily)). + WithLabelValues(info[TagIPFamily]). Dec() - if info.V(TagTelegramIP) != "" { + if telegramIP, ok := info[TagTelegramIP]; ok { p.factory.metricTelegramConnections. - WithLabelValues(info.V(TagTelegramIP), info.V(TagDC)). + WithLabelValues(telegramIP, info[TagDC]). Dec() } } @@ -83,20 +89,20 @@ func (p prometheusProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { } func (p prometheusProcessor) Shutdown() { - p.streams = make(map[string]*streamInfo) + p.streams = make(map[string]streamInfo) } type PrometheusFactory struct { httpServer *http.Server - metricClientConnections *prometheus.GaugeVec - metricTelegramConnections *prometheus.GaugeVec - metricDomainDisguisingConnections *prometheus.GaugeVec + metricClientConnections *prometheus.GaugeVec + metricTelegramConnections *prometheus.GaugeVec + metricDomainFrontingConnections *prometheus.GaugeVec - metricTelegramTraffic *prometheus.CounterVec - metricDomainDisguisingTraffic *prometheus.CounterVec + metricTelegramTraffic *prometheus.CounterVec + metricDomainFrontingTraffic *prometheus.CounterVec - metricDomainDisguising prometheus.Counter + metricDomainFronting prometheus.Counter metricConcurrencyLimited prometheus.Counter metricIPBlocklisted prometheus.Counter metricReplayAttacks prometheus.Counter @@ -104,7 +110,7 @@ type PrometheusFactory struct { func (p *PrometheusFactory) Make() events.Observer { return prometheusProcessor{ - streams: make(map[string]*streamInfo), + streams: make(map[string]streamInfo), factory: p, } } @@ -141,10 +147,10 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint Name: MetricTelegramConnections, Help: "A number of connections to Telegram servers.", }, []string{TagTelegramIP, TagDC}), - metricDomainDisguisingConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + metricDomainFrontingConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: metricPrefix, - Name: MetricDomainDisguisingConnections, - Help: "A number of connections which talk with disguising domain.", + Name: MetricDomainFronting, + Help: "A number of connections which talk with front domain.", }, []string{TagIPFamily}), metricTelegramTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ @@ -152,16 +158,16 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint Name: MetricTelegramTraffic, Help: "Traffic which is generated talking with Telegram servers.", }, []string{TagTelegramIP, TagDC, TagDirection}), - metricDomainDisguisingTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ + metricDomainFrontingTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metricPrefix, - Name: MetricDomainDisguisingTraffic, - Help: "Traffic which is generated talking with disguising domain.", + Name: MetricDomainFrontingTraffic, + Help: "Traffic which is generated talking with front domain.", }, []string{TagDirection}), - metricDomainDisguising: prometheus.NewCounter(prometheus.CounterOpts{ + metricDomainFronting: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metricPrefix, - Name: MetricDomainDisguising, - Help: "A number of routings to disguising domain.", + Name: MetricDomainFronting, + Help: "A number of routings to front domain.", }), metricConcurrencyLimited: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: metricPrefix, @@ -182,12 +188,12 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint registry.MustRegister(factory.metricClientConnections) registry.MustRegister(factory.metricTelegramConnections) - registry.MustRegister(factory.metricDomainDisguisingConnections) + registry.MustRegister(factory.metricDomainFrontingConnections) registry.MustRegister(factory.metricTelegramTraffic) - registry.MustRegister(factory.metricDomainDisguisingTraffic) + registry.MustRegister(factory.metricDomainFrontingTraffic) - registry.MustRegister(factory.metricDomainDisguising) + registry.MustRegister(factory.metricDomainFronting) registry.MustRegister(factory.metricConcurrencyLimited) registry.MustRegister(factory.metricIPBlocklisted) registry.MustRegister(factory.metricReplayAttacks) diff --git a/stats/statsd.go b/stats/statsd.go index b621c73..44567e3 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -2,6 +2,7 @@ package stats import ( "fmt" + "strconv" "strings" "time" @@ -12,19 +13,24 @@ import ( ) type statsdProcessor struct { - streams map[string]*streamInfo + streams map[string]streamInfo client *statsd.Client } func (s statsdProcessor) EventStart(evt mtglib.EventStart) { info := acquireStreamInfo() - info.SetStartTime(evt.CreatedAt) - info.SetClientIP(evt.RemoteIP) + + if evt.RemoteIP.To4() != nil { + info[TagIPFamily] = TagIPFamilyIPv4 + } else { + info[TagIPFamily] = TagIPFamilyIPv6 + } + s.streams[evt.StreamID()] = info s.client.GaugeDelta(MetricClientConnections, 1, - info.TV(TagIPFamily)) + info.T(TagIPFamily)) } func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { @@ -33,13 +39,13 @@ func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { return } - info.SetTelegramIP(evt.RemoteIP) - info.SetDC(evt.DC) + info[TagTelegramIP] = evt.RemoteIP.String() + info[TagDC] = strconv.Itoa(evt.DC) s.client.GaugeDelta(MetricTelegramConnections, 1, - info.TV(TagTelegramIP), - info.TV(TagDC)) + info.T(TagTelegramIP), + info.T(TagDC)) } func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { @@ -50,8 +56,8 @@ func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { s.client.Incr(MetricTelegramTraffic, int64(evt.Traffic), - info.TV(TagTelegramIP), - info.TV(TagDC), + info.T(TagTelegramIP), + info.T(TagDC), statsd.StringTag(TagDirection, getDirection(evt.IsRead))) } @@ -68,13 +74,13 @@ func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { s.client.GaugeDelta(MetricClientConnections, -1, - info.TV(TagIPFamily)) + info.T(TagIPFamily)) - if info.V(TagTelegramIP) != "" { + if _, ok := info[TagTelegramIP]; ok { s.client.GaugeDelta(MetricTelegramConnections, -1, - info.TV(TagTelegramIP), - info.TV(TagDC)) + info.T(TagTelegramIP), + info.T(TagDC)) } } @@ -113,7 +119,7 @@ func (s StatsdFactory) Close() error { func (s StatsdFactory) Make() events.Observer { return statsdProcessor{ client: s.client, - streams: make(map[string]*streamInfo), + streams: make(map[string]streamInfo), } } diff --git a/stats/stream_info.go b/stats/stream_info.go index 0aeb904..9ada4ff 100644 --- a/stats/stream_info.go +++ b/stats/stream_info.go @@ -1,51 +1,16 @@ package stats -import ( - "net" - "strconv" - "time" +import statsd "github.com/smira/go-statsd" - statsd "github.com/smira/go-statsd" -) +type streamInfo map[string]string -type streamInfo struct { - startTime time.Time - tags map[string]string +func (s streamInfo) T(key string) statsd.Tag { + return statsd.StringTag(key, s[key]) } -func (s *streamInfo) SetStartTime(tme time.Time) { - s.startTime = tme -} - -func (s *streamInfo) SetClientIP(ip net.IP) { - if ip.To4() != nil { - s.tags[TagIPFamily] = TagIPFamilyIPv4 - } else { - s.tags[TagIPFamily] = TagIPFamilyIPv6 - } -} - -func (s *streamInfo) SetTelegramIP(ip net.IP) { - s.tags[TagTelegramIP] = ip.String() -} - -func (s *streamInfo) SetDC(dc int) { - s.tags[TagDC] = strconv.Itoa(dc) -} - -func (s *streamInfo) V(key string) string { - return s.tags[key] -} - -func (s *streamInfo) TV(key string) statsd.Tag { - return statsd.StringTag(key, s.tags[key]) -} - -func (s *streamInfo) Reset() { - s.startTime = time.Time{} - - for k := range s.tags { - delete(s.tags, k) +func (s streamInfo) Reset() { + for k := range s { + delete(s, k) } } From bef14bd0094bba9852c8d1db1febd06134f59cf8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 11:56:05 +0300 Subject: [PATCH 141/206] Add support of EventDomainFronting event --- events/event_stream.go | 12 ++++--- events/event_stream_test.go | 23 +++++++++++++ events/init.go | 1 + events/init_test.go | 4 +++ events/multi_observer.go | 15 +++++++++ events/noop.go | 1 + events/noop_test.go | 6 ++++ mtglib/conns.go | 6 ++-- mtglib/events.go | 13 ++++++++ mtglib/events_test.go | 10 ++++++ mtglib/proxy.go | 16 ++++++++-- stats/pools.go | 10 +++--- stats/prometheus.go | 64 +++++++++++++++++++++++++++---------- stats/prometheus_test.go | 61 ++++++++++++++++++++++++++++++++++- stats/statsd.go | 50 ++++++++++++++++++++++------- stats/statsd_test.go | 59 ++++++++++++++++++++++++++++++++-- stats/stream_info.go | 19 +++++++---- 17 files changed, 317 insertions(+), 53 deletions(-) diff --git a/events/event_stream.go b/events/event_stream.go index 1726c81..ef71959 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -60,7 +60,7 @@ func NewEventStream(observerFactories []ObserverFactory) mtglib.EventStream { return rv } -func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, observer Observer) { +func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, observer Observer) { // nolint: cyclop defer observer.Shutdown() for { @@ -69,14 +69,16 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob return case evt := <-eventChan: switch typedEvt := evt.(type) { - case mtglib.EventStart: - observer.EventStart(typedEvt) - case mtglib.EventConnectedToDC: - observer.EventConnectedToDC(typedEvt) case mtglib.EventTraffic: observer.EventTraffic(typedEvt) + case mtglib.EventStart: + observer.EventStart(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) + case mtglib.EventConnectedToDC: + observer.EventConnectedToDC(typedEvt) + case mtglib.EventDomainFronting: + observer.EventDomainFronting(typedEvt) case mtglib.EventIPBlocklisted: observer.EventIPBlocklisted(typedEvt) case mtglib.EventConcurrencyLimited: diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 90dee03..83e6b77 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -90,6 +90,29 @@ func (suite *EventStreamTestSuite) TestEventConnectedToDC() { time.Sleep(100 * time.Millisecond) } +func (suite *EventStreamTestSuite) TestEventDomainFronting() { + evt := mtglib.EventDomainFronting{ + CreatedAt: time.Now(), + ConnID: "connID", + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventDomainFronting", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventDomainFronting) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.ConnID, caught.ConnID) + suite.Equal(evt.StreamID(), caught.StreamID()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + func (suite *EventStreamTestSuite) TestEventTraffic() { evt := mtglib.EventTraffic{ CreatedAt: time.Now(), diff --git a/events/init.go b/events/init.go index 4720a3b..b6dbd22 100644 --- a/events/init.go +++ b/events/init.go @@ -6,6 +6,7 @@ type Observer interface { EventStart(mtglib.EventStart) EventFinish(mtglib.EventFinish) EventConnectedToDC(mtglib.EventConnectedToDC) + EventDomainFronting(mtglib.EventDomainFronting) EventTraffic(mtglib.EventTraffic) EventConcurrencyLimited(mtglib.EventConcurrencyLimited) EventIPBlocklisted(mtglib.EventIPBlocklisted) diff --git a/events/init_test.go b/events/init_test.go index 58e3635..5fccfad 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -17,6 +17,10 @@ func (o *ObserverMock) EventConnectedToDC(evt mtglib.EventConnectedToDC) { o.Called(evt) } +func (o *ObserverMock) EventDomainFronting(evt mtglib.EventDomainFronting) { + o.Called(evt) +} + func (o *ObserverMock) EventTraffic(evt mtglib.EventTraffic) { o.Called(evt) } diff --git a/events/multi_observer.go b/events/multi_observer.go index 1dec489..cd74567 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -40,6 +40,21 @@ func (m multiObserver) EventConnectedToDC(evt mtglib.EventConnectedToDC) { wg.Wait() } +func (m multiObserver) EventDomainFronting(evt mtglib.EventDomainFronting) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventDomainFronting(evt) + }(v) + } + + wg.Wait() +} + func (m multiObserver) EventTraffic(evt mtglib.EventTraffic) { wg := &sync.WaitGroup{} wg.Add(len(m.observers)) diff --git a/events/noop.go b/events/noop.go index 8dcbf57..27594d8 100644 --- a/events/noop.go +++ b/events/noop.go @@ -19,6 +19,7 @@ type noopObserver struct{} func (n noopObserver) EventStart(_ mtglib.EventStart) {} func (n noopObserver) EventConnectedToDC(_ mtglib.EventConnectedToDC) {} +func (n noopObserver) EventDomainFronting(_ mtglib.EventDomainFronting) {} func (n noopObserver) EventTraffic(_ mtglib.EventTraffic) {} func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} diff --git a/events/noop_test.go b/events/noop_test.go index 2621e38..9a1d2bd 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -31,6 +31,10 @@ func (suite *NoopTestSuite) SetupSuite() { RemoteIP: net.ParseIP("127.1.0.1"), DC: 2, }, + "domain-fronting": mtglib.EventDomainFronting{ + CreatedAt: time.Now(), + ConnID: "connID", + }, "traffic": mtglib.EventTraffic{ CreatedAt: time.Now(), ConnID: "connID", @@ -76,6 +80,8 @@ func (suite *NoopTestSuite) TestObserver() { observer.EventStart(typedEvt) case mtglib.EventConnectedToDC: observer.EventConnectedToDC(typedEvt) + case mtglib.EventDomainFronting: + observer.EventDomainFronting(typedEvt) case mtglib.EventFinish: observer.EventFinish(typedEvt) case mtglib.EventConcurrencyLimited: diff --git a/mtglib/conns.go b/mtglib/conns.go index 07b7086..e6a64e7 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -9,7 +9,7 @@ import ( "time" ) -type connTelegramTraffic struct { +type connTraffic struct { net.Conn connID string @@ -17,7 +17,7 @@ type connTelegramTraffic struct { ctx context.Context } -func (c connTelegramTraffic) Read(b []byte) (int, error) { +func (c connTraffic) Read(b []byte) (int, error) { n, err := c.Conn.Read(b) if n > 0 { @@ -32,7 +32,7 @@ func (c connTelegramTraffic) Read(b []byte) (int, error) { return n, err // nolint: wrapcheck } -func (c connTelegramTraffic) Write(b []byte) (int, error) { +func (c connTraffic) Write(b []byte) (int, error) { n, err := c.Conn.Write(b) if n > 0 { diff --git a/mtglib/events.go b/mtglib/events.go index 3a19b5f..e7a5eae 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -62,6 +62,19 @@ func (e EventFinish) Timestamp() time.Time { return e.CreatedAt } +type EventDomainFronting struct { + CreatedAt time.Time + ConnID string +} + +func (e EventDomainFronting) StreamID() string { + return e.ConnID +} + +func (e EventDomainFronting) Timestamp() time.Time { + return e.CreatedAt +} + type EventConcurrencyLimited struct { CreatedAt time.Time } diff --git a/mtglib/events_test.go b/mtglib/events_test.go index 5597fae..c3a2fb9 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -58,6 +58,16 @@ func (suite *EventsTestSuite) TestEventTraffic() { suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } +func (suite *EventsTestSuite) TestEventDomainFronting() { + evt := mtglib.EventDomainFronting{ + CreatedAt: time.Now(), + ConnID: "CONNID", + } + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + func (suite *EventsTestSuite) TestEventConcurrencyLimited() { evt := mtglib.EventConcurrencyLimited{ CreatedAt: time.Now(), diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 6baf4a2..731c68c 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -215,7 +215,7 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { } ctx.telegramConn = obfuscated2.Conn{ - Conn: connTelegramTraffic{ + Conn: connTraffic{ Conn: conn, connID: ctx.connID, stream: p.eventStream, @@ -235,7 +235,12 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { return nil } -func (p *Proxy) doDomainFronting(ctx context.Context, conn *connRewind) { +func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { + p.eventStream.Send(p.ctx, EventDomainFronting{ + CreatedAt: time.Now(), + ConnID: ctx.connID, + }) + conn.Rewind() frontConn, err := p.network.DialContext(ctx, "tcp", p.domainFrontAddress) @@ -245,6 +250,13 @@ func (p *Proxy) doDomainFronting(ctx context.Context, conn *connRewind) { return } + frontConn = connTraffic{ + Conn: frontConn, + ctx: ctx, + connID: ctx.connID, + stream: p.eventStream, + } + rel := relay.AcquireRelay(ctx, p.logger.Named("domain-fronting"), p.bufferSize, p.idleTimeout) defer relay.ReleaseRelay(rel) diff --git a/stats/pools.go b/stats/pools.go index ec0505c..b9b6255 100644 --- a/stats/pools.go +++ b/stats/pools.go @@ -4,15 +4,17 @@ import "sync" var streamInfoPool = sync.Pool{ New: func() interface{} { - return streamInfo{} + return &streamInfo{ + tags: make(map[string]string), + } }, } -func acquireStreamInfo() streamInfo { - return streamInfoPool.Get().(streamInfo) +func acquireStreamInfo() *streamInfo { + return streamInfoPool.Get().(*streamInfo) } -func releaseStreamInfo(info streamInfo) { +func releaseStreamInfo(info *streamInfo) { info.Reset() streamInfoPool.Put(info) } diff --git a/stats/prometheus.go b/stats/prometheus.go index 51c4377..2e7fe10 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -13,7 +13,7 @@ import ( ) type prometheusProcessor struct { - streams map[string]streamInfo + streams map[string]*streamInfo factory *PrometheusFactory } @@ -21,15 +21,15 @@ func (p prometheusProcessor) EventStart(evt mtglib.EventStart) { info := acquireStreamInfo() if evt.RemoteIP.To4() != nil { - info[TagIPFamily] = TagIPFamilyIPv4 + info.tags[TagIPFamily] = TagIPFamilyIPv4 } else { - info[TagIPFamily] = TagIPFamilyIPv6 + info.tags[TagIPFamily] = TagIPFamilyIPv6 } p.streams[evt.StreamID()] = info p.factory.metricClientConnections. - WithLabelValues(info[TagIPFamily]). + WithLabelValues(info.tags[TagIPFamily]). Inc() } @@ -39,11 +39,25 @@ func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { return } - info[TagTelegramIP] = evt.RemoteIP.String() - info[TagDC] = strconv.Itoa(evt.DC) + info.tags[TagTelegramIP] = evt.RemoteIP.String() + info.tags[TagDC] = strconv.Itoa(evt.DC) p.factory.metricTelegramConnections. - WithLabelValues(info[TagTelegramIP], info[TagDC]). + WithLabelValues(info.tags[TagTelegramIP], info.tags[TagDC]). + Inc() +} + +func (p prometheusProcessor) EventDomainFronting(evt mtglib.EventDomainFronting) { + info, ok := p.streams[evt.StreamID()] + if !ok { + return + } + + info.isDomainFronted = true + + p.factory.metricDomainFronting.Inc() + p.factory.metricDomainFrontingConnections. + WithLabelValues(info.tags[TagIPFamily]). Inc() } @@ -53,9 +67,17 @@ func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) { return } - p.factory.metricTelegramTraffic. - WithLabelValues(info[TagTelegramIP], info[TagDC], getDirection(evt.IsRead)). - Add(float64(evt.Traffic)) + direction := getDirection(evt.IsRead) + + if info.isDomainFronted { + p.factory.metricDomainFrontingTraffic. + WithLabelValues(direction). + Add(float64(evt.Traffic)) + } else { + p.factory.metricTelegramTraffic. + WithLabelValues(info.tags[TagTelegramIP], info.tags[TagDC], direction). + Add(float64(evt.Traffic)) + } } func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { @@ -70,12 +92,16 @@ func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) { }() p.factory.metricClientConnections. - WithLabelValues(info[TagIPFamily]). + WithLabelValues(info.tags[TagIPFamily]). Dec() - if telegramIP, ok := info[TagTelegramIP]; ok { + if info.isDomainFronted { + p.factory.metricDomainFrontingConnections. + WithLabelValues(info.tags[TagIPFamily]). + Dec() + } else if telegramIP, ok := info.tags[TagTelegramIP]; ok { p.factory.metricTelegramConnections. - WithLabelValues(telegramIP, info[TagDC]). + WithLabelValues(telegramIP, info.tags[TagDC]). Dec() } } @@ -89,7 +115,11 @@ func (p prometheusProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { } func (p prometheusProcessor) Shutdown() { - p.streams = make(map[string]streamInfo) + for _, v := range p.streams { + releaseStreamInfo(v) + } + + p.streams = make(map[string]*streamInfo) } type PrometheusFactory struct { @@ -110,7 +140,7 @@ type PrometheusFactory struct { func (p *PrometheusFactory) Make() events.Observer { return prometheusProcessor{ - streams: make(map[string]streamInfo), + streams: make(map[string]*streamInfo), factory: p, } } @@ -149,8 +179,8 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint }, []string{TagTelegramIP, TagDC}), metricDomainFrontingConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: metricPrefix, - Name: MetricDomainFronting, - Help: "A number of connections which talk with front domain.", + Name: MetricDomainFrontingConnections, + Help: "A number of connections which talk to front domain.", }, []string{TagIPFamily}), metricTelegramTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{ diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index b4a2335..242fe75 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -54,7 +54,7 @@ func (suite *PrometheusTestSuite) TearDownTest() { suite.httpListener.Close() } -func (suite *PrometheusTestSuite) TestEventStartFinish() { +func (suite *PrometheusTestSuite) TestTelegramPath() { suite.prometheus.EventStart(mtglib.EventStart{ CreatedAt: time.Now(), ConnID: "connID", @@ -114,6 +114,65 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() { suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 0`) } +func (suite *PrometheusTestSuite) TestDomainFrontingPath() { + suite.prometheus.EventStart(mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.10"), + }) + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 1`) + + suite.prometheus.EventDomainFronting(mtglib.EventDomainFronting{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_domain_fronting 1`) + suite.Contains(data, `mtg_domain_fronting_connections{ip_family="ipv4"} 1`) + + suite.prometheus.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 200, + IsRead: true, + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_domain_fronting_traffic{direction="to_client"} 200`) + + suite.prometheus.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 100, + IsRead: false, + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_domain_fronting_traffic{direction="from_client"} 100`) + + suite.prometheus.EventFinish(mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + time.Sleep(100 * time.Millisecond) + + data, err = suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 0`) + suite.Contains(data, `mtg_domain_fronting_connections{ip_family="ipv4"} 0`) +} + func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { suite.prometheus.EventConcurrencyLimited(mtglib.EventConcurrencyLimited{ CreatedAt: time.Now(), diff --git a/stats/statsd.go b/stats/statsd.go index 44567e3..5bb2a38 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -13,7 +13,7 @@ import ( ) type statsdProcessor struct { - streams map[string]streamInfo + streams map[string]*streamInfo client *statsd.Client } @@ -21,9 +21,9 @@ func (s statsdProcessor) EventStart(evt mtglib.EventStart) { info := acquireStreamInfo() if evt.RemoteIP.To4() != nil { - info[TagIPFamily] = TagIPFamilyIPv4 + info.tags[TagIPFamily] = TagIPFamilyIPv4 } else { - info[TagIPFamily] = TagIPFamilyIPv6 + info.tags[TagIPFamily] = TagIPFamilyIPv6 } s.streams[evt.StreamID()] = info @@ -39,8 +39,8 @@ func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { return } - info[TagTelegramIP] = evt.RemoteIP.String() - info[TagDC] = strconv.Itoa(evt.DC) + info.tags[TagTelegramIP] = evt.RemoteIP.String() + info.tags[TagDC] = strconv.Itoa(evt.DC) s.client.GaugeDelta(MetricTelegramConnections, 1, @@ -48,17 +48,39 @@ func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) { info.T(TagDC)) } +func (s statsdProcessor) EventDomainFronting(evt mtglib.EventDomainFronting) { + info, ok := s.streams[evt.StreamID()] + if !ok { + return + } + + info.isDomainFronted = true + + s.client.Incr(MetricDomainFronting, 1) + s.client.GaugeDelta(MetricDomainFrontingConnections, + 1, + info.T(TagIPFamily)) +} + func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) { info, ok := s.streams[evt.StreamID()] if !ok { return } - s.client.Incr(MetricTelegramTraffic, - int64(evt.Traffic), - info.T(TagTelegramIP), - info.T(TagDC), - statsd.StringTag(TagDirection, getDirection(evt.IsRead))) + directionTag := statsd.StringTag(TagDirection, getDirection(evt.IsRead)) + + if info.isDomainFronted { + s.client.Incr(MetricDomainFrontingTraffic, + int64(evt.Traffic), + directionTag) + } else { + s.client.Incr(MetricTelegramTraffic, + int64(evt.Traffic), + info.T(TagTelegramIP), + info.T(TagDC), + directionTag) + } } func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { @@ -76,7 +98,11 @@ func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) { -1, info.T(TagIPFamily)) - if _, ok := info[TagTelegramIP]; ok { + if info.isDomainFronted { + s.client.GaugeDelta(MetricDomainFrontingConnections, + -1, + info.T(TagIPFamily)) + } else if _, ok := info.tags[TagTelegramIP]; ok { s.client.GaugeDelta(MetricTelegramConnections, -1, info.T(TagTelegramIP), @@ -119,7 +145,7 @@ func (s StatsdFactory) Close() error { func (s StatsdFactory) Make() events.Observer { return statsdProcessor{ client: s.client, - streams: make(map[string]streamInfo), + streams: make(map[string]*streamInfo), } } diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 5141eca..ee55ad8 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/suite" ) -const statsdSleepTime = 3 * statsd.DefaultFlushInterval +const statsdSleepTime = 4 * statsd.DefaultFlushInterval type statsdFakeServer struct { conn *net.UDPConn @@ -104,7 +104,7 @@ func (suite *StatsdTestSuite) TearDownTest() { suite.statsdServer.Close() } -func (suite *StatsdTestSuite) TestEventStartFinish() { +func (suite *StatsdTestSuite) TestTelegramPath() { suite.statsd.EventStart(mtglib.EventStart{ CreatedAt: time.Now(), ConnID: "connID", @@ -152,6 +152,61 @@ func (suite *StatsdTestSuite) TestEventStartFinish() { "mtg.telegram_connections:-1|g|#telegram_ip:10.1.0.10,dc:2") suite.Contains(suite.statsdServer.String(), "mtg.client_connections:-1|g|#ip_family:ipv4") + + suite.NotContains(suite.statsdServer.String(), "domain_fronting_traffic") + suite.NotContains(suite.statsdServer.String(), "domain_fronting_connections") +} + +func (suite *StatsdTestSuite) TestDomainFrontingPath() { + suite.statsd.EventStart(mtglib.EventStart{ + CreatedAt: time.Now(), + ConnID: "connID", + RemoteIP: net.ParseIP("10.0.0.10"), + }) + time.Sleep(statsdSleepTime) + suite.Equal("mtg.client_connections:+1|g|#ip_family:ipv4", suite.statsdServer.String()) + + suite.statsd.EventDomainFronting(mtglib.EventDomainFronting{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), "mtg.domain_fronting:1|c") + suite.Contains(suite.statsdServer.String(), + `mtg.domain_fronting_connections:+1|g|#ip_family:ipv4`) + + suite.statsd.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 30, + IsRead: true, + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + `mtg.domain_fronting_traffic:30|c|#direction:to_client`) + + suite.statsd.EventTraffic(mtglib.EventTraffic{ + CreatedAt: time.Now(), + ConnID: "connID", + Traffic: 90, + IsRead: false, + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + `mtg.domain_fronting_traffic:90|c|#direction:from_client`) + + suite.statsd.EventFinish(mtglib.EventFinish{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + time.Sleep(statsdSleepTime) + suite.Contains(suite.statsdServer.String(), + "mtg.domain_fronting_connections:-1|g|#ip_family:ipv4") + suite.Contains(suite.statsdServer.String(), + "mtg.client_connections:-1|g|#ip_family:ipv4") + + suite.NotContains(suite.statsdServer.String(), "telegram_traffic") + suite.NotContains(suite.statsdServer.String(), "telegram_connections") } func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { diff --git a/stats/stream_info.go b/stats/stream_info.go index 9ada4ff..b28bccb 100644 --- a/stats/stream_info.go +++ b/stats/stream_info.go @@ -2,15 +2,20 @@ package stats import statsd "github.com/smira/go-statsd" -type streamInfo map[string]string - -func (s streamInfo) T(key string) statsd.Tag { - return statsd.StringTag(key, s[key]) +type streamInfo struct { + isDomainFronted bool + tags map[string]string } -func (s streamInfo) Reset() { - for k := range s { - delete(s, k) +func (s streamInfo) T(key string) statsd.Tag { + return statsd.StringTag(key, s.tags[key]) +} + +func (s *streamInfo) Reset() { + s.isDomainFronted = false + + for k := range s.tags { + delete(s.tags, k) } } From daa8b9c798fd894646a45c945e950055fae72435 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 12:07:18 +0300 Subject: [PATCH 142/206] Add EventReplayAttack --- events/event_stream.go | 2 ++ events/event_stream_test.go | 22 ++++++++++++++++++++++ events/init.go | 1 + events/init_test.go | 4 ++++ events/multi_observer.go | 15 +++++++++++++++ events/noop.go | 1 + events/noop_test.go | 10 +++++++++- mtglib/events.go | 13 +++++++++++++ mtglib/events_test.go | 10 ++++++++++ mtglib/proxy.go | 4 ++++ stats/prometheus.go | 6 +++++- stats/prometheus_test.go | 13 +++++++++++++ stats/statsd.go | 6 +++++- stats/statsd_test.go | 10 ++++++++++ 14 files changed, 114 insertions(+), 3 deletions(-) diff --git a/events/event_stream.go b/events/event_stream.go index ef71959..891382c 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -83,6 +83,8 @@ func eventStreamProcessor(ctx context.Context, eventChan <-chan mtglib.Event, ob observer.EventIPBlocklisted(typedEvt) case mtglib.EventConcurrencyLimited: observer.EventConcurrencyLimited(typedEvt) + case mtglib.EventReplayAttack: + observer.EventReplayAttack(typedEvt) } } } diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 83e6b77..179e29f 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -206,6 +206,28 @@ func (suite *EventStreamTestSuite) TestEventIPBlocklisted() { time.Sleep(100 * time.Millisecond) } +func (suite *EventStreamTestSuite) TestEventReplayAttack() { + evt := mtglib.EventReplayAttack{ + CreatedAt: time.Now(), + ConnID: "CONNID", + } + + for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { + v. + On("EventReplayAttack", mock.Anything). + Once(). + Run(func(args mock.Arguments) { + caught := args.Get(0).(mtglib.EventReplayAttack) + + suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.StreamID(), caught.StreamID()) + }) + } + + suite.stream.Send(suite.ctx, evt) + time.Sleep(100 * time.Millisecond) +} + func (suite *EventStreamTestSuite) TearDownTest() { suite.stream.Shutdown() suite.ctxCancel() diff --git a/events/init.go b/events/init.go index b6dbd22..1204652 100644 --- a/events/init.go +++ b/events/init.go @@ -10,6 +10,7 @@ type Observer interface { EventTraffic(mtglib.EventTraffic) EventConcurrencyLimited(mtglib.EventConcurrencyLimited) EventIPBlocklisted(mtglib.EventIPBlocklisted) + EventReplayAttack(mtglib.EventReplayAttack) Shutdown() } diff --git a/events/init_test.go b/events/init_test.go index 5fccfad..1b2e0cd 100644 --- a/events/init_test.go +++ b/events/init_test.go @@ -37,6 +37,10 @@ func (o *ObserverMock) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { o.Called(evt) } +func (o *ObserverMock) EventReplayAttack(evt mtglib.EventReplayAttack) { + o.Called(evt) +} + func (o *ObserverMock) Shutdown() { o.Called() } diff --git a/events/multi_observer.go b/events/multi_observer.go index cd74567..eeeacae 100644 --- a/events/multi_observer.go +++ b/events/multi_observer.go @@ -115,6 +115,21 @@ func (m multiObserver) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { wg.Wait() } +func (m multiObserver) EventReplayAttack(evt mtglib.EventReplayAttack) { + wg := &sync.WaitGroup{} + wg.Add(len(m.observers)) + + for _, v := range m.observers { + go func(obs Observer) { + defer wg.Done() + + obs.EventReplayAttack(evt) + }(v) + } + + wg.Wait() +} + func (m multiObserver) Shutdown() { for _, v := range m.observers { v.Shutdown() diff --git a/events/noop.go b/events/noop.go index 27594d8..afc2c66 100644 --- a/events/noop.go +++ b/events/noop.go @@ -24,6 +24,7 @@ func (n noopObserver) EventTraffic(_ mtglib.EventTraffic) func (n noopObserver) EventFinish(_ mtglib.EventFinish) {} func (n noopObserver) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {} func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) {} +func (n noopObserver) EventReplayAttack(_ mtglib.EventReplayAttack) {} func (n noopObserver) Shutdown() {} func NewNoopObserver() Observer { diff --git a/events/noop_test.go b/events/noop_test.go index 9a1d2bd..024febc 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -45,11 +45,17 @@ func (suite *NoopTestSuite) SetupSuite() { CreatedAt: time.Now(), ConnID: "connID", }, - "concurrency-limited": mtglib.EventConcurrencyLimited{}, + "concurrency-limited": mtglib.EventConcurrencyLimited{ + CreatedAt: time.Now(), + }, "ip-blacklisted": mtglib.EventIPBlocklisted{ RemoteIP: net.ParseIP("10.0.0.10"), CreatedAt: time.Now(), }, + "replay-attack": mtglib.EventReplayAttack{ + CreatedAt: time.Now(), + ConnID: "connID", + }, } suite.ctx = context.Background() } @@ -88,6 +94,8 @@ func (suite *NoopTestSuite) TestObserver() { observer.EventConcurrencyLimited(typedEvt) case mtglib.EventIPBlocklisted: observer.EventIPBlocklisted(typedEvt) + case mtglib.EventReplayAttack: + observer.EventReplayAttack(typedEvt) } }) } diff --git a/mtglib/events.go b/mtglib/events.go index e7a5eae..e4c57d1 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -99,3 +99,16 @@ func (e EventIPBlocklisted) StreamID() string { func (e EventIPBlocklisted) Timestamp() time.Time { return e.CreatedAt } + +type EventReplayAttack struct { + CreatedAt time.Time + ConnID string +} + +func (e EventReplayAttack) StreamID() string { + return e.ConnID +} + +func (e EventReplayAttack) Timestamp() time.Time { + return e.CreatedAt +} diff --git a/mtglib/events_test.go b/mtglib/events_test.go index c3a2fb9..1f4814e 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -87,6 +87,16 @@ func (suite *EventsTestSuite) TestEventIPBlocklisted() { suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } +func (suite *EventsTestSuite) TestEventReplayAttack() { + evt := mtglib.EventReplayAttack{ + CreatedAt: time.Now(), + ConnID: "CONNID", + } + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) +} + func TestEvents(t *testing.T) { t.Parallel() suite.Run(t, &EventsTestSuite{}) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 731c68c..e0d5974 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -166,6 +166,10 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { if p.antiReplayCache.SeenBefore(hello.SessionID) { p.logger.Warning("replay attack has been detected!") + p.eventStream.Send(p.ctx, EventReplayAttack{ + CreatedAt: time.Now(), + ConnID: ctx.connID, + }) p.doDomainFronting(ctx, rewind) return false diff --git a/stats/prometheus.go b/stats/prometheus.go index 2e7fe10..aa2933d 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -110,10 +110,14 @@ func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLi p.factory.metricConcurrencyLimited.Inc() } -func (p prometheusProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { +func (p prometheusProcessor) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) { p.factory.metricIPBlocklisted.Inc() } +func (p prometheusProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { + p.factory.metricReplayAttacks.Inc() +} + func (p prometheusProcessor) Shutdown() { for _, v := range p.streams { releaseStreamInfo(v) diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index 242fe75..abb3bec 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -198,6 +198,19 @@ func (suite *PrometheusTestSuite) TestEventIPBlocklisted() { suite.Contains(data, `mtg_ip_blocklisted 1`) } +func (suite *PrometheusTestSuite) TestEventReplayAttack() { + suite.prometheus.EventReplayAttack(mtglib.EventReplayAttack{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + + time.Sleep(100 * time.Millisecond) + + data, err := suite.Get() + suite.NoError(err) + suite.Contains(data, `mtg_replay_attacks 1`) +} + func TestPrometheus(t *testing.T) { t.Parallel() suite.Run(t, &PrometheusTestSuite{}) diff --git a/stats/statsd.go b/stats/statsd.go index 5bb2a38..a1f0ca4 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -114,10 +114,14 @@ func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimite s.client.Incr(MetricConcurrencyLimited, 1) } -func (s statsdProcessor) EventIPBlocklisted(evt mtglib.EventIPBlocklisted) { +func (s statsdProcessor) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) { s.client.Incr(MetricIPBlocklisted, 1) } +func (s statsdProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { + s.client.Incr(MetricReplayAttacks, 1) +} + func (s statsdProcessor) Shutdown() { now := time.Now() events := make([]mtglib.EventFinish, 0, len(s.streams)) diff --git a/stats/statsd_test.go b/stats/statsd_test.go index ee55ad8..02b34ec 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -228,6 +228,16 @@ func (suite *StatsdTestSuite) TestEventIPBlocklisted() { suite.Equal("mtg.ip_blocklisted:1|c", suite.statsdServer.String()) } +func (suite *StatsdTestSuite) TestEventReplayAttack() { + suite.statsd.EventReplayAttack(mtglib.EventReplayAttack{ + CreatedAt: time.Now(), + ConnID: "connID", + }) + + time.Sleep(statsdSleepTime) + suite.Equal("mtg.replay_attacks:1|c", suite.statsdServer.String()) +} + func TestStatsd(t *testing.T) { t.Parallel() suite.Run(t, &StatsdTestSuite{}) From 75b05b9b1457eb7d8f0054f2822bf2f4b34b5154 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 12:09:23 +0300 Subject: [PATCH 143/206] Add public method DomainFrontingAddress --- mtglib/proxy.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index e0d5974..6ecd236 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -24,7 +24,7 @@ type Proxy struct { idleTimeout time.Duration bufferSize int - domainFrontAddress string + domainFrontingPort int workerPool *ants.PoolWithFunc telegram *telegram.Telegram @@ -37,6 +37,10 @@ type Proxy struct { logger Logger } +func (p *Proxy) DomainFrontingAddress() string { + return net.JoinHostPort(p.secret.Host, strconv.Itoa(p.domainFrontingPort)) +} + func (p *Proxy) ServeConn(conn net.Conn) { ctx := newStreamContext(p.ctx, p.logger, conn) defer ctx.Close() @@ -247,7 +251,7 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { conn.Rewind() - frontConn, err := p.network.DialContext(ctx, "tcp", p.domainFrontAddress) + frontConn, err := p.network.DialContext(ctx, "tcp", p.DomainFrontingAddress()) if err != nil { p.logger.WarningError("cannot dial to the fronting domain", err) @@ -324,11 +328,10 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen ipBlocklist: opts.IPBlocklist, eventStream: opts.EventStream, logger: opts.Logger.Named("proxy"), - domainFrontAddress: net.JoinHostPort(opts.Secret.Host, - strconv.Itoa(domainFrontingPort)), - idleTimeout: idleTimeout, - bufferSize: int(bufferSize), - telegram: tg, + domainFrontingPort: int(domainFrontingPort), + idleTimeout: idleTimeout, + bufferSize: int(bufferSize), + telegram: tg, } pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { From 71d4c6c42f55c67821d3cf4d93ce1d904bbf2a8f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 12:12:23 +0300 Subject: [PATCH 144/206] Set worker pool to non-blocking mode --- mtglib/proxy.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 6ecd236..de76a45 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -328,7 +328,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen ipBlocklist: opts.IPBlocklist, eventStream: opts.EventStream, logger: opts.Logger.Named("proxy"), - domainFrontingPort: int(domainFrontingPort), + domainFrontingPort: domainFrontingPort, idleTimeout: idleTimeout, bufferSize: int(bufferSize), telegram: tg, @@ -336,7 +336,9 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { proxy.ServeConn(arg.(net.Conn)) - }, ants.WithLogger(opts.Logger.Named("ants"))) + }, + ants.WithLogger(opts.Logger.Named("ants")), + ants.WithNonblocking(true)) if err != nil { return nil, fmt.Errorf("cannot initialize a pool: %w", err) } From 3cacd74e12de717ac815c8359e71650265b9d08c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 14:21:05 +0300 Subject: [PATCH 145/206] Correctly reset relay state --- mtglib/internal/relay/pools.go | 7 +------ mtglib/internal/relay/relay.go | 33 ++++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/mtglib/internal/relay/pools.go b/mtglib/internal/relay/pools.go index dc51c0f..1fae858 100644 --- a/mtglib/internal/relay/pools.go +++ b/mtglib/internal/relay/pools.go @@ -36,11 +36,6 @@ func AcquireRelay(ctx context.Context, logger Logger, bufferSize int, idleTimeou } func ReleaseRelay(r *Relay) { - r.ctxCancel() - - r.ctx = nil - r.ctxCancel = nil - r.logger = nil - + r.Reset() relayPool.Put(r) } diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go index bbd92df..a9e105a 100644 --- a/mtglib/internal/relay/relay.go +++ b/mtglib/internal/relay/relay.go @@ -11,6 +11,7 @@ type Relay struct { ctx context.Context ctxCancel context.CancelFunc logger Logger + processMutex sync.Mutex eastBuffer []byte westBuffer []byte tickChannel chan struct{} @@ -18,7 +19,23 @@ type Relay struct { tickTimeout time.Duration } +func (r *Relay) Reset() { + r.processMutex.Lock() + defer r.processMutex.Unlock() + + if r.ctxCancel != nil { + r.ctxCancel() + } + + r.ctx = nil + r.ctxCancel = nil + r.logger = nil +} + func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { + r.processMutex.Lock() + defer r.processMutex.Unlock() + eastConn = conn{ ReadWriteCloser: eastConn, ctx: r.ctx, @@ -30,16 +47,10 @@ func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { tickChannel: r.tickChannel, } - defer func() { - r.ctxCancel() - eastConn.Close() - westConn.Close() - }() - wg := &sync.WaitGroup{} wg.Add(3) // nolint: gomnd - go r.runObserver(wg) + go r.runObserver(eastConn, westConn, wg) go r.transmit(eastConn, westConn, r.westBuffer, "west", wg) @@ -58,9 +69,10 @@ func (r *Relay) Process(eastConn, westConn io.ReadWriteCloser) error { func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, buffer []byte, direction string, wg *sync.WaitGroup) { defer func() { - wg.Done() src.Close() dst.Close() + wg.Done() + r.ctxCancel() }() if _, err := io.CopyBuffer(dst, src, buffer); err != nil { @@ -79,10 +91,13 @@ func (r *Relay) transmit(src io.ReadCloser, dst io.WriteCloser, } } -func (r *Relay) runObserver(wg *sync.WaitGroup) { +func (r *Relay) runObserver(one, another io.Closer, wg *sync.WaitGroup) { ticker := time.NewTicker(time.Second) defer func() { + one.Close() + another.Close() + ticker.Stop() select { From 841a4d222762ef7eed2c4aa2f8e9ea2adf2c4e88 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 29 Mar 2021 18:49:32 +0300 Subject: [PATCH 146/206] Get rid of unrequired buffering for tls records --- mtglib/internal/faketls/record/pools.go | 27 +++++------------------- mtglib/internal/faketls/record/record.go | 23 +++++++++++--------- mtglib/internal/obfuscated2/conn.go | 4 +--- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/mtglib/internal/faketls/record/pools.go b/mtglib/internal/faketls/record/pools.go index 50b0fec..16fa81f 100644 --- a/mtglib/internal/faketls/record/pools.go +++ b/mtglib/internal/faketls/record/pools.go @@ -1,22 +1,14 @@ package record import ( - "bytes" "sync" ) -var ( - recordPool = sync.Pool{ - New: func() interface{} { - return &Record{} - }, - } - bytesBufferPool = sync.Pool{ - New: func() interface{} { - return &bytes.Buffer{} - }, - } -) +var recordPool = sync.Pool{ + New: func() interface{} { + return &Record{} + }, +} func AcquireRecord() *Record { return recordPool.Get().(*Record) @@ -26,12 +18,3 @@ func ReleaseRecord(r *Record) { r.Reset() recordPool.Put(r) } - -func acquireBytesBuffer() *bytes.Buffer { - return bytesBufferPool.Get().(*bytes.Buffer) -} - -func releaseBytesBuffer(buf *bytes.Buffer) { - buf.Reset() - bytesBufferPool.Put(buf) -} diff --git a/mtglib/internal/faketls/record/record.go b/mtglib/internal/faketls/record/record.go index bce03b0..2c8b23e 100644 --- a/mtglib/internal/faketls/record/record.go +++ b/mtglib/internal/faketls/record/record.go @@ -61,21 +61,24 @@ func (r *Record) Read(reader io.Reader) error { } func (r *Record) Dump(writer io.Writer) error { - buf := acquireBytesBuffer() - defer releaseBytesBuffer(buf) + buf := [2]byte{byte(r.Type), 0} + if _, err := writer.Write(buf[:1]); err != nil { + return fmt.Errorf("cannot dump record type: %w", err) + } - bufSlice := [2]byte{byte(r.Type), 0} - buf.Write(bufSlice[:1]) + binary.BigEndian.PutUint16(buf[:], uint16(r.Version)) - binary.BigEndian.PutUint16(bufSlice[:], uint16(r.Version)) - buf.Write(bufSlice[:]) + if _, err := writer.Write(buf[:]); err != nil { + return fmt.Errorf("cannot dump version: %w", err) + } - binary.BigEndian.PutUint16(bufSlice[:], uint16(r.Payload.Len())) - buf.Write(bufSlice[:]) + binary.BigEndian.PutUint16(buf[:], uint16(r.Payload.Len())) - buf.Write(r.Payload.Bytes()) + if _, err := writer.Write(buf[:]); err != nil { + return fmt.Errorf("cannot dump payload length: %w", err) + } - if _, err := buf.WriteTo(writer); err != nil { + if _, err := writer.Write(r.Payload.Bytes()); err != nil { return fmt.Errorf("cannot dump record: %w", err) } diff --git a/mtglib/internal/obfuscated2/conn.go b/mtglib/internal/obfuscated2/conn.go index c3a4c05..87221c3 100644 --- a/mtglib/internal/obfuscated2/conn.go +++ b/mtglib/internal/obfuscated2/conn.go @@ -32,7 +32,5 @@ func (c Conn) Write(p []byte) (int, error) { payload := buf.Bytes() c.Encryptor.XORKeyStream(payload, payload) - n, err := buf.WriteTo(c.Conn) - - return int(n), err // nolint: wrapcheck + return c.Conn.Write(payload) } From 10b78322a3d216b28d421b60cc213f3c2fdb3a38 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 14:13:42 +0300 Subject: [PATCH 147/206] Implement caching dns resolver --- antireplay/noop.go | 1 - antireplay/noop_test.go | 2 - antireplay/stable_bloom_filter.go | 2 - antireplay/stable_bloom_filter_test.go | 2 - cli/proxy.go | 6 -- events/event_stream.go | 10 +-- events/event_stream_test.go | 2 +- events/noop.go | 1 - events/noop_test.go | 2 - go.mod | 1 + go.sum | 7 ++ ipblocklist/noop.go | 1 - mtglib/init.go | 3 - network/dns_resolver.go | 88 ++++++++++++++++++++++++++ network/dns_resolver_internal_test.go | 50 +++++++++++++++ network/network.go | 32 ++++------ 16 files changed, 163 insertions(+), 47 deletions(-) create mode 100644 network/dns_resolver.go create mode 100644 network/dns_resolver_internal_test.go diff --git a/antireplay/noop.go b/antireplay/noop.go index b07d34f..8613e40 100644 --- a/antireplay/noop.go +++ b/antireplay/noop.go @@ -5,7 +5,6 @@ import "github.com/9seconds/mtg/v2/mtglib" type noop struct{} func (n noop) SeenBefore(_ []byte) bool { return false } -func (n noop) Shutdown() {} func NewNoop() mtglib.AntiReplayCache { return noop{} diff --git a/antireplay/noop_test.go b/antireplay/noop_test.go index 42f39eb..b78c48d 100644 --- a/antireplay/noop_test.go +++ b/antireplay/noop_test.go @@ -18,8 +18,6 @@ func (suite *NoopTestSuite) TestOp() { suite.False(filter.SeenBefore([]byte{4, 5, 6})) suite.False(filter.SeenBefore([]byte{1, 2, 3})) suite.False(filter.SeenBefore([]byte{4, 5, 6})) - - filter.Shutdown() } func TestNoop(t *testing.T) { diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index ea4f5a7..b52631e 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -20,8 +20,6 @@ func (s *stableBloomFilter) SeenBefore(digest []byte) bool { return s.filter.TestAndAdd(digest) } -func (s *stableBloomFilter) Shutdown() {} - func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd sf.SetHash(xxhash.New64()) diff --git a/antireplay/stable_bloom_filter_test.go b/antireplay/stable_bloom_filter_test.go index 9b2517e..accb590 100644 --- a/antireplay/stable_bloom_filter_test.go +++ b/antireplay/stable_bloom_filter_test.go @@ -18,8 +18,6 @@ func (suite *StableBloomFilterTestSuite) TestOp() { suite.False(filter.SeenBefore([]byte{4, 5, 6})) suite.True(filter.SeenBefore([]byte{1, 2, 3})) suite.True(filter.SeenBefore([]byte{4, 5, 6})) - - filter.Shutdown() } func TestStableBloomFilter(t *testing.T) { diff --git a/cli/proxy.go b/cli/proxy.go index 22ebc38..94b228f 100644 --- a/cli/proxy.go +++ b/cli/proxy.go @@ -59,12 +59,6 @@ func (c *Proxy) Execute() error { // nolint: funlen PreferIP: c.Config.PreferIP.Value(mtglib.DefaultPreferIP), } - defer func() { - opts.AntiReplayCache.Shutdown() - opts.IPBlocklist.Shutdown() - opts.EventStream.Shutdown() - }() - if opts.Concurrency == 0 { opts.Concurrency = mtglib.DefaultConcurrency } diff --git a/events/event_stream.go b/events/event_stream.go index 891382c..9995eb0 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -9,13 +9,13 @@ import ( "github.com/OneOfOne/xxhash" ) -type eventStream struct { +type EventStream struct { ctx context.Context ctxCancel context.CancelFunc chans []chan mtglib.Event } -func (e eventStream) Send(ctx context.Context, evt mtglib.Event) { +func (e EventStream) Send(ctx context.Context, evt mtglib.Event) { var chanNo uint32 if streamID := evt.StreamID(); streamID != "" { @@ -31,17 +31,17 @@ func (e eventStream) Send(ctx context.Context, evt mtglib.Event) { } } -func (e eventStream) Shutdown() { +func (e EventStream) Shutdown() { e.ctxCancel() } -func NewEventStream(observerFactories []ObserverFactory) mtglib.EventStream { +func NewEventStream(observerFactories []ObserverFactory) EventStream { if len(observerFactories) == 0 { observerFactories = append(observerFactories, NewNoopObserver) } ctx, cancel := context.WithCancel(context.Background()) - rv := eventStream{ + rv := EventStream{ ctx: ctx, ctxCancel: cancel, chans: make([]chan mtglib.Event, runtime.NumCPU()), diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 179e29f..599d839 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -19,7 +19,7 @@ type EventStreamTestSuite struct { ctxCancel context.CancelFunc observerMock1 *ObserverMock observerMock2 *ObserverMock - stream mtglib.EventStream + stream events.EventStream } func (suite *EventStreamTestSuite) SetupTest() { diff --git a/events/noop.go b/events/noop.go index afc2c66..2fdb238 100644 --- a/events/noop.go +++ b/events/noop.go @@ -9,7 +9,6 @@ import ( type noop struct{} func (n noop) Send(ctx context.Context, evt mtglib.Event) {} -func (n noop) Shutdown() {} func NewNoopStream() mtglib.EventStream { return noop{} diff --git a/events/noop_test.go b/events/noop_test.go index 024febc..4a7b20b 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -70,8 +70,6 @@ func (suite *NoopTestSuite) TestStream() { stream.Send(suite.ctx, value) }) } - - stream.Shutdown() } func (suite *NoopTestSuite) TestObserver() { diff --git a/go.mod b/go.mod index ad324f1..9112aba 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/d4l3k/messagediff v1.2.1 // indirect + github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/jarcoal/httpmock v1.0.8 github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 github.com/libp2p/go-reuseport v0.0.2 diff --git a/go.sum b/go.sum index 0cf083c..12d7451 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -38,6 +39,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -55,7 +58,10 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= +github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -277,6 +283,7 @@ github.com/smira/go-statsd v1.3.2 h1:1EeuzxNZ/TD9apbTOFSM9nulqfcsQFmT4u1A2DREabI github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go index 4b29449..dcbed19 100644 --- a/ipblocklist/noop.go +++ b/ipblocklist/noop.go @@ -9,7 +9,6 @@ import ( type noop struct{} func (n noop) Contains(ip net.IP) bool { return false } -func (n noop) Shutdown() {} func NewNoop() mtglib.IPBlocklist { return noop{} diff --git a/mtglib/init.go b/mtglib/init.go index 0660e9e..010a78c 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -35,12 +35,10 @@ type Network interface { type AntiReplayCache interface { SeenBefore(data []byte) bool - Shutdown() } type IPBlocklist interface { Contains(net.IP) bool - Shutdown() } type Event interface { @@ -50,7 +48,6 @@ type Event interface { type EventStream interface { Send(context.Context, Event) - Shutdown() } type TimeAttackDetector interface { diff --git a/network/dns_resolver.go b/network/dns_resolver.go new file mode 100644 index 0000000..3205b8f --- /dev/null +++ b/network/dns_resolver.go @@ -0,0 +1,88 @@ +package network + +import ( + "net/http" + "time" + + doh "github.com/babolivier/go-doh-client" + "github.com/dgraph-io/ristretto" +) + +const ( + dnsResolverSize = 1024 * 1024 // 1mb + dnsResolverKeepTime = 10 * time.Minute +) + +type dnsResolver struct { + resolver doh.Resolver + cache *ristretto.Cache +} + +func (d dnsResolver) LookupA(hostname string) []string { + key := "\x00." + hostname + + if value, ok := d.cache.Get(key); ok { + return value.([]string) + } + + var ips []string + + if recs, _, err := d.resolver.LookupA(hostname); err == nil { + for _, v := range recs { + ips = append(ips, v.IP4) + } + + d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime) + } + + return ips +} + +func (d dnsResolver) LookupAAAA(hostname string) []string { + key := "\x01." + hostname + + if value, ok := d.cache.Get(key); ok { + return value.([]string) + } + + var ips []string + + if recs, _, err := d.resolver.LookupAAAA(hostname); err == nil { + for _, v := range recs { + ips = append(ips, v.IP6) + } + + d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime) + } + + return ips +} + +func newDNSResolver(hostname string, httpClient *http.Client) dnsResolver { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 10 * dnsResolverSize, // nolint: gomnd // taken from official doc as a best practice value + MaxCost: dnsResolverSize, + BufferItems: 64, // nolint: gomnd // taken from official doc as a best practice value + Cost: func(value interface{}) int64 { + var cost int64 + + for _, v := range value.([]string) { + cost += int64(len([]byte(v))) + } + + return cost + }, + }) + if err != nil { + panic(err) + } + + return dnsResolver{ + resolver: doh.Resolver{ + Host: hostname, + Class: doh.IN, + HTTPClient: httpClient, + }, + cache: cache, + } +} diff --git a/network/dns_resolver_internal_test.go b/network/dns_resolver_internal_test.go new file mode 100644 index 0000000..a5934e8 --- /dev/null +++ b/network/dns_resolver_internal_test.go @@ -0,0 +1,50 @@ +package network + +import ( + "net" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/suite" +) + +type DNSResolverTestSuite struct { + suite.Suite + + d dnsResolver +} + +func (suite *DNSResolverTestSuite) TestLookupA() { + suite.d.LookupA("google.com") + time.Sleep(10 * time.Millisecond) + + addrs := suite.d.LookupA("google.com") + + for _, v := range addrs { + suite.NotEmpty(v) + suite.NotNil(net.ParseIP(v).To4()) + } +} + +func (suite *DNSResolverTestSuite) TestLookupAAAA() { + suite.d.LookupAAAA("google.com") + time.Sleep(10 * time.Millisecond) + + addrs := suite.d.LookupAAAA("google.com") + + for _, v := range addrs { + suite.NotEmpty(v) + suite.Nil(net.ParseIP(v).To4()) + suite.NotNil(net.ParseIP(v).To16()) + } +} + +func (suite *DNSResolverTestSuite) SetupTest() { + suite.d = newDNSResolver("1.1.1.1", &http.Client{}) +} + +func TestDNSResolver(t *testing.T) { + t.Parallel() + suite.Run(t, &DNSResolverTestSuite{}) +} diff --git a/network/network.go b/network/network.go index 9df70a0..c65c282 100644 --- a/network/network.go +++ b/network/network.go @@ -10,7 +10,6 @@ import ( "time" "github.com/9seconds/mtg/v2/mtglib" - doh "github.com/babolivier/go-doh-client" ) type networkHTTPTransport struct { @@ -26,7 +25,7 @@ func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, erro type network struct { dialer Dialer - dns doh.Resolver + dns dnsResolver httpTimeout time.Duration userAgent string } @@ -84,14 +83,11 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { go func() { defer wg.Done() - if recs, _, err := n.dns.LookupA(address); err == nil { - mutex.Lock() - defer mutex.Unlock() + resolved := n.dns.LookupA(address) - for _, v := range recs { - ips = append(ips, v.IP4) - } - } + mutex.Lock() + ips = append(ips, resolved...) + mutex.Unlock() }() } @@ -102,14 +98,11 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { go func() { defer wg.Done() - if recs, _, err := n.dns.LookupAAAA(address); err == nil { - mutex.Lock() - defer mutex.Unlock() + resolved := n.dns.LookupAAAA(address) - for _, v := range recs { - ips = append(ips, v.IP6) - } - } + mutex.Lock() + ips = append(ips, resolved...) + mutex.Unlock() }() } @@ -140,11 +133,8 @@ func NewNetwork(dialer Dialer, dialer: dialer, httpTimeout: httpTimeout, userAgent: userAgent, - dns: doh.Resolver{ - Host: dohHostname, - Class: doh.IN, - HTTPClient: makeHTTPClient(userAgent, DNSTimeout, dialer.DialContext), - }, + dns: newDNSResolver(dohHostname, + makeHTTPClient(userAgent, DNSTimeout, dialer.DialContext)), }, nil } From 87a720dd58ff0c1fccaac61ff46c4286e0022450 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 14:29:39 +0300 Subject: [PATCH 148/206] Do not use ristretto Memory usage is too big --- go.mod | 5 +- go.sum | 7 --- network/dns_resolver.go | 85 +++++++++++++++------------ network/dns_resolver_internal_test.go | 2 +- network/network.go | 2 +- 5 files changed, 50 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 9112aba..b9f9f50 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/babolivier/go-doh-client v0.0.0-20201028162107-a76cff4cb8b6 github.com/d4l3k/messagediff v1.2.1 // indirect - github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/jarcoal/httpmock v1.0.8 github.com/kentik/patricia v0.0.0-20201202224819-f9447a6e25f1 github.com/libp2p/go-reuseport v0.0.2 @@ -23,7 +22,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/tylertreat/BoomFilters v0.0.0-20200520150052-42a7b4300c0c github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect + golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 - golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 // indirect + golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 ) diff --git a/go.sum b/go.sum index 12d7451..0cf083c 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -39,8 +38,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -58,10 +55,7 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= -github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -283,7 +277,6 @@ github.com/smira/go-statsd v1.3.2 h1:1EeuzxNZ/TD9apbTOFSM9nulqfcsQFmT4u1A2DREabI github.com/smira/go-statsd v1.3.2/go.mod h1:1srXJ9/pbnN04G8f4F1jUzsGOnwkPKXciyqpewGlkC4= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= diff --git a/network/dns_resolver.go b/network/dns_resolver.go index 3205b8f..23a4c37 100644 --- a/network/dns_resolver.go +++ b/network/dns_resolver.go @@ -2,27 +2,38 @@ package network import ( "net/http" + "sync" "time" doh "github.com/babolivier/go-doh-client" - "github.com/dgraph-io/ristretto" ) -const ( - dnsResolverSize = 1024 * 1024 // 1mb - dnsResolverKeepTime = 10 * time.Minute -) +const dnsResolverKeepTime = 10 * time.Minute -type dnsResolver struct { - resolver doh.Resolver - cache *ristretto.Cache +type dnsResolverCacheEntry struct { + ips []string + createdAt time.Time } -func (d dnsResolver) LookupA(hostname string) []string { - key := "\x00." + hostname +func (c dnsResolverCacheEntry) Ok() bool { + return time.Since(c.createdAt) < dnsResolverKeepTime +} - if value, ok := d.cache.Get(key); ok { - return value.([]string) +type dnsResolver struct { + resolver doh.Resolver + cache map[string]dnsResolverCacheEntry + cacheMutex sync.RWMutex +} + +func (d *dnsResolver) LookupA(hostname string) []string { + key := "\x00" + hostname + + d.cacheMutex.RLock() + entry, ok := d.cache[key] + d.cacheMutex.RUnlock() + + if ok && entry.Ok() { + return entry.ips } var ips []string @@ -32,17 +43,26 @@ func (d dnsResolver) LookupA(hostname string) []string { ips = append(ips, v.IP4) } - d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime) + d.cacheMutex.Lock() + d.cache[key] = dnsResolverCacheEntry{ + ips: ips, + createdAt: time.Now(), + } + d.cacheMutex.Unlock() } return ips } -func (d dnsResolver) LookupAAAA(hostname string) []string { - key := "\x01." + hostname +func (d *dnsResolver) LookupAAAA(hostname string) []string { + key := "\x01" + hostname - if value, ok := d.cache.Get(key); ok { - return value.([]string) + d.cacheMutex.RLock() + entry, ok := d.cache[key] + d.cacheMutex.RUnlock() + + if ok && entry.Ok() { + return entry.ips } var ips []string @@ -52,37 +72,24 @@ func (d dnsResolver) LookupAAAA(hostname string) []string { ips = append(ips, v.IP6) } - d.cache.SetWithTTL(key, ips, 0, dnsResolverKeepTime) + d.cacheMutex.Lock() + d.cache[key] = dnsResolverCacheEntry{ + ips: ips, + createdAt: time.Now(), + } + d.cacheMutex.Unlock() } return ips } -func newDNSResolver(hostname string, httpClient *http.Client) dnsResolver { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 10 * dnsResolverSize, // nolint: gomnd // taken from official doc as a best practice value - MaxCost: dnsResolverSize, - BufferItems: 64, // nolint: gomnd // taken from official doc as a best practice value - Cost: func(value interface{}) int64 { - var cost int64 - - for _, v := range value.([]string) { - cost += int64(len([]byte(v))) - } - - return cost - }, - }) - if err != nil { - panic(err) - } - - return dnsResolver{ +func newDNSResolver(hostname string, httpClient *http.Client) *dnsResolver { + return &dnsResolver{ resolver: doh.Resolver{ Host: hostname, Class: doh.IN, HTTPClient: httpClient, }, - cache: cache, + cache: map[string]dnsResolverCacheEntry{}, } } diff --git a/network/dns_resolver_internal_test.go b/network/dns_resolver_internal_test.go index a5934e8..ba54c0c 100644 --- a/network/dns_resolver_internal_test.go +++ b/network/dns_resolver_internal_test.go @@ -12,7 +12,7 @@ import ( type DNSResolverTestSuite struct { suite.Suite - d dnsResolver + d *dnsResolver } func (suite *DNSResolverTestSuite) TestLookupA() { diff --git a/network/network.go b/network/network.go index c65c282..147290b 100644 --- a/network/network.go +++ b/network/network.go @@ -25,9 +25,9 @@ func (n networkHTTPTransport) RoundTrip(req *http.Request) (*http.Response, erro type network struct { dialer Dialer - dns dnsResolver httpTimeout time.Duration userAgent string + dns *dnsResolver } func (n *network) Dial(protocol, address string) (net.Conn, error) { From 23c42def4f90e73d50d4bff190ecd09ee7984625 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 16:45:07 +0300 Subject: [PATCH 149/206] Add test for faketls type and version --- mtglib/internal/faketls/record/init.go | 2 +- mtglib/internal/faketls/record/init_test.go | 79 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 mtglib/internal/faketls/record/init_test.go diff --git a/mtglib/internal/faketls/record/init.go b/mtglib/internal/faketls/record/init.go index 764c29e..0ea3b5f 100644 --- a/mtglib/internal/faketls/record/init.go +++ b/mtglib/internal/faketls/record/init.go @@ -55,7 +55,7 @@ func (v Version) String() string { return "tls1.3" } - return fmt.Sprintf("tls(%d)", uint16(v)) + return fmt.Sprintf("tls?(%d)", uint16(v)) } func (v Version) Valid() error { diff --git a/mtglib/internal/faketls/record/init_test.go b/mtglib/internal/faketls/record/init_test.go new file mode 100644 index 0000000..7922777 --- /dev/null +++ b/mtglib/internal/faketls/record/init_test.go @@ -0,0 +1,79 @@ +package record_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/suite" +) + +type TypeTestSuite struct { + suite.Suite +} + +func (suite *TypeTestSuite) TestChangeCipherSpec() { + suite.Contains(record.TypeChangeCipherSpec.String(), "changeCipher") + suite.Contains(record.TypeChangeCipherSpec.String(), "0x14") + suite.NoError(record.TypeChangeCipherSpec.Valid()) +} + +func (suite *TypeTestSuite) TestHandshake() { + suite.Contains(record.TypeHandshake.String(), "handshake") + suite.Contains(record.TypeHandshake.String(), "0x16") + suite.NoError(record.TypeHandshake.Valid()) +} + +func (suite *TypeTestSuite) TestApplicationData() { + suite.Contains(record.TypeApplicationData.String(), "applicationData") + suite.Contains(record.TypeApplicationData.String(), "0x17") + suite.NoError(record.TypeApplicationData.Valid()) +} + +func (suite *TypeTestSuite) TestUnknown() { + value := record.Type(0x20) + + suite.Contains(value.String(), "unknown") + suite.Contains(value.String(), "0x20") + suite.Error(value.Valid()) +} + +type VersionTestSuite struct { + suite.Suite +} + +func (suite *VersionTestSuite) Test10() { + suite.Equal("tls1.0", record.Version10.String()) + suite.NoError(record.Version10.Valid()) +} + +func (suite *VersionTestSuite) Test11() { + suite.Equal("tls1.1", record.Version11.String()) + suite.NoError(record.Version11.Valid()) +} + +func (suite *VersionTestSuite) Test12() { + suite.Equal("tls1.2", record.Version12.String()) + suite.NoError(record.Version12.Valid()) +} + +func (suite *VersionTestSuite) Test13() { + suite.Equal("tls1.3", record.Version13.String()) + suite.NoError(record.Version13.Valid()) +} + +func (suite *VersionTestSuite) TestUnknown() { + value := record.Version(900) + + suite.Equal("tls?(900)", value.String()) + suite.Error(value.Valid()) +} + +func TestType(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeTestSuite{}) +} + +func TestVersion(t *testing.T) { + t.Parallel() + suite.Run(t, &VersionTestSuite{}) +} From 64754d1039636d27f8ac3dbd5f5ea7065b540898 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 17:46:08 +0300 Subject: [PATCH 150/206] Add tests for faketls record --- mtglib/internal/faketls/record/record_test.go | 110 ++++++++++++++++++ .../record/testdata/05eb6b71f87b6802.json | 6 + .../record/testdata/4eef4abc15b206b6.json | 6 + .../record/testdata/736f358216afe91f.json | 6 + .../record/testdata/8405d94222bd0b6a.json | 6 + .../record/testdata/9036f76e517f0cd1.json | 6 + .../record/testdata/9244766a0fe4a02a.json | 6 + .../record/testdata/9255c73d3de76e7b.json | 6 + .../record/testdata/aeb65b9924315cf8.json | 6 + .../record/testdata/b0acd44296056b54.json | 6 + .../record/testdata/c0545a13fd9a3fa3.json | 6 + .../record/testdata/f083f4501668b759.json | 6 + .../record/testdata/f5696bcdffd11706.json | 6 + 13 files changed, 182 insertions(+) create mode 100644 mtglib/internal/faketls/record/record_test.go create mode 100644 mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json create mode 100644 mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json create mode 100644 mtglib/internal/faketls/record/testdata/736f358216afe91f.json create mode 100644 mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json create mode 100644 mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json create mode 100644 mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json create mode 100644 mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json create mode 100644 mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json create mode 100644 mtglib/internal/faketls/record/testdata/b0acd44296056b54.json create mode 100644 mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json create mode 100644 mtglib/internal/faketls/record/testdata/f083f4501668b759.json create mode 100644 mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json diff --git a/mtglib/internal/faketls/record/record_test.go b/mtglib/internal/faketls/record/record_test.go new file mode 100644 index 0000000..e29b5ba --- /dev/null +++ b/mtglib/internal/faketls/record/record_test.go @@ -0,0 +1,110 @@ +package record_test + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type RecordTestSnapshot struct { + Type int `json:"type"` + Version int `json:"version"` + Payload string `json:"payload"` + Record string `json:"record"` +} + +func (r RecordTestSnapshot) RecordBytes() []byte { + data, _ := base64.StdEncoding.DecodeString(r.Record) + + return data +} + +func (r RecordTestSnapshot) PayloadBytes() []byte { + data, _ := base64.StdEncoding.DecodeString(r.Payload) + + return data +} + +type RecordTestSuite struct { + suite.Suite + + r *record.Record + buf *bytes.Buffer +} + +func (suite *RecordTestSuite) SetupTest() { + suite.r = record.AcquireRecord() + suite.buf = &bytes.Buffer{} +} + +func (suite *RecordTestSuite) TearDownTest() { + record.ReleaseRecord(suite.r) + suite.buf.Reset() +} + +func (suite *RecordTestSuite) TestIdempotent() { + suite.r.Type = record.TypeApplicationData + suite.r.Version = record.Version13 + + suite.r.Payload.Write([]byte{1, 2, 3}) + suite.NoError(suite.r.Dump(suite.buf)) + + suite.r.Reset() + suite.NoError(suite.r.Read(suite.buf)) + + suite.Equal(0, suite.buf.Len()) + suite.Equal(record.TypeApplicationData, suite.r.Type) + suite.Equal(record.Version13, suite.r.Version) + suite.Equal([]byte{1, 2, 3}, suite.r.Payload.Bytes()) +} + +func (suite *RecordTestSuite) TestString() { + _ = suite.r.String() +} + +func (suite *RecordTestSuite) TestSnapshot() { + files, err := os.ReadDir("testdata") + suite.NoError(err) + + testData := map[string]string{} + + for _, f := range files { + testData[f.Name()] = filepath.Join("testdata", f.Name()) + } + + for name, pathV := range testData { + path := pathV + + suite.T().Run(name, func(t *testing.T) { + data, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &RecordTestSnapshot{} + assert.NoError(t, json.Unmarshal(data, snapshot)) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + assert.NoError(t, rec.Read(bytes.NewReader(snapshot.RecordBytes()))) + assert.Equal(t, snapshot.Type, int(rec.Type)) + assert.Equal(t, snapshot.Version, int(rec.Version)) + assert.Equal(t, snapshot.PayloadBytes(), rec.Payload.Bytes()) + + buf := &bytes.Buffer{} + assert.NoError(t, rec.Dump(buf)) + assert.Equal(t, snapshot.RecordBytes(), buf.Bytes()) + }) + } +} + +func TestRecord(t *testing.T) { + t.Parallel() + suite.Run(t, &RecordTestSuite{}) +} diff --git a/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json b/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json new file mode 100644 index 0000000..6cac5ae --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/05eb6b71f87b6802.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 772, + "payload": "sxS+0oAyk+NBv0LLVtQOp9WSx4CweyUZPz01tQ0o4oyp8aaBl6/kMFvLq3q52KE8lCiKejLw2NxVBUkE+4izCf2gLx9qfr81opWnqJTChWzcDijvttbq9cmtDFNL+odKsS3v1/TfYEFtPsoRPrJRmOHRAnqnf49Y5Q==", + "record": "FAMEAHmzFL7SgDKT40G/QstW1A6n1ZLHgLB7JRk/PTW1DSjijKnxpoGXr+QwW8urernYoTyUKIp6MvDY3FUFSQT7iLMJ/aAvH2p+vzWilaeolMKFbNwOKO+21ur1ya0MU0v6h0qxLe/X9N9gQW0+yhE+slGY4dECeqd/j1jl" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json b/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json new file mode 100644 index 0000000..875d64f --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/4eef4abc15b206b6.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 772, + "payload": "waNH223htyxCBKAb6hm0u/SK/9mhI8Ck91nfWob7QMOaIREogrDYREJH4Djcp47XrpAlEaUIDiCvoFLVJ/LK1nYs4swzfHSSl/+Aj1eqPA63XqPa8EG4FAbf0DwjwXxV9qVIhvP9b2TafKbzr4Yb5GCygzFRb/zawA==", + "record": "FgMEAHnBo0fbbeG3LEIEoBvqGbS79Ir/2aEjwKT3Wd9ahvtAw5ohESiCsNhEQkfgONynjteukCURpQgOIK+gUtUn8srWdizizDN8dJKX/4CPV6o8Drdeo9rwQbgUBt/QPCPBfFX2pUiG8/1vZNp8pvOvhhvkYLKDMVFv/NrA" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/736f358216afe91f.json b/mtglib/internal/faketls/record/testdata/736f358216afe91f.json new file mode 100644 index 0000000..5b322d4 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/736f358216afe91f.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 769, + "payload": "jmJ0o1E5+ehAHHYAbCo4AMV03X7RSivYl250s06nD9CO44fyjaoGELz0N7IeCg1jFKcRVSCRmYYmiIY9wydn2fXOJhKif8B0BlM3qhbethYgyP+l1S8hyyETpIiOtiiiOnAJwl1D1j9OryFiJFSdRRXReIMZ4CPqPg==", + "record": "FwMBAHmOYnSjUTn56EAcdgBsKjgAxXTdftFKK9iXbnSzTqcP0I7jh/KNqgYQvPQ3sh4KDWMUpxFVIJGZhiaIhj3DJ2fZ9c4mEqJ/wHQGUzeqFt62FiDI/6XVLyHLIROkiI62KKI6cAnCXUPWP06vIWIkVJ1FFdF4gxngI+o+" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json b/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json new file mode 100644 index 0000000..16a4ba7 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/8405d94222bd0b6a.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 769, + "payload": "hBnpBnNUdlqe/rKXa7Judcz79u7AkUgSGOycn8EqvbkZpVxnI31rNOvAsPZqG+GF7DWJ3R7H2ETmFmrpnyyng32MjSs1jptmV1oAs63zTADD7sVipgid9AJHwfl4CrC3FIQr43IPMYd29JPOl5bqu/SfrgI16PBiJw==", + "record": "FgMBAHmEGekGc1R2Wp7+spdrsm51zPv27sCRSBIY7JyfwSq9uRmlXGcjfWs068Cw9mob4YXsNYndHsfYROYWaumfLKeDfYyNKzWOm2ZXWgCzrfNMAMPuxWKmCJ30AkfB+XgKsLcUhCvjcg8xh3b0k86Xluq79J+uAjXo8GIn" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json b/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json new file mode 100644 index 0000000..5d60155 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/9036f76e517f0cd1.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 770, + "payload": "Vm/C+DO56czlbtR915aHzsugSyDtp8CtojF9w1jKY0efyyfcLrNuhNg/pZm3gQ7v2BBbL1UJ97v/RIjST+5gRIfg3bBN1BE9hkf+N2AYY2lHLi0yeInHB0zFWPeHscsDopDFadIi5KtC8HvbEMuK+kK8POVk5tN9UQ==", + "record": "FwMCAHlWb8L4M7npzOVu1H3XlofOy6BLIO2nwK2iMX3DWMpjR5/LJ9wus26E2D+lmbeBDu/YEFsvVQn3u/9EiNJP7mBEh+DdsE3UET2GR/43YBhjaUcuLTJ4iccHTMVY94exywOikMVp0iLkq0Lwe9sQy4r6Qrw85WTm031R" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json b/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json new file mode 100644 index 0000000..459986b --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/9244766a0fe4a02a.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 770, + "payload": "ajPzpsgk4gwm2stRQKbllvKRLdI7vmyaj1uxEJ/kKoQnQSPumdDNKD618U2Cq6PVd0/b+9YtH67Uzx1QxtpKuby5fUXqw06WUuDAQsmjq7F26EkE5FND6rQUjUPC+e1U0dF4TQzOUSS4IAkFQPAaVehUVTRxVWa/0g==", + "record": "FgMCAHlqM/OmyCTiDCbay1FApuWW8pEt0ju+bJqPW7EQn+QqhCdBI+6Z0M0oPrXxTYKro9V3T9v71i0frtTPHVDG2kq5vLl9RerDTpZS4MBCyaOrsXboSQTkU0PqtBSNQ8L57VTR0XhNDM5RJLggCQVA8BpV6FRVNHFVZr/S" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json b/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json new file mode 100644 index 0000000..e49ea0a --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/9255c73d3de76e7b.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 771, + "payload": "d1Hiv1NYVgEDR9mtJyv9j8mg3dWqfUpeKfOsL+jzSDfVIxeDiJZFLDT50TjNW44/yEOVEX/Y/pk+wnc7E8aCEiwGwAvB+Insw1UCJ2ejt689VWLo2u4klGVKTHuOpUvdGVTc7Lo4FAt91KQSPLYB5iqxomjEv5e3Vg==", + "record": "FAMDAHl3UeK/U1hWAQNH2a0nK/2PyaDd1ap9Sl4p86wv6PNIN9UjF4OIlkUsNPnROM1bjj/IQ5URf9j+mT7CdzsTxoISLAbAC8H4iezDVQInZ6O3rz1VYuja7iSUZUpMe46lS90ZVNzsujgUC33UpBI8tgHmKrGiaMS/l7dW" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json b/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json new file mode 100644 index 0000000..7eac0da --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/aeb65b9924315cf8.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 771, + "payload": "wbdU1CbrzuAJDsh6CFjGyE+AFArJj/Wmsa2wtDyW0kRuE2vUO8gg+nXkg0kkoz0WnvQEOdaswfJIaVrloD78yoyeQVfBB+VUP/63vqn60v5ccaQEn0jLdxgLjiTAxKDQDxCTMRoLnFE2ZZf28zw+HfqpIxiOZs8LhQ==", + "record": "FwMDAHnBt1TUJuvO4AkOyHoIWMbIT4AUCsmP9aaxrbC0PJbSRG4Ta9Q7yCD6deSDSSSjPRae9AQ51qzB8khpWuWgPvzKjJ5BV8EH5VQ//re+qfrS/lxxpASfSMt3GAuOJMDEoNAPEJMxGgucUTZll/bzPD4d+qkjGI5mzwuF" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json b/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json new file mode 100644 index 0000000..3fffda7 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/b0acd44296056b54.json @@ -0,0 +1,6 @@ +{ + "type": 23, + "version": 772, + "payload": "qqnBMb1Af3zZt4DPHpVRuIiON9ODGJUNFicFjranORh67L/HI4D6HnHyycZFUSBOw2FjMBF6UialY8snOYaRKrQmQzuUNg1Ztq7yAZ+Lgj3TBarR6OMlYhEAY0Px9Xv1UuJ0YcvQx33gdM1skJ5HBR3yZvEKNJV1LA==", + "record": "FwMEAHmqqcExvUB/fNm3gM8elVG4iI4304MYlQ0WJwWOtqc5GHrsv8cjgPoecfLJxkVRIE7DYWMwEXpSJqVjyyc5hpEqtCZDO5Q2DVm2rvIBn4uCPdMFqtHo4yViEQBjQ/H1e/VS4nRhy9DHfeB0zWyQnkcFHfJm8Qo0lXUs" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json b/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json new file mode 100644 index 0000000..aa11362 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/c0545a13fd9a3fa3.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 769, + "payload": "NEe735TuQFp7bWpFQhASas/e1XaySvus0ovXmkfCbFq334MyFHq2eDMadziXsfu/GfBjoYggvk0LgYUeoAkBNKR0dfSovjSndaqmIUonoWl+6sZObiGZkRIMwuY2q4Eaw4/iuDu/pZhjRW/iAIH+YH7cyk/1tgdJDg==", + "record": "FAMBAHk0R7vflO5AWnttakVCEBJqz97VdrJK+6zSi9eaR8JsWrffgzIUerZ4Mxp3OJex+78Z8GOhiCC+TQuBhR6gCQE0pHR19Ki+NKd1qqYhSiehaX7qxk5uIZmREgzC5jargRrDj+K4O7+lmGNFb+IAgf5gftzKT/W2B0kO" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/f083f4501668b759.json b/mtglib/internal/faketls/record/testdata/f083f4501668b759.json new file mode 100644 index 0000000..2518193 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/f083f4501668b759.json @@ -0,0 +1,6 @@ +{ + "type": 22, + "version": 771, + "payload": "wrXjZrPm3OSyzO0klv6/G+z2PDloR/colS/RlWwQE31Vb2xm8YkEchDDKwlc/KPLD73qMoz3MQOQLtSLc8LhVYp+l7L9jz49yTaVKtBI5UuGbo09snsKxFCgCyYUBETKabATBQtiaEu/D8dmF4Yk/2ww4sEb8DwKLQ==", + "record": "FgMDAHnCteNms+bc5LLM7SSW/r8b7PY8OWhH9yiVL9GVbBATfVVvbGbxiQRyEMMrCVz8o8sPveoyjPcxA5Au1ItzwuFVin6Xsv2PPj3JNpUq0EjlS4ZujT2yewrEUKALJhQERMppsBMFC2JoS78Px2YXhiT/bDDiwRvwPAot" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json b/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json new file mode 100644 index 0000000..055a7a6 --- /dev/null +++ b/mtglib/internal/faketls/record/testdata/f5696bcdffd11706.json @@ -0,0 +1,6 @@ +{ + "type": 20, + "version": 770, + "payload": "OU5s8Sa11hpXWEarWzFlX55IZt3Eo+F4AMbQ/2RwB4rfHS/JNl8n63OR4oYs9QXw3RfCrYJuU9n6Xn+I/+7ZzAgZ0PbLSXW1PrLtttdfmhTErK90b49YEWdY9na4g++NMkKykwgXvY1hNxZIHX/qawEWJgxXUR3DdQ==", + "record": "FAMCAHk5TmzxJrXWGldYRqtbMWVfnkhm3cSj4XgAxtD/ZHAHit8dL8k2Xyfrc5Hihiz1BfDdF8Ktgm5T2fpef4j/7tnMCBnQ9stJdbU+su2211+aFMSsr3Rvj1gRZ1j2driD740yQrKTCBe9jWE3Fkgdf+prARYmDFdRHcN1" +} \ No newline at end of file From 5b0d54e74d2142962ed6cceaa8afcf0673ab3282 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 17:58:42 +0300 Subject: [PATCH 151/206] Add prefix for codecov --- .codecov.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..d0f97cb --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,4 @@ +--- + +fixes: + - "github.com/9seconds/mtg/v2/::" From fdbee86d5e5dd282283181138d1632d9622836d9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 30 Mar 2021 18:14:28 +0300 Subject: [PATCH 152/206] Ignore coverge.txt --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cb7ca4d..ef266e3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ mtg vendor/ ccbuilds/ .bin/ +coverage.txt From 491b674e702e805ebffb0950c393b57641c4dd21 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 31 Mar 2021 12:24:04 +0300 Subject: [PATCH 153/206] Add tests for client hello --- mtglib/internal/faketls/client_hello_test.go | 159 ++++++++++++++++++ .../client-hello-bad-fa2e46cdb33e2a1b.json | 8 + .../client-hello-ok-19dfe38384b9884b.json | 8 + .../client-hello-ok-48f8a72a56f3174a.json | 8 + .../client-hello-ok-651054256093c6cd.json | 8 + .../client-hello-ok-79d01ef18a9d2621.json | 8 + .../client-hello-ok-7a5569f05b118145.json | 8 + 7 files changed, 207 insertions(+) create mode 100644 mtglib/internal/faketls/client_hello_test.go create mode 100644 mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json create mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json create mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json create mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json create mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json create mode 100644 mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json diff --git a/mtglib/internal/faketls/client_hello_test.go b/mtglib/internal/faketls/client_hello_test.go new file mode 100644 index 0000000..f088cf2 --- /dev/null +++ b/mtglib/internal/faketls/client_hello_test.go @@ -0,0 +1,159 @@ +package faketls_test + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type ClientHelloSnapshot struct { + Time int `json:"time"` + Random string `json:"random"` + SessionID string `json:"session-id"` + Host string `json:"host"` + CipherSuite int `json:"cipher-suite"` + Full string `json:"full"` +} + +func (c ClientHelloSnapshot) GetTime() time.Time { + return time.Unix(int64(c.Time), 0) +} + +func (c ClientHelloSnapshot) GetRandom() []byte { + data, _ := base64.StdEncoding.DecodeString(c.Random) + + return data +} + +func (c ClientHelloSnapshot) GetSessionID() []byte { + data, _ := base64.StdEncoding.DecodeString(c.SessionID) + + return data +} + +func (c ClientHelloSnapshot) GetHost() string { + return c.Host +} + +func (c ClientHelloSnapshot) GetCipherSuite() uint16 { + return uint16(c.CipherSuite) +} + +func (c ClientHelloSnapshot) GetFull() []byte { + data, _ := base64.StdEncoding.DecodeString(c.Full) + + return data +} + +type ClientHelloTestSuite struct { + suite.Suite + + secret mtglib.Secret +} + +func (suite *ClientHelloTestSuite) SetupSuite() { + parsed, err := mtglib.ParseSecret("ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d") + if err != nil { + panic(err) + } + + suite.secret = parsed +} + +func (suite *ClientHelloTestSuite) TestEmptyHandshake() { + _, err := faketls.ParseClientHello(suite.secret.Key[:], nil) + suite.Error(err) +} + +func (suite *ClientHelloTestSuite) TestIncorrectHandshakeType() { + data := make([]byte, 1024) + data[0] = 0x02 + + _, err := faketls.ParseClientHello(suite.secret.Key[:], data) + suite.Error(err) +} + +func (suite *ClientHelloTestSuite) TestIncorrectLength() { + data := make([]byte, 1024) + data[0] = 0x01 + data[1] = 0xff + data[2] = 0xff + + _, err := faketls.ParseClientHello(suite.secret.Key[:], data) + suite.Error(err) +} + +func (suite *ClientHelloTestSuite) TestSnapshotOk() { + files, err := os.ReadDir("testdata") + suite.NoError(err) + + testData := []string{} + + for _, v := range files { + if strings.HasPrefix(v.Name(), "client-hello-ok") { + testData = append(testData, v.Name()) + } + } + + for _, name := range testData { + path := filepath.Join("testdata", name) + + suite.T().Run(name, func(t *testing.T) { + fileData, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &ClientHelloSnapshot{} + assert.NoError(t, json.Unmarshal(fileData, snapshot)) + + hello, err := faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull()) + assert.NoError(t, err) + assert.WithinDuration(t, snapshot.GetTime(), hello.Time, time.Second) + assert.Equal(t, snapshot.GetRandom(), hello.Random[:]) + assert.Equal(t, snapshot.GetSessionID(), hello.SessionID) + assert.Equal(t, snapshot.GetHost(), hello.Host) + assert.Equal(t, snapshot.GetCipherSuite(), hello.CipherSuite) + }) + } +} + +func (suite *ClientHelloTestSuite) TestSnapshotBad() { + files, err := os.ReadDir("testdata") + suite.NoError(err) + + testData := []string{} + + for _, v := range files { + if strings.HasPrefix(v.Name(), "client-hello-bad") { + testData = append(testData, v.Name()) + } + } + + for _, name := range testData { + path := filepath.Join("testdata", name) + + suite.T().Run(name, func(t *testing.T) { + fileData, err := os.ReadFile(path) + assert.NoError(t, err) + + snapshot := &ClientHelloSnapshot{} + assert.NoError(t, json.Unmarshal(fileData, snapshot)) + + _, err = faketls.ParseClientHello(suite.secret.Key[:], snapshot.GetFull()) + assert.Error(t, err) + }) + } +} + +func TestClientHello(t *testing.T) { + t.Parallel() + suite.Run(t, &ClientHelloTestSuite{}) +} diff --git a/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json b/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json new file mode 100644 index 0000000..046a122 --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-bad-fa2e46cdb33e2a1b.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", + "session-id": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCACq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFANAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json b/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json new file mode 100644 index 0000000..0a47b92 --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-19dfe38384b9884b.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "XvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPs=", + "session-id": "St2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDXvCPc3aAbHbhRLv0kUmy6BfPZOGvsused5/HNsKXEPsgSt2BZ2uHMFn3B2trD1jfdtpjoJOOg6JBeLhFcyCMCq4ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgB/7oLx9JElIALsLJS91H2QNyU1H0osKwIUelVndsLyIALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json b/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json new file mode 100644 index 0000000..adc7cc7 --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-48f8a72a56f3174a.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "oYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12os=", + "session-id": "FGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8g=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDoYEu33jl+zQbUKMtQbV1OHB0gXIM2y2aq9iY0QX12osgFGqA3ZFYrSlj//xl7lammNn64K9/MK2mQ3HJUGvP+8gANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAga6CocpFP8Qd4YCFR9pkaCr97po2ALj0P5nI9Nnb3UWMALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json b/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json new file mode 100644 index 0000000..c5165fb --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-651054256093c6cd.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1ro=", + "session-id": "jxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMD5V5sSprk/tFIgy+x1BeKNGhLlFkqfggLpgN7GYOA1rogjxr4d6PXPDk+Lwx3WUp9wvj8TGlOxEdrRJ0ydyJ9+H8ANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgrulAaqUdKeVYM0F+pu6on/h6LBpOyzOKG4xFIKcoFk4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json b/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json new file mode 100644 index 0000000..a642fec --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-79d01ef18a9d2621.json @@ -0,0 +1,8 @@ +{ + "time": 1617181365, + "random": "8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQ=", + "session-id": "00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAU=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMD8xljlOhkDlkafEF5vu3e1r3fWvh8AX548wC3hLZ3szQg00uvDYKnFyZFKyf3HlLwWGCOyeHsPFiU5UZ+Fs5pDAUANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAg/9P7140NtKzjyDwBf99mOy1+FjRPAPHTNQ9WxHOKpV4ALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file diff --git a/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json b/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json new file mode 100644 index 0000000..5b803da --- /dev/null +++ b/mtglib/internal/faketls/testdata/client-hello-ok-7a5569f05b118145.json @@ -0,0 +1,8 @@ +{ + "time": 1617181352, + "random": "zja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4=", + "session-id": "qPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnA=", + "host": "storage.googleapis.com", + "cipher-suite": 4867, + "full": "AQAB/AMDzja3MLZ8WGSfsQRtPV75+tY6gbK3zKPi1Sy7SBBafg4gqPut2yMqXa9zGLII/872SQ3d4Tfqo0uoDb7tpkRfBnAANBMDEwETAsAswCvAJMAjwArACcypwDDAL8AowCfAFMATzKgAnQCcAD0APAA1AC/ACMASAAoBAAF//wEAAQAAAAAbABkAABZzdG9yYWdlLmdvb2dsZWFwaXMuY29tABcAAAANABgAFgQDCAQEAQUDAgMIBQgFBQEIBgYBAgEABQAFAQAAAAAzdAAAABIAAAAQADAALgJoMgVoMi0xNgVoMi0xNQVoMi0xNAhzcGR5LzMuMQZzcGR5LzMIaHR0cC8xLjEACwACAQAAMwAmACQAHQAgXviLRAqAYJ8xOLdlcsUhldI4Xl0g/s9+y2Qrd8raPEgALQACAQEAKwAJCAMEAwMDAgMBAAoACgAIAB0AFwAYABkAFQChAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +} \ No newline at end of file From 96bcb6cd8e4fb5b6899a802df133c12389d265be Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 11:03:42 +0300 Subject: [PATCH 154/206] Add tests for welcome packet --- mtglib/internal/faketls/welcome_test.go | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 mtglib/internal/faketls/welcome_test.go diff --git a/mtglib/internal/faketls/welcome_test.go b/mtglib/internal/faketls/welcome_test.go new file mode 100644 index 0000000..33129e2 --- /dev/null +++ b/mtglib/internal/faketls/welcome_test.go @@ -0,0 +1,82 @@ +package faketls_test + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "math/rand" + "testing" + "time" + + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/stretchr/testify/suite" +) + +type WelcomeTestSuite struct { + suite.Suite + + h *faketls.ClientHello + buf *bytes.Buffer + secret mtglib.Secret +} + +func (suite *WelcomeTestSuite) SetupTest() { + suite.h = &faketls.ClientHello{ + Time: time.Now(), + Host: "google.com", + CipherSuite: 4867, + SessionID: make([]byte, 32), + } + + _, err := rand.Read(suite.h.SessionID) + suite.NoError(err) + + _, err = rand.Read(suite.h.Random[:]) + suite.NoError(err) + + suite.buf = &bytes.Buffer{} + + suite.secret = mtglib.GenerateSecret("google.com") +} + +func (suite *WelcomeTestSuite) TestOk() { + suite.NoError(faketls.SendWelcomePacket(suite.buf, suite.secret.Key[:], *suite.h)) + + welcomePacket := []byte{} + welcomePacket = append(welcomePacket, suite.buf.Bytes()...) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + suite.NoError(rec.Read(suite.buf)) + suite.Equal(record.TypeHandshake, rec.Type) + suite.Equal(record.Version12, rec.Version) + + suite.NoError(rec.Read(suite.buf)) + suite.Equal(record.TypeChangeCipherSpec, rec.Type) + suite.Equal(record.Version12, rec.Version) + + suite.NoError(rec.Read(suite.buf)) + suite.Equal(record.TypeApplicationData, rec.Type) + suite.Equal(record.Version12, rec.Version) + suite.Empty(suite.buf.Bytes()) + + random := make([]byte, 32) + copy(random, welcomePacket[11:]) + + empty := make([]byte, 32) + copy(welcomePacket[11:], empty) + + mac := hmac.New(sha256.New, suite.secret.Key[:]) + mac.Write(suite.h.Random[:]) + mac.Write(welcomePacket) + + suite.Equal(random, mac.Sum(nil)) +} + +func TestWelcome(t *testing.T) { + t.Parallel() + suite.Run(t, &WelcomeTestSuite{}) +} From 54f4b397b6d3105bbc35c41c22368bd8f6766325 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 11:17:45 +0300 Subject: [PATCH 155/206] Add tests for relay conn --- mtglib/internal/relay/conn_internal_test.go | 125 ++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 mtglib/internal/relay/conn_internal_test.go diff --git a/mtglib/internal/relay/conn_internal_test.go b/mtglib/internal/relay/conn_internal_test.go new file mode 100644 index 0000000..0514767 --- /dev/null +++ b/mtglib/internal/relay/conn_internal_test.go @@ -0,0 +1,125 @@ +package relay + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnTestSuite struct { + suite.Suite + + ctxCancel context.CancelFunc + connMock *testlib.NetConnMock + tickChannel chan struct{} + buf []byte + c conn +} + +func (suite *ConnTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + + suite.tickChannel = make(chan struct{}, 1) + suite.connMock = &testlib.NetConnMock{} + suite.ctxCancel = cancel + suite.buf = make([]byte, 5) + + suite.c = conn{ + ReadWriteCloser: suite.connMock, + ctx: ctx, + tickChannel: suite.tickChannel, + } +} + +func (suite *ConnTestSuite) TestReadOk() { + suite.connMock.On("Read", mock.Anything).Once().Return(len(suite.buf), nil) + + n, err := suite.c.Read(suite.buf) + suite.NoError(err) + suite.Equal(len(suite.buf), n) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestReadErr() { + suite.connMock.On("Read", mock.Anything).Once().Return(0, io.EOF) + + _, err := suite.c.Read(suite.buf) + suite.True(errors.Is(err, io.EOF)) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestReadContextDone() { + suite.connMock.On("Read", mock.Anything).Once().Return(len(suite.buf), nil) + suite.ctxCancel() + + suite.tickChannel <- struct{}{} + + suite.c.Read(suite.buf) +} + +func (suite *ConnTestSuite) TestWriteOk() { + suite.connMock.On("Write", mock.Anything).Once().Return(len(suite.buf), nil) + + n, err := suite.c.Write(suite.buf) + suite.NoError(err) + suite.Equal(len(suite.buf), n) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestWriteErr() { + suite.connMock.On("Write", mock.Anything).Once().Return(0, io.EOF) + + _, err := suite.c.Write(suite.buf) + suite.True(errors.Is(err, io.EOF)) + + select { + case <-suite.tickChannel: + default: + suite.FailNow("cannot find a tick event") + } +} + +func (suite *ConnTestSuite) TestWriteContextDone() { + suite.connMock.On("Write", mock.Anything).Once().Return(len(suite.buf), nil) + suite.ctxCancel() + + suite.tickChannel <- struct{}{} + + suite.c.Write(suite.buf) +} + +func (suite *ConnTestSuite) TearDownTest() { + select { + case <-suite.tickChannel: + default: + } + + close(suite.tickChannel) + + suite.connMock.AssertExpectations(suite.T()) +} + +func TestConn(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTestSuite{}) +} From dc28677086636a3bf3654ccce3ae2553f88d1f23 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 11:20:24 +0300 Subject: [PATCH 156/206] Fix lint --- mtglib/internal/faketls/welcome_test.go | 4 ++-- mtglib/internal/relay/conn_internal_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mtglib/internal/faketls/welcome_test.go b/mtglib/internal/faketls/welcome_test.go index 33129e2..4f1b990 100644 --- a/mtglib/internal/faketls/welcome_test.go +++ b/mtglib/internal/faketls/welcome_test.go @@ -70,8 +70,8 @@ func (suite *WelcomeTestSuite) TestOk() { copy(welcomePacket[11:], empty) mac := hmac.New(sha256.New, suite.secret.Key[:]) - mac.Write(suite.h.Random[:]) - mac.Write(welcomePacket) + mac.Write(suite.h.Random[:]) // nolint: errcheck + mac.Write(welcomePacket) // nolint: errcheck suite.Equal(random, mac.Sum(nil)) } diff --git a/mtglib/internal/relay/conn_internal_test.go b/mtglib/internal/relay/conn_internal_test.go index 0514767..26ed696 100644 --- a/mtglib/internal/relay/conn_internal_test.go +++ b/mtglib/internal/relay/conn_internal_test.go @@ -69,7 +69,7 @@ func (suite *ConnTestSuite) TestReadContextDone() { suite.tickChannel <- struct{}{} - suite.c.Read(suite.buf) + suite.c.Read(suite.buf) // nolint: errcheck } func (suite *ConnTestSuite) TestWriteOk() { @@ -105,7 +105,7 @@ func (suite *ConnTestSuite) TestWriteContextDone() { suite.tickChannel <- struct{}{} - suite.c.Write(suite.buf) + suite.c.Write(suite.buf) // nolint: errcheck } func (suite *ConnTestSuite) TearDownTest() { From 03d98ee2566d41d70ab9f430b9372a25066d6088 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 11:28:58 +0300 Subject: [PATCH 157/206] Add relay test for timeout --- mtglib/internal/relay/relay_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mtglib/internal/relay/relay_test.go b/mtglib/internal/relay/relay_test.go index 54b257d..55b7732 100644 --- a/mtglib/internal/relay/relay_test.go +++ b/mtglib/internal/relay/relay_test.go @@ -2,10 +2,13 @@ package relay_test import ( "context" + "io" "testing" "time" "github.com/9seconds/mtg/v2/mtglib/internal/relay" + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -52,6 +55,22 @@ func (suite *RelayTestSuite) TestCopyFine() { _ = suite.r.Process(eastConn, westConn) } +func (suite *RelayTestSuite) TestTimeout() { + eastConn := &rwcMock{} + eastConn.Write([]byte{1, 2, 3, 4, 5}) // nolint: errcheck + + westConn := &testlib.NetConnMock{} + westConn.On("Close").Return(nil) + westConn.On("Read", mock.Anything).Return(0, io.EOF).Run(func(_ mock.Arguments) { + time.Sleep(2 * time.Second) + }) + westConn.On("Write", mock.Anything).Return(0, io.EOF).Run(func(_ mock.Arguments) { + time.Sleep(2 * time.Second) + }) + + suite.Error(suite.r.Process(eastConn, westConn)) +} + func TestRelay(t *testing.T) { t.Parallel() suite.Run(t, &RelayTestSuite{}) From 26070d5b3e536e4a4feb10ad8042f81873a3b9e2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 14:12:29 +0300 Subject: [PATCH 158/206] More correct calculation of dc for obfuscated2 frame --- .../internal/obfuscated2/handshake_frame.go | 7 +---- .../handshake_frame_internal_test.go | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/mtglib/internal/obfuscated2/handshake_frame.go b/mtglib/internal/obfuscated2/handshake_frame.go index fde59ae..fff548f 100644 --- a/mtglib/internal/obfuscated2/handshake_frame.go +++ b/mtglib/internal/obfuscated2/handshake_frame.go @@ -1,7 +1,5 @@ package obfuscated2 -import "encoding/binary" - const ( DefaultDC = 2 @@ -10,14 +8,12 @@ const ( handshakeFrameLenKey = 32 handshakeFrameLenIV = 16 handshakeFrameLenConnectionType = 4 - handshakeFrameLenDC = 2 handshakeFrameOffsetStart = 8 handshakeFrameOffsetKey = handshakeFrameOffsetStart handshakeFrameOffsetIV = handshakeFrameOffsetKey + handshakeFrameLenKey handshakeFrameOffsetConnectionType = handshakeFrameOffsetIV + handshakeFrameLenIV handshakeFrameOffsetDC = handshakeFrameOffsetConnectionType + handshakeFrameLenConnectionType - handshakeFrameOffsetEnd = handshakeFrameOffsetDC + handshakeFrameLenDC ) // Connection-Type: Secure. We support only fake tls. @@ -38,8 +34,7 @@ type handshakeFrame struct { } func (h *handshakeFrame) dc() int { - data := h.data[handshakeFrameOffsetDC:handshakeFrameOffsetEnd] - idx := int16(binary.LittleEndian.Uint16(data)) + idx := int16(h.data[handshakeFrameOffsetDC]) | int16(h.data[handshakeFrameOffsetDC+1])<<8 // nolint: gomnd, lll // little endian for int16 is here switch { case idx > 0: diff --git a/mtglib/internal/obfuscated2/handshake_frame_internal_test.go b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go index 4fedb3f..7b197db 100644 --- a/mtglib/internal/obfuscated2/handshake_frame_internal_test.go +++ b/mtglib/internal/obfuscated2/handshake_frame_internal_test.go @@ -1,9 +1,12 @@ package obfuscated2 import ( + "crypto/rand" "encoding/base64" + "strconv" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -40,6 +43,30 @@ func (suite *HandshakeFrameTestSuite) TestOk() { suite.EqualValues(2094, inverted.dc()) } +func (suite *HandshakeFrameTestSuite) TestDC() { + testData := map[int16]int{ + 1: 1, + -1: 1, + 0: DefaultDC, + } + + for k, v := range testData { + incoming := k + expected := v + + suite.T().Run(strconv.Itoa(int(incoming)), func(t *testing.T) { + frame := handshakeFrame{} + + rand.Read(frame.data[:]) // nolint: errcheck + + frame.data[handshakeFrameOffsetDC] = byte(incoming) + frame.data[handshakeFrameOffsetDC+1] = byte(incoming >> 8) + + assert.Equal(t, expected, frame.dc()) + }) + } +} + func TestHandshakeFrame(t *testing.T) { t.Parallel() suite.Run(t, &HandshakeFrameTestSuite{}) From 38ff62fddcab66017c7e1cde08383e2da5a32de5 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 15:41:51 +0300 Subject: [PATCH 159/206] Add tests for faketls conn --- mtglib/internal/faketls/conn_test.go | 153 +++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 mtglib/internal/faketls/conn_test.go diff --git a/mtglib/internal/faketls/conn_test.go b/mtglib/internal/faketls/conn_test.go new file mode 100644 index 0000000..41bff3d --- /dev/null +++ b/mtglib/internal/faketls/conn_test.go @@ -0,0 +1,153 @@ +package faketls_test + +import ( + "bytes" + "errors" + "io" + "math/rand" + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/faketls" + "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnMock struct { + testlib.NetConnMock + + readBuffer bytes.Buffer + writeBuffer bytes.Buffer +} + +func (m *ConnMock) Read(p []byte) (int, error) { + m.Called(p) + + return m.readBuffer.Read(p) +} + +func (m *ConnMock) Write(p []byte) (int, error) { + m.Called(p) + + return m.writeBuffer.Write(p) +} + +type ConnTestSuite struct { + suite.Suite + + connMock *ConnMock + c *faketls.Conn +} + +func (suite *ConnTestSuite) SetupTest() { + suite.connMock = &ConnMock{} + suite.c = &faketls.Conn{ + Conn: suite.connMock, + } +} + +func (suite *ConnTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnTestSuite) TestRead() { + suite.connMock.On("Read", mock.Anything).Return(0, nil) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeChangeCipherSpec + rec.Version = record.Version12 + + rec.Payload.WriteByte(0x01) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + rec.Reset() + + rec.Type = record.TypeApplicationData + rec.Version = record.Version12 + + rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + + resultBuffer := &bytes.Buffer{} + buf := make([]byte, 2) + + for { + n, err := suite.c.Read(buf) + if errors.Is(err, io.EOF) { + break + } + + resultBuffer.Write(buf[:n]) + } + + suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, resultBuffer.Bytes()) +} + +func (suite *ConnTestSuite) TestReadUnexpected() { + suite.connMock.On("Read", mock.Anything).Return(0, nil) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + rec.Type = record.TypeChangeCipherSpec + rec.Version = record.Version12 + + rec.Payload.WriteByte(0x01) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + rec.Reset() + + rec.Type = record.TypeHandshake + rec.Version = record.Version12 + + rec.Payload.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + rec.Dump(&suite.connMock.readBuffer) // nolint: errcheck + + buf := make([]byte, 2) + + for { + _, err := suite.c.Read(buf) + + switch { + case err == nil: + case errors.Is(err, io.EOF): + suite.FailNow("unexpected to finish") + default: + return + } + } +} + +func (suite *ConnTestSuite) TestWrite() { + suite.connMock.On("Write", mock.Anything).Return(0, nil) + + dataToRec := make([]byte, record.TLSMaxRecordSize*2) + rand.Read(dataToRec) + + n, err := suite.c.Write(dataToRec) + suite.NoError(err) + suite.Equal(len(dataToRec), n) + + rec := record.AcquireRecord() + defer record.ReleaseRecord(rec) + + buf := &bytes.Buffer{} + + for { + if err := rec.Read(&suite.connMock.writeBuffer); err != nil { + break + } + + suite.Equal(record.TypeApplicationData, rec.Type) + suite.Equal(record.Version12, rec.Version) + rec.Payload.WriteTo(buf) // nolint: errcheck + } + + suite.Equal(dataToRec, buf.Bytes()) +} + +func TestConn(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTestSuite{}) +} From 21d4f3d10113a7e26ae33431b56235a2e8d53d0d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 16:51:24 +0300 Subject: [PATCH 160/206] Add string test for config --- config/config_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index 8ef99df..940775c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -42,6 +42,12 @@ func (suite *ConfigTestSuite) TestParseMinimalConfig() { suite.Equal("0.0.0.0:3128", conf.BindTo.String()) } +func (suite *ConfigTestSuite) TestString() { + conf, err := config.Parse(suite.ReadConfig("minimal.toml")) + suite.NoError(err) + suite.NotEmpty(conf.String()) +} + func TestConfig(t *testing.T) { t.Parallel() suite.Run(t, &ConfigTestSuite{}) From 9562a8603892731b8e70744c496c1d4909075856 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 16:51:38 +0300 Subject: [PATCH 161/206] Update test coverage for typeblocklisturi --- config/type_blocklist_uri_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/config/type_blocklist_uri_test.go b/config/type_blocklist_uri_test.go index e0b8974..6ced8ac 100644 --- a/config/type_blocklist_uri_test.go +++ b/config/type_blocklist_uri_test.go @@ -22,6 +22,22 @@ type TypeBlocklistURITestSuite struct { suite.Suite } +func (suite *TypeBlocklistURITestSuite) TestUnmarshalNil() { + typ := &config.TypeBlocklistURI{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + +func (suite *TypeBlocklistURITestSuite) TestUnknownSchema() { + typ := &config.TypeBlocklistURI{} + suite.Error(typ.UnmarshalText([]byte("gopher://lalala"))) +} + +func (suite *TypeBlocklistURITestSuite) TestEmptyHost() { + typ := &config.TypeBlocklistURI{} + suite.Error(typ.UnmarshalText([]byte("https:///path"))) +} + func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() { rnd := make([]byte, 48) From 1925efc2ec459799d83dc864dff6298930b9f505 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 16:53:48 +0300 Subject: [PATCH 162/206] Test unmarshal nil for type bytes --- config/type_blocklist_uri_test.go | 4 ++-- config/type_bytes_test.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/type_blocklist_uri_test.go b/config/type_blocklist_uri_test.go index 6ced8ac..6e0b260 100644 --- a/config/type_blocklist_uri_test.go +++ b/config/type_blocklist_uri_test.go @@ -30,12 +30,12 @@ func (suite *TypeBlocklistURITestSuite) TestUnmarshalNil() { func (suite *TypeBlocklistURITestSuite) TestUnknownSchema() { typ := &config.TypeBlocklistURI{} - suite.Error(typ.UnmarshalText([]byte("gopher://lalala"))) + suite.Error(typ.UnmarshalText([]byte("gopher://lalala"))) } func (suite *TypeBlocklistURITestSuite) TestEmptyHost() { typ := &config.TypeBlocklistURI{} - suite.Error(typ.UnmarshalText([]byte("https:///path"))) + suite.Error(typ.UnmarshalText([]byte("https:///path"))) } func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() { diff --git a/config/type_bytes_test.go b/config/type_bytes_test.go index 6baec1c..c9ff55a 100644 --- a/config/type_bytes_test.go +++ b/config/type_bytes_test.go @@ -17,6 +17,12 @@ type TypeBytesTestSuite struct { suite.Suite } +func (suite *TypeBytesTestSuite) TestUnmarshalNil() { + typ := &config.TypeBytes{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + func (suite *TypeBytesTestSuite) TestUnmarshalFail() { testData := []string{ "1m", From e78c4d3bdec3553bbcaaa4f3b475a1566b4f14e2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 16:56:07 +0300 Subject: [PATCH 163/206] Test unmarshal nil for type duration --- config/type_duration_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/type_duration_test.go b/config/type_duration_test.go index be9a8fe..c31d3ef 100644 --- a/config/type_duration_test.go +++ b/config/type_duration_test.go @@ -18,6 +18,12 @@ type TypeDurationTestSuite struct { suite.Suite } +func (suite *TypeDurationTestSuite) TestUnmarshalNil() { + typ := &config.TypeDuration{} + suite.NoError(typ.UnmarshalText(nil)) + suite.EqualValues(0, typ.Value(0)) +} + func (suite *TypeDurationTestSuite) TestUnmarshalFail() { testData := []string{ "1t", From 102e7610be09c41a7b0fa0d95c073ae334328f49 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 16:58:34 +0300 Subject: [PATCH 164/206] Add missed case for error rate test --- config/type_error_rate_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/type_error_rate_test.go b/config/type_error_rate_test.go index 971b787..166d4c0 100644 --- a/config/type_error_rate_test.go +++ b/config/type_error_rate_test.go @@ -35,6 +35,8 @@ func (suite *TypeErrorRateTestSuite) TestUnmarshalFail() { assert.Error(t, json.Unmarshal(data, &typeErrorRateTestStruct{})) }) } + + suite.Error(json.Unmarshal([]byte("test"), &typeErrorRateTestStruct{})) } func (suite *TypeErrorRateTestSuite) TestUnmarshalOk() { From a36897595917a9c10406c87a7615821846080cc7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 17:08:26 +0300 Subject: [PATCH 165/206] Add missed case for ip --- config/type_bytes.go | 4 ++++ config/type_ip_test.go | 1 + 2 files changed, 5 insertions(+) diff --git a/config/type_bytes.go b/config/type_bytes.go index deed46e..4c7d7d6 100644 --- a/config/type_bytes.go +++ b/config/type_bytes.go @@ -38,6 +38,10 @@ func (c TypeBytes) MarshalText() ([]byte, error) { } func (c TypeBytes) String() string { + if c.value == 0 { + return "" + } + return strings.ToLower(c.value.String()) } diff --git a/config/type_ip_test.go b/config/type_ip_test.go index 8b23fc8..85189f4 100644 --- a/config/type_ip_test.go +++ b/config/type_ip_test.go @@ -94,6 +94,7 @@ func (suite *TypeIPTestSuite) TestMarshalOk() { func (suite *TypeIPTestSuite) TestValue() { testStruct := &typeIPTestStruct{} + suite.Empty(testStruct.Value.String()) suite.Nil(testStruct.Value.Value(nil)) suite.Equal("127.1.0.1", testStruct.Value.Value(net.ParseIP("127.1.0.1")).String()) From 339840c14ad3660ab9cc1ea6a9101d0aa6c9ee42 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 17:21:38 +0300 Subject: [PATCH 166/206] Add test for broken uri --- config/type_blocklist_uri_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/type_blocklist_uri_test.go b/config/type_blocklist_uri_test.go index 6e0b260..3d53135 100644 --- a/config/type_blocklist_uri_test.go +++ b/config/type_blocklist_uri_test.go @@ -38,6 +38,11 @@ func (suite *TypeBlocklistURITestSuite) TestEmptyHost() { suite.Error(typ.UnmarshalText([]byte("https:///path"))) } +func (suite *TypeBlocklistURITestSuite) TestIncorrectURL() { + typ := &config.TypeBlocklistURI{} + suite.Error(typ.UnmarshalText([]byte("h:/--"))) +} + func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() { rnd := make([]byte, 48) From b91f28d7d9e9479a85ad4410b277cce88c1b3377 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 17:25:49 +0300 Subject: [PATCH 167/206] Add test for nil for metric prefix --- config/type_metric_prefix_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/type_metric_prefix_test.go b/config/type_metric_prefix_test.go index 1954f62..5598de0 100644 --- a/config/type_metric_prefix_test.go +++ b/config/type_metric_prefix_test.go @@ -17,6 +17,12 @@ type TypeMetricPrefixTestSuite struct { suite.Suite } +func (suite *TypeMetricPrefixTestSuite) TestUnmarshalNil() { + typ := &config.TypeMetricPrefix{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + func (suite *TypeMetricPrefixTestSuite) TestUnmarshalFail() { testData := []string{ "aaa.aaa", From 132a01c27ec1f210a026f84669ec49ce8b4b4089 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 17:28:45 +0300 Subject: [PATCH 168/206] Correct float test for error rate --- config/type_error_rate_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/type_error_rate_test.go b/config/type_error_rate_test.go index 166d4c0..f3a1ed1 100644 --- a/config/type_error_rate_test.go +++ b/config/type_error_rate_test.go @@ -36,7 +36,11 @@ func (suite *TypeErrorRateTestSuite) TestUnmarshalFail() { }) } - suite.Error(json.Unmarshal([]byte("test"), &typeErrorRateTestStruct{})) + data, err := json.Marshal(map[string]string{ + "value": "hello", + }) + suite.NoError(err) + suite.Error(json.Unmarshal(data, &typeErrorRateTestStruct{})) } func (suite *TypeErrorRateTestSuite) TestUnmarshalOk() { From 6c92ec314dab178fa485485f9337e4aa5e8d1e97 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 21:14:51 +0300 Subject: [PATCH 169/206] Test unmarshal nil for port --- config/type_port.go | 2 +- config/type_port_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config/type_port.go b/config/type_port.go index ea02fbd..892558e 100644 --- a/config/type_port.go +++ b/config/type_port.go @@ -14,7 +14,7 @@ func (c *TypePort) UnmarshalJSON(data []byte) error { return nil } - intValue, err := strconv.ParseUint(string(data), 10, 16) + intValue, err := strconv.ParseUint(string(data), 10, 64) if err != nil { return fmt.Errorf("port number is not a number: %w", err) } diff --git a/config/type_port_test.go b/config/type_port_test.go index 0bc4bfc..98ab187 100644 --- a/config/type_port_test.go +++ b/config/type_port_test.go @@ -18,6 +18,12 @@ type TypePortTestSuite struct { suite.Suite } +func (suite *TypePortTestSuite) TestUnmarshalNil() { + typ := &config.TypePort{} + suite.NoError(typ.UnmarshalJSON(nil)) + suite.Equal("0", typ.String()) +} + func (suite *TypePortTestSuite) TestUnmarshalFail() { testData := []int{ -1, From 3d9a63a1f03f952cec917ee098b4efe3bb94a314 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 21:20:35 +0300 Subject: [PATCH 170/206] Add unmarshal nil test for prefer ip --- config/type_prefer_ip_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/type_prefer_ip_test.go b/config/type_prefer_ip_test.go index 30eb5ea..6a93b53 100644 --- a/config/type_prefer_ip_test.go +++ b/config/type_prefer_ip_test.go @@ -18,6 +18,12 @@ type TypePreferIPTestSuite struct { suite.Suite } +func (suite *TypePreferIPTestSuite) TestUnmarshalNil() { + typ := &config.TypePreferIP{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) +} + func (suite *TypePreferIPTestSuite) TestUnmarshalFail() { testData := []string{ "p", From 988f61718c7a04e7c552e8cd4510250469e714c9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 21:29:48 +0300 Subject: [PATCH 171/206] Add unmarshal nil for typestatsdtagsformat --- config/type_statsd_tag_format_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/config/type_statsd_tag_format_test.go b/config/type_statsd_tag_format_test.go index 2c7074f..e3975af 100644 --- a/config/type_statsd_tag_format_test.go +++ b/config/type_statsd_tag_format_test.go @@ -14,11 +14,17 @@ type typeStatsdTagFormatTestStruct struct { Value config.TypeStatsdTagFormat `json:"value"` } -type TypeStatsdTagFormat struct { +type TypeStatsdTagFormatTestSuite struct { suite.Suite } -func (suite *TypeStatsdTagFormat) TestUnmarshalFail() { +func (suite *TypeStatsdTagFormatTestSuite) TestUnmarshalNil() { + typ := &config.TypeStatsdTagFormat{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Equal("lalala", typ.Value("lalala")) +} + +func (suite *TypeStatsdTagFormatTestSuite) TestUnmarshalFail() { testData := []string{ "p", "ipv4", @@ -38,7 +44,7 @@ func (suite *TypeStatsdTagFormat) TestUnmarshalFail() { } } -func (suite *TypeStatsdTagFormat) TestUnmarshalOk() { +func (suite *TypeStatsdTagFormatTestSuite) TestUnmarshalOk() { testData := []string{ config.TypeStatsdTagFormatDatadog, config.TypeStatsdTagFormatInfluxdb, @@ -70,7 +76,7 @@ func (suite *TypeStatsdTagFormat) TestUnmarshalOk() { } } -func (suite *TypeStatsdTagFormat) TestMarshalOk() { +func (suite *TypeStatsdTagFormatTestSuite) TestMarshalOk() { testData := []string{ config.TypeStatsdTagFormatDatadog, config.TypeStatsdTagFormatInfluxdb, @@ -104,7 +110,7 @@ func (suite *TypeStatsdTagFormat) TestMarshalOk() { } } -func (suite *TypeStatsdTagFormat) TestValue() { +func (suite *TypeStatsdTagFormatTestSuite) TestValue() { testStruct := &typePreferIPTestStruct{} suite.EqualValues(config.TypePreferIPPreferIPv4, @@ -126,5 +132,5 @@ func (suite *TypeStatsdTagFormat) TestValue() { func TestTypeStatsdTagFormat(t *testing.T) { t.Parallel() - suite.Run(t, &TypeStatsdTagFormat{}) + suite.Run(t, &TypeStatsdTagFormatTestSuite{}) } From 1fd0ebea9b6f8d3fa152c96263ac4e642c4c4805 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 21:33:14 +0300 Subject: [PATCH 172/206] Add unmarshal nil for type url --- config/type_url_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/type_url_test.go b/config/type_url_test.go index 0ab8e5a..1e7ccdb 100644 --- a/config/type_url_test.go +++ b/config/type_url_test.go @@ -18,6 +18,15 @@ type TypeURLTestSuite struct { suite.Suite } +func (suite *TypeURLTestSuite) TestUnmarshalNil() { + u, _ := url.Parse("https://google.com") + + typ := &config.TypeURL{} + suite.NoError(typ.UnmarshalText(nil)) + suite.Empty(typ.String()) + suite.Equal("https://google.com", typ.Value(u).String()) +} + func (suite *TypeURLTestSuite) TestUnmarshalFail() { testData := []string{ "http:/aaa.com", @@ -26,6 +35,7 @@ func (suite *TypeURLTestSuite) TestUnmarshalFail() { "://111", "http://aaa.com:xxx", "gopher://aaa.com:888", + "gopher://aaa.com", } for _, v := range testData { From 155a97c25d48a2369a17d9a01b381a2f7184c839 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 1 Apr 2021 21:56:35 +0300 Subject: [PATCH 173/206] Add test for network --- network/network_test.go | 91 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 network/network_test.go diff --git a/network/network_test.go b/network/network_test.go new file mode 100644 index 0000000..86b5f0c --- /dev/null +++ b/network/network_test.go @@ -0,0 +1,91 @@ +package network_test + +import ( + "encoding/json" + "io" + "net/http" + "testing" + "time" + + "github.com/9seconds/mtg/v2/network" + "github.com/stretchr/testify/suite" +) + +type NetworkTestSuite struct { + suite.Suite + HTTPServerTestSuite + + dialer network.Dialer +} + +func (suite *NetworkTestSuite) SetupTest() { + dialer, err := network.NewDefaultDialer(0, 0) + suite.NoError(err) + + suite.dialer = dialer +} + +func (suite *NetworkTestSuite) TestLocalHTTPRequest() { + ntw, err := network.NewNetwork(suite.dialer, "itsme", "1.1.1.1", 0) + suite.NoError(err) + + client := ntw.MakeHTTPClient(nil) + + resp, err := client.Get(suite.httpServer.URL + "/headers") // nolint: noctx + suite.NoError(err) + + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) + + jsonStruct := struct { + Headers struct { + UserAgent []string `json:"User-Agent"` + } `json:"headers"` + }{} + + suite.NoError(json.Unmarshal(data, &jsonStruct)) + suite.Equal([]string{"itsme"}, jsonStruct.Headers.UserAgent) +} + +func (suite *NetworkTestSuite) TestRealHTTPRequest() { + ntw, err := network.NewNetwork(suite.dialer, "itsme", "1.1.1.1", 0) + suite.NoError(err) + + client := ntw.MakeHTTPClient(nil) + + resp, err := client.Get("https://httpbin.org/headers") // nolint: noctx + suite.NoError(err) + + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + suite.NoError(err) + suite.Equal(http.StatusOK, resp.StatusCode) + + jsonStruct := struct { + Headers struct { + UserAgent string `json:"User-Agent"` + } `json:"headers"` + }{} + + suite.NoError(json.Unmarshal(data, &jsonStruct)) + suite.Equal("itsme", jsonStruct.Headers.UserAgent) +} + +func (suite *NetworkTestSuite) TestIncorrectTimeout() { + _, err := network.NewNetwork(suite.dialer, "itsme", "1.1.1.1", -time.Second) + suite.Error(err) +} + +func (suite *NetworkTestSuite) TestIncorrectDOHHostname() { + _, err := network.NewNetwork(suite.dialer, "itsme", "doh.com", 0) + suite.Error(err) +} + +func TestNetwork(t *testing.T) { + t.Parallel() + suite.Run(t, &NetworkTestSuite{}) +} From 5941f0674a2ecd6773500e06e4045c0b23dcbc6c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 10:51:27 +0300 Subject: [PATCH 174/206] Enforce code coverage of secret --- mtglib/secret.go | 38 ++++++++++++++++---------------------- mtglib/secret_test.go | 12 +++++++++++- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/mtglib/secret.go b/mtglib/secret.go index a5b438f..71c4710 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -5,13 +5,12 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "strings" ) const ( SecretKeyLength = 16 - secretFakeTLSFirstByte byte = 238 + secretFakeTLSFirstByte byte = 0xee ) var secretEmptyKey [SecretKeyLength]byte @@ -35,31 +34,26 @@ func (s *Secret) UnmarshalText(data []byte) error { return ErrSecretEmpty } - var ( - decoded []byte - err error - ) - - if strings.HasPrefix(text, "ee") { - decoded, err = hex.DecodeString(strings.TrimPrefix(text, "ee")) + decoded, err := hex.DecodeString(text) + if err != nil { + decoded, err = base64.RawURLEncoding.DecodeString(text) } - if err != nil || len(decoded) <= SecretKeyLength { - decoded, err = base64.RawURLEncoding.DecodeString(text) + if err != nil { + return fmt.Errorf("incorrect secret format: %w", err) + } - if err != nil { - return fmt.Errorf("incorrect secret format: %w", err) - } + if len(decoded) < 2 { // nolint: gomnd // we need at least 1 byte here + return fmt.Errorf("secret is truncated, length=%d", len(decoded)) + } - if len(decoded) <= SecretKeyLength { - return fmt.Errorf("secret has incorrect length %d", len(text)) - } + if decoded[0] != secretFakeTLSFirstByte { + return fmt.Errorf("incorrect first byte of secret: %#x", decoded[0]) + } - if decoded[0] != secretFakeTLSFirstByte { - return fmt.Errorf("incorrect first byte: %v", decoded[0]) - } - - decoded = decoded[1:] + decoded = decoded[1:] + if len(decoded) < SecretKeyLength { + return fmt.Errorf("secret has incorrect length %d", len(decoded)) } copy(s.Key[:], decoded[:SecretKeyLength]) diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go index 49f80cd..3ca6a92 100644 --- a/mtglib/secret_test.go +++ b/mtglib/secret_test.go @@ -45,8 +45,14 @@ func (suite *SecretTestSuite) TestParseSecret() { } func (suite *SecretTestSuite) TestSerialize() { + s := mtglib.Secret{} + + data, err := s.MarshalText() + suite.NoError(err) + suite.Empty(data) + secretData, _ := hex.DecodeString("d11c6cbbd9efe7fed5bc0db220b09665") - s := mtglib.Secret{ + s = mtglib.Secret{ Host: "google.com", } @@ -77,6 +83,10 @@ func (suite *SecretTestSuite) TestIncorrectSecret() { "+ueJ0q91t5XOnFYP8Xac3A", "eed11c6cbbd9efe7fed5bc0db220b09665", "ed11c6cbbd9efe7fed5bc0db220b09665", + "", + "+**", + "ee", + "efd11c6cbbd9efe7fed5bc0db220b09665", } for _, v := range testData { From 75b95995a9143c8c1a457ec6296d574f67845b94 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 11:39:18 +0300 Subject: [PATCH 175/206] Add tests for stream context --- mtglib/init_internal_test.go | 14 ++++ mtglib/stream_context_internal_test.go | 95 ++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 mtglib/init_internal_test.go create mode 100644 mtglib/stream_context_internal_test.go diff --git a/mtglib/init_internal_test.go b/mtglib/init_internal_test.go new file mode 100644 index 0000000..a35b663 --- /dev/null +++ b/mtglib/init_internal_test.go @@ -0,0 +1,14 @@ +package mtglib + +type NoopLogger struct{} + +func (n NoopLogger) Named(_ string) Logger { return n } +func (n NoopLogger) BindInt(_ string, _ int) Logger { return n } +func (n NoopLogger) BindStr(_, _ string) Logger { return n } +func (n NoopLogger) Printf(_ string, _ ...interface{}) {} +func (n NoopLogger) Info(_ string) {} +func (n NoopLogger) Warning(_ string) {} +func (n NoopLogger) Debug(_ string) {} +func (n NoopLogger) InfoError(_ string, _ error) {} +func (n NoopLogger) WarningError(_ string, _ error) {} +func (n NoopLogger) DebugError(_ string, _ error) {} diff --git a/mtglib/stream_context_internal_test.go b/mtglib/stream_context_internal_test.go new file mode 100644 index 0000000..6d447cc --- /dev/null +++ b/mtglib/stream_context_internal_test.go @@ -0,0 +1,95 @@ +package mtglib + +import ( + "context" + "net" + "testing" + + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/suite" +) + +type StreamContextTestSuite struct { + suite.Suite + + connMock *testlib.NetConnMock + logger NoopLogger + ctx *streamContext + ctxCancel context.CancelFunc +} + +func (suite *StreamContextTestSuite) SetupSuite() { + suite.logger = NoopLogger{} +} + +func (suite *StreamContextTestSuite) SetupTest() { + ctx, cancel := context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, "key", "value") // nolint: golint, revive, staticcheck + + suite.ctxCancel = cancel + suite.connMock = &testlib.NetConnMock{} + + addr := &net.TCPAddr{ + IP: net.ParseIP("10.0.0.10"), + Port: 6676, + } + suite.connMock.On("RemoteAddr").Return(addr) + + suite.ctx = newStreamContext(ctx, suite.logger, suite.connMock) +} + +func (suite *StreamContextTestSuite) TearDownTest() { + suite.ctxCancel() + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *StreamContextTestSuite) TestContextInterface() { + _, ok := suite.ctx.Deadline() + suite.False(ok) + + select { + case <-suite.ctx.Done(): + suite.FailNow("unexpectedly done") + default: + } + + suite.NoError(suite.ctx.Err()) + suite.Equal("value", suite.ctx.Value("key")) + + suite.ctxCancel() + + select { + case <-suite.ctx.Done(): + suite.Error(suite.ctx.Err()) + default: + suite.FailNow("unexpectedly not done") + } +} + +func (suite *StreamContextTestSuite) TestClientIP() { + suite.Equal("10.0.0.10", suite.ctx.ClientIP().String()) +} + +func (suite *StreamContextTestSuite) TestClose() { + suite.connMock.On("Close").Once().Return(nil) + + tgConnMock := &testlib.NetConnMock{} + tgConnMock.On("Close").Once().Return(nil) + + suite.ctx.telegramConn = tgConnMock + suite.ctx.Close() + + select { + case <-suite.ctx.Done(): + suite.Error(suite.ctx.Err()) + default: + suite.FailNow("unexpectedly not done") + } + + tgConnMock.AssertExpectations(suite.T()) +} + +func TestStreamContext(t *testing.T) { + t.Parallel() + suite.Run(t, &StreamContextTestSuite{}) +} From 784a5818c8084ef9a30ec87f478ecac8654dd16a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 11:49:07 +0300 Subject: [PATCH 176/206] Cover last case in secret --- mtglib/secret_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/mtglib/secret_test.go b/mtglib/secret_test.go index 3ca6a92..c1f2923 100644 --- a/mtglib/secret_test.go +++ b/mtglib/secret_test.go @@ -82,6 +82,7 @@ func (suite *SecretTestSuite) TestIncorrectSecret() { "ddd11c6cbbd9efe7fed5bc0db220b09665", "+ueJ0q91t5XOnFYP8Xac3A", "eed11c6cbbd9efe7fed5bc0db220b09665", + "eed11c6cbbd9efe7fed5bc0db220b096", "ed11c6cbbd9efe7fed5bc0db220b09665", "", "+**", From 7e7823118ec91c1389f968408653887ca26d4a03 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 12:40:22 +0300 Subject: [PATCH 177/206] Add test conntraffic --- mtglib/conns_internal_test.go | 150 ++++++++++++++++++++++++++++++++++ mtglib/init_internal_test.go | 14 ++++ 2 files changed, 164 insertions(+) create mode 100644 mtglib/conns_internal_test.go diff --git a/mtglib/conns_internal_test.go b/mtglib/conns_internal_test.go new file mode 100644 index 0000000..12fecb4 --- /dev/null +++ b/mtglib/conns_internal_test.go @@ -0,0 +1,150 @@ +package mtglib + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/9seconds/mtg/v2/testlib" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" +) + +type ConnTrafficTestSuite struct { + suite.Suite + + eventStreamMock *EventStreamMock + connMock *testlib.NetConnMock + conn io.ReadWriter +} + +func (suite *ConnTrafficTestSuite) SetupTest() { + suite.eventStreamMock = &EventStreamMock{} + suite.connMock = &testlib.NetConnMock{} + suite.conn = connTraffic{ + Conn: suite.connMock, + connID: "CONNID", + ctx: context.Background(), + stream: suite.eventStreamMock, + } +} + +func (suite *ConnTrafficTestSuite) TearDownTest() { + suite.eventStreamMock.AssertExpectations(suite.T()) + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnTrafficTestSuite) TestReadOk() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.True(evt.IsRead) + }) + suite.connMock.On("Read", mock.Anything).Once().Return(10, nil) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.NoError(err) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestReadErr() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.True(evt.IsRead) + }) + suite.connMock.On("Read", mock.Anything).Once().Return(10, io.EOF) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestReadNothingOk() { + suite.connMock.On("Read", mock.Anything).Once().Return(0, nil) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.NoError(err) + suite.Equal(0, n) +} + +func (suite *ConnTrafficTestSuite) TestReadNothingErr() { + suite.connMock.On("Read", mock.Anything).Once().Return(0, io.EOF) + + n, err := suite.conn.Read(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(0, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteOk() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.False(evt.IsRead) + }) + suite.connMock.On("Write", mock.Anything).Once().Return(10, nil) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.NoError(err) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteErr() { + suite.eventStreamMock. + On("Send", mock.Anything, mock.Anything). + Once(). + Run(func(args mock.Arguments) { + evt := args.Get(1).(EventTraffic) + + suite.Equal("CONNID", evt.StreamID()) + suite.WithinDuration(time.Now(), evt.Timestamp(), time.Second) + suite.EqualValues(10, evt.Traffic) + suite.False(evt.IsRead) + }) + suite.connMock.On("Write", mock.Anything).Once().Return(10, io.EOF) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(10, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteNothingOk() { + suite.connMock.On("Write", mock.Anything).Once().Return(0, nil) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.NoError(err) + suite.Equal(0, n) +} + +func (suite *ConnTrafficTestSuite) TestWriteNothingErr() { + suite.connMock.On("Write", mock.Anything).Once().Return(0, io.EOF) + + n, err := suite.conn.Write(make([]byte, 10)) + suite.True(errors.Is(err, io.EOF)) + suite.Equal(0, n) +} + +func TestConnTraffic(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnTrafficTestSuite{}) +} diff --git a/mtglib/init_internal_test.go b/mtglib/init_internal_test.go index a35b663..98127dd 100644 --- a/mtglib/init_internal_test.go +++ b/mtglib/init_internal_test.go @@ -1,5 +1,11 @@ package mtglib +import ( + "context" + + "github.com/stretchr/testify/mock" +) + type NoopLogger struct{} func (n NoopLogger) Named(_ string) Logger { return n } @@ -12,3 +18,11 @@ func (n NoopLogger) Debug(_ string) {} func (n NoopLogger) InfoError(_ string, _ error) {} func (n NoopLogger) WarningError(_ string, _ error) {} func (n NoopLogger) DebugError(_ string, _ error) {} + +type EventStreamMock struct { + mock.Mock +} + +func (e *EventStreamMock) Send(ctx context.Context, evt Event) { + e.Called(ctx, evt) +} From 766800700a1247542a638e962b3de51ba29ebb70 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 14:27:02 +0300 Subject: [PATCH 178/206] Add test for connrewind --- mtglib/conns_internal_test.go | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/mtglib/conns_internal_test.go b/mtglib/conns_internal_test.go index 12fecb4..6bd778e 100644 --- a/mtglib/conns_internal_test.go +++ b/mtglib/conns_internal_test.go @@ -1,6 +1,7 @@ package mtglib import ( + "bytes" "context" "errors" "io" @@ -12,6 +13,18 @@ import ( "github.com/stretchr/testify/suite" ) +type ConnRewindBaseConn struct { + testlib.NetConnMock + + readBuffer bytes.Buffer +} + +func (c *ConnRewindBaseConn) Read(p []byte) (int, error) { + c.Called(p) + + return c.readBuffer.Read(p) +} + type ConnTrafficTestSuite struct { suite.Suite @@ -144,7 +157,51 @@ func (suite *ConnTrafficTestSuite) TestWriteNothingErr() { suite.Equal(0, n) } +type ConnRewindTestSuite struct { + suite.Suite + + connMock *ConnRewindBaseConn + conn *connRewind +} + +func (suite *ConnRewindTestSuite) SetupTest() { + suite.connMock = &ConnRewindBaseConn{} + suite.conn = newConnRewind(suite.connMock) +} + +func (suite *ConnRewindTestSuite) TearDownTest() { + suite.connMock.AssertExpectations(suite.T()) +} + +func (suite *ConnRewindTestSuite) TestRead() { + suite.connMock.On("Read", mock.Anything) + suite.connMock.readBuffer.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}) + + buf := make([]byte, 2) + + n, err := suite.conn.Read(buf) + suite.NoError(err) + suite.Equal(2, n) + suite.Equal([]byte{1, 2}, buf) + + n, err = suite.conn.Read(buf) + suite.NoError(err) + suite.Equal(2, n) + suite.Equal([]byte{3, 4}, buf) + + suite.conn.Rewind() + + data, err := io.ReadAll(suite.conn) + suite.NoError(err) + suite.Equal([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, data) +} + func TestConnTraffic(t *testing.T) { t.Parallel() suite.Run(t, &ConnTrafficTestSuite{}) } + +func TestConnRewind(t *testing.T) { + t.Parallel() + suite.Run(t, &ConnRewindTestSuite{}) +} From 2cdff017e2a87636fd0780ba5e0925424459ec96 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 17:07:17 +0300 Subject: [PATCH 179/206] Add base tests for proxy --- mtglib/proxy.go | 40 +++++++--- mtglib/proxy_test.go | 184 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 12 deletions(-) create mode 100644 mtglib/proxy_test.go diff --git a/mtglib/proxy.go b/mtglib/proxy.go index de76a45..36de1a8 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -42,6 +42,9 @@ func (p *Proxy) DomainFrontingAddress() string { } func (p *Proxy) ServeConn(conn net.Conn) { + p.streamWaitGroup.Add(1) + defer p.streamWaitGroup.Done() + ctx := newStreamContext(p.ctx, p.logger, conn) defer ctx.Close() @@ -91,20 +94,24 @@ func (p *Proxy) ServeConn(conn net.Conn) { } func (p *Proxy) Serve(listener net.Listener) error { + p.streamWaitGroup.Add(1) + defer p.streamWaitGroup.Done() + for { conn, err := listener.Accept() if err != nil { return fmt.Errorf("cannot accept a new connection: %w", err) } - if addr := conn.RemoteAddr().(*net.TCPAddr).IP; p.ipBlocklist.Contains(addr) { + ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP + logger := p.logger.BindStr("ip", ipAddr.String()) + + if p.ipBlocklist.Contains(ipAddr) { conn.Close() - p.logger. - BindStr("ip", conn.RemoteAddr().(*net.TCPAddr).IP.String()). - Info("ip was blacklisted") + logger.Info("ip was blacklisted") p.eventStream.Send(p.ctx, EventIPBlocklisted{ CreatedAt: time.Now(), - RemoteIP: addr, + RemoteIP: ipAddr, }) continue @@ -117,13 +124,17 @@ func (p *Proxy) Serve(listener net.Listener) error { case errors.Is(err, ants.ErrPoolClosed): return nil case errors.Is(err, ants.ErrPoolOverload): - p.logger. - BindStr("ip", conn.RemoteAddr().(*net.TCPAddr).IP.String()). - Info("connection was concurrency limited") + logger.Info("connection was concurrency limited") p.eventStream.Send(p.ctx, EventConcurrencyLimited{ CreatedAt: time.Now(), }) } + + select { + case <-p.ctx.Done(): + return p.ctx.Err() + default: + } } } @@ -292,9 +303,9 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen return nil, ErrSecretInvalid } - tg, err := telegram.New(opts.Network, opts.PreferIP) - if err != nil { - return nil, fmt.Errorf("cannot build telegram dialer: %w", err) + preferIP := opts.PreferIP + if preferIP == "" { + preferIP = DefaultPreferIP } concurrency := opts.Concurrency @@ -317,6 +328,11 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen domainFrontingPort = DefaultDomainFrontingPort } + tg, err := telegram.New(opts.Network, preferIP) + if err != nil { + return nil, fmt.Errorf("cannot build telegram dialer: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) proxy := &Proxy{ ctx: ctx, @@ -340,7 +356,7 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen ants.WithLogger(opts.Logger.Named("ants")), ants.WithNonblocking(true)) if err != nil { - return nil, fmt.Errorf("cannot initialize a pool: %w", err) + panic(err) } proxy.workerPool = pool diff --git a/mtglib/proxy_test.go b/mtglib/proxy_test.go new file mode 100644 index 0000000..6e767ee --- /dev/null +++ b/mtglib/proxy_test.go @@ -0,0 +1,184 @@ +package mtglib_test + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "testing" + "time" + + "github.com/9seconds/mtg/v2/antireplay" + "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/ipblocklist" + "github.com/9seconds/mtg/v2/logger" + "github.com/9seconds/mtg/v2/mtglib" + "github.com/9seconds/mtg/v2/network" + "github.com/9seconds/mtg/v2/timeattack" + "github.com/stretchr/testify/suite" +) + +type ProxyTestSuite struct { + suite.Suite + + opts *mtglib.ProxyOpts + p *mtglib.Proxy + listener net.Listener +} + +func (suite *ProxyTestSuite) ProxyAddress() string { + _, port, _ := net.SplitHostPort(suite.listener.Addr().String()) + + return net.JoinHostPort("127.0.0.1", port) +} + +func (suite *ProxyTestSuite) ProxySecret() string { + return suite.opts.Secret.Hex() +} + +func (suite *ProxyTestSuite) SetupSuite() { + dialer, err := network.NewDefaultDialer(0, 0) + suite.NoError(err) + + ntw, err := network.NewNetwork(dialer, "mtgtest", "1.1.1.1", 0) + suite.NoError(err) + + suite.opts = &mtglib.ProxyOpts{ + Secret: mtglib.GenerateSecret("httpbin.org"), + Network: ntw, + AntiReplayCache: antireplay.NewNoop(), + TimeAttackDetector: timeattack.NewNoop(), + IPBlocklist: ipblocklist.NewNoop(), + EventStream: events.NewNoopStream(), + Logger: logger.NewNoopLogger(), + } + + proxy, err := mtglib.NewProxy(*suite.opts) + suite.NoError(err) + + suite.p = proxy + + listener, err := net.Listen("tcp", ":0") + suite.NoError(err) + + suite.listener = listener + + go suite.p.Serve(suite.listener) // nolint: errcheck +} + +func (suite *ProxyTestSuite) TearDownSuite() { + if suite.listener != nil { + suite.listener.Close() + } + + if suite.p != nil { + suite.p.Shutdown() + } +} + +func (suite *ProxyTestSuite) TestCannotInitNoSecret() { + opts := *suite.opts + opts.Secret = mtglib.Secret{} + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoNetwork() { + opts := *suite.opts + opts.Network = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoAntiReplayCache() { + opts := *suite.opts + opts.AntiReplayCache = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoIPBlocklist() { + opts := *suite.opts + opts.IPBlocklist = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoEventStream() { + opts := *suite.opts + opts.EventStream = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoTimeAttackDetector() { + opts := *suite.opts + opts.TimeAttackDetector = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitNoLogger() { + opts := *suite.opts + opts.Logger = nil + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestCannotInitIncorrectPreferIP() { + opts := *suite.opts + opts.PreferIP = "xxx" + + _, err := mtglib.NewProxy(opts) + suite.Error(err) +} + +func (suite *ProxyTestSuite) TestDomainFrontingAddress() { + suite.Equal("httpbin.org:443", suite.p.DomainFrontingAddress()) +} + +func (suite *ProxyTestSuite) TestHTTPSRequest() { + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + Timeout: 5 * time.Second, + } + + addr := fmt.Sprintf("https://%s/headers", suite.ProxyAddress()) + + resp, err := client.Get(addr) // nolint: noctx + suite.NoError(err) + + defer resp.Body.Close() + + suite.Equal(http.StatusOK, resp.StatusCode) + + data, err := io.ReadAll(resp.Body) + suite.NoError(err) + + jsonStruct := struct { + Headers struct { + TraceID string `json:"X-Amzn-Trace-Id"` + } `json:"headers"` + }{} + + suite.NoError(json.Unmarshal(data, &jsonStruct)) + suite.NotEmpty(jsonStruct.Headers.TraceID) +} + +func TestProxy(t *testing.T) { + t.Parallel() + suite.Run(t, &ProxyTestSuite{}) +} From 2dc30c5837fe2032629023babc4b1532f303ab45 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 17:12:24 +0300 Subject: [PATCH 180/206] Simplify cli --- cli/proxy.go | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/cli/proxy.go b/cli/proxy.go index 94b228f..b8e0a8e 100644 --- a/cli/proxy.go +++ b/cli/proxy.go @@ -18,10 +18,6 @@ import ( type Proxy struct { base - - prometheusListener net.Listener - prometheus *stats.PrometheusFactory - statsdFactory *stats.StatsdFactory } func (c *Proxy) Run(cli *CLI, version string) error { @@ -32,7 +28,7 @@ func (c *Proxy) Run(cli *CLI, version string) error { return c.Execute() } -func (c *Proxy) Execute() error { // nolint: funlen +func (c *Proxy) Execute() error { zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs zerolog.TimestampFieldName = "timestamp" zerolog.LevelFieldName = "level" @@ -59,10 +55,6 @@ func (c *Proxy) Execute() error { // nolint: funlen PreferIP: c.Config.PreferIP.Value(mtglib.DefaultPreferIP), } - if opts.Concurrency == 0 { - opts.Concurrency = mtglib.DefaultConcurrency - } - opts.Logger.BindStr("configuration", c.Config.String()).Debug("configuration") c.setupAntiReplayCache(&opts) @@ -89,20 +81,8 @@ func (c *Proxy) Execute() error { // nolint: funlen go proxy.Serve(listener) // nolint: errcheck <-ctx.Done() - listener.Close() - - if c.prometheusListener != nil { - c.prometheusListener.Close() - } - - if c.prometheus != nil { - c.prometheus.Close() - } - - if c.statsdFactory != nil { - c.statsdFactory.Close() - } + proxy.Shutdown() return nil } @@ -173,8 +153,6 @@ func (c *Proxy) setupEventStream(opts *mtglib.ProxyOpts) error { return fmt.Errorf("cannot build statsd observer: %w", err) } - c.statsdFactory = &statsdFactory - factories = append(factories, statsdFactory.Make) } @@ -191,9 +169,6 @@ func (c *Proxy) setupEventStream(opts *mtglib.ProxyOpts) error { go prometheus.Serve(listener) // nolint: errcheck - c.prometheusListener = listener - c.prometheus = prometheus - factories = append(factories, prometheus.Make) } From 8566cede74a68c3b82b5ee18989db4d4f8c3d242 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 17:37:24 +0300 Subject: [PATCH 181/206] Update build flags in Makefile --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d5c15ff..fff4788 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ GOLANGCI_LINT_VERSION := v1.37.1 VERSION_GO := $(shell go version) VERSION_DATE := $(shell date -Ru) VERSION_TAG := $(shell git describe --tags --always) -COMMON_BUILD_FLAGS := -mod=readonly -ldflags="-s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'" +COMMON_BUILD_FLAGS := -mod=readonly -ldflags="-extldflags '-static' -s -w -X 'main.version=$(VERSION_TAG) ($(VERSION_GO)) [$(VERSION_DATE)]'" GOBIN := $(ROOT_DIR)/.bin GOTOOL := env "GOBIN=$(GOBIN)" "PATH=$(ROOT_DIR)/.bin:$(PATH)" @@ -27,7 +27,11 @@ $(APP_NAME): build .PHONY: static static: - @env CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo $(COMMON_BUILD_FLAGS) -o "$(APP_NAME)" + @env CGO_ENABLED=0 GOOS=linux go build \ + $(COMMON_BUILD_FLAGS) \ + -tags netgo \ + -a \ + -o "$(APP_NAME)" $(APP_NAME)-%: GOOS=$(shell echo -n "$@" | sed 's?$(APP_NAME)-??' | cut -f1 -d-) $(APP_NAME)-%: GOARCH=$(shell echo -n "$@" | sed 's?$(APP_NAME)-??' | cut -f2 -d-) @@ -35,6 +39,8 @@ $(APP_NAME)-%: ccbuilds @env "GOOS=$(GOOS)" "GOARCH=$(GOARCH)" \ go build \ $(COMMON_BUILD_FLAGS) \ + -tags netgo \ + -a \ -o "./ccbuilds/$(APP_NAME)-$(GOOS)-$(GOARCH)" .PHONY: ccbuilds From 7b5cb8e14bcab136d4e2fadf10a7d9364524625e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 17:41:33 +0300 Subject: [PATCH 182/206] Small updates for docker file --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 506f222..4a71943 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,8 @@ RUN set -x \ FROM scratch ENTRYPOINT ["/mtg"] -ENV MTG_BIND=0.0.0.0:3128 \ - MTG_STATS_BIND=0.0.0.0:3129 -EXPOSE 3128 3129 +CMD ["run", "/config.toml"] COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=build /app/mtg /mtg +COPY --from=build /app/example.config.toml /config.toml From 2fe04c7c9b321f293c86586c376bc9e13fa726f2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 21:27:34 +0300 Subject: [PATCH 183/206] Move utils to internal --- cli/proxy.go | 2 +- {utils => internal/utils}/rlimit.go | 0 {utils => internal/utils}/rlimit_windows.go | 0 {utils => internal/utils}/root_context.go | 0 {utils => internal/utils}/root_context_windows.go | 0 main.go | 2 +- 6 files changed, 2 insertions(+), 2 deletions(-) rename {utils => internal/utils}/rlimit.go (100%) rename {utils => internal/utils}/rlimit_windows.go (100%) rename {utils => internal/utils}/root_context.go (100%) rename {utils => internal/utils}/root_context_windows.go (100%) diff --git a/cli/proxy.go b/cli/proxy.go index b8e0a8e..4b54d8d 100644 --- a/cli/proxy.go +++ b/cli/proxy.go @@ -7,12 +7,12 @@ import ( "github.com/9seconds/mtg/v2/antireplay" "github.com/9seconds/mtg/v2/events" + "github.com/9seconds/mtg/v2/internal/utils" "github.com/9seconds/mtg/v2/ipblocklist" "github.com/9seconds/mtg/v2/logger" "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/stats" "github.com/9seconds/mtg/v2/timeattack" - "github.com/9seconds/mtg/v2/utils" "github.com/rs/zerolog" ) diff --git a/utils/rlimit.go b/internal/utils/rlimit.go similarity index 100% rename from utils/rlimit.go rename to internal/utils/rlimit.go diff --git a/utils/rlimit_windows.go b/internal/utils/rlimit_windows.go similarity index 100% rename from utils/rlimit_windows.go rename to internal/utils/rlimit_windows.go diff --git a/utils/root_context.go b/internal/utils/root_context.go similarity index 100% rename from utils/root_context.go rename to internal/utils/root_context.go diff --git a/utils/root_context_windows.go b/internal/utils/root_context_windows.go similarity index 100% rename from utils/root_context_windows.go rename to internal/utils/root_context_windows.go diff --git a/main.go b/main.go index d554166..55c4afa 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,7 @@ import ( "time" "github.com/9seconds/mtg/v2/cli" - "github.com/9seconds/mtg/v2/utils" + "github.com/9seconds/mtg/v2/internal/utils" "github.com/alecthomas/kong" ) From 24fdec669463a116757f4925cf3dffca210f3615 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 21:32:06 +0300 Subject: [PATCH 184/206] Move testlib to internal --- cli/access_test.go | 2 +- cli/generate_secret_test.go | 2 +- cli/init_test.go | 2 +- {testlib => internal/testlib}/capture_output.go | 0 {testlib => internal/testlib}/events_observer_mock.go | 0 {testlib => internal/testlib}/mtglib_antireplay_cache_mock.go | 0 {testlib => internal/testlib}/mtglib_network_mock.go | 0 {testlib => internal/testlib}/net_conn_mock.go | 0 ipblocklist/firehol_test.go | 2 +- logger/noop_test.go | 2 +- mtglib/conns_internal_test.go | 2 +- mtglib/internal/faketls/conn_test.go | 2 +- mtglib/internal/obfuscated2/client_handshake_test.go | 2 +- mtglib/internal/obfuscated2/server_handshake_test.go | 2 +- mtglib/internal/relay/conn_internal_test.go | 2 +- mtglib/internal/relay/relay_test.go | 2 +- mtglib/internal/telegram/telegram_internal_test.go | 2 +- mtglib/stream_context_internal_test.go | 2 +- network/circuit_breaker_internal_test.go | 2 +- 19 files changed, 14 insertions(+), 14 deletions(-) rename {testlib => internal/testlib}/capture_output.go (100%) rename {testlib => internal/testlib}/events_observer_mock.go (100%) rename {testlib => internal/testlib}/mtglib_antireplay_cache_mock.go (100%) rename {testlib => internal/testlib}/mtglib_network_mock.go (100%) rename {testlib => internal/testlib}/net_conn_mock.go (100%) diff --git a/cli/access_test.go b/cli/access_test.go index 0cf0579..1a82e54 100644 --- a/cli/access_test.go +++ b/cli/access_test.go @@ -6,8 +6,8 @@ import ( "testing" "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib" - "github.com/9seconds/mtg/v2/testlib" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/suite" "github.com/xeipuuv/gojsonschema" diff --git a/cli/generate_secret_test.go b/cli/generate_secret_test.go index 5cbe18d..64c6002 100644 --- a/cli/generate_secret_test.go +++ b/cli/generate_secret_test.go @@ -4,8 +4,8 @@ import ( "strings" "testing" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib" - "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/suite" ) diff --git a/cli/init_test.go b/cli/init_test.go index 77304e9..29d3c74 100644 --- a/cli/init_test.go +++ b/cli/init_test.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/9seconds/mtg/v2/cli" - "github.com/9seconds/mtg/v2/testlib" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/testlib/capture_output.go b/internal/testlib/capture_output.go similarity index 100% rename from testlib/capture_output.go rename to internal/testlib/capture_output.go diff --git a/testlib/events_observer_mock.go b/internal/testlib/events_observer_mock.go similarity index 100% rename from testlib/events_observer_mock.go rename to internal/testlib/events_observer_mock.go diff --git a/testlib/mtglib_antireplay_cache_mock.go b/internal/testlib/mtglib_antireplay_cache_mock.go similarity index 100% rename from testlib/mtglib_antireplay_cache_mock.go rename to internal/testlib/mtglib_antireplay_cache_mock.go diff --git a/testlib/mtglib_network_mock.go b/internal/testlib/mtglib_network_mock.go similarity index 100% rename from testlib/mtglib_network_mock.go rename to internal/testlib/mtglib_network_mock.go diff --git a/testlib/net_conn_mock.go b/internal/testlib/net_conn_mock.go similarity index 100% rename from testlib/net_conn_mock.go rename to internal/testlib/net_conn_mock.go diff --git a/ipblocklist/firehol_test.go b/ipblocklist/firehol_test.go index 9879afd..9d26c6d 100644 --- a/ipblocklist/firehol_test.go +++ b/ipblocklist/firehol_test.go @@ -10,10 +10,10 @@ import ( "testing" "time" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/ipblocklist" "github.com/9seconds/mtg/v2/logger" "github.com/9seconds/mtg/v2/network" - "github.com/9seconds/mtg/v2/testlib" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/logger/noop_test.go b/logger/noop_test.go index ca05901..dca71a8 100644 --- a/logger/noop_test.go +++ b/logger/noop_test.go @@ -4,8 +4,8 @@ import ( "io" "testing" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/logger" - "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/conns_internal_test.go b/mtglib/conns_internal_test.go index 6bd778e..de31f60 100644 --- a/mtglib/conns_internal_test.go +++ b/mtglib/conns_internal_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - "github.com/9seconds/mtg/v2/testlib" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/internal/faketls/conn_test.go b/mtglib/internal/faketls/conn_test.go index 41bff3d..73c0e65 100644 --- a/mtglib/internal/faketls/conn_test.go +++ b/mtglib/internal/faketls/conn_test.go @@ -7,9 +7,9 @@ import ( "math/rand" "testing" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib/internal/faketls" "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" - "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/internal/obfuscated2/client_handshake_test.go b/mtglib/internal/obfuscated2/client_handshake_test.go index ef7402a..6d4b95a 100644 --- a/mtglib/internal/obfuscated2/client_handshake_test.go +++ b/mtglib/internal/obfuscated2/client_handshake_test.go @@ -4,8 +4,8 @@ import ( "bytes" "testing" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" - "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/mtglib/internal/obfuscated2/server_handshake_test.go b/mtglib/internal/obfuscated2/server_handshake_test.go index 9279295..418af0c 100644 --- a/mtglib/internal/obfuscated2/server_handshake_test.go +++ b/mtglib/internal/obfuscated2/server_handshake_test.go @@ -7,8 +7,8 @@ import ( "encoding/base64" "testing" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" - "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/internal/relay/conn_internal_test.go b/mtglib/internal/relay/conn_internal_test.go index 26ed696..86bd81b 100644 --- a/mtglib/internal/relay/conn_internal_test.go +++ b/mtglib/internal/relay/conn_internal_test.go @@ -6,7 +6,7 @@ import ( "io" "testing" - "github.com/9seconds/mtg/v2/testlib" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/internal/relay/relay_test.go b/mtglib/internal/relay/relay_test.go index 55b7732..9e3f19c 100644 --- a/mtglib/internal/relay/relay_test.go +++ b/mtglib/internal/relay/relay_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib/internal/relay" - "github.com/9seconds/mtg/v2/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) diff --git a/mtglib/internal/telegram/telegram_internal_test.go b/mtglib/internal/telegram/telegram_internal_test.go index 64f1d6b..fe21ce2 100644 --- a/mtglib/internal/telegram/telegram_internal_test.go +++ b/mtglib/internal/telegram/telegram_internal_test.go @@ -8,7 +8,7 @@ import ( "strconv" "testing" - "github.com/9seconds/mtg/v2/testlib" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" diff --git a/mtglib/stream_context_internal_test.go b/mtglib/stream_context_internal_test.go index 6d447cc..17f90ea 100644 --- a/mtglib/stream_context_internal_test.go +++ b/mtglib/stream_context_internal_test.go @@ -5,7 +5,7 @@ import ( "net" "testing" - "github.com/9seconds/mtg/v2/testlib" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/stretchr/testify/suite" ) diff --git a/network/circuit_breaker_internal_test.go b/network/circuit_breaker_internal_test.go index a29eec3..a26f594 100644 --- a/network/circuit_breaker_internal_test.go +++ b/network/circuit_breaker_internal_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/9seconds/mtg/v2/testlib" + "github.com/9seconds/mtg/v2/internal/testlib" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) From ea7033d569512e9f3e332d3a0d0012bc42645999 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 21:36:15 +0300 Subject: [PATCH 185/206] Move cli to internal --- {cli => internal/cli}/access.go | 0 {cli => internal/cli}/access_test.go | 0 {cli => internal/cli}/base.go | 0 {cli => internal/cli}/base_internal_test.go | 0 {cli => internal/cli}/cli.go | 0 {cli => internal/cli}/generate_secret.go | 0 {cli => internal/cli}/generate_secret_test.go | 0 {cli => internal/cli}/init_test.go | 2 +- {cli => internal/cli}/proxy.go | 0 {cli => internal/cli}/testdata/minimal.toml | 0 main.go | 2 +- 11 files changed, 2 insertions(+), 2 deletions(-) rename {cli => internal/cli}/access.go (100%) rename {cli => internal/cli}/access_test.go (100%) rename {cli => internal/cli}/base.go (100%) rename {cli => internal/cli}/base_internal_test.go (100%) rename {cli => internal/cli}/cli.go (100%) rename {cli => internal/cli}/generate_secret.go (100%) rename {cli => internal/cli}/generate_secret_test.go (100%) rename {cli => internal/cli}/init_test.go (94%) rename {cli => internal/cli}/proxy.go (100%) rename {cli => internal/cli}/testdata/minimal.toml (100%) diff --git a/cli/access.go b/internal/cli/access.go similarity index 100% rename from cli/access.go rename to internal/cli/access.go diff --git a/cli/access_test.go b/internal/cli/access_test.go similarity index 100% rename from cli/access_test.go rename to internal/cli/access_test.go diff --git a/cli/base.go b/internal/cli/base.go similarity index 100% rename from cli/base.go rename to internal/cli/base.go diff --git a/cli/base_internal_test.go b/internal/cli/base_internal_test.go similarity index 100% rename from cli/base_internal_test.go rename to internal/cli/base_internal_test.go diff --git a/cli/cli.go b/internal/cli/cli.go similarity index 100% rename from cli/cli.go rename to internal/cli/cli.go diff --git a/cli/generate_secret.go b/internal/cli/generate_secret.go similarity index 100% rename from cli/generate_secret.go rename to internal/cli/generate_secret.go diff --git a/cli/generate_secret_test.go b/internal/cli/generate_secret_test.go similarity index 100% rename from cli/generate_secret_test.go rename to internal/cli/generate_secret_test.go diff --git a/cli/init_test.go b/internal/cli/init_test.go similarity index 94% rename from cli/init_test.go rename to internal/cli/init_test.go index 29d3c74..e94e829 100644 --- a/cli/init_test.go +++ b/internal/cli/init_test.go @@ -3,7 +3,7 @@ package cli_test import ( "net/http" - "github.com/9seconds/mtg/v2/cli" + "github.com/9seconds/mtg/v2/internal/cli" "github.com/9seconds/mtg/v2/internal/testlib" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/mock" diff --git a/cli/proxy.go b/internal/cli/proxy.go similarity index 100% rename from cli/proxy.go rename to internal/cli/proxy.go diff --git a/cli/testdata/minimal.toml b/internal/cli/testdata/minimal.toml similarity index 100% rename from cli/testdata/minimal.toml rename to internal/cli/testdata/minimal.toml diff --git a/main.go b/main.go index 55c4afa..b081d8c 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "math/rand" "time" - "github.com/9seconds/mtg/v2/cli" + "github.com/9seconds/mtg/v2/internal/cli" "github.com/9seconds/mtg/v2/internal/utils" "github.com/alecthomas/kong" ) From 7bffdcdf4ad21e3705ddd4b253a22ca7f1de30e3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 21:41:02 +0300 Subject: [PATCH 186/206] Move config to internal --- internal/cli/access_test.go | 2 +- internal/cli/base.go | 2 +- {config => internal/config}/config.go | 0 {config => internal/config}/config_test.go | 2 +- {config => internal/config}/testdata/broken.toml | 0 {config => internal/config}/testdata/minimal.toml | 0 {config => internal/config}/testdata/only_secret.toml | 0 {config => internal/config}/type_blocklist_uri.go | 0 {config => internal/config}/type_blocklist_uri_test.go | 2 +- {config => internal/config}/type_bytes.go | 0 {config => internal/config}/type_bytes_test.go | 2 +- {config => internal/config}/type_duration.go | 0 {config => internal/config}/type_duration_test.go | 2 +- {config => internal/config}/type_error_rate.go | 0 {config => internal/config}/type_error_rate_test.go | 2 +- {config => internal/config}/type_hostport.go | 0 {config => internal/config}/type_hostport_test.go | 2 +- {config => internal/config}/type_http_path.go | 0 {config => internal/config}/type_http_path_test.go | 2 +- {config => internal/config}/type_ip.go | 0 {config => internal/config}/type_ip_test.go | 2 +- {config => internal/config}/type_metric_prefix.go | 0 {config => internal/config}/type_metric_prefix_test.go | 2 +- {config => internal/config}/type_port.go | 0 {config => internal/config}/type_port_test.go | 2 +- {config => internal/config}/type_prefer_ip.go | 0 {config => internal/config}/type_prefer_ip_test.go | 2 +- {config => internal/config}/type_statsd_tag_format.go | 0 {config => internal/config}/type_statsd_tag_format_test.go | 2 +- {config => internal/config}/type_url.go | 0 {config => internal/config}/type_url_test.go | 2 +- 31 files changed, 15 insertions(+), 15 deletions(-) rename {config => internal/config}/config.go (100%) rename {config => internal/config}/config_test.go (96%) rename {config => internal/config}/testdata/broken.toml (100%) rename {config => internal/config}/testdata/minimal.toml (100%) rename {config => internal/config}/testdata/only_secret.toml (100%) rename {config => internal/config}/type_blocklist_uri.go (100%) rename {config => internal/config}/type_blocklist_uri_test.go (98%) rename {config => internal/config}/type_bytes.go (100%) rename {config => internal/config}/type_bytes_test.go (98%) rename {config => internal/config}/type_duration.go (100%) rename {config => internal/config}/type_duration_test.go (98%) rename {config => internal/config}/type_error_rate.go (100%) rename {config => internal/config}/type_error_rate_test.go (98%) rename {config => internal/config}/type_hostport.go (100%) rename {config => internal/config}/type_hostport_test.go (98%) rename {config => internal/config}/type_http_path.go (100%) rename {config => internal/config}/type_http_path_test.go (97%) rename {config => internal/config}/type_ip.go (100%) rename {config => internal/config}/type_ip_test.go (98%) rename {config => internal/config}/type_metric_prefix.go (100%) rename {config => internal/config}/type_metric_prefix_test.go (98%) rename {config => internal/config}/type_port.go (100%) rename {config => internal/config}/type_port_test.go (98%) rename {config => internal/config}/type_prefer_ip.go (100%) rename {config => internal/config}/type_prefer_ip_test.go (98%) rename {config => internal/config}/type_statsd_tag_format.go (100%) rename {config => internal/config}/type_statsd_tag_format_test.go (98%) rename {config => internal/config}/type_url.go (100%) rename {config => internal/config}/type_url_test.go (98%) diff --git a/internal/cli/access_test.go b/internal/cli/access_test.go index 1a82e54..b55d40a 100644 --- a/internal/cli/access_test.go +++ b/internal/cli/access_test.go @@ -5,7 +5,7 @@ import ( "net/http" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/9seconds/mtg/v2/internal/testlib" "github.com/9seconds/mtg/v2/mtglib" "github.com/jarcoal/httpmock" diff --git a/internal/cli/base.go b/internal/cli/base.go index 6b6f892..95c4410 100644 --- a/internal/cli/base.go +++ b/internal/cli/base.go @@ -6,7 +6,7 @@ import ( "net/url" "os" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/network" ) diff --git a/config/config.go b/internal/config/config.go similarity index 100% rename from config/config.go rename to internal/config/config.go diff --git a/config/config_test.go b/internal/config/config_test.go similarity index 96% rename from config/config_test.go rename to internal/config/config_test.go index 940775c..e6b19e0 100644 --- a/config/config_test.go +++ b/internal/config/config_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/suite" ) diff --git a/config/testdata/broken.toml b/internal/config/testdata/broken.toml similarity index 100% rename from config/testdata/broken.toml rename to internal/config/testdata/broken.toml diff --git a/config/testdata/minimal.toml b/internal/config/testdata/minimal.toml similarity index 100% rename from config/testdata/minimal.toml rename to internal/config/testdata/minimal.toml diff --git a/config/testdata/only_secret.toml b/internal/config/testdata/only_secret.toml similarity index 100% rename from config/testdata/only_secret.toml rename to internal/config/testdata/only_secret.toml diff --git a/config/type_blocklist_uri.go b/internal/config/type_blocklist_uri.go similarity index 100% rename from config/type_blocklist_uri.go rename to internal/config/type_blocklist_uri.go diff --git a/config/type_blocklist_uri_test.go b/internal/config/type_blocklist_uri_test.go similarity index 98% rename from config/type_blocklist_uri_test.go rename to internal/config/type_blocklist_uri_test.go index 3d53135..cdba55a 100644 --- a/config/type_blocklist_uri_test.go +++ b/internal/config/type_blocklist_uri_test.go @@ -9,7 +9,7 @@ import ( "strconv" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_bytes.go b/internal/config/type_bytes.go similarity index 100% rename from config/type_bytes.go rename to internal/config/type_bytes.go diff --git a/config/type_bytes_test.go b/internal/config/type_bytes_test.go similarity index 98% rename from config/type_bytes_test.go rename to internal/config/type_bytes_test.go index c9ff55a..e395274 100644 --- a/config/type_bytes_test.go +++ b/internal/config/type_bytes_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_duration.go b/internal/config/type_duration.go similarity index 100% rename from config/type_duration.go rename to internal/config/type_duration.go diff --git a/config/type_duration_test.go b/internal/config/type_duration_test.go similarity index 98% rename from config/type_duration_test.go rename to internal/config/type_duration_test.go index c31d3ef..ad2dd4e 100644 --- a/config/type_duration_test.go +++ b/internal/config/type_duration_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_error_rate.go b/internal/config/type_error_rate.go similarity index 100% rename from config/type_error_rate.go rename to internal/config/type_error_rate.go diff --git a/config/type_error_rate_test.go b/internal/config/type_error_rate_test.go similarity index 98% rename from config/type_error_rate_test.go rename to internal/config/type_error_rate_test.go index f3a1ed1..de99f09 100644 --- a/config/type_error_rate_test.go +++ b/internal/config/type_error_rate_test.go @@ -5,7 +5,7 @@ import ( "strconv" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_hostport.go b/internal/config/type_hostport.go similarity index 100% rename from config/type_hostport.go rename to internal/config/type_hostport.go diff --git a/config/type_hostport_test.go b/internal/config/type_hostport_test.go similarity index 98% rename from config/type_hostport_test.go rename to internal/config/type_hostport_test.go index 5cf1994..6972e1b 100644 --- a/config/type_hostport_test.go +++ b/internal/config/type_hostport_test.go @@ -5,7 +5,7 @@ import ( "net" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_http_path.go b/internal/config/type_http_path.go similarity index 100% rename from config/type_http_path.go rename to internal/config/type_http_path.go diff --git a/config/type_http_path_test.go b/internal/config/type_http_path_test.go similarity index 97% rename from config/type_http_path_test.go rename to internal/config/type_http_path_test.go index 4c456ef..01843db 100644 --- a/config/type_http_path_test.go +++ b/internal/config/type_http_path_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_ip.go b/internal/config/type_ip.go similarity index 100% rename from config/type_ip.go rename to internal/config/type_ip.go diff --git a/config/type_ip_test.go b/internal/config/type_ip_test.go similarity index 98% rename from config/type_ip_test.go rename to internal/config/type_ip_test.go index 85189f4..86f14ed 100644 --- a/config/type_ip_test.go +++ b/internal/config/type_ip_test.go @@ -5,7 +5,7 @@ import ( "net" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_metric_prefix.go b/internal/config/type_metric_prefix.go similarity index 100% rename from config/type_metric_prefix.go rename to internal/config/type_metric_prefix.go diff --git a/config/type_metric_prefix_test.go b/internal/config/type_metric_prefix_test.go similarity index 98% rename from config/type_metric_prefix_test.go rename to internal/config/type_metric_prefix_test.go index 5598de0..0c3727d 100644 --- a/config/type_metric_prefix_test.go +++ b/internal/config/type_metric_prefix_test.go @@ -4,7 +4,7 @@ import ( "encoding/json" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_port.go b/internal/config/type_port.go similarity index 100% rename from config/type_port.go rename to internal/config/type_port.go diff --git a/config/type_port_test.go b/internal/config/type_port_test.go similarity index 98% rename from config/type_port_test.go rename to internal/config/type_port_test.go index 98ab187..75fc920 100644 --- a/config/type_port_test.go +++ b/internal/config/type_port_test.go @@ -5,7 +5,7 @@ import ( "strconv" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_prefer_ip.go b/internal/config/type_prefer_ip.go similarity index 100% rename from config/type_prefer_ip.go rename to internal/config/type_prefer_ip.go diff --git a/config/type_prefer_ip_test.go b/internal/config/type_prefer_ip_test.go similarity index 98% rename from config/type_prefer_ip_test.go rename to internal/config/type_prefer_ip_test.go index 6a93b53..917b918 100644 --- a/config/type_prefer_ip_test.go +++ b/internal/config/type_prefer_ip_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_statsd_tag_format.go b/internal/config/type_statsd_tag_format.go similarity index 100% rename from config/type_statsd_tag_format.go rename to internal/config/type_statsd_tag_format.go diff --git a/config/type_statsd_tag_format_test.go b/internal/config/type_statsd_tag_format_test.go similarity index 98% rename from config/type_statsd_tag_format_test.go rename to internal/config/type_statsd_tag_format_test.go index e3975af..b639ed4 100644 --- a/config/type_statsd_tag_format_test.go +++ b/internal/config/type_statsd_tag_format_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) diff --git a/config/type_url.go b/internal/config/type_url.go similarity index 100% rename from config/type_url.go rename to internal/config/type_url.go diff --git a/config/type_url_test.go b/internal/config/type_url_test.go similarity index 98% rename from config/type_url_test.go rename to internal/config/type_url_test.go index 1e7ccdb..4288172 100644 --- a/config/type_url_test.go +++ b/internal/config/type_url_test.go @@ -5,7 +5,7 @@ import ( "net/url" "testing" - "github.com/9seconds/mtg/v2/config" + "github.com/9seconds/mtg/v2/internal/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) From 5d5b77d73bf02bea05f3d6308383d6859cbaa187 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 2 Apr 2021 21:52:04 +0300 Subject: [PATCH 187/206] Tune defaults --- antireplay/init.go | 4 ++-- example.config.toml | 4 ++-- timeattack/init.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/antireplay/init.go b/antireplay/init.go index 4585a65..84fd82e 100644 --- a/antireplay/init.go +++ b/antireplay/init.go @@ -1,6 +1,6 @@ package antireplay const ( - DefaultMaxSize = 10 * 1024 * 1024 // 10mib - DefaultErrorRate = 0.0001 + DefaultMaxSize = 1024 * 1024 // 1MiB + DefaultErrorRate = 0.001 ) diff --git a/example.config.toml b/example.config.toml index 7402a57..fe3c883 100644 --- a/example.config.toml +++ b/example.config.toml @@ -137,10 +137,10 @@ enabled = true # approximate we try hard to store data quite dense but it is possible # that we can go over this limit for 10-20% under some conditions and # architectures. -max-size = "16mb" +max-size = "1mib" # we use stable bloom filters for anti-replay cache. This helps # to maintain a desired error ratio. -error-rate = 0.0001 +error-rate = 0.001 # You can protect proxies by using different blocklists. If client has # ip from the given range, we do not try to do a proper handshake. We diff --git a/timeattack/init.go b/timeattack/init.go index 385b9d7..5bf3ab4 100644 --- a/timeattack/init.go +++ b/timeattack/init.go @@ -3,5 +3,5 @@ package timeattack import "time" const ( - DefaultDuration = time.Second + DefaultDuration = 5 * time.Second ) From 3661fe108dec558ef55407dfee1700e384743f26 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 11:31:05 +0300 Subject: [PATCH 188/206] Add documentation for antireplay package --- antireplay/doc.go | 7 +++++++ antireplay/init.go | 9 +++++++-- antireplay/noop.go | 3 +++ antireplay/stable_bloom_filter.go | 12 ++++++++++++ internal/cli/proxy.go | 4 ++-- mtglib/init.go | 32 +++++++++++++++++++++++++++++++ 6 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 antireplay/doc.go diff --git a/antireplay/doc.go b/antireplay/doc.go new file mode 100644 index 0000000..7e86edb --- /dev/null +++ b/antireplay/doc.go @@ -0,0 +1,7 @@ +// Antireplay package has cache implementations that are effective +// against replay attacks. +// +// To understand more about replay attacks, please read documentation +// for mtglib.AntiReplayCache interface. This package has a list of some +// implementations of this interface. +package antireplay diff --git a/antireplay/init.go b/antireplay/init.go index 84fd82e..1f17681 100644 --- a/antireplay/init.go +++ b/antireplay/init.go @@ -1,6 +1,11 @@ package antireplay const ( - DefaultMaxSize = 1024 * 1024 // 1MiB - DefaultErrorRate = 0.001 + // DefaultStableBloomFilterMaxSize is a recommended byte size for a + // stable bloom filter. + DefaultStableBloomFilterMaxSize = 1024 * 1024 // 1MiB + + // DefaultStableBloomFilterErrorRate is a recommended default error + // rate for a stable bloom filter. + DefaultStableBloomFilterErrorRate = 0.001 ) diff --git a/antireplay/noop.go b/antireplay/noop.go index 8613e40..e4e7cf4 100644 --- a/antireplay/noop.go +++ b/antireplay/noop.go @@ -6,6 +6,9 @@ type noop struct{} func (n noop) SeenBefore(_ []byte) bool { return false } +// NewNoop returns an implementation that does nothing. A corresponding +// method always returns false, so this cache accepts everything you +// pass to it. func NewNoop() mtglib.AntiReplayCache { return noop{} } diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index b52631e..4e075b7 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -20,6 +20,18 @@ func (s *stableBloomFilter) SeenBefore(digest []byte) bool { return s.filter.TestAndAdd(digest) } +// NewStableBloomFilter returns an implementation of AntiReplayCache +// based on stable bloom filter. +// +// http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf +// +// The basic idea of a stable bloom filter is quite simple: each time +// when you set a new element, you randomly reset P elements. There is a +// hardcore math which proves that if you choose this P correctly, you +// can maintain the same error rate for a stream of elements. +// +// byteSize is the number of bytes you want to give to a bloom filter . +// errorRate is desired false-positive error rate . func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd sf.SetHash(xxhash.New64()) diff --git a/internal/cli/proxy.go b/internal/cli/proxy.go index 4b54d8d..fed863a 100644 --- a/internal/cli/proxy.go +++ b/internal/cli/proxy.go @@ -93,8 +93,8 @@ func (c *Proxy) setupAntiReplayCache(opts *mtglib.ProxyOpts) { } opts.AntiReplayCache = antireplay.NewStableBloomFilter( - c.Config.Defense.AntiReplay.MaxSize.Value(antireplay.DefaultMaxSize), - c.Config.Defense.AntiReplay.ErrorRate.Value(antireplay.DefaultErrorRate), + c.Config.Defense.AntiReplay.MaxSize.Value(antireplay.DefaultStableBloomFilterMaxSize), + c.Config.Defense.AntiReplay.ErrorRate.Value(antireplay.DefaultStableBloomFilterErrorRate), ) } diff --git a/mtglib/init.go b/mtglib/init.go index 010a78c..321da03 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -33,7 +33,39 @@ type Network interface { MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client } +// AntiReplayCache is an interface that is used to detect replay attacks +// based on some traffic fingerprints. +// +// Replay attacks are probe attacks whose main goal is to identify if +// server software can be classified in some way. For example, if you +// send some HTTP request to a web server, then you can expect that this +// server will respond with HTTP response back. +// +// There is a problem though. Let's imagine, that connection is +// encrypted. Let's imagine, that it is encrypted with some static key +// like ShadowSocks (https://shadowsocks.org/assets/whitepaper.pdf). +// In that case, in theory, if you repeat the same bytes, you can get +// the same responses. Let's imagine, that you've cracked the key. then +// if you send the same bytes, you can decrypt a response and see its +// structure. Based on its structure you can identify if this server is +// SOCKS5, MTPROTO proxy etc. +// +// This is just one example, maybe not the best or not the most +// relevant. In real life, different organizations use such replay +// attacks to perform some reverse engineering of the proxy, do some +// statical analysis to identify server software. +// +// There are many ways how to protect your proxy against them. One +// is domain fronting which is a core part of mtg. Another one is to +// collect some 'handshake fingerprints' and forbid duplication. +// +// So, it one is sending the same byte flow right after you (or a couple +// of hours after), mtg should detect that and reject this connection +// (or redirect to fronting domain). type AntiReplayCache interface { + // Seen before checks if this set of bytes was observed before or + // not. If it is required to store this information somewhere else, + // then it has to do that. SeenBefore(data []byte) bool } From b3112c5c6714dd04a4b7f1bd6939bebebd4284bc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 11:39:09 +0300 Subject: [PATCH 189/206] Move documentation to init.go --- antireplay/doc.go | 7 ------- antireplay/init.go | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 antireplay/doc.go diff --git a/antireplay/doc.go b/antireplay/doc.go deleted file mode 100644 index 7e86edb..0000000 --- a/antireplay/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// Antireplay package has cache implementations that are effective -// against replay attacks. -// -// To understand more about replay attacks, please read documentation -// for mtglib.AntiReplayCache interface. This package has a list of some -// implementations of this interface. -package antireplay diff --git a/antireplay/init.go b/antireplay/init.go index 1f17681..eb4eeb1 100644 --- a/antireplay/init.go +++ b/antireplay/init.go @@ -1,3 +1,9 @@ +// Antireplay package has cache implementations that are effective +// against replay attacks. +// +// To understand more about replay attacks, please read documentation +// for mtglib.AntiReplayCache interface. This package has a list of some +// implementations of this interface. package antireplay const ( From 04b88cc8645ed081b0b4eadfc3154cbfd5074553 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 15:12:46 +0300 Subject: [PATCH 190/206] Add documentation for event stream --- events/event_stream.go | 17 ++++++++++++++ events/init.go | 51 ++++++++++++++++++++++++++++++++++++++++++ events/noop.go | 2 ++ mtglib/init.go | 15 +++++++++++++ 4 files changed, 85 insertions(+) diff --git a/events/event_stream.go b/events/event_stream.go index 9995eb0..cc971a6 100644 --- a/events/event_stream.go +++ b/events/event_stream.go @@ -9,12 +9,23 @@ import ( "github.com/OneOfOne/xxhash" ) +// EventStream is a default implementation of the mtglib.EventStream +// interface. +// +// EventStream manages a set of goroutines, observers. Main +// responsibility of the event stream is to route an event to relevant +// observer based on some hash so each observer will have all events +// which belong to some stream id. +// +// Thus, EventStream can spawn many observers. type EventStream struct { ctx context.Context ctxCancel context.CancelFunc chans []chan mtglib.Event } +// Send starts delivering of the message to observer with respect to a +// given context If context is closed, message could be not delivered. func (e EventStream) Send(ctx context.Context, evt mtglib.Event) { var chanNo uint32 @@ -31,10 +42,16 @@ func (e EventStream) Send(ctx context.Context, evt mtglib.Event) { } } +// Shutdown stops an event stream pipeline. func (e EventStream) Shutdown() { e.ctxCancel() } +// NewEventStream builds a new default event stream. +// +// If you give an empty array of observers, then NoopObserver is going +// to be used. If you give many observers, then they will process a +// message concurrently. func NewEventStream(observerFactories []ObserverFactory) EventStream { if len(observerFactories) == 0 { observerFactories = append(observerFactories, NewNoopObserver) diff --git a/events/init.go b/events/init.go index 1204652..f754ede 100644 --- a/events/init.go +++ b/events/init.go @@ -1,18 +1,69 @@ +// Events has a default implementations of EventStream for mtglib. +// +// Please see documentation for mtglib.EventStream interface to get an +// idea of such an abstraction. This package has implementations for the +// default event stream. +// +// Default event stream has a list of its own concepts. First, all it +// does is a routing of messages to known observers. It takes an event, +// defines its type and pass this message to a method of the observer. +// +// There might be many observers, but default event stream has a +// guarantee though. It uses StreamID as a sharding key and guarantees +// that a message with the same StreamID will be devlivered to the same +// observer instance. So, each producer is guarateed to get all relevant +// messages related to the same session. It is not possible that it will +// get EventFinish if it has not seen EventStart for that session yet. package events import "github.com/9seconds/mtg/v2/mtglib" +// Observer is an instance that listens for the incoming events. +// +// As it is said in the package description, the default event stream +// guarantees that all events with the same StreamID are going to be +// routed to the same instance of the observer. So, there is no need +// to synchronize information about streams between many observers +// instances, they can have their local storage. type Observer interface { + // EventStart reacts on incoming mtglib.EventStart event. EventStart(mtglib.EventStart) + + // EventFinish reacts on incoming mtglib.EventFinish event. EventFinish(mtglib.EventFinish) + + // EventConnectedToDC reacts on incoming mtglib.EventConnectedToDC + // event. EventConnectedToDC(mtglib.EventConnectedToDC) + + // EventDomainFronting reacts on incoming mtglib.EventDomainFronting + // event. EventDomainFronting(mtglib.EventDomainFronting) + + // EventTraffic reacts on incoming mtglib.EventTraffic event. EventTraffic(mtglib.EventTraffic) + + // EventConcurrencyLimited reacts on incoming + // mtglib.EventConcurrencyLimited event. EventConcurrencyLimited(mtglib.EventConcurrencyLimited) + + // EventIPBlocklisted reacts on incoming mtglib.EventIPBlocklisted event. EventIPBlocklisted(mtglib.EventIPBlocklisted) + + // EventReplayAttack reacts on incoming mtglib.EventReplayAttack event. EventReplayAttack(mtglib.EventReplayAttack) + // Shutdown stop observer. Default event stream guarantees: + // 1. If shutdown is executed, it is executed only once + // 2. Observer won't receieve any new message after this + // function call. Shutdown() } +// ObserverFactory creates a new instance of the observer. +// +// Default event stream creates a small set of goroutines to manage +// incoming messages. Each message is routed to an appropriate observer +// based on a sharding key, stream id. So, it is possible that an +// instance of mtg will have many observer instances, not a single one. type ObserverFactory func() Observer diff --git a/events/noop.go b/events/noop.go index 2fdb238..0582d9f 100644 --- a/events/noop.go +++ b/events/noop.go @@ -10,6 +10,7 @@ type noop struct{} func (n noop) Send(ctx context.Context, evt mtglib.Event) {} +// NewNoopStream creates a stream which discards each message. func NewNoopStream() mtglib.EventStream { return noop{} } @@ -26,6 +27,7 @@ func (n noopObserver) EventIPBlocklisted(_ mtglib.EventIPBlocklisted) func (n noopObserver) EventReplayAttack(_ mtglib.EventReplayAttack) {} func (n noopObserver) Shutdown() {} +// NewNoopObserver creates an observer which discards each message. func NewNoopObserver() Observer { return noopObserver{} } diff --git a/mtglib/init.go b/mtglib/init.go index 321da03..860e334 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -78,7 +78,22 @@ type Event interface { Timestamp() time.Time } +// EventStream is an abstraction that accepts a set of events produced +// by mtg. Its main goal is to inject your logging or monitoring system. +// +// The idea is simple. When mtg works, it emits a set of events during +// a lifecycle of the requestor: EventStart, EventFinish etc. mtg is a +// producer which puts these events into a stream. Responsibility of +// the stream is to deliver this event to consumers/observers. There +// might be many different observers (for example, you want to have both +// statsd and prometheus), mtg should know nothing about them. type EventStream interface { + // Send delivers an event to observers. Given context has to be + // respected. If the context is closed, all blocking operations should + // be released ASAP. + // + // It is possible that context is closed but the message is delivered. + // EventStream implementations should solve this issue somehow. Send(context.Context, Event) } From 0274b3436a6d27cd90ad4ef1cd64148edcd9a9d2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 16:09:21 +0300 Subject: [PATCH 191/206] Add documentation for ipblocklist --- antireplay/stable_bloom_filter.go | 13 +++++- internal/cli/proxy.go | 2 +- ipblocklist/firehol.go | 73 +++++++++++++++++++++---------- ipblocklist/init.go | 14 +++++- ipblocklist/noop.go | 2 + mtglib/init.go | 6 +++ 6 files changed, 82 insertions(+), 28 deletions(-) diff --git a/antireplay/stable_bloom_filter.go b/antireplay/stable_bloom_filter.go index 4e075b7..61f9934 100644 --- a/antireplay/stable_bloom_filter.go +++ b/antireplay/stable_bloom_filter.go @@ -30,9 +30,18 @@ func (s *stableBloomFilter) SeenBefore(digest []byte) bool { // hardcore math which proves that if you choose this P correctly, you // can maintain the same error rate for a stream of elements. // -// byteSize is the number of bytes you want to give to a bloom filter . -// errorRate is desired false-positive error rate . +// byteSize is the number of bytes you want to give to a bloom filter. +// errorRate is desired false-positive error rate. If you want to use +// default values, please pass 0 for byteSize and <0 for errorRate. func NewStableBloomFilter(byteSize uint, errorRate float64) mtglib.AntiReplayCache { + if byteSize == 0 { + byteSize = DefaultStableBloomFilterMaxSize + } + + if errorRate < 0 { + errorRate = DefaultStableBloomFilterErrorRate + } + sf := boom.NewDefaultStableBloomFilter(byteSize*8, errorRate) // nolint: gomnd sf.SetHash(xxhash.New64()) diff --git a/internal/cli/proxy.go b/internal/cli/proxy.go index fed863a..713afcb 100644 --- a/internal/cli/proxy.go +++ b/internal/cli/proxy.go @@ -133,7 +133,7 @@ func (c *Proxy) setupIPBlocklist(opts *mtglib.ProxyOpts) error { return err // nolint: wrapcheck } - go firehol.Run(c.Config.Defense.Blocklist.UpdateEach.Value(ipblocklist.DefaultUpdateEach)) + go firehol.Run(c.Config.Defense.Blocklist.UpdateEach.Value(ipblocklist.DefaultFireholUpdateEach)) opts.IPBlocklist = firehol diff --git a/ipblocklist/firehol.go b/ipblocklist/firehol.go index 20dd0d8..947134b 100644 --- a/ipblocklist/firehol.go +++ b/ipblocklist/firehol.go @@ -27,6 +27,19 @@ const ( var fireholRegexpComment = regexp.MustCompile(`\s*#.*?$`) +// Firehol is IPBlocklist which uses lists from FireHOL: +// https://iplists.firehol.org/ +// +// It can use both local files and remote URLs. This is not necessary +// that blocklists should be taken from this website, we expect only +// compatible formats here. +// +// Example of the format: +// +// # this is a comment +// # to ignore +// 127.0.0.1 # you can specify an IP +// 10.0.0.0/8 # or cidr type Firehol struct { ctx context.Context ctxCancel context.CancelFunc @@ -44,6 +57,12 @@ type Firehol struct { treeV6 *bool_tree.TreeV6 } +// Shutdown stop a background update process. +func (f *Firehol) Shutdown() { + f.ctxCancel() +} + +// Contains is given IP list can be found in FireHOL blocklists. func (f *Firehol) Contains(ip net.IP) bool { if ip == nil { return true @@ -61,27 +80,15 @@ func (f *Firehol) Contains(ip net.IP) bool { return f.containsIPv6(ip.To16()) } -func (f *Firehol) containsIPv4(addr net.IP) bool { - ip := patricia.NewIPv4AddressFromBytes(addr, 32) - - if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { - return true - } - - return false -} - -func (f *Firehol) containsIPv6(addr net.IP) bool { - ip := patricia.NewIPv6Address(addr, 128) - - if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { - return true - } - - return false -} - +// Run starts a background update process. +// +// This is a blocking method so you probably want to run it in a +// goroutine. func (f *Firehol) Run(updateEach time.Duration) { + if updateEach == 0 { + updateEach = DefaultFireholUpdateEach + } + ticker := time.NewTicker(updateEach) defer func() { @@ -113,8 +120,24 @@ func (f *Firehol) Run(updateEach time.Duration) { } } -func (f *Firehol) Shutdown() { - f.ctxCancel() +func (f *Firehol) containsIPv4(addr net.IP) bool { + ip := patricia.NewIPv4AddressFromBytes(addr, 32) + + if ok, _, err := f.treeV4.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false +} + +func (f *Firehol) containsIPv6(addr net.IP) bool { + ip := patricia.NewIPv6Address(addr, 128) + + if ok, _, err := f.treeV6.FindDeepestTag(ip); ok && err == nil { + return true + } + + return false } func (f *Firehol) update() error { // nolint: funlen, cyclop @@ -302,6 +325,10 @@ func (f *Firehol) updateAddToTrees(ip net.IP, cidr uint, return nil } +// NewFirehol creates a new instance of FireHOL IP blocklist. +// +// This method does not start an update process so please execute Run +// when it is necessary. func NewFirehol(logger mtglib.Logger, network mtglib.Network, downloadConcurrency uint, remoteURLs []string, @@ -326,7 +353,7 @@ func NewFirehol(logger mtglib.Logger, network mtglib.Network, } if downloadConcurrency == 0 { - downloadConcurrency = DefaultDownloadConcurrency + downloadConcurrency = DefaultFireholDownloadConcurrency } workerPool, _ := ants.NewPool(int(downloadConcurrency)) diff --git a/ipblocklist/init.go b/ipblocklist/init.go index cfa8df8..f5a06d3 100644 --- a/ipblocklist/init.go +++ b/ipblocklist/init.go @@ -1,8 +1,18 @@ +// Package ipblocklist contains default implementation of the +// IPBlocklist for mtg. +// +// Please check documentation for mtglib.IPBlocklist interface to get an +// idea of this abstraction. package ipblocklist import "time" const ( - DefaultDownloadConcurrency = 1 - DefaultUpdateEach = 12 * time.Hour + // DefaultFireholDownloadConcurrency defines a default max number of + // concurrent downloads of ip blocklists for Firehol. + DefaultFireholDownloadConcurrency = 1 + + // DefaultFireholUpdateEach defines a default time period when + // Firehol requests updates of the blocklists. + DefaultFireholUpdateEach = 6 * time.Hour ) diff --git a/ipblocklist/noop.go b/ipblocklist/noop.go index dcbed19..2df31cb 100644 --- a/ipblocklist/noop.go +++ b/ipblocklist/noop.go @@ -10,6 +10,8 @@ type noop struct{} func (n noop) Contains(ip net.IP) bool { return false } +// NewNoop returns a dummy ipblocklist which allows all incoming +// connections. func NewNoop() mtglib.IPBlocklist { return noop{} } diff --git a/mtglib/init.go b/mtglib/init.go index 860e334..ca69ed5 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -69,6 +69,12 @@ type AntiReplayCache interface { SeenBefore(data []byte) bool } +// IPBlocklist filters requests based on IP address. +// +// If this filter has an IP address, then mtg closes a request without +// reading anything from a socket. It also does not give such request to +// a worker pool, so in worst cases you can expect that you invoke this +// object more frequent than defined proxy concurrency. type IPBlocklist interface { Contains(net.IP) bool } From 0dd890a09d8a6c3eeb9e78f178a9de4581d5e399 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 16:55:26 +0300 Subject: [PATCH 192/206] Add documentation for logger --- logger/init.go | 9 +++++++++ logger/noop.go | 1 + logger/zerolog.go | 1 + mtglib/init.go | 22 ++++++++++++++++++++++ 4 files changed, 33 insertions(+) diff --git a/logger/init.go b/logger/init.go index afcaf1d..8ff8ddc 100644 --- a/logger/init.go +++ b/logger/init.go @@ -1,5 +1,14 @@ +// Package logger has implementation of loggers for mtglib.Logger +// interface. +// +// Please see a description of that interface to get some agreements +// which are used by mtglib. package logger +// StdLikeLogger is an interface which is close to log.Logger. This is +// commonly used by many 3pp tools. While mtglib itself does not need +// it, it is always a good idea to support it and have a transient end +// to end logging. type StdLikeLogger interface { Printf(format string, args ...interface{}) } diff --git a/logger/noop.go b/logger/noop.go index d416542..61470ac 100644 --- a/logger/noop.go +++ b/logger/noop.go @@ -15,6 +15,7 @@ func (n noopLogger) InfoError(_ string, _ error) {} func (n noopLogger) WarningError(_ string, _ error) {} func (n noopLogger) DebugError(_ string, _ error) {} +// NewNoopLogger returns a logger which discards all events. func NewNoopLogger() mtglib.Logger { return noopLogger{} } diff --git a/logger/zerolog.go b/logger/zerolog.go index d504afa..b0dc0b6 100644 --- a/logger/zerolog.go +++ b/logger/zerolog.go @@ -114,6 +114,7 @@ func (z *zeroLogContext) attachCtx(evt *zerolog.Event) { } } +// NewZeroLogger returns a logger which is using rs/zerolog library. func NewZeroLogger(log zerolog.Logger) mtglib.Logger { return &zeroLogContext{ log: &log, diff --git a/mtglib/init.go b/mtglib/init.go index ca69ed5..570cc1d 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -76,6 +76,8 @@ type AntiReplayCache interface { // a worker pool, so in worst cases you can expect that you invoke this // object more frequent than defined proxy concurrency. type IPBlocklist interface { + // Contains checks if given IP address belongs to this blocklist If. + // it is, a connection is terminated . Contains(net.IP) bool } @@ -107,6 +109,26 @@ type TimeAttackDetector interface { Valid(time.Time) error } +// Logger defines an interface of the logger used by mtglib. +// +// Each logger has a name. It is possible to stack names to organize +// poor-man namespaces. Also, each logger must be able to bind +// parameters to avoid pushing them all the time. +// +// Example +// +// logger := SomeLogger{} +// logger = logger.BindStr("ip", net.IP{127, 0, 0, 1}) +// logger.Info("Hello") +// +// In that case, ip is bound as a parameter. It is a great idea to +// put this parameter somewhere in a log message. +// +// logger1 = logger.BindStr("param1", "11") +// logger2 = logger.BindInt("param2", 11) +// +// logger1 should see no param2 and vice versa, logger2 should not see param1 +// If you attach a parameter to a logger, parents should not know about that. type Logger interface { Named(name string) Logger From 372c703ad6a9f0c2dbe571506a7fc1c167e28e31 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 5 Apr 2021 18:46:15 +0300 Subject: [PATCH 193/206] Add documentation for network --- mtglib/init.go | 24 +++++++++++++ network/default.go | 5 +++ network/init.go | 63 ++++++++++++++++++++++++++++++--- network/load_balanced_socks5.go | 10 ++++++ network/network.go | 4 +++ network/socks5.go | 4 +++ 6 files changed, 105 insertions(+), 5 deletions(-) diff --git a/mtglib/init.go b/mtglib/init.go index 570cc1d..34e9e4e 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -27,9 +27,33 @@ const ( DefaultPreferIP = "prefer-ipv6" ) +// Network defines a knowledge how to work with a network. It may sound +// fun but it encapsulates all the knowledge how to properly establish +// connections to remote hosts and configure HTTP clients. +// +// For example, if you want to use SOCKS5 proxy, you probably want to +// have all traffic routed to this proxy: telegram connections, http +// requests and so on. This knowledge is encapsulated into instances of +// such interface. +// +// mtglib uses Network for: +// +// 1. Dialing to Telegram +// +// 2. Dialing to front domain +// +// 3. Doing HTTP requests (for example, for FireHOL ipblocklist). type Network interface { + // Dial establishes context-free TCP connections. Dial(network, address string) (net.Conn, error) + + // DialContext dials using a context. This is a preferrable + // way of establishing TCP connections. DialContext(ctx context.Context, network, address string) (net.Conn, error) + + // MakeHTTPClient build an HTTP client with given dial function. If + // nothing is provided, then DialContext of this interface is going + // to be used. MakeHTTPClient(func(ctx context.Context, network, address string) (net.Conn, error)) *http.Client } diff --git a/network/default.go b/network/default.go index 72b3da5..cad8799 100644 --- a/network/default.go +++ b/network/default.go @@ -60,6 +60,11 @@ func (d *defaultDialer) DialContext(ctx context.Context, network, address string return tcpConn, nil } +// NewDefaultDialer build a new dialer which dials bypassing proxies +// etc. +// +// The most default one you can imagine. But it has tunes TCP +// connections and setups SO_REUSEPORT. func NewDefaultDialer(timeout time.Duration, bufferSize int) (Dialer, error) { switch { case timeout < 0: diff --git a/network/init.go b/network/init.go index 57053af..0dc9660 100644 --- a/network/init.go +++ b/network/init.go @@ -1,3 +1,20 @@ +// Network contains a default implementation of the network. +// +// Please see mtglib.Network interface to get some basic idea behind +// this abstraction. +// +// Some notable feature of this implementation: +// +// 1. It detaches dialer from a network. Dialer is something which +// implements a real dialer and network completes it with more higher +// level details. +// +// 2. It uses only TCP connections. Even for DNS it uses DNS-Over-HTTPS +// +// 3. It has some simple implementation of DNS cache which is good +// enough for our purpose. +// +// 4. It sets uses SO_REUSEPORT port if applicable. package network import ( @@ -8,23 +25,59 @@ import ( ) const ( - DefaultTimeout = 10 * time.Second - DefaultHTTPTimeout = 10 * time.Second - DefaultBufferSize = 16 * 1024 // 16 kib + // DefaultTimeout is a default timeout for establishing TCP + // connection. + DefaultTimeout = 10 * time.Second + // DefaultHTTPTimeout defines a default timeout for making HTTP + // request. + DefaultHTTPTimeout = 10 * time.Second + + // DefaultBufferSize defines a TCP buffer size. Both read and write, so + // for real size, please multiply this number by 2. + DefaultBufferSize = 16 * 1024 // 16 kib + + // ProxyDialerOpenThreshold is used for load balancing SOCKS5 dialer + // only. + // + // This dialer uses circuit breaker with of 3 stages: OPEN, + // HALF_OPEN and CLOSED. If state is CLOSED, all requests go in + // a normal mode. If you get more that ProxyDialerOpenThreshold + // errors, circuit breaker goes into OPEN mode. + // + // When circuit breaker is in OPEN mode, it forbids all request to + // a given proxy. But after ProxyDialerHalfOpenTimeout it gives a + // second chance and opens an access for a SINGLE request. If this + // request success, then circuit breaker closes, otherwise opens + // again. + // + // When circuit breaker is closed, it clears an error states each + // ProxyDialerResetFailuresTimeout. ProxyDialerOpenThreshold = 5 ProxyDialerHalfOpenTimeout = time.Minute ProxyDialerResetFailuresTimeout = 10 * time.Second + // DefaultDOHHostname defines a default IP address for DOH host. + // Since mtg is simple, please pass IP address here. We do not + // have bootstrap servers here embedded. DefaultDOHHostname = "9.9.9.9" - DNSTimeout = 5 * time.Second + + // DNSTimeout defines a timeout for DNS queries. + DNSTimeout = 5 * time.Second ) var ( - ErrCircuitBreakerOpened = errors.New("circuit breaker is opened") + // ErrCircuitBreakerOpened is returned when proxy is being accessed + // but circuit breaker is opened. + ErrCircuitBreakerOpened = errors.New("circuit breaker is opened") + + // ErrCannotDialWithAllProxies is returned when load balancing + // client is trying to access proxies but all of them are failed. ErrCannotDialWithAllProxies = errors.New("cannot dial with all proxies") ) +// Dialer defines an interface which is required to bootstrap a network +// instance from. type Dialer interface { Dial(network, address string) (net.Conn, error) DialContext(ctx context.Context, network, address string) (net.Conn, error) diff --git a/network/load_balanced_socks5.go b/network/load_balanced_socks5.go index 2a60d69..a52004d 100644 --- a/network/load_balanced_socks5.go +++ b/network/load_balanced_socks5.go @@ -32,6 +32,16 @@ func (l loadBalancedSocks5Dialer) DialContext(ctx context.Context, network, addr return nil, ErrCannotDialWithAllProxies } +// NewLoadBalancedSocks5Dialer builds a new load balancing SOCKS5 +// dialer. +// +// The main difference from one which is made by NewSocks5Dialer is that +// we actually have a list of these proxies. When dial is requested, +// any proxy is picked and used. If proxy fails for some reason, we try +// another one. +// +// So, it is mostly useful if you have some routes with proxies which +// are not always online or having buggy network. func NewLoadBalancedSocks5Dialer(baseDialer Dialer, proxyURLs []*url.URL) (Dialer, error) { dialers := make([]Dialer, 0, len(proxyURLs)) diff --git a/network/network.go b/network/network.go index 147290b..3ff9af6 100644 --- a/network/network.go +++ b/network/network.go @@ -115,6 +115,10 @@ func (n *network) dnsResolve(protocol, address string) ([]string, error) { return ips, nil } +// NewNetwork assembles an mtglib.Network compatible structure +// based on a dialer and given params. +// +// It brings simple DNS cache and DNS-Over-HTTPS when necessary. func NewNetwork(dialer Dialer, userAgent, dohHostname string, httpTimeout time.Duration) (mtglib.Network, error) { diff --git a/network/socks5.go b/network/socks5.go index 91c9c61..43aac4a 100644 --- a/network/socks5.go +++ b/network/socks5.go @@ -7,6 +7,10 @@ import ( "golang.org/x/net/proxy" ) +// NewSocks5Dialer build a new dialer from a given one (so, in theory +// you can chain here). Proxy parameters are passed with URI in a form of: +// +// socks5://[user:[password]]@host:port func NewSocks5Dialer(baseDialer Dialer, proxyURL *url.URL) (Dialer, error) { rv, err := proxy.FromURL(proxyURL, baseDialer) if err != nil { From a3bae795c4eb9da3e5bc3937085e677598ffe343 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 6 Apr 2021 11:03:51 +0300 Subject: [PATCH 194/206] Add documentation for timeattack --- mtglib/init.go | 8 ++++++++ timeattack/detector.go | 3 +++ timeattack/init.go | 9 ++++++--- timeattack/noop.go | 1 + 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/mtglib/init.go b/mtglib/init.go index 34e9e4e..9f3ba08 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -129,7 +129,15 @@ type EventStream interface { Send(context.Context, Event) } +// TimeAttackDetector is an abstraction that checks a time, taken from +// the faketls client hello message. This timestamp is encoded into +// client-generated random bytes and can be extracted after some client +// hello verification. +// +// This is mostly to prevent replay attacks. type TimeAttackDetector interface { + // Valid returns an error if timestamp is invalid or should not be + // accepted. Valid(time.Time) error } diff --git a/timeattack/detector.go b/timeattack/detector.go index ba70d3b..4ed1179 100644 --- a/timeattack/detector.go +++ b/timeattack/detector.go @@ -29,6 +29,9 @@ func (d detector) Valid(then time.Time) error { return nil } +// NewDetector returns a new TimeAttackDetector which validates that +// timestamp belongs to intervar [X-duration, X+duration], so a small +// timeshift is acceptable. func NewDetector(duration time.Duration) mtglib.TimeAttackDetector { return detector{ Duration: duration, diff --git a/timeattack/init.go b/timeattack/init.go index 5bf3ab4..1c66919 100644 --- a/timeattack/init.go +++ b/timeattack/init.go @@ -1,7 +1,10 @@ +// TimeAttack has implementation of mtglib.TimeAttackDetector> package timeattack import "time" -const ( - DefaultDuration = 5 * time.Second -) +// DefaultDuration is a default duration when timestamps are acceptable. +// +// It means that all timestamps which are X-DefaultDuration <= X <= +// X+DefaultDuration are fine. +const DefaultDuration = 5 * time.Second diff --git a/timeattack/noop.go b/timeattack/noop.go index 43537c5..e3827b8 100644 --- a/timeattack/noop.go +++ b/timeattack/noop.go @@ -10,6 +10,7 @@ type noop struct{} func (n noop) Valid(_ time.Time) error { return nil } +// NewNoop returns TimeAttackDetector which accepts all timestamps. func NewNoop() mtglib.TimeAttackDetector { return noop{} } From 54a7c6a2a5321eb10473aac0e2edcd85b4523efb Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 6 Apr 2021 11:09:37 +0300 Subject: [PATCH 195/206] Small subtle optimizations of faketls --- mtglib/internal/faketls/client_hello.go | 12 ++++-------- mtglib/internal/faketls/init.go | 6 +++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go index 62d3278..51e1fe5 100644 --- a/mtglib/internal/faketls/client_hello.go +++ b/mtglib/internal/faketls/client_hello.go @@ -3,6 +3,7 @@ package faketls import ( "crypto/hmac" "crypto/sha256" + "crypto/subtle" "encoding/binary" "fmt" "time" @@ -39,10 +40,7 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { } copy(hello.Random[:], handshake[ClientHelloRandomOffset:]) - - for i := ClientHelloRandomOffset; i < ClientHelloRandomOffset+RandomLen; i++ { - handshake[i] = 0 - } + copy(handshake[ClientHelloRandomOffset:], clientHelloEmptyRandom) rec := record.AcquireRecord() defer record.ReleaseRecord(rec) @@ -62,10 +60,8 @@ func ParseClientHello(secret, handshake []byte) (ClientHello, error) { computedRandom[i] ^= hello.Random[i] } - for i := 0; i < RandomLen-4; i++ { - if computedRandom[i] != 0 { - return hello, ErrBadDigest - } + if subtle.ConstantTimeCompare(clientHelloEmptyRandom[:RandomLen-4], computedRandom[:RandomLen-4]) != 1 { + return hello, ErrBadDigest } timestamp := int64(binary.LittleEndian.Uint32(computedRandom[RandomLen-4:])) diff --git a/mtglib/internal/faketls/init.go b/mtglib/internal/faketls/init.go index dfa269b..5bbf6f3 100644 --- a/mtglib/internal/faketls/init.go +++ b/mtglib/internal/faketls/init.go @@ -1,6 +1,9 @@ package faketls -import "errors" +import ( + "bytes" + "errors" +) const ( RandomLen = 32 @@ -34,4 +37,5 @@ var ( 0x00, 0x1d, // x25519 curve 0x00, 0x20, // 32 bytes of key } + clientHelloEmptyRandom = bytes.Repeat([]byte{0}, RandomLen) ) From b748603096e6f27a8acb8f25e2dfd4905aee57a0 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 6 Apr 2021 11:59:46 +0300 Subject: [PATCH 196/206] Add docs for stats --- stats/init.go | 91 +++++++++++++++++++++++++++++++++++++++++---- stats/prometheus.go | 12 ++++++ stats/statsd.go | 13 +++++++ timeattack/init.go | 2 +- 4 files changed, 109 insertions(+), 9 deletions(-) diff --git a/stats/init.go b/stats/init.go index d9d64b9..3a2bae7 100644 --- a/stats/init.go +++ b/stats/init.go @@ -1,31 +1,106 @@ +// Stats package has implementations of events.Observers for different +// monitoring systems. +// +// Observer is a consumer of events produced by mtg. Consumers, defined +// in this package, process these events and provide information used by +// different monitoring system or time series databases. package stats const ( + // DefaultMetricPrefix defines a base prefix for all metrics. DefaultMetricPrefix = "mtg" + // DefaultStatsdMetricPrefix defines a base prefix for metrics + // which are passed to statsd. DefaultStatsdMetricPrefix = DefaultMetricPrefix + "." - DefaultStatsdTagFormat = "datadog" - MetricClientConnections = "client_connections" - MetricTelegramConnections = "telegram_connections" + // DefaultStatsdTagFormat defines a format of tags for statsd + // observer. + DefaultStatsdTagFormat = "datadog" + + // MetricClientConnections defines a metric which is responsible for a + // number of currently active connections established by client. + // + // Type: gauge + // Tags: + // ip_family | A type of ip (ipv4 or ipv6) of the client. + MetricClientConnections = "client_connections" + + // MetricTelegramConnections defines a metric which is responsible for + // a count of active connections to Telegram servers. + // + // Type: gauge + // Tags: + // telegram_ip | IP address of the telegram server. + // dc | Index of the datacenter to connect to. + MetricTelegramConnections = "telegram_connections" + + // MetricDomainFrontingConnections defines a metric which is + // responsible for a count of active connections to a fronting domain. + // Fronting domain is that one that is encoded in a secret. + // + // Type: gauge + // Tags: + // ip_family | A type of IP (ipv4 or ipv6) that was used. MetricDomainFrontingConnections = "domain_fronting_connections" - MetricTelegramTraffic = "telegram_traffic" + // MetricTelegramTraffic defines a metric for traffic (in bytes) that + // is sent to and from Telegram servers. + // + // Type: counter + // Tags: + // telegram_ip | IP address of the telegram server. + // dc | Index of the datacenter + // direction | Direction of the traffc flow. Values are + // | 'to_client' and 'from_client' + MetricTelegramTraffic = "telegram_traffic" + + // MetricDomainFrontingTraffic defines a metric for traffic (in bytes) + // that is sent to and from fronting domain. + // + // Type: counter + // Tags: + // direction | Direction of the traffc flow. Values are + // | 'to_client' and 'from_client' MetricDomainFrontingTraffic = "domain_fronting_traffic" - MetricDomainFronting = "domain_fronting" - MetricConcurrencyLimited = "concurrency_limited" - MetricIPBlocklisted = "ip_blocklisted" - MetricReplayAttacks = "replay_attacks" + // MetricDomainFronting defines a metric for a number of domain + // fronting routing events. + // + // Type: counter + MetricDomainFronting = "domain_fronting" + // MetricConcurrencyLimited defines a metric for a count of events, + // when the client was blocked due to the concurrency limit. + // + // Type: counter + MetricConcurrencyLimited = "concurrency_limited" + + // MetricIPBlocklisted defines a metric for a count of events, when + // client was blocked because her IP address was found in blocklists. + // + // Type: counter + MetricIPBlocklisted = "ip_blocklisted" + + // MetricReplayAttacks defines a metric for a count of events, when + // mtg has detected a replay attack. Just a reminder: mtg immediately + // routes a connection to a fronting domain if such event is detected. + // + // Type: counter + MetricReplayAttacks = "replay_attacks" + + // TagIPFamily defines a name of the 'ip_family' tag and all values. TagIPFamily = "ip_family" TagIPFamilyIPv4 = "ipv4" TagIPFamilyIPv6 = "ipv6" + // TagTelegramIP defines a name of the 'telegram_ip' tag. TagTelegramIP = "telegram_ip" + // TagDC defines a name of the 'dc' tag. TagDC = "dc" + // TagDirection defines a name of the 'direction' tag. TagDirection = "direction" TagDirectionToClient = "to_client" TagDirectionFromClient = "from_client" diff --git a/stats/prometheus.go b/stats/prometheus.go index aa2933d..634be95 100644 --- a/stats/prometheus.go +++ b/stats/prometheus.go @@ -126,6 +126,12 @@ func (p prometheusProcessor) Shutdown() { p.streams = make(map[string]*streamInfo) } +// PrometheusFactory is a factory of events.Observers which collect +// information in a format suitable for Prometheus. +// +// This factory can also serve on a given listener. In that case it +// starts HTTP server with a single endpoint - a Prometheus-compatible +// scrape output. type PrometheusFactory struct { httpServer *http.Server @@ -142,6 +148,7 @@ type PrometheusFactory struct { metricReplayAttacks prometheus.Counter } +// Make builds a new observer. func (p *PrometheusFactory) Make() events.Observer { return prometheusProcessor{ streams: make(map[string]*streamInfo), @@ -149,14 +156,19 @@ func (p *PrometheusFactory) Make() events.Observer { } } +// Serve starts an HTTP server on a given listener. func (p *PrometheusFactory) Serve(listener net.Listener) error { return p.httpServer.Serve(listener) } +// Close stops a factory. Please pay attention that underlying listener +// is not closed. func (p *PrometheusFactory) Close() error { return p.httpServer.Shutdown(context.Background()) } +// NewPrometheus builds an events.ObserverFactory which can serve HTTP +// endpoint with Prometheus scrape data. func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint: funlen registry := prometheus.NewPedanticRegistry() httpHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ diff --git a/stats/statsd.go b/stats/statsd.go index a1f0ca4..63bc005 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -138,14 +138,23 @@ func (s statsdProcessor) Shutdown() { } } +// StatsdFactory is a factory of events.Observers which dumps +// information to statsd. +// +// Please beware that we support ONLY UDP endpoints there. And this +// factory won't use mtglib.Network so it won't use a proxy if you +// provide any. If you need it, I would recommend starting a local +// statsd and route metrics further by features of the chosen server. type StatsdFactory struct { client *statsd.Client } +// Close stops sending requests to statsd. func (s StatsdFactory) Close() error { return s.client.Close() } +// Make build a new observer. func (s StatsdFactory) Make() events.Observer { return statsdProcessor{ client: s.client, @@ -153,6 +162,10 @@ func (s StatsdFactory) Make() events.Observer { } } +// NewStatsd builds an events.ObserverFactory that sends events +// to statsd. +// +// Valid tagFormats are 'datadog', 'influxdb' and 'graphite'. func NewStatsd(address string, log logger.StdLikeLogger, metricPrefix, tagFormat string) (StatsdFactory, error) { options := []statsd.Option{ diff --git a/timeattack/init.go b/timeattack/init.go index 1c66919..e2f9788 100644 --- a/timeattack/init.go +++ b/timeattack/init.go @@ -1,4 +1,4 @@ -// TimeAttack has implementation of mtglib.TimeAttackDetector> +// TimeAttack has implementation of mtglib.TimeAttackDetector. package timeattack import "time" From 463af648ce986da105a979133c0cafa2b78071cb Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 7 Apr 2021 14:40:54 +0300 Subject: [PATCH 197/206] Rework events --- events/event_stream_test.go | 66 ++++--------- events/noop_test.go | 45 ++------- mtglib/conns.go | 21 +--- mtglib/conns_internal_test.go | 8 +- mtglib/events.go | 175 +++++++++++++++++++--------------- mtglib/events_test.go | 44 ++------- mtglib/init.go | 82 +++++++++++++--- mtglib/proxy.go | 55 ++++------- mtglib/secret.go | 6 +- mtglib/stream_context.go | 8 +- stats/prometheus_test.go | 84 ++++------------ stats/statsd.go | 7 +- stats/statsd_test.go | 84 ++++------------ 13 files changed, 275 insertions(+), 410 deletions(-) diff --git a/events/event_stream_test.go b/events/event_stream_test.go index 599d839..93152c6 100644 --- a/events/event_stream_test.go +++ b/events/event_stream_test.go @@ -39,11 +39,7 @@ func (suite *EventStreamTestSuite) SetupTest() { } func (suite *EventStreamTestSuite) TestEventStart() { - evt := mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.1"), - } + evt := mtglib.NewEventStart("connID", net.ParseIP("10.0.0.1")) for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -52,10 +48,9 @@ func (suite *EventStreamTestSuite) TestEventStart() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventStart) - suite.Equal(evt.CreatedAt, caught.CreatedAt) - suite.Equal(evt.ConnID, caught.ConnID) suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) }) } @@ -64,12 +59,7 @@ func (suite *EventStreamTestSuite) TestEventStart() { } func (suite *EventStreamTestSuite) TestEventConnectedToDC() { - evt := mtglib.EventConnectedToDC{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.1"), - DC: 3, - } + evt := mtglib.NewEventConnectedToDC("connID", net.ParseIP("10.0.0.1"), 3) for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -78,11 +68,10 @@ func (suite *EventStreamTestSuite) TestEventConnectedToDC() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventConnectedToDC) - suite.Equal(evt.CreatedAt, caught.CreatedAt) - suite.Equal(evt.ConnID, caught.ConnID) suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) suite.Equal(evt.StreamID(), caught.StreamID()) suite.Equal(evt.DC, caught.DC) + suite.Equal(evt.Timestamp(), caught.Timestamp()) }) } @@ -91,10 +80,7 @@ func (suite *EventStreamTestSuite) TestEventConnectedToDC() { } func (suite *EventStreamTestSuite) TestEventDomainFronting() { - evt := mtglib.EventDomainFronting{ - CreatedAt: time.Now(), - ConnID: "connID", - } + evt := mtglib.NewEventDomainFronting("connID") for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -103,9 +89,8 @@ func (suite *EventStreamTestSuite) TestEventDomainFronting() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventDomainFronting) - suite.Equal(evt.CreatedAt, caught.CreatedAt) - suite.Equal(evt.ConnID, caught.ConnID) suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) }) } @@ -114,12 +99,7 @@ func (suite *EventStreamTestSuite) TestEventDomainFronting() { } func (suite *EventStreamTestSuite) TestEventTraffic() { - evt := mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 1024, - IsRead: true, - } + evt := mtglib.NewEventTraffic("connID", 1024, true) for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -128,9 +108,8 @@ func (suite *EventStreamTestSuite) TestEventTraffic() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventTraffic) - suite.Equal(evt.CreatedAt, caught.CreatedAt) - suite.Equal(evt.ConnID, caught.ConnID) suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) suite.Equal(evt.Traffic, caught.Traffic) suite.Equal(evt.IsRead, caught.IsRead) }) @@ -141,10 +120,7 @@ func (suite *EventStreamTestSuite) TestEventTraffic() { } func (suite *EventStreamTestSuite) TestEventFinish() { - evt := mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "connID", - } + evt := mtglib.NewEventFinish("connID") for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -153,9 +129,8 @@ func (suite *EventStreamTestSuite) TestEventFinish() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventFinish) - suite.Equal(evt.CreatedAt, caught.CreatedAt) - suite.Equal(evt.ConnID, caught.ConnID) suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) }) } @@ -164,9 +139,7 @@ func (suite *EventStreamTestSuite) TestEventFinish() { } func (suite *EventStreamTestSuite) TestEventConcurrencyLimited() { - evt := mtglib.EventConcurrencyLimited{ - CreatedAt: time.Now(), - } + evt := mtglib.NewEventConcurrencyLimited() for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -175,7 +148,8 @@ func (suite *EventStreamTestSuite) TestEventConcurrencyLimited() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventConcurrencyLimited) - suite.Equal(evt.CreatedAt, caught.CreatedAt) + suite.Equal(evt.Timestamp(), caught.Timestamp()) + suite.Empty(evt.StreamID()) }) } @@ -184,10 +158,7 @@ func (suite *EventStreamTestSuite) TestEventConcurrencyLimited() { } func (suite *EventStreamTestSuite) TestEventIPBlocklisted() { - evt := mtglib.EventIPBlocklisted{ - CreatedAt: time.Now(), - RemoteIP: net.ParseIP("10.0.0.10"), - } + evt := mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")) for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -196,8 +167,8 @@ func (suite *EventStreamTestSuite) TestEventIPBlocklisted() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventIPBlocklisted) - suite.Equal(evt.CreatedAt, caught.CreatedAt) suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) suite.Equal(evt.RemoteIP.String(), caught.RemoteIP.String()) }) } @@ -207,10 +178,7 @@ func (suite *EventStreamTestSuite) TestEventIPBlocklisted() { } func (suite *EventStreamTestSuite) TestEventReplayAttack() { - evt := mtglib.EventReplayAttack{ - CreatedAt: time.Now(), - ConnID: "CONNID", - } + evt := mtglib.NewEventReplayAttack("CONNID") for _, v := range []*ObserverMock{suite.observerMock1, suite.observerMock2} { v. @@ -219,8 +187,8 @@ func (suite *EventStreamTestSuite) TestEventReplayAttack() { Run(func(args mock.Arguments) { caught := args.Get(0).(mtglib.EventReplayAttack) - suite.Equal(evt.CreatedAt, caught.CreatedAt) suite.Equal(evt.StreamID(), caught.StreamID()) + suite.Equal(evt.Timestamp(), caught.Timestamp()) }) } diff --git a/events/noop_test.go b/events/noop_test.go index 4a7b20b..369a061 100644 --- a/events/noop_test.go +++ b/events/noop_test.go @@ -4,7 +4,6 @@ import ( "context" "net" "testing" - "time" "github.com/9seconds/mtg/v2/events" "github.com/9seconds/mtg/v2/mtglib" @@ -20,42 +19,14 @@ type NoopTestSuite struct { func (suite *NoopTestSuite) SetupSuite() { suite.testData = map[string]mtglib.Event{ - "start": mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("127.0.0.1"), - }, - "connected-to-dc": mtglib.EventConnectedToDC{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("127.1.0.1"), - DC: 2, - }, - "domain-fronting": mtglib.EventDomainFronting{ - CreatedAt: time.Now(), - ConnID: "connID", - }, - "traffic": mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 1000, - IsRead: true, - }, - "finish": mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "connID", - }, - "concurrency-limited": mtglib.EventConcurrencyLimited{ - CreatedAt: time.Now(), - }, - "ip-blacklisted": mtglib.EventIPBlocklisted{ - RemoteIP: net.ParseIP("10.0.0.10"), - CreatedAt: time.Now(), - }, - "replay-attack": mtglib.EventReplayAttack{ - CreatedAt: time.Now(), - ConnID: "connID", - }, + "start": mtglib.NewEventStart("connID", net.ParseIP("127.0.0.1")), + "connected-to-dc": mtglib.NewEventConnectedToDC("connID", net.ParseIP("127.1.0.1"), 2), + "domain-fronting": mtglib.NewEventDomainFronting("connID"), + "traffic": mtglib.NewEventTraffic("connID", 1000, true), + "finish": mtglib.NewEventFinish("connID"), + "concurrency-limited": mtglib.NewEventConcurrencyLimited(), + "ip-blacklisted": mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")), + "replay-attack": mtglib.NewEventReplayAttack("connID"), } suite.ctx = context.Background() } diff --git a/mtglib/conns.go b/mtglib/conns.go index e6a64e7..da0b046 100644 --- a/mtglib/conns.go +++ b/mtglib/conns.go @@ -6,27 +6,21 @@ import ( "io" "net" "sync" - "time" ) type connTraffic struct { net.Conn - connID string - stream EventStream - ctx context.Context + streamID string + stream EventStream + ctx context.Context } func (c connTraffic) Read(b []byte) (int, error) { n, err := c.Conn.Read(b) if n > 0 { - c.stream.Send(c.ctx, EventTraffic{ - CreatedAt: time.Now(), - ConnID: c.connID, - Traffic: uint(n), - IsRead: true, - }) + c.stream.Send(c.ctx, NewEventTraffic(c.streamID, uint(n), true)) } return n, err // nolint: wrapcheck @@ -36,12 +30,7 @@ func (c connTraffic) Write(b []byte) (int, error) { n, err := c.Conn.Write(b) if n > 0 { - c.stream.Send(c.ctx, EventTraffic{ - CreatedAt: time.Now(), - ConnID: c.connID, - Traffic: uint(n), - IsRead: false, - }) + c.stream.Send(c.ctx, NewEventTraffic(c.streamID, uint(n), false)) } return n, err // nolint: wrapcheck diff --git a/mtglib/conns_internal_test.go b/mtglib/conns_internal_test.go index de31f60..0c73759 100644 --- a/mtglib/conns_internal_test.go +++ b/mtglib/conns_internal_test.go @@ -37,10 +37,10 @@ func (suite *ConnTrafficTestSuite) SetupTest() { suite.eventStreamMock = &EventStreamMock{} suite.connMock = &testlib.NetConnMock{} suite.conn = connTraffic{ - Conn: suite.connMock, - connID: "CONNID", - ctx: context.Background(), - stream: suite.eventStreamMock, + Conn: suite.connMock, + streamID: "CONNID", + ctx: context.Background(), + stream: suite.eventStreamMock, } } diff --git a/mtglib/events.go b/mtglib/events.go index e4c57d1..77c4c76 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -5,110 +5,133 @@ import ( "time" ) +type eventBase struct { + streamID string + timestamp time.Time +} + +func (e eventBase) StreamID() string { + return e.streamID +} + +func (e eventBase) Timestamp() time.Time { + return e.timestamp +} + type EventStart struct { - CreatedAt time.Time - ConnID string - RemoteIP net.IP -} + eventBase -func (e EventStart) StreamID() string { - return e.ConnID -} - -func (e EventStart) Timestamp() time.Time { - return e.CreatedAt + RemoteIP net.IP } type EventConnectedToDC struct { - CreatedAt time.Time - ConnID string - RemoteIP net.IP - DC int -} + eventBase -func (e EventConnectedToDC) StreamID() string { - return e.ConnID -} - -func (e EventConnectedToDC) Timestamp() time.Time { - return e.CreatedAt + RemoteIP net.IP + DC int } type EventTraffic struct { - CreatedAt time.Time - ConnID string - Traffic uint - IsRead bool -} + eventBase -func (e EventTraffic) StreamID() string { - return e.ConnID -} - -func (e EventTraffic) Timestamp() time.Time { - return e.CreatedAt + Traffic uint + IsRead bool } type EventFinish struct { - CreatedAt time.Time - ConnID string -} - -func (e EventFinish) StreamID() string { - return e.ConnID -} - -func (e EventFinish) Timestamp() time.Time { - return e.CreatedAt + eventBase } type EventDomainFronting struct { - CreatedAt time.Time - ConnID string -} - -func (e EventDomainFronting) StreamID() string { - return e.ConnID -} - -func (e EventDomainFronting) Timestamp() time.Time { - return e.CreatedAt + eventBase } type EventConcurrencyLimited struct { - CreatedAt time.Time -} - -func (e EventConcurrencyLimited) StreamID() string { - return "" -} - -func (e EventConcurrencyLimited) Timestamp() time.Time { - return e.CreatedAt + eventBase } type EventIPBlocklisted struct { - CreatedAt time.Time - RemoteIP net.IP -} + eventBase -func (e EventIPBlocklisted) StreamID() string { - return "" -} - -func (e EventIPBlocklisted) Timestamp() time.Time { - return e.CreatedAt + RemoteIP net.IP } type EventReplayAttack struct { - CreatedAt time.Time - ConnID string + eventBase } -func (e EventReplayAttack) StreamID() string { - return e.ConnID +func NewEventStart(streamID string, remoteIP net.IP) EventStart { + return EventStart{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + RemoteIP: remoteIP, + } } -func (e EventReplayAttack) Timestamp() time.Time { - return e.CreatedAt +func NewEventConnectedToDC(streamID string, remoteIP net.IP, dc int) EventConnectedToDC { + return EventConnectedToDC{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + RemoteIP: remoteIP, + DC: dc, + } +} + +func NewEventTraffic(streamID string, traffic uint, isRead bool) EventTraffic { + return EventTraffic{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + Traffic: traffic, + IsRead: isRead, + } +} + +func NewEventFinish(streamID string) EventFinish { + return EventFinish{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + } +} + +func NewEventDomainFronting(streamID string) EventDomainFronting { + return EventDomainFronting{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + } +} + +func NewEventConcurrencyLimited() EventConcurrencyLimited { + return EventConcurrencyLimited{ + eventBase: eventBase{ + timestamp: time.Now(), + }, + } +} + +func NewEventIPBlocklisted(remoteIP net.IP) EventIPBlocklisted { + return EventIPBlocklisted{ + eventBase: eventBase{ + timestamp: time.Now(), + }, + RemoteIP: remoteIP, + } +} + +func NewEventReplayAttack(streamID string) EventReplayAttack { + return EventReplayAttack{ + eventBase: eventBase{ + timestamp: time.Now(), + streamID: streamID, + }, + } } diff --git a/mtglib/events_test.go b/mtglib/events_test.go index 1f4814e..bb88b0a 100644 --- a/mtglib/events_test.go +++ b/mtglib/events_test.go @@ -14,84 +14,56 @@ type EventsTestSuite struct { } func (suite *EventsTestSuite) TestEventStart() { - evt := mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "CONNID", - RemoteIP: net.ParseIP("10.0.0.10"), - } + evt := mtglib.NewEventStart("CONNID", net.ParseIP("10.0.0.10")) suite.Equal("CONNID", evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventFinish() { - evt := mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "CONNID", - } + evt := mtglib.NewEventFinish("CONNID") suite.Equal("CONNID", evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventConnectedToDC() { - evt := mtglib.EventConnectedToDC{ - CreatedAt: time.Now(), - ConnID: "CONNID", - RemoteIP: net.ParseIP("10.0.0.10"), - DC: 3, - } + evt := mtglib.NewEventConnectedToDC("CONNID", net.ParseIP("10.0.0.10"), 3) suite.Equal("CONNID", evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventTraffic() { - evt := mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "CONNID", - Traffic: 3, - IsRead: true, - } + evt := mtglib.NewEventTraffic("CONNID", 1000, true) suite.Equal("CONNID", evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventDomainFronting() { - evt := mtglib.EventDomainFronting{ - CreatedAt: time.Now(), - ConnID: "CONNID", - } + evt := mtglib.NewEventDomainFronting("CONNID") suite.Equal("CONNID", evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventConcurrencyLimited() { - evt := mtglib.EventConcurrencyLimited{ - CreatedAt: time.Now(), - } + evt := mtglib.NewEventConcurrencyLimited() suite.Empty(evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventIPBlocklisted() { - evt := mtglib.EventIPBlocklisted{ - CreatedAt: time.Now(), - RemoteIP: net.ParseIP("10.0.0.10"), - } + evt := mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10")) suite.Empty(evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) } func (suite *EventsTestSuite) TestEventReplayAttack() { - evt := mtglib.EventReplayAttack{ - CreatedAt: time.Now(), - ConnID: "CONNID", - } + evt := mtglib.NewEventReplayAttack("CONNID") suite.Equal("CONNID", evt.StreamID()) suite.WithinDuration(time.Now(), evt.Timestamp(), 10*time.Millisecond) diff --git a/mtglib/init.go b/mtglib/init.go index 9f3ba08..cd1b204 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -9,22 +9,66 @@ import ( ) var ( - ErrSecretEmpty = errors.New("secret is empty") - ErrSecretInvalid = errors.New("secret is invalid") - ErrNetworkIsNotDefined = errors.New("network is not defined") - ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") + // ErrSecretEmpty is returned if you are trying to create a proxy + // but do not provide a secret. + ErrSecretEmpty = errors.New("secret is empty") + + // ErrSecretInvalid is returned if you are trying to create a proxy + // but secret value is invalid (no host or payload are zeroes). + ErrSecretInvalid = errors.New("secret is invalid") + + // ErrNetworkIsNotDefined is returned if you are trying to create a + // proxy but network value is undefined. + ErrNetworkIsNotDefined = errors.New("network is not defined") + + // ErrAntiReplayCacheIsNotDefined is returned if you are trying to + // create a proxy but anti replay cache value is undefined. + ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") + + // ErrTimeAttackDetectorIsNotDefined is returned if you are trying to + // create a proxy but time attack detector is not defined. ErrTimeAttackDetectorIsNotDefined = errors.New("time attack detector is not defined") - ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") - ErrEventStreamIsNotDefined = errors.New("event stream is not defined") - ErrLoggerIsNotDefined = errors.New("logger is not defined") + + // ErrIPBlocklistIsNotDefined is returned if you are trying to + // create a proxy but ip blocklist instance is not defined. + ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") + + // ErrEventStreamIsNotDefined is returned if you are trying to create a + // proxy but event stream instance is not defined. + ErrEventStreamIsNotDefined = errors.New("event stream is not defined") + + // ErrLoggerIsNotDefined is returned if you are trying to + // create a proxy but logger is not defined. + ErrLoggerIsNotDefined = errors.New("logger is not defined") ) const ( - DefaultConcurrency = 4096 - DefaultBufferSize = 16 * 1024 // 16 kib + // DefaultConcurrency is a default max count of simultaneously + // connected clients. + DefaultConcurrency = 4096 + + // DefaultBufferSize is a default size of a copy buffer. + DefaultBufferSize = 16 * 1024 // 16 kib + + // DefaultDomainFrontingPort is a default port (HTTPS) to connect to in + // case of probe-resistance activity. DefaultDomainFrontingPort = 443 - DefaultIdleTimeout = time.Minute - DefaultPreferIP = "prefer-ipv6" + + // DefaultIdleTimeout is a default timeout for closing a connection + // in case of idling. + DefaultIdleTimeout = time.Minute + + // DefaultPreferIP is a default value for Telegram IP connectivity + // preference. + DefaultPreferIP = "prefer-ipv6" + + // SecretKeyLength defines a length of the secret bytes used + // by Telegram and a proxy. + SecretKeyLength = 16 + + // ConnectionIDBytesLength defines a count of random bytes + // used to generate a stream/connection ids. + ConnectionIDBytesLength = 16 ) // Network defines a knowledge how to work with a network. It may sound @@ -105,8 +149,24 @@ type IPBlocklist interface { Contains(net.IP) bool } +// Event is a data structure which is populated during mtg request +// processing lifecycle. Each request popluates many events: +// +// 1. Client connected +// +// 2. Request is finished +// +// 3. Connection to Telegram server is established +// +// and so on. All these events are data structures but all of them +// must conform the same interface. type Event interface { + // StreamID returns an identifier of the stream, connection, + // request, you name it. All events within the same stream returns + // the same stream id. StreamID() string + + // Timestamp returns a timestamp when this event was generated. Timestamp() time.Time } diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 36de1a8..9400df5 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -53,18 +53,11 @@ func (p *Proxy) ServeConn(conn net.Conn) { ctx.Close() }() - p.eventStream.Send(ctx, EventStart{ - CreatedAt: time.Now(), - ConnID: ctx.connID, - RemoteIP: ctx.ClientIP(), - }) + p.eventStream.Send(ctx, NewEventStart(ctx.streamID, ctx.ClientIP())) ctx.logger.Info("Stream has been started") defer func() { - p.eventStream.Send(ctx, EventFinish{ - CreatedAt: time.Now(), - ConnID: ctx.connID, - }) + p.eventStream.Send(ctx, NewEventFinish(ctx.streamID)) ctx.logger.Info("Stream has been finished") }() @@ -109,10 +102,7 @@ func (p *Proxy) Serve(listener net.Listener) error { if p.ipBlocklist.Contains(ipAddr) { conn.Close() logger.Info("ip was blacklisted") - p.eventStream.Send(p.ctx, EventIPBlocklisted{ - CreatedAt: time.Now(), - RemoteIP: ipAddr, - }) + p.eventStream.Send(p.ctx, NewEventIPBlocklisted(ipAddr)) continue } @@ -125,9 +115,7 @@ func (p *Proxy) Serve(listener net.Listener) error { return nil case errors.Is(err, ants.ErrPoolOverload): logger.Info("connection was concurrency limited") - p.eventStream.Send(p.ctx, EventConcurrencyLimited{ - CreatedAt: time.Now(), - }) + p.eventStream.Send(p.ctx, NewEventConcurrencyLimited()) } select { @@ -181,10 +169,7 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { if p.antiReplayCache.SeenBefore(hello.SessionID) { p.logger.Warning("replay attack has been detected!") - p.eventStream.Send(p.ctx, EventReplayAttack{ - CreatedAt: time.Now(), - ConnID: ctx.connID, - }) + p.eventStream.Send(p.ctx, NewEventReplayAttack(ctx.streamID)) p.doDomainFronting(ctx, rewind) return false @@ -235,31 +220,23 @@ func (p *Proxy) doTelegramCall(ctx *streamContext) error { ctx.telegramConn = obfuscated2.Conn{ Conn: connTraffic{ - Conn: conn, - connID: ctx.connID, - stream: p.eventStream, - ctx: ctx, + Conn: conn, + streamID: ctx.streamID, + stream: p.eventStream, + ctx: ctx, }, Encryptor: encryptor, Decryptor: decryptor, } - p.eventStream.Send(ctx, EventConnectedToDC{ - CreatedAt: time.Now(), - ConnID: ctx.connID, - RemoteIP: conn.RemoteAddr().(*net.TCPAddr).IP, - DC: ctx.dc, - }) + p.eventStream.Send(ctx, + NewEventConnectedToDC(ctx.streamID, conn.RemoteAddr().(*net.TCPAddr).IP, ctx.dc)) return nil } func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { - p.eventStream.Send(p.ctx, EventDomainFronting{ - CreatedAt: time.Now(), - ConnID: ctx.connID, - }) - + p.eventStream.Send(p.ctx, NewEventDomainFronting(ctx.streamID)) conn.Rewind() frontConn, err := p.network.DialContext(ctx, "tcp", p.DomainFrontingAddress()) @@ -270,10 +247,10 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { } frontConn = connTraffic{ - Conn: frontConn, - ctx: ctx, - connID: ctx.connID, - stream: p.eventStream, + Conn: frontConn, + ctx: ctx, + streamID: ctx.streamID, + stream: p.eventStream, } rel := relay.AcquireRelay(ctx, diff --git a/mtglib/secret.go b/mtglib/secret.go index 71c4710..c76eed2 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -7,11 +7,7 @@ import ( "fmt" ) -const ( - SecretKeyLength = 16 - - secretFakeTLSFirstByte byte = 0xee -) +const secretFakeTLSFirstByte byte = 0xee var secretEmptyKey [SecretKeyLength]byte diff --git a/mtglib/stream_context.go b/mtglib/stream_context.go index 9b1027d..f704f37 100644 --- a/mtglib/stream_context.go +++ b/mtglib/stream_context.go @@ -8,14 +8,12 @@ import ( "time" ) -const ConnectionIDBytesLength = 16 - type streamContext struct { ctx context.Context ctxCancel context.CancelFunc clientConn net.Conn telegramConn net.Conn - connID string + streamID string dc int logger Logger } @@ -64,10 +62,10 @@ func newStreamContext(ctx context.Context, logger Logger, clientConn net.Conn) * ctx: ctx, ctxCancel: cancel, clientConn: clientConn, - connID: base64.RawURLEncoding.EncodeToString(connIDBytes), + streamID: base64.RawURLEncoding.EncodeToString(connIDBytes), } streamCtx.logger = logger. - BindStr("stream-id", streamCtx.connID). + BindStr("stream-id", streamCtx.streamID). BindStr("client-ip", streamCtx.ClientIP().String()) return streamCtx diff --git a/stats/prometheus_test.go b/stats/prometheus_test.go index abb3bec..6e44c0b 100644 --- a/stats/prometheus_test.go +++ b/stats/prometheus_test.go @@ -55,57 +55,39 @@ func (suite *PrometheusTestSuite) TearDownTest() { } func (suite *PrometheusTestSuite) TestTelegramPath() { - suite.prometheus.EventStart(mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.10"), - }) + suite.prometheus.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) time.Sleep(100 * time.Millisecond) data, err := suite.Get() suite.NoError(err) suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 1`) - suite.prometheus.EventConnectedToDC(mtglib.EventConnectedToDC{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.1"), - DC: 4, - }) + suite.prometheus.EventConnectedToDC( + mtglib.NewEventConnectedToDC("connID", net.ParseIP("10.0.0.1"), 4)) time.Sleep(100 * time.Millisecond) data, err = suite.Get() suite.NoError(err) suite.Contains(data, `mtg_telegram_connections{dc="4",telegram_ip="10.0.0.1"} 1`) - suite.prometheus.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 200, - IsRead: true, - }) + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 200, true)) time.Sleep(100 * time.Millisecond) data, err = suite.Get() suite.NoError(err) suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="to_client",telegram_ip="10.0.0.1"} 200`) - suite.prometheus.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 100, - IsRead: false, - }) + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 100, false)) time.Sleep(100 * time.Millisecond) data, err = suite.Get() suite.NoError(err) suite.Contains(data, `mtg_telegram_traffic{dc="4",direction="from_client",telegram_ip="10.0.0.1"} 100`) - suite.prometheus.EventFinish(mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.prometheus.EventFinish(mtglib.NewEventFinish("connID")) time.Sleep(100 * time.Millisecond) data, err = suite.Get() @@ -115,21 +97,15 @@ func (suite *PrometheusTestSuite) TestTelegramPath() { } func (suite *PrometheusTestSuite) TestDomainFrontingPath() { - suite.prometheus.EventStart(mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.10"), - }) + suite.prometheus.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) time.Sleep(100 * time.Millisecond) data, err := suite.Get() suite.NoError(err) suite.Contains(data, `mtg_client_connections{ip_family="ipv4"} 1`) - suite.prometheus.EventDomainFronting(mtglib.EventDomainFronting{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.prometheus.EventDomainFronting(mtglib.NewEventDomainFronting("connID")) time.Sleep(100 * time.Millisecond) data, err = suite.Get() @@ -137,34 +113,23 @@ func (suite *PrometheusTestSuite) TestDomainFrontingPath() { suite.Contains(data, `mtg_domain_fronting 1`) suite.Contains(data, `mtg_domain_fronting_connections{ip_family="ipv4"} 1`) - suite.prometheus.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 200, - IsRead: true, - }) + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 200, true)) time.Sleep(100 * time.Millisecond) data, err = suite.Get() suite.NoError(err) suite.Contains(data, `mtg_domain_fronting_traffic{direction="to_client"} 200`) - suite.prometheus.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 100, - IsRead: false, - }) + suite.prometheus.EventTraffic( + mtglib.NewEventTraffic("connID", 100, false)) time.Sleep(100 * time.Millisecond) data, err = suite.Get() suite.NoError(err) suite.Contains(data, `mtg_domain_fronting_traffic{direction="from_client"} 100`) - suite.prometheus.EventFinish(mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.prometheus.EventFinish(mtglib.NewEventFinish("connID")) time.Sleep(100 * time.Millisecond) data, err = suite.Get() @@ -174,9 +139,7 @@ func (suite *PrometheusTestSuite) TestDomainFrontingPath() { } func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { - suite.prometheus.EventConcurrencyLimited(mtglib.EventConcurrencyLimited{ - CreatedAt: time.Now(), - }) + suite.prometheus.EventConcurrencyLimited(mtglib.NewEventConcurrencyLimited()) time.Sleep(100 * time.Millisecond) @@ -186,10 +149,8 @@ func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() { } func (suite *PrometheusTestSuite) TestEventIPBlocklisted() { - suite.prometheus.EventIPBlocklisted(mtglib.EventIPBlocklisted{ - CreatedAt: time.Now(), - RemoteIP: net.ParseIP("2001:db8::68"), - }) + suite.prometheus.EventIPBlocklisted( + mtglib.NewEventIPBlocklisted(net.ParseIP("2001:db8::68"))) time.Sleep(100 * time.Millisecond) @@ -199,10 +160,7 @@ func (suite *PrometheusTestSuite) TestEventIPBlocklisted() { } func (suite *PrometheusTestSuite) TestEventReplayAttack() { - suite.prometheus.EventReplayAttack(mtglib.EventReplayAttack{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.prometheus.EventReplayAttack(mtglib.NewEventReplayAttack("connID")) time.Sleep(100 * time.Millisecond) diff --git a/stats/statsd.go b/stats/statsd.go index 63bc005..f394ec6 100644 --- a/stats/statsd.go +++ b/stats/statsd.go @@ -4,7 +4,6 @@ import ( "fmt" "strconv" "strings" - "time" "github.com/9seconds/mtg/v2/events" "github.com/9seconds/mtg/v2/logger" @@ -123,14 +122,10 @@ func (s statsdProcessor) EventReplayAttack(_ mtglib.EventReplayAttack) { } func (s statsdProcessor) Shutdown() { - now := time.Now() events := make([]mtglib.EventFinish, 0, len(s.streams)) for k := range s.streams { - events = append(events, mtglib.EventFinish{ - CreatedAt: now, - ConnID: k, - }) + events = append(events, mtglib.NewEventFinish(k)) } for i := range events { diff --git a/stats/statsd_test.go b/stats/statsd_test.go index 02b34ec..4383509 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -105,48 +105,30 @@ func (suite *StatsdTestSuite) TearDownTest() { } func (suite *StatsdTestSuite) TestTelegramPath() { - suite.statsd.EventStart(mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.10"), - }) + suite.statsd.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) time.Sleep(statsdSleepTime) suite.Equal("mtg.client_connections:+1|g|#ip_family:ipv4", suite.statsdServer.String()) - suite.statsd.EventConnectedToDC(mtglib.EventConnectedToDC{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.1.0.10"), - DC: 2, - }) + suite.statsd.EventConnectedToDC( + mtglib.NewEventConnectedToDC("connID", net.ParseIP("10.1.0.10"), 2)) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.telegram_connections:+1|g|#telegram_ip:10.1.0.10,dc:2") - suite.statsd.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 30, - IsRead: true, - }) + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 30, true)) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.telegram_traffic:30|c|#telegram_ip:10.1.0.10,dc:2,direction:to_client") - suite.statsd.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 90, - IsRead: false, - }) + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 90, false)) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.telegram_traffic:90|c|#telegram_ip:10.1.0.10,dc:2,direction:from_client") - suite.statsd.EventFinish(mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.statsd.EventFinish(mtglib.NewEventFinish("connID")) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.telegram_connections:-1|g|#telegram_ip:10.1.0.10,dc:2") @@ -158,47 +140,30 @@ func (suite *StatsdTestSuite) TestTelegramPath() { } func (suite *StatsdTestSuite) TestDomainFrontingPath() { - suite.statsd.EventStart(mtglib.EventStart{ - CreatedAt: time.Now(), - ConnID: "connID", - RemoteIP: net.ParseIP("10.0.0.10"), - }) + suite.statsd.EventStart( + mtglib.NewEventStart("connID", net.ParseIP("10.0.0.10"))) time.Sleep(statsdSleepTime) suite.Equal("mtg.client_connections:+1|g|#ip_family:ipv4", suite.statsdServer.String()) - suite.statsd.EventDomainFronting(mtglib.EventDomainFronting{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.statsd.EventDomainFronting(mtglib.NewEventDomainFronting("connID")) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.domain_fronting:1|c") suite.Contains(suite.statsdServer.String(), `mtg.domain_fronting_connections:+1|g|#ip_family:ipv4`) - suite.statsd.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 30, - IsRead: true, - }) + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 30, true)) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), `mtg.domain_fronting_traffic:30|c|#direction:to_client`) - suite.statsd.EventTraffic(mtglib.EventTraffic{ - CreatedAt: time.Now(), - ConnID: "connID", - Traffic: 90, - IsRead: false, - }) + suite.statsd.EventTraffic( + mtglib.NewEventTraffic("connID", 90, false)) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), `mtg.domain_fronting_traffic:90|c|#direction:from_client`) - suite.statsd.EventFinish(mtglib.EventFinish{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.statsd.EventFinish(mtglib.NewEventFinish("connID")) time.Sleep(statsdSleepTime) suite.Contains(suite.statsdServer.String(), "mtg.domain_fronting_connections:-1|g|#ip_family:ipv4") @@ -210,29 +175,22 @@ func (suite *StatsdTestSuite) TestDomainFrontingPath() { } func (suite *StatsdTestSuite) TestEventConcurrencyLimited() { - suite.statsd.EventConcurrencyLimited(mtglib.EventConcurrencyLimited{ - CreatedAt: time.Now(), - }) + suite.statsd.EventConcurrencyLimited(mtglib.NewEventConcurrencyLimited()) time.Sleep(statsdSleepTime) suite.Equal("mtg.concurrency_limited:1|c", suite.statsdServer.String()) } func (suite *StatsdTestSuite) TestEventIPBlocklisted() { - suite.statsd.EventIPBlocklisted(mtglib.EventIPBlocklisted{ - CreatedAt: time.Now(), - RemoteIP: net.ParseIP("10.0.0.10"), - }) + suite.statsd.EventIPBlocklisted( + mtglib.NewEventIPBlocklisted(net.ParseIP("10.0.0.10"))) time.Sleep(statsdSleepTime) suite.Equal("mtg.ip_blocklisted:1|c", suite.statsdServer.String()) } func (suite *StatsdTestSuite) TestEventReplayAttack() { - suite.statsd.EventReplayAttack(mtglib.EventReplayAttack{ - CreatedAt: time.Now(), - ConnID: "connID", - }) + suite.statsd.EventReplayAttack(mtglib.NewEventReplayAttack("connID")) time.Sleep(statsdSleepTime) suite.Equal("mtg.replay_attacks:1|c", suite.statsdServer.String()) From 611583ba88974c4bd2b5320fec485e53d6173d7d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 7 Apr 2021 15:22:58 +0300 Subject: [PATCH 198/206] Add doc strings to mtglib events --- mtglib/events.go | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/mtglib/events.go b/mtglib/events.go index 77c4c76..1bc6fb7 100644 --- a/mtglib/events.go +++ b/mtglib/events.go @@ -10,56 +10,89 @@ type eventBase struct { timestamp time.Time } +// StreamID returns a ID of the stream this event belongs to. func (e eventBase) StreamID() string { return e.streamID } +// Timestamp return a time when this event was generated. func (e eventBase) Timestamp() time.Time { return e.timestamp } +// EventStart is emitted when mtg proxy starts to process a new +// connection. type EventStart struct { eventBase + // RemoteIP is an IP address of the client. RemoteIP net.IP } +// EventConnectedToDC is emitted when mtg proxy has connected to a +// Telegram server. type EventConnectedToDC struct { eventBase + // RemoteIP is an IP address of the Telegram server proxy has been + // connected to. RemoteIP net.IP - DC int + + // DC is an index of the datacenter proxy has been connected to. + DC int } +// EventTraffic is emitted when we read/write some bytes on a connection. type EventTraffic struct { eventBase + // Traffic is a count of bytes which were transmitted. Traffic uint - IsRead bool + + // IsRead defines if we _read_ or _write_ to connection. A rule of + // thumb is simple: EventTraffic is bound to a remote connection. Not + // to a client one, but either to Telegram or front domain one. + // + // In the case of Telegram, isRead means that we've fetched some bytes + // from Telegram to send it to a client. + // + // In the case of the front domain, it means that we've fetched some + // bytes from this domain to send it to a client. + IsRead bool } +// EventFinish is emitted when we stop to manage a connection. type EventFinish struct { eventBase } +// EventDomainFronting is emitted when we connect to a front domain +// instead of Telegram server. type EventDomainFronting struct { eventBase } +// EventConcurrencyLimited is emitted when connection was declined +// because of the concurrency limit of the worker pool. type EventConcurrencyLimited struct { eventBase } +// EventIPBlocklisted is emitted when connection was declined because +// IP address was found in IP blocklist. type EventIPBlocklisted struct { eventBase RemoteIP net.IP } +// EventReplayAttack is emitted when mtg detects a replay attack on a +// connection. type EventReplayAttack struct { eventBase } +// NewEventStart creates a new EventStart event. func NewEventStart(streamID string, remoteIP net.IP) EventStart { return EventStart{ eventBase: eventBase{ @@ -70,6 +103,7 @@ func NewEventStart(streamID string, remoteIP net.IP) EventStart { } } +// NewEventConnectedToDC creates a new EventConnectedToDC event. func NewEventConnectedToDC(streamID string, remoteIP net.IP, dc int) EventConnectedToDC { return EventConnectedToDC{ eventBase: eventBase{ @@ -81,6 +115,7 @@ func NewEventConnectedToDC(streamID string, remoteIP net.IP, dc int) EventConnec } } +// NewEventTraffic creates a new EventTraffic event. func NewEventTraffic(streamID string, traffic uint, isRead bool) EventTraffic { return EventTraffic{ eventBase: eventBase{ @@ -92,6 +127,7 @@ func NewEventTraffic(streamID string, traffic uint, isRead bool) EventTraffic { } } +// NewEventFinish creates a new EventFinish event. func NewEventFinish(streamID string) EventFinish { return EventFinish{ eventBase: eventBase{ @@ -101,6 +137,7 @@ func NewEventFinish(streamID string) EventFinish { } } +// NewEventDomainFronting creates a new EventDomainFronting event. func NewEventDomainFronting(streamID string) EventDomainFronting { return EventDomainFronting{ eventBase: eventBase{ @@ -110,6 +147,8 @@ func NewEventDomainFronting(streamID string) EventDomainFronting { } } +// NewEventConcurrencyLimited creates a new EventConcurrencyLimited +// event. func NewEventConcurrencyLimited() EventConcurrencyLimited { return EventConcurrencyLimited{ eventBase: eventBase{ @@ -118,6 +157,7 @@ func NewEventConcurrencyLimited() EventConcurrencyLimited { } } +// NewEventIPBlocklisted creates a new EventIPBlocklisted event. func NewEventIPBlocklisted(remoteIP net.IP) EventIPBlocklisted { return EventIPBlocklisted{ eventBase: eventBase{ @@ -127,6 +167,7 @@ func NewEventIPBlocklisted(remoteIP net.IP) EventIPBlocklisted { } } +// NewEventReplayAttack creates a new EventReplayAttack event. func NewEventReplayAttack(streamID string) EventReplayAttack { return EventReplayAttack{ eventBase: eventBase{ From e6d444546f53e46137c9feaf8394881f16e144f6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 7 Apr 2021 18:02:16 +0300 Subject: [PATCH 199/206] Add documentation for a secret --- mtglib/secret.go | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/mtglib/secret.go b/mtglib/secret.go index c76eed2..d9c1ff3 100644 --- a/mtglib/secret.go +++ b/mtglib/secret.go @@ -11,11 +11,43 @@ const secretFakeTLSFirstByte byte = 0xee var secretEmptyKey [SecretKeyLength]byte +// Secret is a data structure that presents a secret. +// +// Telegram secret is not a simple string like +// "ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d". +// Actually, this is a serialized datastructure of 2 parts: key and host. +// +// ee367a189aee18fa31c190054efd4a8e9573746f726167652e676f6f676c65617069732e636f6d +// |-|-------------------------------|------------------------------------------- +// p key hostname +// +// Serialized secret starts with 'ee'. Actually, in the past we also had +// 'dd' secrets and prefixless ones. But this is history. Currently, +// we do have only 'ee' secrets which mean faketls + protection from +// statistical attacks on a length. 'ee' is a byte 238 (0xee). +// +// After that, we have 16 bytes of the key. This is a random generated +// secret data of the proxy and this data is used to derive +// authentication schemas. These secrets are mixed into hmacs and sha256 +// checksums which are used to build AEAD ciphers for obfuscated2 +// protocol and ensure faketls handshake. +// +// Host is a domain fronting hostname in latin1 (ASCII) encoding. This +// hostname should be used for SNI in faketls and MTG verifies it. Also, +// this is when mtg gets about a domain fronting hostname. +// +// Secrets can be serialized into 2 forms: hex and base64. If +// you decode both forms into bytes, you'll get the same byte array. +// Telegram clients nowadays accept all forms. type Secret struct { - Key [SecretKeyLength]byte + // Key is a set of bytes used for traffic authentication. + Key [SecretKeyLength]byte + + // Host is a domain fronting hostname. Host string } +// MarshalText is to support text.Marshaller interface. func (s Secret) MarshalText() ([]byte, error) { if s.Valid() { return []byte(s.String()), nil @@ -24,6 +56,7 @@ func (s Secret) MarshalText() ([]byte, error) { return nil, nil } +// MarshalText is to support text.Unmarshaller interface. func (s *Secret) UnmarshalText(data []byte) error { text := string(data) if text == "" { @@ -62,18 +95,22 @@ func (s *Secret) UnmarshalText(data []byte) error { return nil } +// Valid checks if this secret is valid and can be used in proxy. func (s Secret) Valid() bool { return s.Key != secretEmptyKey && s.Host != "" } +// String is to support fmt.Stringer interface. func (s Secret) String() string { return s.Base64() } +// Base64 returns a base64-encoded form of this secret. func (s Secret) Base64() string { return base64.RawURLEncoding.EncodeToString(s.makeBytes()) } +// Hex returns a hex-encoded form of this secret (ee-secret). func (s Secret) Hex() string { return hex.EncodeToString(s.makeBytes()) } @@ -85,6 +122,7 @@ func (s *Secret) makeBytes() []byte { return data } +// GenerateSecret makes a new secret with a given hostname. func GenerateSecret(hostname string) Secret { s := Secret{ Host: hostname, @@ -97,6 +135,7 @@ func GenerateSecret(hostname string) Secret { return s } +// ParseSecret parses a secret (both hex and base64 forms). func ParseSecret(secret string) (Secret, error) { s := Secret{} From 4c38ea2b110f653d6c664b96c8d712a85207226a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 8 Apr 2021 11:00:54 +0300 Subject: [PATCH 200/206] Refactor some configuration to proxy_opts --- mtglib/proxy.go | 63 +++++++++--------------------------------- mtglib/proxy_opts.go | 65 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 50 deletions(-) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 9400df5..0317553 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -262,50 +262,12 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { } } -func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen - switch { - case opts.Network == nil: - return nil, ErrNetworkIsNotDefined - case opts.AntiReplayCache == nil: - return nil, ErrAntiReplayCacheIsNotDefined - case opts.IPBlocklist == nil: - return nil, ErrIPBlocklistIsNotDefined - case opts.EventStream == nil: - return nil, ErrEventStreamIsNotDefined - case opts.TimeAttackDetector == nil: - return nil, ErrTimeAttackDetectorIsNotDefined - case opts.Logger == nil: - return nil, ErrLoggerIsNotDefined - case !opts.Secret.Valid(): - return nil, ErrSecretInvalid +func NewProxy(opts ProxyOpts) (*Proxy, error) { + if err := opts.valid(); err != nil { + return nil, fmt.Errorf("invalid settings: %w", err) } - preferIP := opts.PreferIP - if preferIP == "" { - preferIP = DefaultPreferIP - } - - concurrency := opts.Concurrency - if concurrency == 0 { - concurrency = DefaultConcurrency - } - - idleTimeout := opts.IdleTimeout - if idleTimeout < 1 { - idleTimeout = DefaultIdleTimeout - } - - bufferSize := opts.BufferSize - if bufferSize < 1 { - bufferSize = DefaultBufferSize - } - - domainFrontingPort := int(opts.DomainFrontingPort) - if domainFrontingPort == 0 { - domainFrontingPort = DefaultDomainFrontingPort - } - - tg, err := telegram.New(opts.Network, preferIP) + tg, err := telegram.New(opts.Network, opts.getPreferIP()) if err != nil { return nil, fmt.Errorf("cannot build telegram dialer: %w", err) } @@ -320,17 +282,18 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { // nolint: cyclop, funlen timeAttackDetector: opts.TimeAttackDetector, ipBlocklist: opts.IPBlocklist, eventStream: opts.EventStream, - logger: opts.Logger.Named("proxy"), - domainFrontingPort: domainFrontingPort, - idleTimeout: idleTimeout, - bufferSize: int(bufferSize), + logger: opts.getLogger("proxy"), + domainFrontingPort: opts.getDomainFrontingPort(), + idleTimeout: opts.getIdleTimeout(), + bufferSize: opts.getBufferSize(), telegram: tg, } - pool, err := ants.NewPoolWithFunc(int(concurrency), func(arg interface{}) { - proxy.ServeConn(arg.(net.Conn)) - }, - ants.WithLogger(opts.Logger.Named("ants")), + pool, err := ants.NewPoolWithFunc(opts.getConcurrency(), + func(arg interface{}) { + proxy.ServeConn(arg.(net.Conn)) + }, + ants.WithLogger(opts.getLogger("ants")), ants.WithNonblocking(true)) if err != nil { panic(err) diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index a20caaa..c7bff07 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -17,3 +17,68 @@ type ProxyOpts struct { IdleTimeout time.Duration PreferIP string } + +func (p ProxyOpts) valid() error { + switch { + case p.Network == nil: + return ErrNetworkIsNotDefined + case p.AntiReplayCache == nil: + return ErrAntiReplayCacheIsNotDefined + case p.IPBlocklist == nil: + return ErrIPBlocklistIsNotDefined + case p.EventStream == nil: + return ErrEventStreamIsNotDefined + case p.TimeAttackDetector == nil: + return ErrTimeAttackDetectorIsNotDefined + case p.Logger == nil: + return ErrLoggerIsNotDefined + case !p.Secret.Valid(): + return ErrSecretInvalid + } + + return nil +} + +func (p ProxyOpts) getBufferSize() int { + if p.BufferSize < 1 { + return DefaultBufferSize + } + + return int(p.BufferSize) +} + +func (p ProxyOpts) getConcurrency() int { + if p.Concurrency == 0 { + return DefaultConcurrency + } + + return int(p.Concurrency) +} + +func (p ProxyOpts) getDomainFrontingPort() int { + if p.DomainFrontingPort == 0 { + return DefaultDomainFrontingPort + } + + return int(p.DomainFrontingPort) +} + +func (p ProxyOpts) getIdleTimeout() time.Duration { + if p.IdleTimeout == 0 { + return DefaultIdleTimeout + } + + return p.IdleTimeout +} + +func (p ProxyOpts) getPreferIP() string { + if p.PreferIP == "" { + return DefaultPreferIP + } + + return p.PreferIP +} + +func (p ProxyOpts) getLogger(name string) Logger { + return p.Logger.Named(name) +} From 40c6cf294a44304a106662b3c0ea3f763bb033db Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 8 Apr 2021 11:47:44 +0300 Subject: [PATCH 201/206] Add documentation for the objects of mtglib --- mtglib/proxy.go | 21 ++++++---- mtglib/proxy_opts.go | 91 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 94 insertions(+), 18 deletions(-) diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 0317553..8a550fb 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -17,6 +17,7 @@ import ( "github.com/panjf2000/ants/v2" ) +// Proxy is an MTPROTO proxy structure. type Proxy struct { ctx context.Context ctxCancel context.CancelFunc @@ -37,10 +38,13 @@ type Proxy struct { logger Logger } +// DomainFrontingAddress returns a host:port pair for a fronting domain. func (p *Proxy) DomainFrontingAddress() string { return net.JoinHostPort(p.secret.Host, strconv.Itoa(p.domainFrontingPort)) } +// ServeConn serves a connection. We do not check IP blocklist and +// concurrency limit here. func (p *Proxy) ServeConn(conn net.Conn) { p.streamWaitGroup.Add(1) defer p.streamWaitGroup.Done() @@ -86,6 +90,7 @@ func (p *Proxy) ServeConn(conn net.Conn) { } } +// Serve starts a proxy on a given listener. func (p *Proxy) Serve(listener net.Listener) error { p.streamWaitGroup.Add(1) defer p.streamWaitGroup.Done() @@ -93,7 +98,12 @@ func (p *Proxy) Serve(listener net.Listener) error { for { conn, err := listener.Accept() if err != nil { - return fmt.Errorf("cannot accept a new connection: %w", err) + select { + case <-p.ctx.Done(): + return nil + default: + return fmt.Errorf("cannot accept a new connection: %w", err) + } } ipAddr := conn.RemoteAddr().(*net.TCPAddr).IP @@ -117,15 +127,11 @@ func (p *Proxy) Serve(listener net.Listener) error { logger.Info("connection was concurrency limited") p.eventStream.Send(p.ctx, NewEventConcurrencyLimited()) } - - select { - case <-p.ctx.Done(): - return p.ctx.Err() - default: - } } } +// Shutdown 'gracefully' shutdowns all connections. Please remember that +// it does not close an underlying listener. func (p *Proxy) Shutdown() { p.ctxCancel() p.streamWaitGroup.Wait() @@ -262,6 +268,7 @@ func (p *Proxy) doDomainFronting(ctx *streamContext, conn *connRewind) { } } +// NewProxy makes a new proxy instance. func NewProxy(opts ProxyOpts) (*Proxy, error) { if err := opts.valid(); err != nil { return nil, fmt.Errorf("invalid settings: %w", err) diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index c7bff07..8da37f8 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -2,20 +2,89 @@ package mtglib import "time" +// ProxyOpts is a structure with settings to mtg proxy. +// +// This is not required per se, but this is to shorten function +// signature and give an ability to conveniently provide default values. type ProxyOpts struct { - Secret Secret - Network Network - AntiReplayCache AntiReplayCache - TimeAttackDetector TimeAttackDetector - IPBlocklist IPBlocklist - EventStream EventStream - Logger Logger + // Secret defines a secret which should be used by a proxy. + // + // This is a mandatory setting. + Secret Secret - BufferSize uint - Concurrency uint + // Network defines a network instance which should be used for all + // network communications made by proxies. + // + // This is a mandatory setting. + Network Network + + // AntiReplayCache defines an instance of antireplay cache. + // + // This is a mandatory setting. + AntiReplayCache AntiReplayCache + + // TimeAttackDetector defines an instance of timeattack detector. + // + // This is a mandatory setting. + TimeAttackDetector TimeAttackDetector + + // IPBlocklist defines an instance of IP blocklist. + // + // This is a mandatory setting. + IPBlocklist IPBlocklist + + // EventStream defines an instance of event stream. + // + // This ia a mandatory setting. + EventStream EventStream + + // Logger defines an instance of the logger. + // + // This is a mandatory setting. + Logger Logger + + // BufferSize is a size of the copy buffer in bytes. + // + // Please remember that we multiply this number in 2, because when + // we relay between proxies, we have to create 2 intermediate + // buffers: to and from. + // + // This is an optional setting. + BufferSize uint + + // Concurrency is a size of the worker pool for connection management. + // + // If we have more connections than this number, they are going to be + // rejected. + // + // This is an optional setting. + Concurrency uint + + // DomainFrontingPort is a port we use to connect to a fronting + // domain. + // + // This is required because secret does not specify a port. It + // specifies a hostname only. + // + // This is an optional setting. DomainFrontingPort uint - IdleTimeout time.Duration - PreferIP string + + // IdleTimeout is a timeout for relay when we have to break a + // stream. + // + // This is a timeout for any activity. So, if we have any message + // which will pass to either direction, a timer is reset. If we have + // no any reads or writes for this timeout, a connection will be + // aborted. + // + // This is an optional setting. + IdleTimeout time.Duration + + // PreferIP defines an IP connectivity preference. Valid values are: + // 'prefer-ipv4', 'prefer-ipv6', 'only-ipv4', 'only-ipv6'. + // + // This is an optional setting. + PreferIP string } func (p ProxyOpts) valid() error { From d3aa0f2de5891379cc8ca5f5af7cde7faa08e4c5 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 8 Apr 2021 11:54:49 +0300 Subject: [PATCH 202/206] Add mtglib-level documentation --- mtglib/init.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mtglib/init.go b/mtglib/init.go index cd1b204..92eb9e6 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -1,3 +1,20 @@ +// mtglib defines a package with MTPROTO proxy. +// +// Since mtg itself is build as an example of how to work with mtglib, +// it worth to telling a couple of words about a project organization. +// +// A core object of the project is mtglib.Proxy. This is a proxy you +// expect: that one which you configure, set to serve on a listener +// and/or shutdown on application termination. +// +// But it also has a core logic unrelated to Telegram per se: anti +// replay cache, network connectivity (who knows, maybe you want to have +// a native VMESS integration) and so on. +// +// You can supply such parts to a proxy with interfaces. The rest of +// the packages in mtg define some default implementations of these +// interfaces. But if you want to integrate it with, let say, influxdb, +// you can do it easily. package mtglib import ( From 0652f322fa12356664e9b1924e09eb88e1f56a4c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 8 Apr 2021 14:39:01 +0300 Subject: [PATCH 203/206] Add docstrings to a logger --- mtglib/init.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mtglib/init.go b/mtglib/init.go index 92eb9e6..e6807d7 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -239,16 +239,38 @@ type TimeAttackDetector interface { // logger1 should see no param2 and vice versa, logger2 should not see param1 // If you attach a parameter to a logger, parents should not know about that. type Logger interface { + // Named returns a new logger with a bound name. Name chaining is + // allowed and appreciated. Named(name string) Logger + // BindInt binds new integer parameter to a new logger instance. BindInt(name string, value int) Logger + + // BindStr binds new string parameter to a new logger instance. BindStr(name, value string) Logger + // Printf is to support log.Logger behavior. Printf(format string, args ...interface{}) + + // Info puts a message about some normal situation. Info(msg string) + + // InfoError puts a message about some normal situation but this + // situation is related to a given error. InfoError(msg string, err error) + + // Warning puts a message about some extraordinary situation + // worth to look at. Warning(msg string) + + // WarningError puts a message about some extraordinary situation + // worth to look at. This situation is related to a given error. WarningError(msg string, err error) + + // Debug puts a message useful for debugging only. Debug(msg string) + + // Debug puts a message useful for debugging only. This message is + // related to a given error. DebugError(msg string, err error) } From e2073f05850c98f638389e73dbd65988d02a20f4 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 8 Apr 2021 14:58:14 +0300 Subject: [PATCH 204/206] Remove timeattack module --- example.config.toml | 21 ++++---- internal/cli/proxy.go | 23 ++------- internal/config/config.go | 42 +++++++--------- mtglib/init.go | 20 ++------ mtglib/internal/faketls/client_hello.go | 20 ++++++++ mtglib/proxy.go | 64 ++++++++++++------------- mtglib/proxy_opts.go | 24 +++++++--- mtglib/proxy_test.go | 22 +++------ timeattack/detector.go | 39 --------------- timeattack/detector_test.go | 28 ----------- timeattack/init.go | 10 ---- timeattack/noop.go | 16 ------- timeattack/noop_test.go | 26 ---------- 13 files changed, 108 insertions(+), 247 deletions(-) delete mode 100644 timeattack/detector.go delete mode 100644 timeattack/detector_test.go delete mode 100644 timeattack/init.go delete mode 100644 timeattack/noop.go delete mode 100644 timeattack/noop_test.go diff --git a/example.config.toml b/example.config.toml index fe3c883..c96e354 100644 --- a/example.config.toml +++ b/example.config.toml @@ -48,6 +48,14 @@ prefer-ip = "prefer-ipv6" # access. domain-fronting-port = 443 +# FakeTLS can compare timestamps to prevent probes. Each message has +# encrypted timestamp. So, mtg can compare this timestamp and decide if +# we need to proceed with connection or not. +# +# Sometimes time can be skewed so we accept all messages within a +# time range of this parameter. +tolerate-time-skewness = "5s" + # network defines different network-related settings [network] # please be aware that mtg needs to do some external requests. For @@ -111,19 +119,6 @@ tcp = "5s" http = "10s" idle = "1m" -# FakeTLS can compare timestamps to prevent probes. Each message has -# encrypted timestamp. So, mtg can compare this timestamp and decide if -# we need to proceed with connection or not. -# -# Please ensure that you have some ntp active on this host. Otherwise, -# you can endup with badly performing proxy. -[defense.time] -# You can enable/disable that. A good idea is always enable. -enabled = true -# Time can be skewed by many reasons. So, this is a time interval -# when message is cosidered as a good one. -allow-skewness = "5s" - # Some countries do active probing on Telegram connections. This technique # allows to protect from such effort. # diff --git a/internal/cli/proxy.go b/internal/cli/proxy.go index 713afcb..b20b39d 100644 --- a/internal/cli/proxy.go +++ b/internal/cli/proxy.go @@ -12,7 +12,6 @@ import ( "github.com/9seconds/mtg/v2/logger" "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/stats" - "github.com/9seconds/mtg/v2/timeattack" "github.com/rs/zerolog" ) @@ -41,12 +40,11 @@ func (c *Proxy) Execute() error { ctx := utils.RootContext() opts := mtglib.ProxyOpts{ - Logger: logger.NewZeroLogger(zerolog.New(os.Stdout).With().Timestamp().Logger()), - Network: c.Network, - AntiReplayCache: antireplay.NewNoop(), - IPBlocklist: ipblocklist.NewNoop(), - TimeAttackDetector: timeattack.NewNoop(), - EventStream: events.NewNoopStream(), + Logger: logger.NewZeroLogger(zerolog.New(os.Stdout).With().Timestamp().Logger()), + Network: c.Network, + AntiReplayCache: antireplay.NewNoop(), + IPBlocklist: ipblocklist.NewNoop(), + EventStream: events.NewNoopStream(), Secret: c.Config.Secret, BufferSize: c.Config.TCPBuffer.Value(mtglib.DefaultBufferSize), @@ -58,7 +56,6 @@ func (c *Proxy) Execute() error { opts.Logger.BindStr("configuration", c.Config.String()).Debug("configuration") c.setupAntiReplayCache(&opts) - c.setupTimeAttackDetector(&opts) if err := c.setupIPBlocklist(&opts); err != nil { return fmt.Errorf("cannot setup ipblocklist: %w", err) @@ -98,16 +95,6 @@ func (c *Proxy) setupAntiReplayCache(opts *mtglib.ProxyOpts) { ) } -func (c *Proxy) setupTimeAttackDetector(opts *mtglib.ProxyOpts) { - if !c.Config.Defense.Time.Enabled { - return - } - - opts.TimeAttackDetector = timeattack.NewDetector( - c.Config.Defense.Time.AllowSkewness.Value(timeattack.DefaultDuration), - ) -} - func (c *Proxy) setupIPBlocklist(opts *mtglib.ProxyOpts) error { if !c.Config.Defense.Blocklist.Enabled { return nil diff --git a/internal/config/config.go b/internal/config/config.go index 8414b9a..a634c7b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,18 +10,15 @@ import ( ) type Config struct { - Debug bool `json:"debug"` - Secret mtglib.Secret `json:"secret"` - BindTo TypeHostPort `json:"bind-to"` - TCPBuffer TypeBytes `json:"tcp-buffer"` - PreferIP TypePreferIP `json:"prefer-ip"` - DomainFrontingPort TypePort `json:"domain-fronting-port"` - Concurrency uint `json:"concurrency"` - Defense struct { - Time struct { - Enabled bool `json:"enabled"` - AllowSkewness TypeDuration `json:"allow-skewness"` - } `json:"time"` + Debug bool `json:"debug"` + Secret mtglib.Secret `json:"secret"` + BindTo TypeHostPort `json:"bind-to"` + TCPBuffer TypeBytes `json:"tcp-buffer"` + PreferIP TypePreferIP `json:"prefer-ip"` + DomainFrontingPort TypePort `json:"domain-fronting-port"` + TolerateTimeSkewness TypeDuration `json:"tolerate-time-skewness"` + Concurrency uint `json:"concurrency"` + Defense struct { AntiReplay struct { Enabled bool `json:"enabled"` MaxSize TypeBytes `json:"max-size"` @@ -85,18 +82,15 @@ func (c *Config) String() string { } type configRaw struct { - Debug bool `toml:"debug" json:"debug,omitempty"` - Secret string `toml:"secret" json:"secret"` - BindTo string `toml:"bind-to" json:"bind-to"` - TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` - PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` - DomainFrontingPort uint `toml:"domain-fronting-port" json:"domain-fronting-port,omitempty"` - Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"` - Defense struct { - Time struct { - Enabled bool `toml:"enabled" json:"enabled,omitempty"` - AllowSkewness string `toml:"allow-skewness" json:"allow-skewness,omitempty"` - } `toml:"time" json:"time,omitempty"` + Debug bool `toml:"debug" json:"debug,omitempty"` + Secret string `toml:"secret" json:"secret"` + BindTo string `toml:"bind-to" json:"bind-to"` + TCPBuffer string `toml:"tcp-buffer" json:"tcp-buffer,omitempty"` + PreferIP string `toml:"prefer-ip" json:"prefer-ip,omitempty"` + DomainFrontingPort uint `toml:"domain-fronting-port" json:"domain-fronting-port,omitempty"` + TolerateTimeSkewness string `toml:"tolerate-time-skewness" json:"tolerate-time-skewness,omitempty"` + Concurrency uint `toml:"concurrency" json:"concurrency,omitempty"` + Defense struct { AntiReplay struct { Enabled bool `toml:"enabled" json:"enabled,omitempty"` MaxSize string `toml:"max-size" json:"max-size,omitempty"` diff --git a/mtglib/init.go b/mtglib/init.go index e6807d7..d4f265b 100644 --- a/mtglib/init.go +++ b/mtglib/init.go @@ -42,10 +42,6 @@ var ( // create a proxy but anti replay cache value is undefined. ErrAntiReplayCacheIsNotDefined = errors.New("anti-replay cache is not defined") - // ErrTimeAttackDetectorIsNotDefined is returned if you are trying to - // create a proxy but time attack detector is not defined. - ErrTimeAttackDetectorIsNotDefined = errors.New("time attack detector is not defined") - // ErrIPBlocklistIsNotDefined is returned if you are trying to // create a proxy but ip blocklist instance is not defined. ErrIPBlocklistIsNotDefined = errors.New("ip blocklist is not defined") @@ -75,6 +71,10 @@ const ( // in case of idling. DefaultIdleTimeout = time.Minute + // DefaultTolerateTimeSkewness is a default timeout for time + // skewness on a faketls timeout verification. + DefaultTolerateTimeSkewness = 3 * time.Second + // DefaultPreferIP is a default value for Telegram IP connectivity // preference. DefaultPreferIP = "prefer-ipv6" @@ -206,18 +206,6 @@ type EventStream interface { Send(context.Context, Event) } -// TimeAttackDetector is an abstraction that checks a time, taken from -// the faketls client hello message. This timestamp is encoded into -// client-generated random bytes and can be extracted after some client -// hello verification. -// -// This is mostly to prevent replay attacks. -type TimeAttackDetector interface { - // Valid returns an error if timestamp is invalid or should not be - // accepted. - Valid(time.Time) error -} - // Logger defines an interface of the logger used by mtglib. // // Each logger has a name. It is possible to stack names to organize diff --git a/mtglib/internal/faketls/client_hello.go b/mtglib/internal/faketls/client_hello.go index 51e1fe5..2062a8a 100644 --- a/mtglib/internal/faketls/client_hello.go +++ b/mtglib/internal/faketls/client_hello.go @@ -19,6 +19,26 @@ type ClientHello struct { CipherSuite uint16 } +func (c ClientHello) Valid(hostname string, tolerateTimeSkewness time.Duration) error { + if c.Host != "" && c.Host != hostname { + return fmt.Errorf("incorrect hostname %s", hostname) + } + + now := time.Now() + + timeDiff := now.Sub(c.Time) + if timeDiff < 0 { + timeDiff = -timeDiff + } + + if timeDiff > tolerateTimeSkewness { + return fmt.Errorf("incorrect timestamp. got=%d, now=%d, diff=%s", + c.Time.Unix(), now.Unix(), timeDiff.String()) + } + + return nil +} + func ParseClientHello(secret, handshake []byte) (ClientHello, error) { hello := ClientHello{} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index 8a550fb..e07a9fe 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -23,19 +23,19 @@ type Proxy struct { ctxCancel context.CancelFunc streamWaitGroup sync.WaitGroup - idleTimeout time.Duration - bufferSize int - domainFrontingPort int - workerPool *ants.PoolWithFunc - telegram *telegram.Telegram + idleTimeout time.Duration + tolerateTimeSkewness time.Duration + bufferSize int + domainFrontingPort int + workerPool *ants.PoolWithFunc + telegram *telegram.Telegram - secret Secret - network Network - antiReplayCache AntiReplayCache - timeAttackDetector TimeAttackDetector - ipBlocklist IPBlocklist - eventStream EventStream - logger Logger + secret Secret + network Network + antiReplayCache AntiReplayCache + ipBlocklist IPBlocklist + eventStream EventStream + logger Logger } // DomainFrontingAddress returns a host:port pair for a fronting domain. @@ -159,15 +159,11 @@ func (p *Proxy) doFakeTLSHandshake(ctx *streamContext) bool { return false } - if hello.Host != "" && hello.Host != p.secret.Host { - p.logger.BindStr("hostname", hello.Host).Info("incorrect domain was found in SNI") - p.doDomainFronting(ctx, rewind) - - return false - } - - if err := p.timeAttackDetector.Valid(hello.Time); err != nil { - p.logger.InfoError("invalid faketls time", err) + if err := hello.Valid(p.secret.Host, p.tolerateTimeSkewness); err != nil { + p.logger. + BindStr("hostname", hello.Host). + BindStr("hello-time", hello.Time.String()). + InfoError("invalid faketls client hello", err) p.doDomainFronting(ctx, rewind) return false @@ -281,19 +277,19 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { ctx, cancel := context.WithCancel(context.Background()) proxy := &Proxy{ - ctx: ctx, - ctxCancel: cancel, - secret: opts.Secret, - network: opts.Network, - antiReplayCache: opts.AntiReplayCache, - timeAttackDetector: opts.TimeAttackDetector, - ipBlocklist: opts.IPBlocklist, - eventStream: opts.EventStream, - logger: opts.getLogger("proxy"), - domainFrontingPort: opts.getDomainFrontingPort(), - idleTimeout: opts.getIdleTimeout(), - bufferSize: opts.getBufferSize(), - telegram: tg, + ctx: ctx, + ctxCancel: cancel, + secret: opts.Secret, + network: opts.Network, + antiReplayCache: opts.AntiReplayCache, + ipBlocklist: opts.IPBlocklist, + eventStream: opts.EventStream, + logger: opts.getLogger("proxy"), + domainFrontingPort: opts.getDomainFrontingPort(), + tolerateTimeSkewness: opts.getTolerateTimeSkewness(), + idleTimeout: opts.getIdleTimeout(), + bufferSize: opts.getBufferSize(), + telegram: tg, } pool, err := ants.NewPoolWithFunc(opts.getConcurrency(), diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index 8da37f8..c2fdc69 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -23,11 +23,6 @@ type ProxyOpts struct { // This is a mandatory setting. AntiReplayCache AntiReplayCache - // TimeAttackDetector defines an instance of timeattack detector. - // - // This is a mandatory setting. - TimeAttackDetector TimeAttackDetector - // IPBlocklist defines an instance of IP blocklist. // // This is a mandatory setting. @@ -80,6 +75,15 @@ type ProxyOpts struct { // This is an optional setting. IdleTimeout time.Duration + // TolerateTimeSkewness is a time boundary that defines a time + // range where faketls timestamp is acceptable. + // + // This means that if if you got a timestamp X, now is Y, then + // if |X-Y| < TolerateTimeSkewness, then you accept a packet. + // + // This is an optional setting. + TolerateTimeSkewness time.Duration + // PreferIP defines an IP connectivity preference. Valid values are: // 'prefer-ipv4', 'prefer-ipv6', 'only-ipv4', 'only-ipv6'. // @@ -97,8 +101,6 @@ func (p ProxyOpts) valid() error { return ErrIPBlocklistIsNotDefined case p.EventStream == nil: return ErrEventStreamIsNotDefined - case p.TimeAttackDetector == nil: - return ErrTimeAttackDetectorIsNotDefined case p.Logger == nil: return ErrLoggerIsNotDefined case !p.Secret.Valid(): @@ -140,6 +142,14 @@ func (p ProxyOpts) getIdleTimeout() time.Duration { return p.IdleTimeout } +func (p ProxyOpts) getTolerateTimeSkewness() time.Duration { + if p.TolerateTimeSkewness == 0 { + return DefaultTolerateTimeSkewness + } + + return p.TolerateTimeSkewness +} + func (p ProxyOpts) getPreferIP() string { if p.PreferIP == "" { return DefaultPreferIP diff --git a/mtglib/proxy_test.go b/mtglib/proxy_test.go index 6e767ee..3d3171b 100644 --- a/mtglib/proxy_test.go +++ b/mtglib/proxy_test.go @@ -16,7 +16,6 @@ import ( "github.com/9seconds/mtg/v2/logger" "github.com/9seconds/mtg/v2/mtglib" "github.com/9seconds/mtg/v2/network" - "github.com/9seconds/mtg/v2/timeattack" "github.com/stretchr/testify/suite" ) @@ -46,13 +45,12 @@ func (suite *ProxyTestSuite) SetupSuite() { suite.NoError(err) suite.opts = &mtglib.ProxyOpts{ - Secret: mtglib.GenerateSecret("httpbin.org"), - Network: ntw, - AntiReplayCache: antireplay.NewNoop(), - TimeAttackDetector: timeattack.NewNoop(), - IPBlocklist: ipblocklist.NewNoop(), - EventStream: events.NewNoopStream(), - Logger: logger.NewNoopLogger(), + Secret: mtglib.GenerateSecret("httpbin.org"), + Network: ntw, + AntiReplayCache: antireplay.NewNoop(), + IPBlocklist: ipblocklist.NewNoop(), + EventStream: events.NewNoopStream(), + Logger: logger.NewNoopLogger(), } proxy, err := mtglib.NewProxy(*suite.opts) @@ -118,14 +116,6 @@ func (suite *ProxyTestSuite) TestCannotInitNoEventStream() { suite.Error(err) } -func (suite *ProxyTestSuite) TestCannotInitNoTimeAttackDetector() { - opts := *suite.opts - opts.TimeAttackDetector = nil - - _, err := mtglib.NewProxy(opts) - suite.Error(err) -} - func (suite *ProxyTestSuite) TestCannotInitNoLogger() { opts := *suite.opts opts.Logger = nil diff --git a/timeattack/detector.go b/timeattack/detector.go deleted file mode 100644 index 4ed1179..0000000 --- a/timeattack/detector.go +++ /dev/null @@ -1,39 +0,0 @@ -package timeattack - -import ( - "fmt" - "time" - - "github.com/9seconds/mtg/v2/mtglib" -) - -type detector struct { - time.Duration -} - -func (d detector) Valid(then time.Time) error { - now := time.Now() - - diff := now.Sub(then) - if diff < 0 { - diff = -diff - } - - if diff > d.Duration { - return fmt.Errorf("time is invalid. now=%d, then=%d, diff=%v", - now.Unix(), - then.Unix(), - diff) - } - - return nil -} - -// NewDetector returns a new TimeAttackDetector which validates that -// timestamp belongs to intervar [X-duration, X+duration], so a small -// timeshift is acceptable. -func NewDetector(duration time.Duration) mtglib.TimeAttackDetector { - return detector{ - Duration: duration, - } -} diff --git a/timeattack/detector_test.go b/timeattack/detector_test.go deleted file mode 100644 index dd4bd9f..0000000 --- a/timeattack/detector_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package timeattack_test - -import ( - "testing" - "time" - - "github.com/9seconds/mtg/v2/timeattack" - "github.com/stretchr/testify/suite" -) - -type DetectorTestSuite struct { - suite.Suite -} - -func (suite *DetectorTestSuite) TestOp() { - d := timeattack.NewDetector(time.Second) - - suite.NoError(d.Valid(time.Now())) - suite.NoError(d.Valid(time.Now().Add(100 * time.Millisecond))) - suite.NoError(d.Valid(time.Now().Add(-100 * time.Millisecond))) - suite.Error(d.Valid(time.Now().Add(time.Hour))) - suite.Error(d.Valid(time.Now().Add(-time.Hour))) -} - -func TestDetector(t *testing.T) { - t.Parallel() - suite.Run(t, &DetectorTestSuite{}) -} diff --git a/timeattack/init.go b/timeattack/init.go deleted file mode 100644 index e2f9788..0000000 --- a/timeattack/init.go +++ /dev/null @@ -1,10 +0,0 @@ -// TimeAttack has implementation of mtglib.TimeAttackDetector. -package timeattack - -import "time" - -// DefaultDuration is a default duration when timestamps are acceptable. -// -// It means that all timestamps which are X-DefaultDuration <= X <= -// X+DefaultDuration are fine. -const DefaultDuration = 5 * time.Second diff --git a/timeattack/noop.go b/timeattack/noop.go deleted file mode 100644 index e3827b8..0000000 --- a/timeattack/noop.go +++ /dev/null @@ -1,16 +0,0 @@ -package timeattack - -import ( - "time" - - "github.com/9seconds/mtg/v2/mtglib" -) - -type noop struct{} - -func (n noop) Valid(_ time.Time) error { return nil } - -// NewNoop returns TimeAttackDetector which accepts all timestamps. -func NewNoop() mtglib.TimeAttackDetector { - return noop{} -} diff --git a/timeattack/noop_test.go b/timeattack/noop_test.go deleted file mode 100644 index 4b3da79..0000000 --- a/timeattack/noop_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package timeattack_test - -import ( - "testing" - "time" - - "github.com/9seconds/mtg/v2/timeattack" - "github.com/stretchr/testify/suite" -) - -type NoopTestSuite struct { - suite.Suite -} - -func (suite *NoopTestSuite) TestOp() { - d := timeattack.NewNoop() - - suite.NoError(d.Valid(time.Now())) - suite.NoError(d.Valid(time.Now().Add(time.Hour))) - suite.NoError(d.Valid(time.Now().Add(-time.Hour))) -} - -func TestNoop(t *testing.T) { - t.Parallel() - suite.Run(t, &NoopTestSuite{}) -} From 43f5d2131ed0d56a27c1ed553664551a9e40f1a8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 8 Apr 2021 15:26:49 +0300 Subject: [PATCH 205/206] Add test for client hello validation --- mtglib/internal/faketls/client_hello_test.go | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/mtglib/internal/faketls/client_hello_test.go b/mtglib/internal/faketls/client_hello_test.go index f088cf2..4d5ad53 100644 --- a/mtglib/internal/faketls/client_hello_test.go +++ b/mtglib/internal/faketls/client_hello_test.go @@ -153,6 +153,38 @@ func (suite *ClientHelloTestSuite) TestSnapshotBad() { } } +func (suite *ClientHelloTestSuite) TestValidateHostname() { + hello := faketls.ClientHello{ + Time: time.Now(), + } + suite.NoError(hello.Valid("hostname", time.Second)) + + hello.Host = "hostname" + suite.Error(hello.Valid("hostname2", time.Second)) + suite.NoError(hello.Valid("hostname", time.Second)) +} + +func (suite *ClientHelloTestSuite) TestValidateTime() { + testData := []time.Duration{ + -2 * time.Second, + 2 * time.Second, + } + + for _, v := range testData { + value := v + + suite.T().Run(value.String(), func(t *testing.T) { + hello := faketls.ClientHello{ + Host: "hostname", + Time: time.Now().Add(value), + } + suite.Error(hello.Valid("hostname", 500*time.Millisecond)) + suite.Error(hello.Valid("hostname", time.Second)) + suite.NoError(hello.Valid("hostname", 3*time.Second)) + }) + } +} + func TestClientHello(t *testing.T) { t.Parallel() suite.Run(t, &ClientHelloTestSuite{}) From 4f048a3b82f7e5b2a7c0fe9040423420ea028e1b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 9 Apr 2021 11:58:46 +0300 Subject: [PATCH 206/206] Update README --- README.md | 493 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 275 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index 10f0b16..d9cd8a4 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # mtg -Bullshit-free MTPROTO proxy for Telegram +Highly-opionated (ex-bullshit-free) MTPROTO proxy for +[Telegram](https://telegram.org/). -[![Build Status](https://travis-ci.org/9seconds/mtg.svg?branch=master)](https://travis-ci.org/9seconds/mtg) -[![Go Report Card](https://goreportcard.com/badge/github.com/9seconds/mtg)](https://goreportcard.com/report/github.com/9seconds/mtg) -[![Docker Build Status](https://img.shields.io/docker/build/nineseconds/mtg.svg)](https://hub.docker.com/r/nineseconds/mtg/) +[![CI](https://github.com/9seconds/mtg/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/9seconds/mtg/actions/workflows/ci.yaml) +[![codecov](https://codecov.io/gh/9seconds/mtg/branch/master/graph/badge.svg?token=JfdDyGVpT4)](https://codecov.io/gh/9seconds/mtg) +[![Go Reference](https://pkg.go.dev/badge/github.com/9seconds/mtg.svg)](https://pkg.go.dev/github.com/9seconds/mtg) -**Please see a guide on upgrading to 1.0 at the end of this README.** +**If you use v1.0 or upgrade broke you proxy, please read the chapter +[Version 2](#version-2)** -# Rationale +## Rationale There are several available proxies for Telegram MTPROTO available. Here are the most notable: @@ -17,267 +19,322 @@ are the most notable: * [Python](https://github.com/alexbers/mtprotoproxy) * [Erlang](https://github.com/seriyps/mtproto_proxy) -Almost all of them follow the way how official proxy was built. This -includes support of multiple secrets, support of promoted channels, etc. +You can use any of these. They work great and all implementations have +feature parity now. This includes support of adtag, replay attack +protection, domain fronting, faketls, and so on. mtg has a similar +goal: to give a possibility to connect to Telegram in a restricted, +censored environment. But it does it slightly differently in details +that probably matter. -mtg is an implementation in golang which is intended to be: +* **Resource-efficient** + + It has to be resource-efficient. It does not mean that you will see + the smallest memory usage. It means that it will try to use allocated + resources in zero-waste mode, reusing as much memory as possible and + so on. -* **Lightweight** - It has to consume as few resources as possible but not by losing - maintainability. * **Easily deployable** + I strongly believe that Telegram proxies should follow the way of - ShadowSocks: promoted channels is a strange way of doing business - I suppose. I think the only viable way is to have a proxy with - minimum configuration which should work everywhere. + [ShadowSocks](https://shadowsocks.org): promoted channels is a strange + way of doing business I suppose. I think the only viable way is to + have a proxy that can be restored anywhere easily. + * **A single secret** - I think that multiple secrets solve no problems and just complexify + + I think that multiple secrets solve no problems and just complex software. I also believe that in the case of throwout proxies, this - feature is a useless luxury. -* **Minimum docker image size** - Official image is less than 3.5 megabytes. Literally. + the feature is a useless luxury. + +* **No adtag support** + + Please read [Version 2](#version-2) chapter. + * **No management WebUI** + This is an implementation of a simple lightweight proxy. I won't do that. -This proxy supports 2 modes of work: direct connection to Telegram and -promoted channel mode. If you do not need promoted channels, I would -recommend you to go with direct mode: this way is more robust. +* **Proxy chaining** -To run a proxy in direct mode, all you need to do is just provide a -secret. If you do not provide ADTag as a second parameter, promoted -channels mode won't be activated. + mtg has the support of [SOCKS5](https://en.wikipedia.org/wiki/SOCKS) + proxies. So, in theory, you can run this proxy as a frontend + and route traffic via [v2ray](https://www.v2ray.com/), + [Gost](https://docs.ginuerzh.xyz/gost/), + [Trojan](https://trojan-gfw.github.io/trojan/), or any other project + you like. -To get promoted channel, please contact -[@MTProxybot](https://t.me/MTProxybot) and provide generated adtag as a -second parameter. +* **Native blocklist support** + Previously, this was delegated to the [FireHOL](https://firehol.org/) + project or similar ones which track attacks and publish a list of + potentially dangerous IPs. mtg has native support of such blocklists. -# Source code organization +* **Can be used as a library** -There are 2 main branches: + mtg v2 was redesigned in a way so it can be embedded into your + software (written in Golang) with a minimum effort + you can replace + some parts with those you want. -1. `master` branch contains potentially unstable features -2. `stable` branch contains stable version. Usually you want to use this branch. +### Version 2 -# How to build +If you use version 1.x before, you are probably noticed some major +backward non-compatible details: + +1. Configuration file +2. Removed support of adtag + +For the configuration file, please check out the full example in this +repository. It has a lot of comments and most of the options are +optional. We do have only `secret` and `bind-to` sections mandatory. +Other sections in the example configuration file are filled with default +values. + +Adtag support was removed completely. This was done to debloat mtg and +keep it simple and obvious. Hopefully, this goal is achieved and the +source code is clean and straightforward enough. + +I always was quite skeptical about adtag. In my POV, a proxy as a fat +big connectivity point for hundreds of clients is an illusion. If you +work in a censored environment, the first thing that authority does is +IP blocking. For us, it means, those big proxies that can benefit from +having a pinned channel are going to be blocked in a minute. + +Proxy has to be intimate. It has to be shared within a small group as +a family or maybe your college friends. It has to have a small number +of connections and never publicly announced its presence. It has to fly +under the radar. If the proxy is detected, you need to be able to give +a rebirth on a new IP address as soon as possible. I do no think that +having some special channel for such a use case makes any sense. + +But other details like replay attack protection, domain fronting, +accurate FakeTLS implementation, IP blacklisting, and proxy +chaining matter here. If you work in censored perimeter like +[GFW](https://en.wikipedia.org/wiki/Great_Firewall)-protected +country, you probably want to have an MTPROTO proxy as +a frontend that transports traffic via cloaked tunnels +made by [Trojan](https://trojan-gfw.github.io/trojan/), +[Shadowsocks](https://shadowsocks.org), [v2ray](https://www.v2ray.com/), +or [Gost](https://docs.ginuerzh.xyz/gost/). That's why you have to have +the support of chaining as a first-class citizen. + +Yes, this is possible and doable with optional adtag support. But the +truth is that the MTPROTO proxy for Telegram is just a thing that either +work as a normal client (direct mode) or doing some RPC calls in [TL +language](https://core.telegram.org/mtproto/TL) (adtag support). I +understand the intention of the developers and I understand that they +were under high pressure fighting with [RKN](https://rkn.gov.ru/) and +doing TON after that. Nothing is ideal. But for the proxy, it means that +source code is full of complex non-trivial code which is required only +to support a feature that we barely need. + +So, to have a reasonable MTPROTO proxy, adtag support was removed. This +is a rare chance in my career where software v2 debloats a previous +version. It feels so good :) + +### Version 1 and 2 + +I do continue to support both versions 1 and 2. But in a different mode. + +Version 1 is now officially in maintenance mode. It means that I won't +make any new features or improvements there. You can consider a feature +freeze there. No bugs are going to be fixed there except for critical +ones. PRs are welcome though. The goal is to keep it working. It will +get some periodical updates like updates to the new Golang version of +dependencies version bump, but that's mostly it. + +**If you want to have mtg with _adtag support_, please use version 1**. + +Version 2 is going to have all my love, active support, bug fixing, etc. +It is under active development and maintenance. + +This project has several main branches + +1. [`master`](https://github.com/9seconds/mtg/tree/master) branch + contains a bleeding edge. It may potentially have some features + which will break your source code. +2. [`stable`](https://github.com/9seconds/mtg/tree/stable) branch contains + dumps of a master branch when we consider it 'stable'. This is a + branch you probably want to pick. +3. [`v2`](https://github.com/9seconds/mtg/tree/v2) has a development + of the v2.x version. In theory, it is the same as `master` but this + will change when we have v3.x. +4. [`v1`](https://github.com/9seconds/mtg/tree/v1) has a version 1.x. + +## Getting started + +### Download a tool + +#### Download binaries + +Binaries can be downloaded from the release page. Also, you can download +docker image. + +For the current version, please download like ```console -make +docker pull nineseconds/mtg:2 ``` -If you want to build for another platform: +For version 1: ```console -make crosscompile +docker pull nineseconds/mtg:1 ``` -If you want to build Docker image (called `mtg`): +You may also check both [Docker +Hub](https://hub.docker.com/r/nineseconds/mtg/tags) and [Github +Registry](https://github.com/users/9seconds/packages/container/package/mtg). +Please do not choose `latest` or `stable` if you want to avoid +surprises. Always choose some version tag. + +Also, if you have `go` installed, you can always download this tool with `go get`: + +```console +go get github.com/9seconds/mtg/v2 +``` + +#### Build from sources + +```console +git clone https://github.com:9seconds/mtg.git +cd mtg +make static +``` + +or for the docker image: ```console make docker ``` -# Docker image +### Generate secret -Docker follows the same policy as the source code organization: - -- `latest` mirrors the master branch -- `stable` mirrors the stable branch -- tags are for tagged releases +If you already have a secret in Base64 format or that, which starts with `ee`, +you can skip this chapter. Otherwise: ```console -docker pull nineseconds/mtg:latest +$ mtg generate-secret google.com +7ibaERuTSGPH1RdztfYnN4tnb29nbGUuY29t ``` +or + ```console -docker pull nineseconds/mtg:stable +$ mtg generate-secret --hex google.com +ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d ``` +This secret is a keystone for a proxy and your password for a client. +You need to keep it secured. + +We recommend choosing a hostname wisely. Here we have a _google.com_ +but in reality, all providers can easily detect that this is not a +Google. Google has a list of networks it officially uses and your IP +address won't probably belong to it. It is a great idea to hide behind +some domain that has some relation to this IP address. + +For example, you've bought a VPS from [Digital +Ocean](https://www.digitalocean.com/). Then it might be a good idea to +generate a secret for _digitalocean.com_ then. + +### Prepare a configuration file + +Please checkout an example configuration file. All options except of +`secret` and `bind-to` are optional. You can safely have this minimal +configuration file: + +```toml +secret = "ee473ce5d4958eb5f968c87680a23854a0676f6f676c652e636f6d" +bind-to = "0.0.0.0:443" +``` + +This is enough to run the whole application. All other +options already have sensible defaults for the app at almost any scale. + +Oh, the configuration is done in [TOML format](https://toml.io/en/). + +### Run a proxy + +Put a binary and a config into your webserver. Just for example, +a binary goes to `/usr/local/bin/mtg` and configuration to `/etc/mtg.toml`. + +Now you can create a systemd unit: + ```console -docker pull nineseconds/mtg:0.10 +$ cat /etc/systemd/system/mtg.service +[Unit] +Description=mtg + +[Service] +ExecStart=/usr/local/bin/mtg run /etc/mtg.toml +Restart=always +RestartSec=3 + +[Install] +WantedBy=multi-user.target +$ sudo systemctl daemon-reload +$ sudo systemctl enable mtg +$ sudo systemctl start mtg ``` -# Ansible role - -You can find unofficial Ansible role for mtg here: https://github.com/rlex/ansible-role-mtg -Also, there is another project on Ansible Galaxy: https://galaxy.ansible.com/ivansible/lin_mtproxy - -# Configuration - -To run this tool you need to configure as less as possible. Telegram -clients support 3 different secret types: - -* Simple - basically, it is just a flow of frames ciphered by AES-CTR stream - cipher. -* Secured - the same stream as simple but with some random noise to prevent - statistical analysis of traffic flow. -* FakeTLS - this mode envelops telegram stream in TLS so it looks (in theory) - the same as any TLS1.3 traffic from DPI point of view. - -If you do not have preferences, go with FakeTLS or at least secured. -Simple mode is a little bit naive and traffic flow can be easily -identified as Telegram one. - -Unlike the rest of implementation, mtg is quite strict about the -execution mode: if you run a proxy instance with FakeTLS secret, you -can't connect to it with simple or secured clients. You can't connect -to the proxy with secured secret with FakeTLS key. It forces one mode -of working. So, unfortunately, there is no way how to connect to the -deployed proxy with another secret (if you know how to construct and -convert them). But at the same time, old clients can't connect so they -won't expose the type of the service. - -First, you need to generate a secret: +or you can run a docker image ```console -$ mtg generate-secret simple -52a493bdfb90eea55739eabff2d92a14 +docker run -d -v /etc/mtg.toml:/config.toml -p 443:3128 --restart=unless-stopped nineseconds/mtg:2 ``` +where _443_ is a host port (a port you want to connect to from a +client), and _3128_ is the one you have in your config in the `bind-to` +section. + +### Access a proxy + +Now you can generate some useful links: + ```console -$ mtg generate-secret secured -ddf05fb7acb549be047a7c585116581418 +$ mtg access /etc/mtg.toml +{ + "ipv4": { + "ip": "x.y.z.a", + "port": 3128, + "tg_url": "tg://proxy?...", + "tg_qrcode": "https://api.qrserver.com/v1/create-qr-code?data...", + "tme_url": "https://t.me/proxy?...", + "tme_qrcode": "https://api.qrserver.com/v1/create-qr-code?data..." + }, + "secret": { + "hex": "...", + "base64": "..." + } +} ``` -```console -$ mtg generate-secret -c google.com tls -ee852380f362a09343efb4690c4e17862e676f6f676c652e636f6d -``` +## Metrics -Or, if you prefer docker: +Out of the box, mtg works with +[statsd](https://github.com/statsd/statsd) and +[Prometheus](https://prometheus.io/). Please check configuration file +example to get how to set this integration up. -```console -$ docker run --rm nineseconds/mtg generate-secret tls -c bing.com -eedf71035a8ed48a623d8e83e66aec4d0562696e672e636f6d -``` +Here goes a list of metrics with their types but without a prefix. -## Antireplay cache +| Name | Type | Tags | Description | +|-----------------------------|---------|----------------------------------|--------------------------------------------------------------------------------------------| +| client_connections | gauge | `ip_family` | Count of processing client connections. | +| telegram_connections | gauge | `telegram_ip`, `dc` | Count of connections to Telegram servers. | +| domain_fronting_connections | gauge | `ip_family` | Count of connections to fronting domain. | +| telegram_traffic | counter | `telegram_ip`, `dc`, `direction` | Count of bytes, transmitted to/from Telegram. | +| domain_fronting_traffic | counter | `direction` | Count of bytes, transmitted to/from fronting domain. | +| domain_fronting | counter | – | Count of domain fronting events. | +| concurrency_limited | counter | – | Count of events, when client connection was rejected due to concurrency limit. | +| ip_blocklisted | counter | – | Count of events when client connection was rejected because IP was found in the blacklist. | +| replay_attacks | counter | – | Count of detected replay attacks. | -To prevent replay attacks, we have internal storage of first frames -messages for connected clients. These frames are generated randomly -by design and we have the negligible possibility of duplication -(probability is 1/(2^64)) but it could be quite effective to prevent -replays. +Tag meaning: -It is possible to disable this cache. To do that, please explicitly set -its size to 0. - - -## FakeTLS - -If you run this a proxy in faketls mode, this proxy will try to hide -itself cloaking a host provided as a part of the generated secret. It -means that if you cloak google.com then you can curl this proxy and -you'll get a google.com response back. - -mtg proxies L3 traffic. In other words, only TCP, without interfering in -TLS, HTTP or any other high-level protocol. - - -## Environment variables - -It is possible to configure this tool using environment variables. You -can configure any flag but not secret or adtag. Here is the list of -supported environment variables: - -| Environment variable | Corresponding flags | Default value | Description | -|-------------------------------|------------------------------|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `MTG_DEBUG` | `-d`, `--debug` | `false` | Run in debug mode. Usually, you need to run in this mode only if you develop this tool or its maintainer is asking you to provide logs with such verbosity. | -| `MTG_VERBOSE` | `-v`, `--verbose` | `false` | Run in verbose mode. This is way less chatty than debug mode. | -| `MTG_BIND` | `-b`, `--bind` | `0.0.0.0:3128` | Which host/port pair should we bind to (listen on). | -| `MTG_IPV4` | `-4`, `--public-ipv4` | [Autodetect](https://ifconfig.co) | IPv4 address:port of this proxy. This is required if you NAT your proxy or run it in a docker container. In that case, you absolutely need to specify public IPv4 address of the proxy, otherwise either URLs will be broken or proxy could not access Telegram middle proxies. | -| `MTG_IPV6` | `-6`, `--public-ipv6` | [Autodetect](https://ifconfig.co) | IPv6 address:port of this proxy. This is required if you NAT your proxy or run it in a docker container. In that case, you absolutely need to specify public IPv6 address of the proxy, otherwise either URLs will be broken or proxy could not access Telegram middle proxies. | -| `MTG_STATS_BIND` | `-t`, `--stats-bind` | `127.0.0.1:3129` | Which hist:port should we bind the internal statistics HTTP server (Prometheus). | -| `MTG_STATS_NAMESPACE` | `--stats-namespace` | `mtg` | Which namespace should be used for prometheus metrics. | -| `MTG_STATSD_ADDR` | `--statsd-addr` | | host:port of statsd service. No defaults, by default we do not send anything there. | -| `MTG_STATSD_PREFIX` | `--statsd-prefix` | `mtg` | Which bucket prefix we should use. For example, if you set `mtg`, then metric `traffic.ingress` would be send as `mtg.traffic.ingress`. | -| `MTG_STATSD_TAGS_FORMAT` | `--statsd-tags-format` | | Which tags format we should use. By default, we are using default vanilla statsd tags format but if you want to send directly to InfluxDB or Datadog, please specify it there. Possible options are `influxdb` and `datadog`. | -| `MTG_STATSD_TAGS` | `--statsd-tags` | | Which tags should we send to statsd with our metrics. Please specify them as `key=value` pairs. | -| `MTG_BUFFER_WRITE` | `-w`, `--write-buffer` | `32KB` | The size of TCP write buffer in bytes. Write buffer is the buffer for messages which are going from client to Telegram. | -| `MTG_BUFFER_READ` | `-r`, `--read-buffer` | `32KB` | The size of TCP read buffer in bytes. Read buffer is the buffer for messages from Telegram to client. | -| `MTG_ANTIREPLAY_MAXSIZE` | `--anti-replay-max-size` | `128MB` | Max size of antireplay cache. | -| `MTG_CLOAK_PORT` | `--cloak-port` | `443` | Which port we should use to connect to cloaked host in FakeTLS mode. | -| `MTG_MULTIPLEX_PERCONNECTION` | `--multiplex-per-connection` | `50` | How many client connections can share a single Telegram connection in adtag mode | -| `MTG_NTP_SERVERS` | `--ntp-server` | default pool | A list of NTP servers to use. | -| `MTG_PREFER_DIRECT_IP` | `--prefer-ip` | `ipv6` | Which IP protocol to prefer if possible. Works mostly in direct mode. | - - -Usually you want to modify only read/write buffer sizes. If you feel -that proxy is slow, try to increase both sizes giving more priority to -read buffer. - -Unfortunately, MTPROTO proxy protocol does not allow us to use splice -or any other neat tricks how to eliminate the need of copying data into -userspace. - -# How to run the tool - -Now run the tool: - -```console -$ mtg run -``` - -How to run the tool with ADTag: - -```console -$ mtg run -``` - -This tool will listen on port 3128 by default with the given secret. - - -# oneliner to run this proxy - -Please ensure that docker is installed. After that just execute - -```console -curl -sfL --compressed https://raw.githubusercontent.com/9seconds/mtg/master/run.sh | bash -``` - - -# statsd integration - -mtg provides an integration with statsd, you can enable it with command -line interface. To enable it, you have to provide IP address of statsd -service. - -Out of the box, mtg supports 2 additional dialects: [InfluxDB](https://www.influxdata.com/blog/getting-started-with-sending-statsd-metrics-to-telegraf-influxdb/) -and [Datadog](https://docs.datadoghq.com/developers/dogstatsd/). - -All metrics are gauges. Here is the list of metrics and their meaning: - -| Metric name | Unit | Description | -|------------------------|---------|--------------------------------------------| -| `connections` | number | The number of active connections. | -| `telegram_connections` | number | The number of active telegram connections. | -| `crashes` | number | An amount of crashes in client handlers. | -| `traffic.egress` | bytes | Traffic from the start of application. | -| `replay_attacks` | number | The number of prevented replay attacks. | - -All metrics are prefixed with given prefix. Default prefix is `mtg`. -Also, metrics provide tags (ipv4/ipv6, dc indexes etc). - - -# Prometheus integration - -[Prometheus](https://prometheus.io) integration comes out of -the box, you do not need to setup anything special. - - -# Upgrade to 1.0 - -Version 1.0 breaks compatibility with previous versions so please read -this chapter carefully: - -1. mtg now uses subcommands. Please use `mtg run` instead of just - `mtg` to run a proxy. -2. Options which set host and port separately were removed in a - favor of fused `host:port` options. -3. Own stats server was removed. Prometheus endpoint is moved to - default stats endpoint. -4. It is possible to connect to this proxy only with a secret which - was used to run it. So, no backward compatibility of clients. -5. Multiplexing involves connectivity with middle proxies and involves - the most complex code path of this proxy. To avoid potential bugs, - we still recommend using direct mode. +| Name | Values | Description | +|-------------|------------------------|-----------------------------------------------| +| ip_family | ipv4, ipv6 | A version of the IP protocol. | +| dc | | A number of the Telegram DC for a connection. | +| telegram_ip | | IP address of the Telegram server. | +| direction | to_client, from_client | A direction of the traffic flow. |