From 67c1194d5b80f98b752776ec70aae0947fb902a2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 24 Jun 2018 20:43:04 +0300 Subject: [PATCH 01/71] Add telegram caller struct --- main.go | 1 + telegram/middle.go | 37 +++++++++ telegram/middle_caller.go | 157 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 telegram/middle.go create mode 100644 telegram/middle_caller.go diff --git a/main.go b/main.go index fa63cfc..18de1f2 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/proxy" + "github.com/9seconds/mtg/telegram" "github.com/juju/errors" ) diff --git a/telegram/middle.go b/telegram/middle.go new file mode 100644 index 0000000..6fb8432 --- /dev/null +++ b/telegram/middle.go @@ -0,0 +1,37 @@ +package telegram + +import ( + "net" + "net/http" + "sync" + + "go.uber.org/zap" + + "github.com/9seconds/mtg/config" +) + +type middleTelegram struct { + middleTelegramCaller +} + +func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram { + tg := &middleTelegram{ + middleTelegramCaller: middleTelegramCaller{ + baseTelegram: baseTelegram{ + dialer: tgDialer{net.Dialer{Timeout: telegramDialTimeout}}, + }, + logger: logger, + httpClient: &http.Client{ + Timeout: middleTelegramHTTPClientTimeout, + }, + dialerMutex: &sync.RWMutex{}, + }, + } + + if err := tg.update(); err != nil { + panic(err) + } + go tg.autoUpdate() + + return tg +} diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go new file mode 100644 index 0000000..27d48fc --- /dev/null +++ b/telegram/middle_caller.go @@ -0,0 +1,157 @@ +package telegram + +import ( + "bufio" + "io" + "io/ioutil" + "net" + "net/http" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/juju/errors" + "go.uber.org/zap" + + "github.com/9seconds/mtg/mtproto" +) + +const ( + middleTelegramAutoUpdateInterval = 6 * time.Hour + middleTelegramHTTPClientTimeout = 30 * time.Second + + tgAddrProxySecret = "https://core.telegram.org/getProxySecret" // nolint: gas + tgAddrProxyV4 = "https://core.telegram.org/getProxyConfig" // nolint: gas + tgAddrProxyV6 = "https://core.telegram.org/getProxyConfigV6" // nolint: gas + tgUserAgent = "mtg" +) + +var middleTelegramProxyConfigSplitter *regexp.Regexp + +type middleTelegramCaller struct { + baseTelegram + + proxySecret []byte + dialerMutex *sync.RWMutex + logger *zap.SugaredLogger + httpClient *http.Client +} + +func (t *middleTelegramCaller) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWriteCloser, error) { + dc := connOpts.DC + if dc == 0 { + dc = 1 + } + t.dialerMutex.RLock() + defer t.dialerMutex.RUnlock() + + return t.baseTelegram.dial(dc) +} + +func (t *middleTelegramCaller) autoUpdate() { + for range time.Tick(middleTelegramAutoUpdateInterval) { + if err := t.update(); err != nil { + t.logger.Warnw("Cannot update from Telegram", "error", err) + } + } +} + +func (t *middleTelegramCaller) update() error { + secret, err := t.getTelegramProxySecret() + if err != nil { + return errors.Annotate(err, "Cannot get proxy secret") + } + + v4Addresses, err := t.getTelegramAddresses(tgAddrProxyV4) + if err != nil { + return errors.Annotate(err, "Cannot get ipv4 addresses") + } + + v6Addresses, err := t.getTelegramAddresses(tgAddrProxyV6) + if err != nil { + return errors.Annotate(err, "Cannot get ipv6 addresses") + } + + t.dialerMutex.Lock() + t.proxySecret = secret + t.v4Addresses = v4Addresses + t.v6Addresses = v6Addresses + t.dialerMutex.Unlock() + + t.logger.Infow("Telegram middle proxy data has been updated") + + return nil +} + +func (t *middleTelegramCaller) getTelegramProxySecret() ([]byte, error) { + resp, err := t.call(tgAddrProxySecret) + if err != nil { + return nil, errors.Annotate(err, "Cannot access telegram server") + } + defer resp.Body.Close() // nolint: errcheck + + secret, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, errors.Annotate(err, "Cannot read response") + } + + return secret, nil +} + +func (t *middleTelegramCaller) getTelegramAddresses(url string) (map[int16][]string, error) { + resp, err := t.call(url) + if err != nil { + return nil, errors.Annotate(err, "Cannot access telegram server") + } + defer resp.Body.Close() + + scanner := bufio.NewScanner(resp.Body) + data := map[int16][]string{} + for scanner.Scan() { + text := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(text, "#") { + continue + } + + chunks := middleTelegramProxyConfigSplitter.Split(text, 3) + if len(chunks) != 3 || chunks[0] != "proxy_for" { + return nil, errors.Errorf("Incorrect config '%s'", text) + } + dcIdx64, err2 := strconv.ParseInt(chunks[1], 10, 16) + if err2 != nil { + return nil, errors.Errorf("Incorrect config '%s'", text) + } + dcIdx := int16(dcIdx64) + + addr := strings.TrimRight(chunks[2], ";") + if _, _, err2 = net.SplitHostPort(addr); err != nil { + return nil, errors.Annotatef(err2, "Incorrect config '%s'", text) + } + + if addresses, ok := data[dcIdx]; ok { + data[dcIdx] = append(addresses, addr) + } else { + data[dcIdx] = []string{addr} + } + } + err = scanner.Err() + if err != nil { + return nil, errors.Annotate(err, "Cannot read response from the telegram") + } + + return data, nil +} + +func (t *middleTelegramCaller) call(url string) (*http.Response, error) { + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Accept", "text/plain") + req.Header.Set("User-Agent", tgUserAgent) + + return t.httpClient.Do(req) +} + +func init() { + middleTelegramProxyConfigSplitter = regexp.MustCompile(`\s+`) +} From f9a2b0513d00f8fa20a21a71736c6d4407349308 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 1 Jul 2018 10:27:36 +0300 Subject: [PATCH 02/71] Propagate connection protocol --- client/direct.go | 1 + main.go | 1 - mtproto/connection_options.go | 13 +++++++++++-- telegram/direct.go | 2 +- telegram/middle.go | 2 +- telegram/middle_caller.go | 2 +- telegram/telegram.go | 15 ++++++++++----- 7 files changed, 25 insertions(+), 11 deletions(-) diff --git a/client/direct.go b/client/direct.go index 537d12a..f4db89c 100644 --- a/client/direct.go +++ b/client/direct.go @@ -35,6 +35,7 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, io if err != nil { return nil, nil, errors.Annotate(err, "Cannot parse obfuscated frame") } + connOpts.ConnectionProto = mtproto.ConnectionProtocolAny socket := wrappers.NewStreamCipherRWC(conn, obfs2.Encryptor, obfs2.Decryptor) diff --git a/main.go b/main.go index 18de1f2..fa63cfc 100644 --- a/main.go +++ b/main.go @@ -16,7 +16,6 @@ import ( "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/proxy" - "github.com/9seconds/mtg/telegram" "github.com/juju/errors" ) diff --git a/mtproto/connection_options.go b/mtproto/connection_options.go index 8510692..29ee283 100644 --- a/mtproto/connection_options.go +++ b/mtproto/connection_options.go @@ -10,11 +10,14 @@ import ( // by the user. type ConnectionType uint8 +type ConnectionProtocol uint8 + // ConnectionOpts presents an options, metadata on connection requested // by the user on handshake. type ConnectionOpts struct { - DC int16 - ConnectionType ConnectionType + DC int16 + ConnectionType ConnectionType + ConnectionProto ConnectionProtocol } // Different connection types which user requests from Telegram. @@ -24,6 +27,12 @@ const ( ConnectionTypeIntermediate ) +const ( + ConnectionProtocolIPv4 ConnectionProtocol = 1 + ConnectionProtocolIPv6 = ConnectionProtocolIPv4 << 1 + ConnectionProtocolAny = ConnectionProtocolIPv4 | ConnectionProtocolIPv6 +) + // Connection tags for mtproto handshakes. var ( ConnectionTagAbridged = []byte{0xef, 0xef, 0xef, 0xef} diff --git a/telegram/direct.go b/telegram/direct.go index 89a7016..f1ee52f 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -41,7 +41,7 @@ func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWriteClo dc = 1 } - return t.baseTelegram.dial(dc - 1) + return t.baseTelegram.dial(dc-1, connOpts.ConnectionProto) } func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn io.ReadWriteCloser) (io.ReadWriteCloser, error) { diff --git a/telegram/middle.go b/telegram/middle.go index 6fb8432..01a3b73 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -14,7 +14,7 @@ type middleTelegram struct { middleTelegramCaller } -func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram { +func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) *middleTelegram { tg := &middleTelegram{ middleTelegramCaller: middleTelegramCaller{ baseTelegram: baseTelegram{ diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go index 27d48fc..2d0d86e 100644 --- a/telegram/middle_caller.go +++ b/telegram/middle_caller.go @@ -47,7 +47,7 @@ func (t *middleTelegramCaller) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWr t.dialerMutex.RLock() defer t.dialerMutex.RUnlock() - return t.baseTelegram.dial(dc) + return t.baseTelegram.dial(dc, connOpts.ConnectionProto) } func (t *middleTelegramCaller) autoUpdate() { diff --git a/telegram/telegram.go b/telegram/telegram.go index 0f71efc..6bbb504 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -24,13 +24,18 @@ type baseTelegram struct { v6Addresses map[int16][]string } -func (b *baseTelegram) dial(dcIdx int16) (io.ReadWriteCloser, error) { +func (b *baseTelegram) dial(dcIdx int16, proto mtproto.ConnectionProtocol) (io.ReadWriteCloser, error) { addrs := make([]string, 2) - if addr, ok := b.v6Addresses[dcIdx]; ok && len(addr) > 0 { - addrs = append(addrs, addr[rand.Intn(len(addr))]) + + if proto&mtproto.ConnectionProtocolIPv6 != 0 { + if addr, ok := b.v6Addresses[dcIdx]; ok && len(addr) > 0 { + addrs = append(addrs, addr[rand.Intn(len(addr))]) + } } - if addr, ok := b.v4Addresses[dcIdx]; ok && len(addr) > 0 { - addrs = append(addrs, addr[rand.Intn(len(addr))]) + if proto&mtproto.ConnectionProtocolIPv4 != 0 { + if addr, ok := b.v4Addresses[dcIdx]; ok && len(addr) > 0 { + addrs = append(addrs, addr[rand.Intn(len(addr))]) + } } for _, addr := range addrs { From 9f90f8775828ae2be1a97adebc5fb80d31fc8b47 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 1 Jul 2018 11:41:10 +0300 Subject: [PATCH 03/71] Add RPC structs for nonce --- mtproto/rpc/rpc.go | 5 +++ mtproto/rpc/rpc_nonce_request.go | 67 +++++++++++++++++++++++++++++++ mtproto/rpc/rpc_nonce_response.go | 58 ++++++++++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 mtproto/rpc/rpc.go create mode 100644 mtproto/rpc/rpc_nonce_request.go create mode 100644 mtproto/rpc/rpc_nonce_response.go diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go new file mode 100644 index 0000000..dc07a21 --- /dev/null +++ b/mtproto/rpc/rpc.go @@ -0,0 +1,5 @@ +package rpc + +type RPC interface { + Bytes() []byte +} diff --git a/mtproto/rpc/rpc_nonce_request.go b/mtproto/rpc/rpc_nonce_request.go new file mode 100644 index 0000000..8dd29e4 --- /dev/null +++ b/mtproto/rpc/rpc_nonce_request.go @@ -0,0 +1,67 @@ +package rpc + +import ( + "bytes" + "crypto/rand" + "encoding/binary" + "time" + + "github.com/juju/errors" +) + +const ( + RPCNonceSeqNo = -2 + + rpcNonceLength = 16 + rpcNonceKeySelectorLength = 4 + rpcNonceCryptoTSLength = 4 + rpcNonceTagLength = 4 + rpcNonceCryptoAESLength = 4 + + rpcNonceRequestLength = rpcNonceTagLength + rpcNonceKeySelectorLength + + rpcNonceCryptoAESLength + rpcNonceCryptoTSLength + rpcNonceLength +) + +var ( + rpcNonceTag = [rpcNonceTagLength]byte{0xaa, 0x87, 0xcb, 0x7a} + rpcNonceCryptoAESTag = [rpcNonceCryptoAESLength]byte{0x01, 0x00, 0x00, 0x00} +) + +type RPCNonceRequest struct { + KeySelector [rpcNonceKeySelectorLength]byte + CryptoTS [rpcNonceCryptoTSLength]byte + Nonce [rpcNonceLength]byte +} + +func (r *RPCNonceRequest) Bytes() []byte { + buf := &bytes.Buffer{} + buf.Grow(rpcNonceRequestLength) + + buf.Write(rpcNonceTag[:]) + buf.Write(r.KeySelector[:]) + buf.Write(rpcNonceCryptoAESTag[:]) + buf.Write(r.CryptoTS[:]) + buf.Write(r.Nonce[:]) + + return buf.Bytes() +} + +func NewRPCNonceRequest(proxySecret []byte) (*RPCNonceRequest, error) { + var nonce [rpcNonceLength]byte + var keySelector [rpcNonceKeySelectorLength]byte + var cryptoTS [rpcNonceCryptoTSLength]byte + + if _, err := rand.Read(nonce[:]); err != nil { + return nil, errors.Annotate(err, "Cannot generate nonce") + } + copy(keySelector[:], proxySecret) + + timestamp := time.Now().Truncate(time.Second).Unix() % 4294967296 // 256 ^ 4 - do not know how to name + binary.LittleEndian.PutUint32(cryptoTS[:], uint32(timestamp)) + + return &RPCNonceRequest{ + KeySelector: keySelector, + CryptoTS: cryptoTS, + Nonce: nonce, + }, nil +} diff --git a/mtproto/rpc/rpc_nonce_response.go b/mtproto/rpc/rpc_nonce_response.go new file mode 100644 index 0000000..7cf90fc --- /dev/null +++ b/mtproto/rpc/rpc_nonce_response.go @@ -0,0 +1,58 @@ +package rpc + +import ( + "bytes" + + "github.com/juju/errors" +) + +const rpcNonceResponseLength = rpcNonceRequestLength + +type RPCNonceResponse struct { + RPCNonceRequest + + RPCType [rpcNonceTagLength]byte + Crypto [rpcNonceCryptoAESLength]byte +} + +func (r *RPCNonceResponse) Bytes() []byte { + buf := &bytes.Buffer{} + buf.Grow(rpcNonceResponseLength) + + buf.Write(r.RPCType[:]) + buf.Write(r.KeySelector[:]) + buf.Write(r.Crypto[:]) + buf.Write(r.CryptoTS[:]) + buf.Write(r.Nonce[:]) + + return buf.Bytes() +} + +func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { + if r.RPCType != rpcNonceTag { + return errors.New("Unexpected RPC type") + } + if r.Crypto != rpcNonceCryptoAESTag { + return errors.New("Unexpected crypto type") + } + if r.KeySelector != req.KeySelector { + return errors.New("Unexpected key selector") + } + + return nil +} + +func NewRPCNonceResponse(data []byte) (*RPCNonceResponse, error) { + if len(data) != rpcNonceResponseLength { + return nil, errors.New("Unexpected message length") + } + + resp := RPCNonceResponse{} + copy(resp.RPCType[:], data[:4]) + copy(resp.KeySelector[:], data[4:8]) + copy(resp.Crypto[:], data[8:12]) + copy(resp.CryptoTS[:], data[12:16]) + copy(resp.Nonce[:], data[16:]) + + return &resp, nil +} From 625364835690bd52db69bd5a574db1f432ad5310 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 1 Jul 2018 12:01:56 +0300 Subject: [PATCH 04/71] Remove frame pool for obfuscated2 frames --- client/direct.go | 1 - obfuscated2/frame.go | 32 +++++++++++++++----------------- obfuscated2/frame_pool.go | 24 ------------------------ obfuscated2/frame_test.go | 8 ++++---- obfuscated2/obfuscated2.go | 18 ++++++++---------- obfuscated2/obfuscated2_test.go | 10 +++++----- telegram/direct.go | 3 +-- 7 files changed, 33 insertions(+), 63 deletions(-) delete mode 100644 obfuscated2/frame_pool.go diff --git a/client/direct.go b/client/direct.go index f4db89c..ed40dbe 100644 --- a/client/direct.go +++ b/client/direct.go @@ -29,7 +29,6 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, io if err != nil { return nil, nil, errors.Annotate(err, "Cannot extract frame") } - defer obfuscated2.ReturnFrame(frame) obfs2, connOpts, err := obfuscated2.ParseObfuscated2ClientFrame(conf.Secret, frame) if err != nil { diff --git a/obfuscated2/frame.go b/obfuscated2/frame.go index ea03241..6c6f6ab 100644 --- a/obfuscated2/frame.go +++ b/obfuscated2/frame.go @@ -66,57 +66,55 @@ func (f Frame) ConnectionType() (mtproto.ConnectionType, error) { // Invert inverts frame for extracting encryption keys. Pkease check that link: // https://blog.susanka.eu/how-telegram-obfuscates-its-mtproto-traffic/ -func (f Frame) Invert() *Frame { - reversed := MakeFrame() - copy(*reversed, f) +func (f Frame) Invert() Frame { + reversed := make(Frame, FrameLen) + copy(reversed, f) for i := 0; i < frameLenKey+frameLenIV; i++ { - (*reversed)[frameOffsetFirst+i] = f[frameOffsetIV-1-i] + reversed[frameOffsetFirst+i] = f[frameOffsetIV-1-i] } return reversed } // ExtractFrame extracts exact obfuscated2 handshake frame from given reader. -func ExtractFrame(conn io.Reader) (*Frame, error) { - frame := MakeFrame() - buf := bytes.NewBuffer(*frame) +func ExtractFrame(conn io.Reader) (Frame, error) { + frame := make(Frame, FrameLen) + buf := bytes.NewBuffer(frame) buf.Reset() if _, err := io.CopyN(buf, conn, FrameLen); err != nil { - ReturnFrame(frame) return nil, errors.Annotate(err, "Cannot extract obfuscated header") } - copy(*frame, buf.Bytes()) + copy(frame, buf.Bytes()) return frame, nil } -func generateFrame(connectionType mtproto.ConnectionType) *Frame { - frame := MakeFrame() - data := *frame +func generateFrame(connectionType mtproto.ConnectionType) Frame { + frame := make(Frame, FrameLen) for { - if _, err := rand.Read(data); err != nil { + if _, err := rand.Read(frame); err != nil { continue } - if data[0] == 0xef { + if frame[0] == 0xef { continue } - val := (uint32(data[3]) << 24) | (uint32(data[2]) << 16) | (uint32(data[1]) << 8) | uint32(data[0]) + val := (uint32(frame[3]) << 24) | (uint32(frame[2]) << 16) | (uint32(frame[1]) << 8) | uint32(frame[0]) if val == 0x44414548 || val == 0x54534f50 || val == 0x20544547 || val == 0x4954504f || val == 0xeeeeeeee { continue } - val = (uint32(data[7]) << 24) | (uint32(data[6]) << 16) | (uint32(data[5]) << 8) | uint32(data[4]) + val = (uint32(frame[7]) << 24) | (uint32(frame[6]) << 16) | (uint32(frame[5]) << 8) | uint32(frame[4]) if val == 0x00000000 { continue } // error has to be checked before calling this function tag, _ := connectionType.Tag() // nolint: errcheck - copy(data.Magic(), tag) + copy(frame.Magic(), tag) return frame } diff --git a/obfuscated2/frame_pool.go b/obfuscated2/frame_pool.go deleted file mode 100644 index 431073f..0000000 --- a/obfuscated2/frame_pool.go +++ /dev/null @@ -1,24 +0,0 @@ -package obfuscated2 - -import "sync" - -var framePool sync.Pool - -// MakeFrame returns new pointer to the handshake frame. -func MakeFrame() *Frame { - return framePool.Get().(*Frame) -} - -// ReturnFrame returns pointer to the handshake frame back to the pool. -func ReturnFrame(f *Frame) { - framePool.Put(f) -} - -func init() { - framePool = sync.Pool{ - New: func() interface{} { - data := make(Frame, FrameLen) - return &data - }, - } -} diff --git a/obfuscated2/frame_test.go b/obfuscated2/frame_test.go index 0c7e692..42d997c 100644 --- a/obfuscated2/frame_test.go +++ b/obfuscated2/frame_test.go @@ -54,21 +54,21 @@ func TestFrameValid(t *testing.T) { func TestFrameDoubleInvert(t *testing.T) { frame := makeFrame() - assert.True(t, bytes.Equal(frame, *frame.Invert().Invert())) + assert.True(t, bytes.Equal(frame, frame.Invert().Invert())) } func TestFrameInvert(t *testing.T) { frame := makeFrame() reversed := frame.Invert() - assert.Exactly(t, frame[:8], (*reversed)[:8]) - assert.Exactly(t, frame[56:], (*reversed)[56:]) + assert.Exactly(t, frame[:8], reversed[:8]) + assert.Exactly(t, frame[56:], reversed[56:]) toCompare := make([]byte, 48) for i := 0; i < 48; i++ { toCompare[i] = frame[55-i] } - assert.Equal(t, []byte((*reversed)[8:56]), toCompare) + assert.Equal(t, []byte(reversed[8:56]), toCompare) } func TestFrameGenerateValid(t *testing.T) { diff --git a/obfuscated2/obfuscated2.go b/obfuscated2/obfuscated2.go index bc027ad..50a34c7 100644 --- a/obfuscated2/obfuscated2.go +++ b/obfuscated2/obfuscated2.go @@ -21,7 +21,7 @@ type Obfuscated2 struct { // details: http://telegra.ph/telegram-blocks-wtf-05-26 // // Beware, link above is in russian. -func ParseObfuscated2ClientFrame(secret []byte, frame *Frame) (*Obfuscated2, *mtproto.ConnectionOpts, error) { +func ParseObfuscated2ClientFrame(secret []byte, frame Frame) (*Obfuscated2, *mtproto.ConnectionOpts, error) { decHasher := sha256.New() decHasher.Write(frame.Key()) // nolint: errcheck decHasher.Write(secret) // nolint: errcheck @@ -33,9 +33,8 @@ func ParseObfuscated2ClientFrame(secret []byte, frame *Frame) (*Obfuscated2, *mt encHasher.Write(secret) // nolint: errcheck encryptor := makeStreamCipher(encHasher.Sum(nil), invertedFrame.IV()) - decryptedFrame := MakeFrame() - defer ReturnFrame(decryptedFrame) - decryptor.XORKeyStream(*decryptedFrame, *frame) + decryptedFrame := make(Frame, FrameLen) + decryptor.XORKeyStream(decryptedFrame, frame) connType, err := decryptedFrame.ConnectionType() if err != nil { return nil, nil, errors.Annotate(err, "Unknown protocol") @@ -56,18 +55,17 @@ func ParseObfuscated2ClientFrame(secret []byte, frame *Frame) (*Obfuscated2, *mt // MakeTelegramObfuscated2Frame creates new handshake frame to send to // Telegram. // https://blog.susanka.eu/how-telegram-obfuscates-its-mtproto-traffic/ -func MakeTelegramObfuscated2Frame(opts *mtproto.ConnectionOpts) (*Obfuscated2, *Frame) { +func MakeTelegramObfuscated2Frame(opts *mtproto.ConnectionOpts) (*Obfuscated2, Frame) { frame := generateFrame(opts.ConnectionType) encryptor := makeStreamCipher(frame.Key(), frame.IV()) decryptorFrame := frame.Invert() decryptor := makeStreamCipher(decryptorFrame.Key(), decryptorFrame.IV()) - copyFrame := MakeFrame() - defer ReturnFrame(copyFrame) - copy((*copyFrame)[:frameOffsetIV], (*frame)[:frameOffsetIV]) - encryptor.XORKeyStream(*frame, *frame) - copy((*frame)[:frameOffsetIV], (*copyFrame)[:frameOffsetIV]) + copyFrame := make(Frame, FrameLen) + copy(copyFrame[:frameOffsetIV], frame[:frameOffsetIV]) + encryptor.XORKeyStream(frame, frame) + copy(frame[:frameOffsetIV], copyFrame[:frameOffsetIV]) obfs := &Obfuscated2{ Decryptor: decryptor, diff --git a/obfuscated2/obfuscated2_test.go b/obfuscated2/obfuscated2_test.go index 0c7b1a8..c6b5c4a 100644 --- a/obfuscated2/obfuscated2_test.go +++ b/obfuscated2/obfuscated2_test.go @@ -18,7 +18,7 @@ func TestObfs2TelegramFrameDecrypt(t *testing.T) { decryptor := makeStreamCipher(frame.Key(), frame.IV()) decrypted := make(Frame, FrameLen) - decryptor.XORKeyStream(decrypted, *frame) + decryptor.XORKeyStream(decrypted, frame) _, err := decrypted.ConnectionType() assert.Nil(t, err) @@ -53,8 +53,8 @@ func TestObfs2Full(t *testing.T) { encryptor := makeStreamCipher(clientKey, clientFrame.IV()) encrypted := make(Frame, FrameLen) - encryptor.XORKeyStream(encrypted, *clientFrame) - copy(encrypted[:56], (*clientFrame)[:56]) + encryptor.XORKeyStream(encrypted, clientFrame) + copy(encrypted[:56], clientFrame[:56]) invertedClientFrame := clientFrame.Invert() clientHasher = sha256.New() @@ -63,7 +63,7 @@ func TestObfs2Full(t *testing.T) { invertedClientKey := clientHasher.Sum(nil) clientDecryptor := makeStreamCipher(invertedClientKey, invertedClientFrame.IV()) - clientObfs, _, err := ParseObfuscated2ClientFrame(secret, &encrypted) + clientObfs, _, err := ParseObfuscated2ClientFrame(secret, encrypted) assert.Nil(t, err) connOpts := &mtproto.ConnectionOpts{ @@ -73,7 +73,7 @@ func TestObfs2Full(t *testing.T) { tgObfs, tgFrame := MakeTelegramObfuscated2Frame(connOpts) tgDecryptor := makeStreamCipher(tgFrame.Key(), tgFrame.IV()) decrypted := make(Frame, FrameLen) - tgDecryptor.XORKeyStream(decrypted, *tgFrame) + tgDecryptor.XORKeyStream(decrypted, tgFrame) _, err = decrypted.ConnectionType() assert.Nil(t, err) diff --git a/telegram/direct.go b/telegram/direct.go index f1ee52f..5681415 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -46,9 +46,8 @@ func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWriteClo func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn io.ReadWriteCloser) (io.ReadWriteCloser, error) { obfs2, frame := obfuscated2.MakeTelegramObfuscated2Frame(connOpts) - defer obfuscated2.ReturnFrame(frame) - if n, err := conn.Write(*frame); err != nil || n != len(*frame) { + if n, err := conn.Write(frame); err != nil || n != obfuscated2.FrameLen { return nil, errors.Annotate(err, "Cannot write hadnshake frame") } From b9ee1e185710556d69aa679e2b4b44c4118e5395 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 1 Jul 2018 12:36:30 +0300 Subject: [PATCH 05/71] Add rpc request/response for handshake --- mtproto/rpc/rpc.go | 9 ++++- mtproto/rpc/rpc_handshake_request.go | 47 ++++++++++++++++++++++++ mtproto/rpc/rpc_handshake_response.go | 53 +++++++++++++++++++++++++++ mtproto/rpc/rpc_nonce_request.go | 6 +-- mtproto/rpc/rpc_nonce_response.go | 4 +- 5 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 mtproto/rpc/rpc_handshake_request.go create mode 100644 mtproto/rpc/rpc_handshake_response.go diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index dc07a21..f72bfdd 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -1,5 +1,12 @@ package rpc +import "bytes" + +const ( + RPCNonceSeqNo = -2 + RPCHandshakeSeqNo = -1 +) + type RPC interface { - Bytes() []byte + Bytes() *bytes.Buffer } diff --git a/mtproto/rpc/rpc_handshake_request.go b/mtproto/rpc/rpc_handshake_request.go new file mode 100644 index 0000000..662d7a7 --- /dev/null +++ b/mtproto/rpc/rpc_handshake_request.go @@ -0,0 +1,47 @@ +package rpc + +import "bytes" + +const ( + rpcHandshakeTagLength = 4 + rpcHandshakeFlagsLength = 4 + rpcHandshakeSenderPIDLength = 12 + rpcHandshakePeerPIDLength = rpcHandshakeSenderPIDLength + + rpcHandshakeRequestLength = rpcHandshakeTagLength + rpcHandshakeFlagsLength + + rpcHandshakeSenderPIDLength + rpcHandshakePeerPIDLength +) + +var ( + rpcHandshakeSenderPID [rpcHandshakeSenderPIDLength]byte + rpcHandshakePeerPID [rpcHandshakePeerPIDLength]byte + + rpcHandshakeTag = [rpcHandshakeTagLength]byte{0xf5, 0xee, 0x82, 0x76} + rpcHandshakeFlags = [rpcHandshakeFlagsLength]byte{0x00, 0x00, 0x00, 0x00} + + rpcHandshakeBuffer *bytes.Buffer +) + +type RPCHandshakeRequest struct { +} + +func (r *RPCHandshakeRequest) Bytes() *bytes.Buffer { + buf := &bytes.Buffer{} + buf.Grow(rpcHandshakeRequestLength) + + buf.Write(rpcHandshakeTag[:]) + buf.Write(rpcHandshakeFlags[:]) + buf.Write(rpcHandshakeSenderPID[:]) + buf.Write(rpcHandshakePeerPID[:]) + + return buf +} + +func init() { + copy(rpcHandshakeSenderPID[:], "IPIPPRPDTIME") + copy(rpcHandshakePeerPID[:], "IPIPPRPDTIME") +} + +func NewRPCHandshakeRequest() *RPCHandshakeRequest { + return &RPCHandshakeRequest{} +} diff --git a/mtproto/rpc/rpc_handshake_response.go b/mtproto/rpc/rpc_handshake_response.go new file mode 100644 index 0000000..9af0984 --- /dev/null +++ b/mtproto/rpc/rpc_handshake_response.go @@ -0,0 +1,53 @@ +package rpc + +import ( + "bytes" + + "github.com/juju/errors" +) + +const rpcHandshakeResponseLength = rpcHandshakeRequestLength + +type RPCHandshakeResponse struct { + Type [rpcHandshakeTagLength]byte + Flags [rpcHandshakeFlagsLength]byte + SenderPID [rpcHandshakeSenderPIDLength]byte + PeerPID [rpcHandshakePeerPIDLength]byte +} + +func (r *RPCHandshakeResponse) Bytes() *bytes.Buffer { + buf := &bytes.Buffer{} + buf.Grow(rpcHandshakeResponseLength) + + buf.Write(r.Type[:]) + buf.Write(r.Flags[:]) + buf.Write(r.SenderPID[:]) + buf.Write(r.PeerPID[:]) + + return buf +} + +func (r *RPCHandshakeResponse) Valid(req *RPCHandshakeRequest) error { + if r.Type != rpcHandshakeTag { + return errors.New("Unexpected handshake tag") + } + if r.PeerPID != rpcHandshakeSenderPID { + return errors.New("Incorrect sender PID") + } + + return nil +} + +func NewRPCHandshakeResponse(data []byte) (*RPCHandshakeResponse, error) { + if len(data) != rpcHandshakeResponseLength { + return nil, errors.New("Incorrect handshake response length") + } + + resp := RPCHandshakeResponse{} + copy(resp.Type[:], data[:4]) + copy(resp.Flags[:], data[4:8]) + copy(resp.SenderPID[:], data[8:20]) + copy(resp.PeerPID[:], data[20:]) + + return &resp, nil +} diff --git a/mtproto/rpc/rpc_nonce_request.go b/mtproto/rpc/rpc_nonce_request.go index 8dd29e4..b94d814 100644 --- a/mtproto/rpc/rpc_nonce_request.go +++ b/mtproto/rpc/rpc_nonce_request.go @@ -10,8 +10,6 @@ import ( ) const ( - RPCNonceSeqNo = -2 - rpcNonceLength = 16 rpcNonceKeySelectorLength = 4 rpcNonceCryptoTSLength = 4 @@ -33,7 +31,7 @@ type RPCNonceRequest struct { Nonce [rpcNonceLength]byte } -func (r *RPCNonceRequest) Bytes() []byte { +func (r *RPCNonceRequest) Bytes() *bytes.Buffer { buf := &bytes.Buffer{} buf.Grow(rpcNonceRequestLength) @@ -43,7 +41,7 @@ func (r *RPCNonceRequest) Bytes() []byte { buf.Write(r.CryptoTS[:]) buf.Write(r.Nonce[:]) - return buf.Bytes() + return buf } func NewRPCNonceRequest(proxySecret []byte) (*RPCNonceRequest, error) { diff --git a/mtproto/rpc/rpc_nonce_response.go b/mtproto/rpc/rpc_nonce_response.go index 7cf90fc..cad02c3 100644 --- a/mtproto/rpc/rpc_nonce_response.go +++ b/mtproto/rpc/rpc_nonce_response.go @@ -15,7 +15,7 @@ type RPCNonceResponse struct { Crypto [rpcNonceCryptoAESLength]byte } -func (r *RPCNonceResponse) Bytes() []byte { +func (r *RPCNonceResponse) Bytes() *bytes.Buffer { buf := &bytes.Buffer{} buf.Grow(rpcNonceResponseLength) @@ -25,7 +25,7 @@ func (r *RPCNonceResponse) Bytes() []byte { buf.Write(r.CryptoTS[:]) buf.Write(r.Nonce[:]) - return buf.Bytes() + return buf } func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { From 5e82dce22e7a7802297a19e25975734552dcf784 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 1 Jul 2018 13:17:03 +0300 Subject: [PATCH 06/71] Add rpc proxy flags --- mtproto/rpc/rpc_proxy_flags.go | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 mtproto/rpc/rpc_proxy_flags.go diff --git a/mtproto/rpc/rpc_proxy_flags.go b/mtproto/rpc/rpc_proxy_flags.go new file mode 100644 index 0000000..8e0f202 --- /dev/null +++ b/mtproto/rpc/rpc_proxy_flags.go @@ -0,0 +1,51 @@ +package rpc + +import ( + "bytes" + "encoding/binary" + + "github.com/9seconds/mtg/mtproto" +) + +type RPCProxyRequestFlags uint32 + +const ( + RPCProxyRequestFlagsHasAdTag RPCProxyRequestFlags = 0x8 + RPCProxyRequestFlagsEncrypted = 0x2 + RPCProxyRequestFlagsMagic = 0x1000 + RPCProxyRequestFlagsExtMode2 = 0x20000 + RPCProxyRequestFlagsIntermediate = 0x20000000 + RPCProxyRequestFlagsAbdridged = 0x40000000 + RPCProxyRequestFlagsQuickAck = 0x80000000 +) + +var rpcProxyRequestFlagsEncryptedPrefix [8]byte + +func (r RPCProxyRequestFlags) Bytes() []byte { + converted := make([]byte, 4) + binary.LittleEndian.PutUint32(converted, uint32(r)) + + return converted +} + +func NewRPCRproxyRequestFlags(connectionType mtproto.ConnectionType, quickAck bool, message []byte) RPCProxyRequestFlags { + flags := RPCProxyRequestFlagsHasAdTag + flags |= RPCProxyRequestFlagsMagic + flags |= RPCProxyRequestFlagsExtMode2 + + switch connectionType { + case mtproto.ConnectionTypeAbridged: + flags |= RPCProxyRequestFlagsAbdridged + case mtproto.ConnectionTypeIntermediate: + flags |= RPCProxyRequestFlagsIntermediate + } + + if quickAck { + flags |= RPCProxyRequestFlagsQuickAck + } + if bytes.HasPrefix(message, rpcProxyRequestFlagsEncryptedPrefix[:]) { + flags |= RPCProxyRequestFlagsEncrypted + } + + return flags +} From 0e2898f6be9ea71980ba8d13aaf28a9c1c94c9e2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 08:09:22 +0300 Subject: [PATCH 07/71] Return back timeoutrwc --- client/direct.go | 7 +++---- config/config.go | 3 +++ telegram/dialer.go | 7 +++---- wrappers/timeoutrwc.go | 31 +++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 wrappers/timeoutrwc.go diff --git a/client/direct.go b/client/direct.go index ed40dbe..23e9613 100644 --- a/client/direct.go +++ b/client/direct.go @@ -13,9 +13,7 @@ import ( "github.com/9seconds/mtg/wrappers" ) -const ( - handshakeTimeout = 10 * time.Second -) +const handshakeTimeout = 10 * time.Second // DirectInit initializes client to access Telegram bypassing middleproxies. func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, io.ReadWriteCloser, error) { @@ -36,7 +34,8 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, io } connOpts.ConnectionProto = mtproto.ConnectionProtocolAny - socket := wrappers.NewStreamCipherRWC(conn, obfs2.Encryptor, obfs2.Decryptor) + socket := wrappers.NewTimeoutRWC(conn) + socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) return connOpts, socket, nil } diff --git a/config/config.go b/config/config.go index 54f4cba..9935fa8 100644 --- a/config/config.go +++ b/config/config.go @@ -16,6 +16,9 @@ const ( BufferReadSize = 32 * 1024 BufferSizeCopy = 32 * 1024 + TimeoutRead = time.Minute + TimeoutWrite = time.Minute + keepAlivePeriod = 20 * time.Second ) diff --git a/telegram/dialer.go b/telegram/dialer.go index ef51f50..f6b2cf7 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -8,11 +8,10 @@ import ( "github.com/juju/errors" "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/wrappers" ) -const ( - telegramDialTimeout = 10 * time.Second -) +const telegramDialTimeout = 10 * time.Second type tgDialer struct { net.Dialer @@ -36,5 +35,5 @@ func (t *tgDialer) dialRWC(addr string) (io.ReadWriteCloser, error) { return nil, err } - return conn, nil + return wrappers.NewTimeoutRWC(conn), nil } diff --git a/wrappers/timeoutrwc.go b/wrappers/timeoutrwc.go new file mode 100644 index 0000000..e4ea9b0 --- /dev/null +++ b/wrappers/timeoutrwc.go @@ -0,0 +1,31 @@ +package wrappers + +import ( + "io" + "net" + "time" + + "github.com/9seconds/mtg/config" +) + +type TimeoutReadWriteCloser struct { + conn net.Conn +} + +func (t *TimeoutReadWriteCloser) Read(p []byte) (int, error) { + t.conn.SetReadDeadline(time.Now().Add(config.TimeoutRead)) + return t.conn.Read(p) +} + +func (t *TimeoutReadWriteCloser) Write(p []byte) (int, error) { + t.conn.SetWriteDeadline(time.Now().Add(config.TimeoutWrite)) + return t.conn.Write(p) +} + +func (t *TimeoutReadWriteCloser) Close() error { + return t.conn.Close() +} + +func NewTimeoutRWC(conn net.Conn) io.ReadWriteCloser { + return &TimeoutReadWriteCloser{conn} +} From 96bb6fbe980fb64d82ae941161c5079a75f9d054 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 09:15:03 +0300 Subject: [PATCH 08/71] Add rpc proxy request --- mtproto/bufferpool/bufferpool.go | 32 +++++++++++ mtproto/extras.go | 6 ++ mtproto/rpc/rpc_proxy_flags.go | 29 +--------- mtproto/rpc/rpc_proxy_request.go | 98 ++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 mtproto/bufferpool/bufferpool.go create mode 100644 mtproto/extras.go create mode 100644 mtproto/rpc/rpc_proxy_request.go diff --git a/mtproto/bufferpool/bufferpool.go b/mtproto/bufferpool/bufferpool.go new file mode 100644 index 0000000..f58a88e --- /dev/null +++ b/mtproto/bufferpool/bufferpool.go @@ -0,0 +1,32 @@ +package bufferpool + +import ( + "bytes" + "sync" +) + +const bufferPoolSize = 4 * 1024 + +var bufferPool sync.Pool + +func Get() *bytes.Buffer { + buf := bufferPool.Get().(*bytes.Buffer) + buf.Reset() + + return buf +} + +func Return(buf *bytes.Buffer) { + bufferPool.Put(buf) +} + +func init() { + bufferPool = sync.Pool{ + New: func() interface{} { + buf := &bytes.Buffer{} + buf.Grow(bufferPoolSize) + + return buf + }, + } +} diff --git a/mtproto/extras.go b/mtproto/extras.go new file mode 100644 index 0000000..5c0ad19 --- /dev/null +++ b/mtproto/extras.go @@ -0,0 +1,6 @@ +package mtproto + +type Extras struct { + QuickAck bool + SimpleAck bool +} diff --git a/mtproto/rpc/rpc_proxy_flags.go b/mtproto/rpc/rpc_proxy_flags.go index 8e0f202..4edd74d 100644 --- a/mtproto/rpc/rpc_proxy_flags.go +++ b/mtproto/rpc/rpc_proxy_flags.go @@ -1,11 +1,6 @@ package rpc -import ( - "bytes" - "encoding/binary" - - "github.com/9seconds/mtg/mtproto" -) +import "encoding/binary" type RPCProxyRequestFlags uint32 @@ -27,25 +22,3 @@ func (r RPCProxyRequestFlags) Bytes() []byte { return converted } - -func NewRPCRproxyRequestFlags(connectionType mtproto.ConnectionType, quickAck bool, message []byte) RPCProxyRequestFlags { - flags := RPCProxyRequestFlagsHasAdTag - flags |= RPCProxyRequestFlagsMagic - flags |= RPCProxyRequestFlagsExtMode2 - - switch connectionType { - case mtproto.ConnectionTypeAbridged: - flags |= RPCProxyRequestFlagsAbdridged - case mtproto.ConnectionTypeIntermediate: - flags |= RPCProxyRequestFlagsIntermediate - } - - if quickAck { - flags |= RPCProxyRequestFlagsQuickAck - } - if bytes.HasPrefix(message, rpcProxyRequestFlagsEncryptedPrefix[:]) { - flags |= RPCProxyRequestFlagsEncrypted - } - - return flags -} diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go new file mode 100644 index 0000000..f9df232 --- /dev/null +++ b/mtproto/rpc/rpc_proxy_request.go @@ -0,0 +1,98 @@ +package rpc + +import ( + "bytes" + "crypto/rand" + "encoding/binary" + "net" + + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/mtproto/bufferpool" + "github.com/juju/errors" +) + +const ( + rpcProxyRequestConnectionIDLength = 8 + rpcProxyRequestIPPortLength = 16 + 4 +) + +var ( + rpcProxyRequestTag = []byte{0xee, 0xf1, 0xce, 0x36} + rpcProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} + rpcProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} +) + +type RPCProxyRequest struct { + Flags RPCProxyRequestFlags + ConnectionID [rpcProxyRequestConnectionIDLength]byte + RemoteIPPort [rpcProxyRequestIPPortLength]byte + LocalIPPort [rpcProxyRequestIPPortLength]byte + ADTag []byte + Message *bytes.Buffer + Extras *mtproto.Extras +} + +func (r *RPCProxyRequest) Bytes() *bytes.Buffer { + buf := bufferpool.Get() + + flags := r.Flags + if r.Extras.QuickAck { + flags |= RPCProxyRequestFlagsQuickAck + } + + messageBytes := r.Message.Bytes() + if bytes.HasPrefix(messageBytes, rpcProxyRequestFlagsEncryptedPrefix[:]) { + flags |= RPCProxyRequestFlagsEncrypted + } + + buf.Write(rpcProxyRequestTag) + buf.Write(flags.Bytes()) + buf.Write(r.ConnectionID[:]) + buf.Write(r.RemoteIPPort[:]) + buf.Write(r.LocalIPPort[:]) + buf.Write(rpcProxyRequestExtraSize) + buf.Write(rpcProxyRequestProxyTag) + buf.WriteByte(byte(len(r.ADTag))) + buf.Write(r.ADTag) + + for i := 0; i < (buf.Len() % 4); i++ { + buf.WriteByte(0x00) + } + if r.Message != nil { + buf.Write(messageBytes) + } + + return buf +} + +func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *net.TCPAddr, adTag []byte, extras *mtproto.Extras) (*RPCProxyRequest, error) { + flags := RPCProxyRequestFlagsHasAdTag | RPCProxyRequestFlagsMagic | RPCProxyRequestFlagsExtMode2 + + switch connectionType { + case mtproto.ConnectionTypeAbridged: + flags |= RPCProxyRequestFlagsAbdridged + case mtproto.ConnectionTypeIntermediate: + flags |= RPCProxyRequestFlagsIntermediate + } + + request := RPCProxyRequest{ + Flags: flags, + ADTag: adTag, + Extras: extras, + } + + if _, err := rand.Read(request.ConnectionID[:]); err != nil { + return nil, errors.Annotate(err, "Cannot generate connection ID") + } + + port := make([]byte, 4) + copy(request.LocalIPPort[:], local.IP.To16()) + binary.LittleEndian.PutUint32(port, uint32(local.Port)) + copy(request.LocalIPPort[16:], port) + + copy(request.RemoteIPPort[:], remote.IP.To16()) + binary.LittleEndian.PutUint32(port, uint32(remote.Port)) + copy(request.RemoteIPPort[16:], port) + + return &request, nil +} From b74cc8d3ea19533a719e3e34754fba88fbd5b766 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 10:20:48 +0300 Subject: [PATCH 09/71] Add blockcipherrwc --- wrappers/blockcipherrwc.go | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 wrappers/blockcipherrwc.go diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go new file mode 100644 index 0000000..386432c --- /dev/null +++ b/wrappers/blockcipherrwc.go @@ -0,0 +1,84 @@ +package wrappers + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "io" + + "github.com/juju/errors" +) + +type BlockCipherReadWriteCloser struct { + buf *bytes.Buffer + + conn io.ReadWriteCloser + encryptor cipher.BlockMode + decryptor cipher.BlockMode +} + +func (c *BlockCipherReadWriteCloser) Read(p []byte) (int, error) { + if c.buf.Len() > 0 { + return c.flush(p) + } + + for c.buf.Len() == 0 || c.buf.Len()%aes.BlockSize != 0 { + n, err := c.conn.Read(p) + if err != nil { + return 0, errors.Annotate(err, "Cannot read from socket") + } + c.buf.Write(p[:n]) + } + + return c.flush(p) +} + +func (c *BlockCipherReadWriteCloser) Write(p []byte) (int, error) { + if len(p)%aes.BlockSize > 0 { + return 0, errors.Errorf("Incorrect block size %d", len(p)) + } + + buf := getBuffer() + defer putBuffer(buf) + buf.Grow(len(p)) + buf.Write(p) + + encrypted := buf.Bytes() + c.encryptor.CryptBlocks(encrypted, p) + + return c.conn.Write(encrypted) +} + +func (c *BlockCipherReadWriteCloser) Close() error { + defer putBuffer(c.buf) + return c.conn.Close() +} + +func (c *BlockCipherReadWriteCloser) flush(p []byte) (int, error) { + sizeToRead := len(p) + if c.buf.Len() < sizeToRead { + sizeToRead = c.buf.Len() + } + sizeToRead = aes.BlockSize * (sizeToRead / aes.BlockSize) + + c.decryptor.CryptBlocks(p, c.buf.Bytes()[:sizeToRead]) + if sizeToRead == c.buf.Len() { + c.buf.Reset() + } else { + leftover := c.buf.Bytes()[sizeToRead:] + putBuffer(c.buf) + c.buf = getBuffer() + c.buf.Write(leftover) + } + + return sizeToRead, nil +} + +func NewBlockCipherRWC(conn io.ReadWriteCloser, encryptor, decryptor cipher.BlockMode) io.ReadWriteCloser { + return &BlockCipherReadWriteCloser{ + buf: getBuffer(), + conn: conn, + encryptor: encryptor, + decryptor: decryptor, + } +} From 2c387cc21344ec69f648661d635afb04ac70aa8e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 10:33:25 +0300 Subject: [PATCH 10/71] Add bytesrwc --- mtproto/{bufferpool => }/bufferpool.go | 6 ++-- mtproto/rpc/rpc_proxy_request.go | 6 ++-- mtproto/rwc.go | 48 ++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) rename mtproto/{bufferpool => }/bufferpool.go (78%) create mode 100644 mtproto/rwc.go diff --git a/mtproto/bufferpool/bufferpool.go b/mtproto/bufferpool.go similarity index 78% rename from mtproto/bufferpool/bufferpool.go rename to mtproto/bufferpool.go index f58a88e..2c8c246 100644 --- a/mtproto/bufferpool/bufferpool.go +++ b/mtproto/bufferpool.go @@ -1,4 +1,4 @@ -package bufferpool +package mtproto import ( "bytes" @@ -9,14 +9,14 @@ const bufferPoolSize = 4 * 1024 var bufferPool sync.Pool -func Get() *bytes.Buffer { +func GetBuffer() *bytes.Buffer { buf := bufferPool.Get().(*bytes.Buffer) buf.Reset() return buf } -func Return(buf *bytes.Buffer) { +func ReturnBuffer(buf *bytes.Buffer) { bufferPool.Put(buf) } diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go index f9df232..e7d0a4a 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/rpc_proxy_request.go @@ -6,9 +6,9 @@ import ( "encoding/binary" "net" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/mtproto/bufferpool" "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" ) const ( @@ -33,7 +33,7 @@ type RPCProxyRequest struct { } func (r *RPCProxyRequest) Bytes() *bytes.Buffer { - buf := bufferpool.Get() + buf := mtproto.GetBuffer() flags := r.Flags if r.Extras.QuickAck { diff --git a/mtproto/rwc.go b/mtproto/rwc.go new file mode 100644 index 0000000..ad14b8c --- /dev/null +++ b/mtproto/rwc.go @@ -0,0 +1,48 @@ +package mtproto + +import ( + "bytes" + "io" +) + +type BytesRWC interface { + Write(*bytes.Buffer) (int, error) + Read([]byte) (int, error) + Close() error +} + +type StartBytesRWC struct { + conn BytesRWC +} + +func (s *StartBytesRWC) Write(p []byte) (int, error) { + buf := GetBuffer() + buf.Write(p) + defer ReturnBuffer(buf) + + return s.conn.Write(buf) +} + +func (s *StartBytesRWC) Read(p []byte) (int, error) { + return s.conn.Read(p) +} + +func (s *StartBytesRWC) Close() error { + return s.conn.Close() +} + +type FinishBytesRWC struct { + conn io.ReadWriteCloser +} + +func (f *FinishBytesRWC) Write(buf *bytes.Buffer) (int, error) { + return f.conn.Write(buf.Bytes()) +} + +func (f *FinishBytesRWC) Read(p []byte) (int, error) { + return f.conn.Read(p) +} + +func (f *FinishBytesRWC) Close() error { + return f.conn.Close() +} From 420159a7d30cc9e8c2201613989eb7adfb5f41e4 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 11:43:11 +0300 Subject: [PATCH 11/71] Add framerwc --- mtproto/wrappers/frame.go | 138 +++++++++++++++++++++++++++++++++++++ wrappers/blockcipherrwc.go | 12 ++-- 2 files changed, 145 insertions(+), 5 deletions(-) create mode 100644 mtproto/wrappers/frame.go diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go new file mode 100644 index 0000000..b03484b --- /dev/null +++ b/mtproto/wrappers/frame.go @@ -0,0 +1,138 @@ +package wrappers + +import ( + "bytes" + "crypto/aes" + "encoding/binary" + "hash/crc32" + "io" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" +) + +// Frame: { MessageLength(4) | SequenceNumber(4) | Message(???) | CRC32(4) [| padding(4), ...] } +const ( + frameRWCMinMessageLength = 12 + frameRWCMaxMessageLength = 16777216 +) + +var frameRWCPadding = [4]byte{0x04, 0x00, 0x00, 0x00} + +type FrameRWC struct { + conn mtproto.BytesRWC + + readSeqNo int32 + writeSeqNo int32 + readBuf *bytes.Buffer +} + +func (f *FrameRWC) Write(buf *bytes.Buffer) (int, error) { + writeBuf := mtproto.GetBuffer() + defer mtproto.ReturnBuffer(writeBuf) + + // 4 - len bytes + // 4 - seq bytes + // . - message + // 4 - crc32 + messageLength := 4 + 4 + buf.Len() + 4 + paddingLength := (aes.BlockSize - messageLength%aes.BlockSize) % aes.BlockSize + writeBuf.Grow(messageLength + paddingLength) + + binary.Write(writeBuf, binary.LittleEndian, uint32(messageLength)) + binary.Write(writeBuf, binary.LittleEndian, f.writeSeqNo) + writeBuf.Write(buf.Bytes()) + f.writeSeqNo++ + + checksum := crc32.ChecksumIEEE(writeBuf.Bytes()) + binary.Write(writeBuf, binary.LittleEndian, checksum) + writeBuf.Write(bytes.Repeat(frameRWCPadding[:], paddingLength/4)) + + return f.conn.Write(writeBuf) +} + +func (f *FrameRWC) Read(p []byte) (int, error) { + if f.readBuf.Len() > 0 { + return f.flush(p) + } + + buf := mtproto.GetBuffer() + defer mtproto.ReturnBuffer(buf) + + for { + buf.Reset() + if _, err := io.CopyN(buf, f.conn, 4); err != nil { + return 0, errors.Annotate(err, "Cannot read frame padding") + } + if !bytes.Equal(buf.Bytes(), frameRWCPadding[:]) { + break + } + } + + messageLength := binary.LittleEndian.Uint32(buf.Bytes()) + if messageLength%4 != 0 || messageLength < frameRWCMinMessageLength || messageLength > frameRWCMaxMessageLength { + return 0, errors.Errorf("Incorrect frame message length %d", messageLength) + } + sum := crc32.NewIEEE() + sum.Write(buf.Bytes()) + + buf.Reset() + buf.Grow(int(messageLength) - 4) // -4 because we already read the first number + if _, err := io.CopyN(buf, f.conn, int64(messageLength)-4); err != nil { + return 0, errors.Annotate(err, "Cannot read the message frame") + } + sum.Write(buf.Bytes()) + + var seqNo int32 + binary.Read(buf, binary.LittleEndian, seqNo) + if seqNo != f.readSeqNo { + return 0, errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, f.readSeqNo) + } + f.readSeqNo++ + + data := buf.Bytes()[:int(messageLength)-4-4-4] + checksum := binary.LittleEndian.Uint32(buf.Bytes()[int(messageLength)-4-4-4:]) + if checksum != sum.Sum32() { + return 0, errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) + + } + f.readBuf.Write(data) + + return f.flush(p) +} + +func (f *FrameRWC) Close() error { + defer mtproto.ReturnBuffer(f.readBuf) + return f.conn.Close() +} + +func (f *FrameRWC) flush(p []byte) (int, error) { + sizeToRead := len(p) + if f.readBuf.Len() < sizeToRead { + sizeToRead = f.readBuf.Len() + } + + data := f.readBuf.Bytes() + copy(p, data[:sizeToRead]) + if sizeToRead == f.readBuf.Len() { + f.readBuf.Reset() + } else { + newBuf := mtproto.GetBuffer() + newBuf.Write(data[sizeToRead:]) + + mtproto.ReturnBuffer(f.readBuf) + f.readBuf = newBuf + } + + return sizeToRead, nil +} + +func NewFrameRWC(conn mtproto.BytesRWC, seqNo int32) mtproto.BytesRWC { + return &FrameRWC{ + conn: conn, + readSeqNo: seqNo, + writeSeqNo: seqNo, + readBuf: mtproto.GetBuffer(), + } +} diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go index 386432c..1645034 100644 --- a/wrappers/blockcipherrwc.go +++ b/wrappers/blockcipherrwc.go @@ -29,6 +29,7 @@ func (c *BlockCipherReadWriteCloser) Read(p []byte) (int, error) { } c.buf.Write(p[:n]) } + c.decryptor.CryptBlocks(c.buf.Bytes(), c.buf.Bytes()) return c.flush(p) } @@ -59,16 +60,17 @@ func (c *BlockCipherReadWriteCloser) flush(p []byte) (int, error) { if c.buf.Len() < sizeToRead { sizeToRead = c.buf.Len() } - sizeToRead = aes.BlockSize * (sizeToRead / aes.BlockSize) - c.decryptor.CryptBlocks(p, c.buf.Bytes()[:sizeToRead]) + data := c.buf.Bytes() + copy(p, data[:sizeToRead]) if sizeToRead == c.buf.Len() { c.buf.Reset() } else { - leftover := c.buf.Bytes()[sizeToRead:] + newBuf := getBuffer() + newBuf.Write(data[sizeToRead:]) + putBuffer(c.buf) - c.buf = getBuffer() - c.buf.Write(leftover) + c.buf = newBuf } return sizeToRead, nil From 5ea0c4f7bf56900f574a1159af5866c0b98b793f Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 13:56:19 +0300 Subject: [PATCH 12/71] Add middleproxycipher --- mtproto/crypt/crypt.go | 97 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 mtproto/crypt/crypt.go diff --git a/mtproto/crypt/crypt.go b/mtproto/crypt/crypt.go new file mode 100644 index 0000000..6530f2b --- /dev/null +++ b/mtproto/crypt/crypt.go @@ -0,0 +1,97 @@ +package crypt + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/sha1" + "encoding/binary" + "io" + "net" + + "github.com/9seconds/mtg/mtproto/rpc" + "github.com/9seconds/mtg/wrappers" +) + +type CipherPurpose uint8 + +const ( + CipherPurposeClient CipherPurpose = iota + CipherPurposeServer +) + +var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} + +func NewMiddleProxyCipherRWC(conn io.ReadWriteCloser, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) io.ReadWriteCloser { + encryptor := newCBCCipher(CipherPurposeClient, req, resp, client, remote, secret) + decryptor := newCBCCipher(CipherPurposeServer, req, resp, client, remote, secret) + + return wrappers.NewBlockCipherRWC(conn, encryptor, decryptor) +} + +func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) cipher.BlockMode { + message := bytes.Buffer{} + message.Write(resp.Nonce[:]) + message.Write(req.Nonce[:]) + message.Write(req.CryptoTS[:]) + + clientIPv4 := emptyIP[:] + serverIPv4 := emptyIP[:] + if client.IP.To4() != nil { + clientIPv4 = reverseBytes(client.IP.To4()) + serverIPv4 = reverseBytes(remote.IP.To4()) + } + message.Write(serverIPv4) + + var port [2]byte + binary.LittleEndian.PutUint16(port[:], uint16(client.Port)) + message.Write(port[:]) + + switch purpose { + case CipherPurposeClient: + message.WriteString("CLIENT") + case CipherPurposeServer: + message.WriteString("SERVER") + default: + panic("Unexpected cipher purpose") + } + + message.Write(clientIPv4) + binary.LittleEndian.PutUint16(port[:], uint16(remote.Port)) + message.Write(port[:]) + message.Write(secret) + message.Write(resp.Nonce[:]) + + if client.IP.To4() == nil { + message.Write(client.IP.To16()) + message.Write(remote.IP.To16()) + } + message.Write(req.Nonce[:]) + + return makeCipher(message.Bytes()) +} + +func makeCipher(message []byte) cipher.BlockMode { + md5sum := md5.Sum(message[1:]) + sha1sum := sha1.Sum(message) + + key := append(md5sum[12:], sha1sum[:]...) + iv := md5.Sum(message[2:]) + + block, err := aes.NewCipher(key) + if err != nil { + panic("Cannot create cipher from the given key") + } + + return cipher.NewCBCEncrypter(block, iv[:]) +} + +func reverseBytes(data []byte) []byte { + rv := make([]byte, len(data)) + for k, v := range data { + rv[len(data)-1-k] = v + } + + return rv +} From 6fa5f31ca8251f2e817e4b99ffa5bf5722d69c3e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 15:44:52 +0300 Subject: [PATCH 13/71] Remove mtproto pool --- mtproto/bufferpool.go | 32 ------------------ mtproto/rpc/rpc.go | 6 ---- mtproto/rpc/rpc_handshake_request.go | 4 +-- mtproto/rpc/rpc_handshake_response.go | 4 +-- mtproto/rpc/rpc_nonce_request.go | 4 +-- mtproto/rpc/rpc_nonce_response.go | 4 +-- mtproto/rpc/rpc_proxy_request.go | 6 ++-- mtproto/rwc.go | 48 --------------------------- mtproto/{crypt => wrappers}/crypt.go | 2 +- mtproto/wrappers/frame.go | 31 ++++++----------- 10 files changed, 23 insertions(+), 118 deletions(-) delete mode 100644 mtproto/bufferpool.go delete mode 100644 mtproto/rwc.go rename mtproto/{crypt => wrappers}/crypt.go (99%) diff --git a/mtproto/bufferpool.go b/mtproto/bufferpool.go deleted file mode 100644 index 2c8c246..0000000 --- a/mtproto/bufferpool.go +++ /dev/null @@ -1,32 +0,0 @@ -package mtproto - -import ( - "bytes" - "sync" -) - -const bufferPoolSize = 4 * 1024 - -var bufferPool sync.Pool - -func GetBuffer() *bytes.Buffer { - buf := bufferPool.Get().(*bytes.Buffer) - buf.Reset() - - return buf -} - -func ReturnBuffer(buf *bytes.Buffer) { - bufferPool.Put(buf) -} - -func init() { - bufferPool = sync.Pool{ - New: func() interface{} { - buf := &bytes.Buffer{} - buf.Grow(bufferPoolSize) - - return buf - }, - } -} diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index f72bfdd..8f2ae99 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -1,12 +1,6 @@ package rpc -import "bytes" - const ( RPCNonceSeqNo = -2 RPCHandshakeSeqNo = -1 ) - -type RPC interface { - Bytes() *bytes.Buffer -} diff --git a/mtproto/rpc/rpc_handshake_request.go b/mtproto/rpc/rpc_handshake_request.go index 662d7a7..12e79e6 100644 --- a/mtproto/rpc/rpc_handshake_request.go +++ b/mtproto/rpc/rpc_handshake_request.go @@ -25,7 +25,7 @@ var ( type RPCHandshakeRequest struct { } -func (r *RPCHandshakeRequest) Bytes() *bytes.Buffer { +func (r *RPCHandshakeRequest) Bytes() []byte { buf := &bytes.Buffer{} buf.Grow(rpcHandshakeRequestLength) @@ -34,7 +34,7 @@ func (r *RPCHandshakeRequest) Bytes() *bytes.Buffer { buf.Write(rpcHandshakeSenderPID[:]) buf.Write(rpcHandshakePeerPID[:]) - return buf + return buf.Bytes() } func init() { diff --git a/mtproto/rpc/rpc_handshake_response.go b/mtproto/rpc/rpc_handshake_response.go index 9af0984..f415840 100644 --- a/mtproto/rpc/rpc_handshake_response.go +++ b/mtproto/rpc/rpc_handshake_response.go @@ -15,7 +15,7 @@ type RPCHandshakeResponse struct { PeerPID [rpcHandshakePeerPIDLength]byte } -func (r *RPCHandshakeResponse) Bytes() *bytes.Buffer { +func (r *RPCHandshakeResponse) Bytes() []byte { buf := &bytes.Buffer{} buf.Grow(rpcHandshakeResponseLength) @@ -24,7 +24,7 @@ func (r *RPCHandshakeResponse) Bytes() *bytes.Buffer { buf.Write(r.SenderPID[:]) buf.Write(r.PeerPID[:]) - return buf + return buf.Bytes() } func (r *RPCHandshakeResponse) Valid(req *RPCHandshakeRequest) error { diff --git a/mtproto/rpc/rpc_nonce_request.go b/mtproto/rpc/rpc_nonce_request.go index b94d814..3fcfe15 100644 --- a/mtproto/rpc/rpc_nonce_request.go +++ b/mtproto/rpc/rpc_nonce_request.go @@ -31,7 +31,7 @@ type RPCNonceRequest struct { Nonce [rpcNonceLength]byte } -func (r *RPCNonceRequest) Bytes() *bytes.Buffer { +func (r *RPCNonceRequest) Bytes() []byte { buf := &bytes.Buffer{} buf.Grow(rpcNonceRequestLength) @@ -41,7 +41,7 @@ func (r *RPCNonceRequest) Bytes() *bytes.Buffer { buf.Write(r.CryptoTS[:]) buf.Write(r.Nonce[:]) - return buf + return buf.Bytes() } func NewRPCNonceRequest(proxySecret []byte) (*RPCNonceRequest, error) { diff --git a/mtproto/rpc/rpc_nonce_response.go b/mtproto/rpc/rpc_nonce_response.go index cad02c3..7cf90fc 100644 --- a/mtproto/rpc/rpc_nonce_response.go +++ b/mtproto/rpc/rpc_nonce_response.go @@ -15,7 +15,7 @@ type RPCNonceResponse struct { Crypto [rpcNonceCryptoAESLength]byte } -func (r *RPCNonceResponse) Bytes() *bytes.Buffer { +func (r *RPCNonceResponse) Bytes() []byte { buf := &bytes.Buffer{} buf.Grow(rpcNonceResponseLength) @@ -25,7 +25,7 @@ func (r *RPCNonceResponse) Bytes() *bytes.Buffer { buf.Write(r.CryptoTS[:]) buf.Write(r.Nonce[:]) - return buf + return buf.Bytes() } func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go index e7d0a4a..84edafb 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/rpc_proxy_request.go @@ -32,8 +32,8 @@ type RPCProxyRequest struct { Extras *mtproto.Extras } -func (r *RPCProxyRequest) Bytes() *bytes.Buffer { - buf := mtproto.GetBuffer() +func (r *RPCProxyRequest) Bytes() []byte { + buf := &bytes.Buffer{} flags := r.Flags if r.Extras.QuickAck { @@ -62,7 +62,7 @@ func (r *RPCProxyRequest) Bytes() *bytes.Buffer { buf.Write(messageBytes) } - return buf + return buf.Bytes() } func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *net.TCPAddr, adTag []byte, extras *mtproto.Extras) (*RPCProxyRequest, error) { diff --git a/mtproto/rwc.go b/mtproto/rwc.go deleted file mode 100644 index ad14b8c..0000000 --- a/mtproto/rwc.go +++ /dev/null @@ -1,48 +0,0 @@ -package mtproto - -import ( - "bytes" - "io" -) - -type BytesRWC interface { - Write(*bytes.Buffer) (int, error) - Read([]byte) (int, error) - Close() error -} - -type StartBytesRWC struct { - conn BytesRWC -} - -func (s *StartBytesRWC) Write(p []byte) (int, error) { - buf := GetBuffer() - buf.Write(p) - defer ReturnBuffer(buf) - - return s.conn.Write(buf) -} - -func (s *StartBytesRWC) Read(p []byte) (int, error) { - return s.conn.Read(p) -} - -func (s *StartBytesRWC) Close() error { - return s.conn.Close() -} - -type FinishBytesRWC struct { - conn io.ReadWriteCloser -} - -func (f *FinishBytesRWC) Write(buf *bytes.Buffer) (int, error) { - return f.conn.Write(buf.Bytes()) -} - -func (f *FinishBytesRWC) Read(p []byte) (int, error) { - return f.conn.Read(p) -} - -func (f *FinishBytesRWC) Close() error { - return f.conn.Close() -} diff --git a/mtproto/crypt/crypt.go b/mtproto/wrappers/crypt.go similarity index 99% rename from mtproto/crypt/crypt.go rename to mtproto/wrappers/crypt.go index 6530f2b..fbff067 100644 --- a/mtproto/crypt/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -1,4 +1,4 @@ -package crypt +package wrappers import ( "bytes" diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index b03484b..9e7a4c7 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -8,8 +8,6 @@ import ( "io" "github.com/juju/errors" - - "github.com/9seconds/mtg/mtproto" ) // Frame: { MessageLength(4) | SequenceNumber(4) | Message(???) | CRC32(4) [| padding(4), ...] } @@ -21,35 +19,35 @@ const ( var frameRWCPadding = [4]byte{0x04, 0x00, 0x00, 0x00} type FrameRWC struct { - conn mtproto.BytesRWC + conn io.ReadWriteCloser readSeqNo int32 writeSeqNo int32 readBuf *bytes.Buffer } -func (f *FrameRWC) Write(buf *bytes.Buffer) (int, error) { - writeBuf := mtproto.GetBuffer() - defer mtproto.ReturnBuffer(writeBuf) +func (f *FrameRWC) Write(buf []byte) (int, error) { + writeBuf := &bytes.Buffer{} // 4 - len bytes // 4 - seq bytes // . - message // 4 - crc32 - messageLength := 4 + 4 + buf.Len() + 4 + messageLength := 4 + 4 + len(buf) + 4 paddingLength := (aes.BlockSize - messageLength%aes.BlockSize) % aes.BlockSize writeBuf.Grow(messageLength + paddingLength) binary.Write(writeBuf, binary.LittleEndian, uint32(messageLength)) binary.Write(writeBuf, binary.LittleEndian, f.writeSeqNo) - writeBuf.Write(buf.Bytes()) + writeBuf.Write(buf) f.writeSeqNo++ checksum := crc32.ChecksumIEEE(writeBuf.Bytes()) binary.Write(writeBuf, binary.LittleEndian, checksum) writeBuf.Write(bytes.Repeat(frameRWCPadding[:], paddingLength/4)) - return f.conn.Write(writeBuf) + _, err := f.conn.Write(writeBuf.Bytes()) + return len(buf), err } func (f *FrameRWC) Read(p []byte) (int, error) { @@ -57,9 +55,7 @@ func (f *FrameRWC) Read(p []byte) (int, error) { return f.flush(p) } - buf := mtproto.GetBuffer() - defer mtproto.ReturnBuffer(buf) - + buf := &bytes.Buffer{} for { buf.Reset() if _, err := io.CopyN(buf, f.conn, 4); err != nil { @@ -103,7 +99,6 @@ func (f *FrameRWC) Read(p []byte) (int, error) { } func (f *FrameRWC) Close() error { - defer mtproto.ReturnBuffer(f.readBuf) return f.conn.Close() } @@ -118,21 +113,17 @@ func (f *FrameRWC) flush(p []byte) (int, error) { if sizeToRead == f.readBuf.Len() { f.readBuf.Reset() } else { - newBuf := mtproto.GetBuffer() - newBuf.Write(data[sizeToRead:]) - - mtproto.ReturnBuffer(f.readBuf) - f.readBuf = newBuf + f.readBuf = bytes.NewBuffer(data[sizeToRead:]) } return sizeToRead, nil } -func NewFrameRWC(conn mtproto.BytesRWC, seqNo int32) mtproto.BytesRWC { +func NewFrameRWC(conn io.ReadWriteCloser, seqNo int32) io.ReadWriteCloser { return &FrameRWC{ conn: conn, readSeqNo: seqNo, writeSeqNo: seqNo, - readBuf: mtproto.GetBuffer(), + readBuf: &bytes.Buffer{}, } } From 09476cc467755b93512315394d75a9d9c1c6e320 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 16:09:59 +0300 Subject: [PATCH 14/71] Change rwc to rwc with addresses --- client/client.go | 4 ++-- client/direct.go | 3 +-- mtproto/wrappers/crypt.go | 8 +++++--- mtproto/wrappers/frame.go | 11 +++++++++-- telegram/dialer.go | 3 +-- telegram/direct.go | 5 ++--- telegram/middle_caller.go | 4 ++-- telegram/telegram.go | 8 ++++---- wrappers/blockcipherrwc.go | 22 +++++++++++++--------- wrappers/ctxrwc.go | 20 ++++++++++++-------- wrappers/logrwc.go | 20 ++++++++++++-------- wrappers/rwcaddr.go | 12 ++++++++++++ wrappers/streamcipherrwc.go | 20 ++++++++++++-------- wrappers/timeoutrwc.go | 17 ++++++++++------- wrappers/trafficrwc.go | 20 ++++++++++++-------- 15 files changed, 109 insertions(+), 68 deletions(-) create mode 100644 wrappers/rwcaddr.go diff --git a/client/client.go b/client/client.go index de5e32a..cb6b291 100644 --- a/client/client.go +++ b/client/client.go @@ -1,12 +1,12 @@ package client import ( - "io" "net" "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" ) // Init has to initialize client connection based on given config. -type Init func(net.Conn, *config.Config) (*mtproto.ConnectionOpts, io.ReadWriteCloser, error) +type Init func(net.Conn, *config.Config) (*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr, error) diff --git a/client/direct.go b/client/direct.go index 23e9613..19d1ede 100644 --- a/client/direct.go +++ b/client/direct.go @@ -1,7 +1,6 @@ package client import ( - "io" "net" "time" @@ -16,7 +15,7 @@ import ( const handshakeTimeout = 10 * time.Second // DirectInit initializes client to access Telegram bypassing middleproxies. -func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, io.ReadWriteCloser, error) { +func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr, error) { if err := config.SetSocketOptions(conn); err != nil { return nil, nil, errors.Annotate(err, "Cannot set socket options") } diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index fbff067..7ae5255 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -7,7 +7,6 @@ import ( "crypto/md5" "crypto/sha1" "encoding/binary" - "io" "net" "github.com/9seconds/mtg/mtproto/rpc" @@ -23,14 +22,17 @@ const ( var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} -func NewMiddleProxyCipherRWC(conn io.ReadWriteCloser, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) io.ReadWriteCloser { +func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, + resp *rpc.RPCNonceResponse, client *net.TCPAddr, remote *net.TCPAddr, + secret []byte) wrappers.ReadWriteCloserWithAddr { encryptor := newCBCCipher(CipherPurposeClient, req, resp, client, remote, secret) decryptor := newCBCCipher(CipherPurposeServer, req, resp, client, remote, secret) return wrappers.NewBlockCipherRWC(conn, encryptor, decryptor) } -func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) cipher.BlockMode { +func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, + client *net.TCPAddr, remote *net.TCPAddr, secret []byte) cipher.BlockMode { message := bytes.Buffer{} message.Write(resp.Nonce[:]) message.Write(req.Nonce[:]) diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index 9e7a4c7..fc3e1d6 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -6,8 +6,11 @@ import ( "encoding/binary" "hash/crc32" "io" + "net" "github.com/juju/errors" + + "github.com/9seconds/mtg/wrappers" ) // Frame: { MessageLength(4) | SequenceNumber(4) | Message(???) | CRC32(4) [| padding(4), ...] } @@ -19,7 +22,7 @@ const ( var frameRWCPadding = [4]byte{0x04, 0x00, 0x00, 0x00} type FrameRWC struct { - conn io.ReadWriteCloser + conn wrappers.ReadWriteCloserWithAddr readSeqNo int32 writeSeqNo int32 @@ -102,6 +105,10 @@ func (f *FrameRWC) Close() error { return f.conn.Close() } +func (f *FrameRWC) Addr() *net.TCPAddr { + return f.conn.Addr() +} + func (f *FrameRWC) flush(p []byte) (int, error) { sizeToRead := len(p) if f.readBuf.Len() < sizeToRead { @@ -119,7 +126,7 @@ func (f *FrameRWC) flush(p []byte) (int, error) { return sizeToRead, nil } -func NewFrameRWC(conn io.ReadWriteCloser, seqNo int32) io.ReadWriteCloser { +func NewFrameRWC(conn wrappers.ReadWriteCloserWithAddr, seqNo int32) wrappers.ReadWriteCloserWithAddr { return &FrameRWC{ conn: conn, readSeqNo: seqNo, diff --git a/telegram/dialer.go b/telegram/dialer.go index f6b2cf7..f0d9cb5 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -1,7 +1,6 @@ package telegram import ( - "io" "net" "time" @@ -29,7 +28,7 @@ func (t *tgDialer) dial(addr string) (net.Conn, error) { return conn, nil } -func (t *tgDialer) dialRWC(addr string) (io.ReadWriteCloser, error) { +func (t *tgDialer) dialRWC(addr string) (wrappers.ReadWriteCloserWithAddr, error) { conn, err := t.dial(addr) if err != nil { return nil, err diff --git a/telegram/direct.go b/telegram/direct.go index 5681415..81fb2eb 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -1,7 +1,6 @@ package telegram import ( - "io" "net" "github.com/juju/errors" @@ -33,7 +32,7 @@ type directTelegram struct { baseTelegram } -func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWriteCloser, error) { +func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { dc := connOpts.DC if dc < 0 { dc = -dc @@ -44,7 +43,7 @@ func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWriteClo return t.baseTelegram.dial(dc-1, connOpts.ConnectionProto) } -func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn io.ReadWriteCloser) (io.ReadWriteCloser, error) { +func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { obfs2, frame := obfuscated2.MakeTelegramObfuscated2Frame(connOpts) if n, err := conn.Write(frame); err != nil || n != obfuscated2.FrameLen { diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go index 2d0d86e..acc1b27 100644 --- a/telegram/middle_caller.go +++ b/telegram/middle_caller.go @@ -2,7 +2,6 @@ package telegram import ( "bufio" - "io" "io/ioutil" "net" "net/http" @@ -16,6 +15,7 @@ import ( "go.uber.org/zap" "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" ) const ( @@ -39,7 +39,7 @@ type middleTelegramCaller struct { httpClient *http.Client } -func (t *middleTelegramCaller) Dial(connOpts *mtproto.ConnectionOpts) (io.ReadWriteCloser, error) { +func (t *middleTelegramCaller) Dial(connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { dc := connOpts.DC if dc == 0 { dc = 1 diff --git a/telegram/telegram.go b/telegram/telegram.go index 6bbb504..5ce16ad 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -1,20 +1,20 @@ package telegram import ( - "io" "math/rand" "github.com/juju/errors" "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" ) // Telegram defines an interface to connect to Telegram. This // encapsulates logic of working with middleproxies or direct // connections. type Telegram interface { - Dial(*mtproto.ConnectionOpts) (io.ReadWriteCloser, error) - Init(*mtproto.ConnectionOpts, io.ReadWriteCloser) (io.ReadWriteCloser, error) + Dial(*mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) + Init(*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) } type baseTelegram struct { @@ -24,7 +24,7 @@ type baseTelegram struct { v6Addresses map[int16][]string } -func (b *baseTelegram) dial(dcIdx int16, proto mtproto.ConnectionProtocol) (io.ReadWriteCloser, error) { +func (b *baseTelegram) dial(dcIdx int16, proto mtproto.ConnectionProtocol) (wrappers.ReadWriteCloserWithAddr, error) { addrs := make([]string, 2) if proto&mtproto.ConnectionProtocolIPv6 != 0 { diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go index 1645034..ad2c9d1 100644 --- a/wrappers/blockcipherrwc.go +++ b/wrappers/blockcipherrwc.go @@ -4,20 +4,20 @@ import ( "bytes" "crypto/aes" "crypto/cipher" - "io" + "net" "github.com/juju/errors" ) -type BlockCipherReadWriteCloser struct { +type BlockCipherReadWriteCloserWithAddr struct { buf *bytes.Buffer - conn io.ReadWriteCloser + conn ReadWriteCloserWithAddr encryptor cipher.BlockMode decryptor cipher.BlockMode } -func (c *BlockCipherReadWriteCloser) Read(p []byte) (int, error) { +func (c *BlockCipherReadWriteCloserWithAddr) Read(p []byte) (int, error) { if c.buf.Len() > 0 { return c.flush(p) } @@ -34,7 +34,7 @@ func (c *BlockCipherReadWriteCloser) Read(p []byte) (int, error) { return c.flush(p) } -func (c *BlockCipherReadWriteCloser) Write(p []byte) (int, error) { +func (c *BlockCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { if len(p)%aes.BlockSize > 0 { return 0, errors.Errorf("Incorrect block size %d", len(p)) } @@ -50,12 +50,16 @@ func (c *BlockCipherReadWriteCloser) Write(p []byte) (int, error) { return c.conn.Write(encrypted) } -func (c *BlockCipherReadWriteCloser) Close() error { +func (c *BlockCipherReadWriteCloserWithAddr) Close() error { defer putBuffer(c.buf) return c.conn.Close() } -func (c *BlockCipherReadWriteCloser) flush(p []byte) (int, error) { +func (c *BlockCipherReadWriteCloserWithAddr) Addr() *net.TCPAddr { + return c.conn.Addr() +} + +func (c *BlockCipherReadWriteCloserWithAddr) flush(p []byte) (int, error) { sizeToRead := len(p) if c.buf.Len() < sizeToRead { sizeToRead = c.buf.Len() @@ -76,8 +80,8 @@ func (c *BlockCipherReadWriteCloser) flush(p []byte) (int, error) { return sizeToRead, nil } -func NewBlockCipherRWC(conn io.ReadWriteCloser, encryptor, decryptor cipher.BlockMode) io.ReadWriteCloser { - return &BlockCipherReadWriteCloser{ +func NewBlockCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.BlockMode) ReadWriteCloserWithAddr { + return &BlockCipherReadWriteCloserWithAddr{ buf: getBuffer(), conn: conn, encryptor: encryptor, diff --git a/wrappers/ctxrwc.go b/wrappers/ctxrwc.go index 26f47be..b378a96 100644 --- a/wrappers/ctxrwc.go +++ b/wrappers/ctxrwc.go @@ -2,21 +2,21 @@ package wrappers import ( "context" - "io" + "net" "github.com/juju/errors" ) // CtxReadWriteCloser wraps underlying connection and does management of the // context and its cancel function. -type CtxReadWriteCloser struct { +type CtxReadWriteCloserWithAddr struct { ctx context.Context - conn io.ReadWriteCloser + conn ReadWriteCloserWithAddr cancel context.CancelFunc } // Read reads from connection -func (c *CtxReadWriteCloser) Read(p []byte) (int, error) { +func (c *CtxReadWriteCloserWithAddr) Read(p []byte) (int, error) { select { case <-c.ctx.Done(): return 0, errors.Annotate(c.ctx.Err(), "Read is failed because of closed context") @@ -30,7 +30,7 @@ func (c *CtxReadWriteCloser) Read(p []byte) (int, error) { } // Write writes into connection. -func (c *CtxReadWriteCloser) Write(p []byte) (int, error) { +func (c *CtxReadWriteCloserWithAddr) Write(p []byte) (int, error) { select { case <-c.ctx.Done(): return 0, errors.Annotate(c.ctx.Err(), "Write is failed because of closed context") @@ -44,14 +44,18 @@ func (c *CtxReadWriteCloser) Write(p []byte) (int, error) { } // Close closes underlying connection. -func (c *CtxReadWriteCloser) Close() error { +func (c *CtxReadWriteCloserWithAddr) Close() error { return c.conn.Close() } +func (c *CtxReadWriteCloserWithAddr) Addr() *net.TCPAddr { + return c.conn.Addr() +} + // NewCtxRWC returns ReadWriteCloser which respects given context, // cancellation etc. -func NewCtxRWC(ctx context.Context, cancel context.CancelFunc, conn io.ReadWriteCloser) io.ReadWriteCloser { - return &CtxReadWriteCloser{ +func NewCtxRWC(ctx context.Context, cancel context.CancelFunc, conn ReadWriteCloserWithAddr) ReadWriteCloserWithAddr { + return &CtxReadWriteCloserWithAddr{ conn: conn, ctx: ctx, cancel: cancel, diff --git a/wrappers/logrwc.go b/wrappers/logrwc.go index 355e239..bab7af7 100644 --- a/wrappers/logrwc.go +++ b/wrappers/logrwc.go @@ -1,44 +1,48 @@ package wrappers import ( - "io" + "net" "go.uber.org/zap" ) // LogReadWriteCloser adds additional logging for reading/writing. All // logging is performed for debug mode only. -type LogReadWriteCloser struct { - conn io.ReadWriteCloser +type LogReadWriteCloserWithAddr struct { + conn ReadWriteCloserWithAddr logger *zap.SugaredLogger sockid string name string } // Read reads from connection -func (l *LogReadWriteCloser) Read(p []byte) (n int, err error) { +func (l *LogReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { n, err = l.conn.Read(p) l.logger.Debugw("Finish reading", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err) return } // Write writes into connection. -func (l *LogReadWriteCloser) Write(p []byte) (n int, err error) { +func (l *LogReadWriteCloserWithAddr) Write(p []byte) (n int, err error) { n, err = l.conn.Write(p) l.logger.Debugw("Finish writing", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err) return } // Close closes underlying connection. -func (l *LogReadWriteCloser) Close() error { +func (l *LogReadWriteCloserWithAddr) Close() error { err := l.conn.Close() l.logger.Debugw("Finish closing socket", "name", l.name, "socketid", l.sockid, "error", err) return err } +func (l *LogReadWriteCloserWithAddr) Addr() *net.TCPAddr { + return l.conn.Addr() +} + // NewLogRWC wraps ReadWriteCloser with logger calls. -func NewLogRWC(conn io.ReadWriteCloser, logger *zap.SugaredLogger, sockid string, name string) io.ReadWriteCloser { - return &LogReadWriteCloser{ +func NewLogRWC(conn ReadWriteCloserWithAddr, logger *zap.SugaredLogger, sockid string, name string) ReadWriteCloserWithAddr { + return &LogReadWriteCloserWithAddr{ conn: conn, logger: logger, sockid: sockid, diff --git a/wrappers/rwcaddr.go b/wrappers/rwcaddr.go new file mode 100644 index 0000000..804bc5a --- /dev/null +++ b/wrappers/rwcaddr.go @@ -0,0 +1,12 @@ +package wrappers + +import ( + "io" + "net" +) + +type ReadWriteCloserWithAddr interface { + io.ReadWriteCloser + + Addr() *net.TCPAddr +} diff --git a/wrappers/streamcipherrwc.go b/wrappers/streamcipherrwc.go index 1d7d73c..d847c72 100644 --- a/wrappers/streamcipherrwc.go +++ b/wrappers/streamcipherrwc.go @@ -2,26 +2,26 @@ package wrappers import ( "crypto/cipher" - "io" + "net" ) // StreamCipherReadWriteCloser is a ReadWriteCloser which ciphers // incoming and outgoing data with givem cipher.Stream instances. -type StreamCipherReadWriteCloser struct { +type StreamCipherReadWriteCloserWithAddr struct { encryptor cipher.Stream decryptor cipher.Stream - conn io.ReadWriteCloser + conn ReadWriteCloserWithAddr } // Read reads from connection -func (c *StreamCipherReadWriteCloser) Read(p []byte) (n int, err error) { +func (c *StreamCipherReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { n, err = c.conn.Read(p) c.decryptor.XORKeyStream(p, p[:n]) return } // Write writes into connection. -func (c *StreamCipherReadWriteCloser) Write(p []byte) (int, error) { +func (c *StreamCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { // This is to decrease an amount of allocations. Unfortunately, escape // analysis in (at least Golang 1.10) is absolutely not perfect. For // example, it understands that we want to have a slice locally, right? @@ -39,14 +39,18 @@ func (c *StreamCipherReadWriteCloser) Write(p []byte) (int, error) { } // Close closes underlying connection. -func (c *StreamCipherReadWriteCloser) Close() error { +func (c *StreamCipherReadWriteCloserWithAddr) Close() error { return c.conn.Close() } +func (c *StreamCipherReadWriteCloserWithAddr) Addr() *net.TCPAddr { + return c.conn.Addr() +} + // NewStreamCipherRWC returns wrapper which transparently // encrypts/decrypts traffic with obfuscated2 protocol. -func NewStreamCipherRWC(conn io.ReadWriteCloser, encryptor, decryptor cipher.Stream) io.ReadWriteCloser { - return &StreamCipherReadWriteCloser{ +func NewStreamCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.Stream) ReadWriteCloserWithAddr { + return &StreamCipherReadWriteCloserWithAddr{ conn: conn, encryptor: encryptor, decryptor: decryptor, diff --git a/wrappers/timeoutrwc.go b/wrappers/timeoutrwc.go index e4ea9b0..e758600 100644 --- a/wrappers/timeoutrwc.go +++ b/wrappers/timeoutrwc.go @@ -1,31 +1,34 @@ package wrappers import ( - "io" "net" "time" "github.com/9seconds/mtg/config" ) -type TimeoutReadWriteCloser struct { +type TimeoutReadWriteCloserWithAddr struct { conn net.Conn } -func (t *TimeoutReadWriteCloser) Read(p []byte) (int, error) { +func (t *TimeoutReadWriteCloserWithAddr) Read(p []byte) (int, error) { t.conn.SetReadDeadline(time.Now().Add(config.TimeoutRead)) return t.conn.Read(p) } -func (t *TimeoutReadWriteCloser) Write(p []byte) (int, error) { +func (t *TimeoutReadWriteCloserWithAddr) Write(p []byte) (int, error) { t.conn.SetWriteDeadline(time.Now().Add(config.TimeoutWrite)) return t.conn.Write(p) } -func (t *TimeoutReadWriteCloser) Close() error { +func (t *TimeoutReadWriteCloserWithAddr) Close() error { return t.conn.Close() } -func NewTimeoutRWC(conn net.Conn) io.ReadWriteCloser { - return &TimeoutReadWriteCloser{conn} +func (t *TimeoutReadWriteCloserWithAddr) Addr() *net.TCPAddr { + return t.conn.RemoteAddr().(*net.TCPAddr) +} + +func NewTimeoutRWC(conn net.Conn) ReadWriteCloserWithAddr { + return &TimeoutReadWriteCloserWithAddr{conn} } diff --git a/wrappers/trafficrwc.go b/wrappers/trafficrwc.go index 485a54c..78518ad 100644 --- a/wrappers/trafficrwc.go +++ b/wrappers/trafficrwc.go @@ -1,37 +1,41 @@ package wrappers -import "io" +import "net" // TrafficReadWriteCloser counts an amount of ingress/egress traffic by // calling given callbacks. -type TrafficReadWriteCloser struct { - conn io.ReadWriteCloser +type TrafficReadWriteCloserWithAddr struct { + conn ReadWriteCloserWithAddr readCallback func(int) writeCallback func(int) } // Read reads from connection -func (t *TrafficReadWriteCloser) Read(p []byte) (n int, err error) { +func (t *TrafficReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { n, err = t.conn.Read(p) t.readCallback(n) return } // Write writes into connection. -func (t *TrafficReadWriteCloser) Write(p []byte) (n int, err error) { +func (t *TrafficReadWriteCloserWithAddr) Write(p []byte) (n int, err error) { n, err = t.conn.Write(p) t.writeCallback(n) return } // Close closes underlying connection. -func (t *TrafficReadWriteCloser) Close() error { +func (t *TrafficReadWriteCloserWithAddr) Close() error { return t.conn.Close() } +func (t *TrafficReadWriteCloserWithAddr) Addr() *net.TCPAddr { + return t.conn.Addr() +} + // NewTrafficRWC wraps ReadWriteCloser to have read/write callbacks. -func NewTrafficRWC(conn io.ReadWriteCloser, readCallback, writeCallback func(int)) io.ReadWriteCloser { - return &TrafficReadWriteCloser{ +func NewTrafficRWC(conn ReadWriteCloserWithAddr, readCallback, writeCallback func(int)) ReadWriteCloserWithAddr { + return &TrafficReadWriteCloserWithAddr{ conn: conn, readCallback: readCallback, writeCallback: writeCallback, From 9f746aa699cad00db2b926ceb38a42aaaed5e857 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 16:14:11 +0300 Subject: [PATCH 15/71] Propagate client addr with connection options --- client/direct.go | 1 + mtproto/connection_options.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/client/direct.go b/client/direct.go index 19d1ede..8ef2dc7 100644 --- a/client/direct.go +++ b/client/direct.go @@ -32,6 +32,7 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wr return nil, nil, errors.Annotate(err, "Cannot parse obfuscated frame") } connOpts.ConnectionProto = mtproto.ConnectionProtocolAny + connOpts.ClientAddr = conn.RemoteAddr().(*net.TCPAddr) socket := wrappers.NewTimeoutRWC(conn) socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) diff --git a/mtproto/connection_options.go b/mtproto/connection_options.go index 29ee283..0f82ec0 100644 --- a/mtproto/connection_options.go +++ b/mtproto/connection_options.go @@ -2,6 +2,7 @@ package mtproto import ( "bytes" + "net" "github.com/juju/errors" ) @@ -18,6 +19,7 @@ type ConnectionOpts struct { DC int16 ConnectionType ConnectionType ConnectionProto ConnectionProtocol + ClientAddr *net.TCPAddr } // Different connection types which user requests from Telegram. From 6db04980ff3da2782086e5524600a93017e96d2a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 16:28:20 +0300 Subject: [PATCH 16/71] Now it is a time to start to make a handshake --- mtproto/extras.go | 6 ------ mtproto/rpc/rpc.go | 5 +++++ mtproto/rpc/rpc_proxy_request.go | 4 ++-- telegram/middle.go | 8 +++++++- 4 files changed, 14 insertions(+), 9 deletions(-) delete mode 100644 mtproto/extras.go diff --git a/mtproto/extras.go b/mtproto/extras.go deleted file mode 100644 index 5c0ad19..0000000 --- a/mtproto/extras.go +++ /dev/null @@ -1,6 +0,0 @@ -package mtproto - -type Extras struct { - QuickAck bool - SimpleAck bool -} diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index 8f2ae99..0ae4d7f 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -4,3 +4,8 @@ const ( RPCNonceSeqNo = -2 RPCHandshakeSeqNo = -1 ) + +type Extras struct { + QuickAck bool + SimpleAck bool +} diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go index 84edafb..2ffa36f 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/rpc_proxy_request.go @@ -28,8 +28,8 @@ type RPCProxyRequest struct { RemoteIPPort [rpcProxyRequestIPPortLength]byte LocalIPPort [rpcProxyRequestIPPortLength]byte ADTag []byte + Extras Extras Message *bytes.Buffer - Extras *mtproto.Extras } func (r *RPCProxyRequest) Bytes() []byte { @@ -65,7 +65,7 @@ func (r *RPCProxyRequest) Bytes() []byte { return buf.Bytes() } -func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *net.TCPAddr, adTag []byte, extras *mtproto.Extras) (*RPCProxyRequest, error) { +func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *net.TCPAddr, adTag []byte, extras Extras) (*RPCProxyRequest, error) { flags := RPCProxyRequestFlagsHasAdTag | RPCProxyRequestFlagsMagic | RPCProxyRequestFlagsExtMode2 switch connectionType { diff --git a/telegram/middle.go b/telegram/middle.go index 01a3b73..4f38b8f 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -8,13 +8,15 @@ import ( "go.uber.org/zap" "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" ) type middleTelegram struct { middleTelegramCaller } -func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) *middleTelegram { +func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram { tg := &middleTelegram{ middleTelegramCaller: middleTelegramCaller{ baseTelegram: baseTelegram{ @@ -35,3 +37,7 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) *middleTe return tg } + +func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { + return nil, nil +} From 5f54a235d28eca7a89065954dfb1dc58305efb07 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 2 Jul 2018 17:51:19 +0300 Subject: [PATCH 17/71] Add base of rpc handshake --- mtproto/wrappers/crypt.go | 4 +- telegram/middle.go | 81 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index 7ae5255..4150ec2 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -23,8 +23,8 @@ const ( var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, - resp *rpc.RPCNonceResponse, client *net.TCPAddr, remote *net.TCPAddr, - secret []byte) wrappers.ReadWriteCloserWithAddr { + resp *rpc.RPCNonceResponse, client *net.TCPAddr, secret []byte) wrappers.ReadWriteCloserWithAddr { + remote := conn.Addr() encryptor := newCBCCipher(CipherPurposeClient, req, resp, client, remote, secret) decryptor := newCBCCipher(CipherPurposeServer, req, resp, client, remote, secret) diff --git a/telegram/middle.go b/telegram/middle.go index 4f38b8f..65e4af2 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -1,6 +1,7 @@ package telegram import ( + "io/ioutil" "net" "net/http" "sync" @@ -9,7 +10,10 @@ import ( "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/mtproto/rpc" + mtwrappers "github.com/9seconds/mtg/mtproto/wrappers" "github.com/9seconds/mtg/wrappers" + "github.com/juju/errors" ) type middleTelegram struct { @@ -39,5 +43,82 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram } func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { + rpcNonceConn := mtwrappers.NewFrameRWC(conn, rpc.RPCNonceSeqNo) + + rpcNonceReq, err := t.sendRPCNonceRequest(rpcNonceConn) + if err != nil { + return nil, err + } + rpcNonceResp, err := t.receiveRPCNonceResponse(rpcNonceConn, rpcNonceReq) + if err != nil { + return nil, err + } + + secureConn := mtwrappers.NewFrameRWC(conn, rpc.RPCHandshakeSeqNo) + secureConn = mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, + rpcNonceResp, connOpts.ClientAddr, t.proxySecret) + + rpcHandshakeReq, err := t.sendRPCHandshakeRequest(secureConn) + if err != nil { + return nil, err + } + _, err = t.receiveRPCHandshakeResponse(secureConn, rpcHandshakeReq) + if err != nil { + return nil, err + } + return nil, nil } + +func (t *middleTelegram) sendRPCNonceRequest(conn wrappers.ReadWriteCloserWithAddr) (*rpc.RPCNonceRequest, error) { + rpcNonceReq, err := rpc.NewRPCNonceRequest(t.proxySecret) + if err != nil { + return nil, errors.Annotate(err, "Cannot create RPC nonce request") + } + if _, err = conn.Write(rpcNonceReq.Bytes()); err != nil { + return nil, errors.Annotate(err, "Cannot send RPC nonce request") + } + + return rpcNonceReq, nil +} + +func (t *middleTelegram) receiveRPCNonceResponse(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest) (*rpc.RPCNonceResponse, error) { + ans, err := ioutil.ReadAll(conn) + if err != nil { + return nil, errors.Annotate(err, "Cannot read RPC nonce response") + } + rpcNonceResp, err := rpc.NewRPCNonceResponse(ans) + if err != nil { + return nil, errors.Annotate(err, "Cannot initialize RPC nonce response") + } + if err = rpcNonceResp.Valid(req); err != nil { + return nil, errors.Annotate(err, "Invalid RPC nonce response") + } + + return rpcNonceResp, nil +} + +func (t *middleTelegram) sendRPCHandshakeRequest(conn wrappers.ReadWriteCloserWithAddr) (*rpc.RPCHandshakeRequest, error) { + req := rpc.NewRPCHandshakeRequest() + if _, err := conn.Write(req.Bytes()); err != nil { + return nil, errors.Annotate(err, "Cannot send RPC handshake request") + } + + return req, nil +} + +func (t *middleTelegram) receiveRPCHandshakeResponse(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCHandshakeRequest) (*rpc.RPCHandshakeResponse, error) { + ans, err := ioutil.ReadAll(conn) + if err != nil { + return nil, errors.Annotate(err, "Cannot read RPC handshake response") + } + rpcHandshakeResp, err := rpc.NewRPCHandshakeResponse(ans) + if err != nil { + return nil, errors.Annotate(err, "Cannot initialize RPC handshake response") + } + if err = rpcHandshakeResp.Valid(req); err != nil { + return nil, errors.Annotate(err, "Invalid RPC handshake response") + } + + return rpcHandshakeResp, nil +} From 586ffab01bdadd3d73bfe90ac5336c41f7868dc7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 3 Jul 2018 10:10:00 +0300 Subject: [PATCH 18/71] Add buffered reader --- mtproto/rpc/rpc_proxy_request.go | 10 +-- mtproto/wrappers/frame.go | 103 +++++++++++++------------------ wrappers/blockcipherrwc.go | 63 +++++-------------- wrappers/buffered_reader.go | 38 ++++++++++++ 4 files changed, 100 insertions(+), 114 deletions(-) create mode 100644 wrappers/buffered_reader.go diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go index 2ffa36f..feaecb3 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/rpc_proxy_request.go @@ -29,10 +29,9 @@ type RPCProxyRequest struct { LocalIPPort [rpcProxyRequestIPPortLength]byte ADTag []byte Extras Extras - Message *bytes.Buffer } -func (r *RPCProxyRequest) Bytes() []byte { +func (r *RPCProxyRequest) Bytes(message []byte) []byte { buf := &bytes.Buffer{} flags := r.Flags @@ -40,8 +39,7 @@ func (r *RPCProxyRequest) Bytes() []byte { flags |= RPCProxyRequestFlagsQuickAck } - messageBytes := r.Message.Bytes() - if bytes.HasPrefix(messageBytes, rpcProxyRequestFlagsEncryptedPrefix[:]) { + if bytes.HasPrefix(message, rpcProxyRequestFlagsEncryptedPrefix[:]) { flags |= RPCProxyRequestFlagsEncrypted } @@ -58,9 +56,7 @@ func (r *RPCProxyRequest) Bytes() []byte { for i := 0; i < (buf.Len() % 4); i++ { buf.WriteByte(0x00) } - if r.Message != nil { - buf.Write(messageBytes) - } + buf.Write(message) return buf.Bytes() } diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index fc3e1d6..9a77c73 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -22,11 +22,11 @@ const ( var frameRWCPadding = [4]byte{0x04, 0x00, 0x00, 0x00} type FrameRWC struct { - conn wrappers.ReadWriteCloserWithAddr + wrappers.BufferedReader + conn wrappers.ReadWriteCloserWithAddr readSeqNo int32 writeSeqNo int32 - readBuf *bytes.Buffer } func (f *FrameRWC) Write(buf []byte) (int, error) { @@ -54,51 +54,49 @@ func (f *FrameRWC) Write(buf []byte) (int, error) { } func (f *FrameRWC) Read(p []byte) (int, error) { - if f.readBuf.Len() > 0 { - return f.flush(p) - } + return f.BufferedRead(p, func() error { + buf := &bytes.Buffer{} + for { + buf.Reset() + if _, err := io.CopyN(buf, f.conn, 4); err != nil { + return errors.Annotate(err, "Cannot read frame padding") + } + if !bytes.Equal(buf.Bytes(), frameRWCPadding[:]) { + break + } + } + + messageLength := binary.LittleEndian.Uint32(buf.Bytes()) + if messageLength%4 != 0 || messageLength < frameRWCMinMessageLength || messageLength > frameRWCMaxMessageLength { + return errors.Errorf("Incorrect frame message length %d", messageLength) + } + sum := crc32.NewIEEE() + sum.Write(buf.Bytes()) - buf := &bytes.Buffer{} - for { buf.Reset() - if _, err := io.CopyN(buf, f.conn, 4); err != nil { - return 0, errors.Annotate(err, "Cannot read frame padding") + buf.Grow(int(messageLength) - 4) // -4 because we already read the first number + if _, err := io.CopyN(buf, f.conn, int64(messageLength)-4); err != nil { + return errors.Annotate(err, "Cannot read the message frame") } - if !bytes.Equal(buf.Bytes(), frameRWCPadding[:]) { - break + sum.Write(buf.Bytes()) + + var seqNo int32 + binary.Read(buf, binary.LittleEndian, seqNo) + if seqNo != f.readSeqNo { + return errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, f.readSeqNo) } - } + f.readSeqNo++ - messageLength := binary.LittleEndian.Uint32(buf.Bytes()) - if messageLength%4 != 0 || messageLength < frameRWCMinMessageLength || messageLength > frameRWCMaxMessageLength { - return 0, errors.Errorf("Incorrect frame message length %d", messageLength) - } - sum := crc32.NewIEEE() - sum.Write(buf.Bytes()) + data := buf.Bytes()[:int(messageLength)-4-4-4] + checksum := binary.LittleEndian.Uint32(buf.Bytes()[int(messageLength)-4-4-4:]) + if checksum != sum.Sum32() { + return errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) - buf.Reset() - buf.Grow(int(messageLength) - 4) // -4 because we already read the first number - if _, err := io.CopyN(buf, f.conn, int64(messageLength)-4); err != nil { - return 0, errors.Annotate(err, "Cannot read the message frame") - } - sum.Write(buf.Bytes()) + } + f.Buffer.Write(data) - var seqNo int32 - binary.Read(buf, binary.LittleEndian, seqNo) - if seqNo != f.readSeqNo { - return 0, errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, f.readSeqNo) - } - f.readSeqNo++ - - data := buf.Bytes()[:int(messageLength)-4-4-4] - checksum := binary.LittleEndian.Uint32(buf.Bytes()[int(messageLength)-4-4-4:]) - if checksum != sum.Sum32() { - return 0, errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) - - } - f.readBuf.Write(data) - - return f.flush(p) + return nil + }) } func (f *FrameRWC) Close() error { @@ -109,28 +107,11 @@ func (f *FrameRWC) Addr() *net.TCPAddr { return f.conn.Addr() } -func (f *FrameRWC) flush(p []byte) (int, error) { - sizeToRead := len(p) - if f.readBuf.Len() < sizeToRead { - sizeToRead = f.readBuf.Len() - } - - data := f.readBuf.Bytes() - copy(p, data[:sizeToRead]) - if sizeToRead == f.readBuf.Len() { - f.readBuf.Reset() - } else { - f.readBuf = bytes.NewBuffer(data[sizeToRead:]) - } - - return sizeToRead, nil -} - func NewFrameRWC(conn wrappers.ReadWriteCloserWithAddr, seqNo int32) wrappers.ReadWriteCloserWithAddr { return &FrameRWC{ - conn: conn, - readSeqNo: seqNo, - writeSeqNo: seqNo, - readBuf: &bytes.Buffer{}, + BufferedReader: wrappers.NewBufferedReader(), + conn: conn, + readSeqNo: seqNo, + writeSeqNo: seqNo, } } diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go index ad2c9d1..4d11d5d 100644 --- a/wrappers/blockcipherrwc.go +++ b/wrappers/blockcipherrwc.go @@ -1,7 +1,6 @@ package wrappers import ( - "bytes" "crypto/aes" "crypto/cipher" "net" @@ -10,7 +9,7 @@ import ( ) type BlockCipherReadWriteCloserWithAddr struct { - buf *bytes.Buffer + BufferedReader conn ReadWriteCloserWithAddr encryptor cipher.BlockMode @@ -18,20 +17,19 @@ type BlockCipherReadWriteCloserWithAddr struct { } func (c *BlockCipherReadWriteCloserWithAddr) Read(p []byte) (int, error) { - if c.buf.Len() > 0 { - return c.flush(p) - } - - for c.buf.Len() == 0 || c.buf.Len()%aes.BlockSize != 0 { - n, err := c.conn.Read(p) - if err != nil { - return 0, errors.Annotate(err, "Cannot read from socket") + return c.BufferedRead(p, func() error { + bufferLength := c.Buffer.Len() + for bufferLength%aes.BlockSize != 0 || bufferLength == 0 { + n, err := c.conn.Read(p) + if err != nil { + return errors.Annotate(err, "Cannot read from socket") + } + c.Buffer.Write(p[:n]) } - c.buf.Write(p[:n]) - } - c.decryptor.CryptBlocks(c.buf.Bytes(), c.buf.Bytes()) + c.decryptor.CryptBlocks(c.Buffer.Bytes(), c.Buffer.Bytes()) - return c.flush(p) + return nil + }) } func (c *BlockCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { @@ -39,19 +37,13 @@ func (c *BlockCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { return 0, errors.Errorf("Incorrect block size %d", len(p)) } - buf := getBuffer() - defer putBuffer(buf) - buf.Grow(len(p)) - buf.Write(p) - - encrypted := buf.Bytes() + encrypted := make([]byte, len(p)) c.encryptor.CryptBlocks(encrypted, p) return c.conn.Write(encrypted) } func (c *BlockCipherReadWriteCloserWithAddr) Close() error { - defer putBuffer(c.buf) return c.conn.Close() } @@ -59,32 +51,11 @@ func (c *BlockCipherReadWriteCloserWithAddr) Addr() *net.TCPAddr { return c.conn.Addr() } -func (c *BlockCipherReadWriteCloserWithAddr) flush(p []byte) (int, error) { - sizeToRead := len(p) - if c.buf.Len() < sizeToRead { - sizeToRead = c.buf.Len() - } - - data := c.buf.Bytes() - copy(p, data[:sizeToRead]) - if sizeToRead == c.buf.Len() { - c.buf.Reset() - } else { - newBuf := getBuffer() - newBuf.Write(data[sizeToRead:]) - - putBuffer(c.buf) - c.buf = newBuf - } - - return sizeToRead, nil -} - func NewBlockCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.BlockMode) ReadWriteCloserWithAddr { return &BlockCipherReadWriteCloserWithAddr{ - buf: getBuffer(), - conn: conn, - encryptor: encryptor, - decryptor: decryptor, + BufferedReader: NewBufferedReader(), + conn: conn, + encryptor: encryptor, + decryptor: decryptor, } } diff --git a/wrappers/buffered_reader.go b/wrappers/buffered_reader.go new file mode 100644 index 0000000..a824805 --- /dev/null +++ b/wrappers/buffered_reader.go @@ -0,0 +1,38 @@ +package wrappers + +import "bytes" + +type BufferedReader struct { + Buffer *bytes.Buffer +} + +func (b *BufferedReader) BufferedRead(p []byte, callback func() error) (int, error) { + if b.Buffer.Len() > 0 { + return b.flush(p) + } + if err := callback(); err != nil { + return 0, err + } + return b.flush(p) +} + +func (b *BufferedReader) flush(p []byte) (int, error) { + sizeToRead := len(p) + if b.Buffer.Len() < sizeToRead { + sizeToRead = b.Buffer.Len() + } + + data := b.Buffer.Bytes() + copy(p, data[:sizeToRead]) + if sizeToRead == b.Buffer.Len() { + b.Buffer.Reset() + } else { + b.Buffer = bytes.NewBuffer(data[sizeToRead:]) + } + + return sizeToRead, nil +} + +func NewBufferedReader() BufferedReader { + return BufferedReader{Buffer: &bytes.Buffer{}} +} From 812b5e6e0bf9f3cff16187f9fbc86bb7bb360ab2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 3 Jul 2018 10:29:17 +0300 Subject: [PATCH 19/71] Correct encryptor/decryptor --- mtproto/wrappers/crypt.go | 10 +++++----- telegram/middle.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index 4150ec2..82ab6e4 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -25,14 +25,14 @@ var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, client *net.TCPAddr, secret []byte) wrappers.ReadWriteCloserWithAddr { remote := conn.Addr() - encryptor := newCBCCipher(CipherPurposeClient, req, resp, client, remote, secret) - decryptor := newCBCCipher(CipherPurposeServer, req, resp, client, remote, secret) + encryptor, _ := newCBCCipher(CipherPurposeClient, req, resp, client, remote, secret) + _, decryptor := newCBCCipher(CipherPurposeServer, req, resp, client, remote, secret) return wrappers.NewBlockCipherRWC(conn, encryptor, decryptor) } func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, - client *net.TCPAddr, remote *net.TCPAddr, secret []byte) cipher.BlockMode { + client *net.TCPAddr, remote *net.TCPAddr, secret []byte) (cipher.BlockMode, cipher.BlockMode) { message := bytes.Buffer{} message.Write(resp.Nonce[:]) message.Write(req.Nonce[:]) @@ -74,7 +74,7 @@ func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPC return makeCipher(message.Bytes()) } -func makeCipher(message []byte) cipher.BlockMode { +func makeCipher(message []byte) (cipher.BlockMode, cipher.BlockMode) { md5sum := md5.Sum(message[1:]) sha1sum := sha1.Sum(message) @@ -86,7 +86,7 @@ func makeCipher(message []byte) cipher.BlockMode { panic("Cannot create cipher from the given key") } - return cipher.NewCBCEncrypter(block, iv[:]) + return cipher.NewCBCEncrypter(block, iv[:]), cipher.NewCBCDecrypter(block, iv[:]) } func reverseBytes(data []byte) []byte { diff --git a/telegram/middle.go b/telegram/middle.go index 65e4af2..33d8326 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -55,7 +55,7 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re } secureConn := mtwrappers.NewFrameRWC(conn, rpc.RPCHandshakeSeqNo) - secureConn = mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, + secureConn = mtwrappers.NewMiddleProxyCipherRWC(secureConn, rpcNonceReq, rpcNonceResp, connOpts.ClientAddr, t.proxySecret) rpcHandshakeReq, err := t.sendRPCHandshakeRequest(secureConn) From daddf50390556d4f1518af8aa82f41f5237564e4 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 3 Jul 2018 10:36:37 +0300 Subject: [PATCH 20/71] Minor cleanups --- mtproto/rpc/rpc_handshake_request.go | 5 +---- mtproto/rpc/rpc_nonce_request.go | 3 +-- telegram/middle.go | 11 ++++++----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/mtproto/rpc/rpc_handshake_request.go b/mtproto/rpc/rpc_handshake_request.go index 12e79e6..c0a30a6 100644 --- a/mtproto/rpc/rpc_handshake_request.go +++ b/mtproto/rpc/rpc_handshake_request.go @@ -8,8 +8,7 @@ const ( rpcHandshakeSenderPIDLength = 12 rpcHandshakePeerPIDLength = rpcHandshakeSenderPIDLength - rpcHandshakeRequestLength = rpcHandshakeTagLength + rpcHandshakeFlagsLength + - rpcHandshakeSenderPIDLength + rpcHandshakePeerPIDLength + rpcHandshakeRequestLength = rpcHandshakeTagLength + rpcHandshakeFlagsLength + rpcHandshakeSenderPIDLength + rpcHandshakePeerPIDLength ) var ( @@ -18,8 +17,6 @@ var ( rpcHandshakeTag = [rpcHandshakeTagLength]byte{0xf5, 0xee, 0x82, 0x76} rpcHandshakeFlags = [rpcHandshakeFlagsLength]byte{0x00, 0x00, 0x00, 0x00} - - rpcHandshakeBuffer *bytes.Buffer ) type RPCHandshakeRequest struct { diff --git a/mtproto/rpc/rpc_nonce_request.go b/mtproto/rpc/rpc_nonce_request.go index 3fcfe15..65a6947 100644 --- a/mtproto/rpc/rpc_nonce_request.go +++ b/mtproto/rpc/rpc_nonce_request.go @@ -16,8 +16,7 @@ const ( rpcNonceTagLength = 4 rpcNonceCryptoAESLength = 4 - rpcNonceRequestLength = rpcNonceTagLength + rpcNonceKeySelectorLength + - rpcNonceCryptoAESLength + rpcNonceCryptoTSLength + rpcNonceLength + rpcNonceRequestLength = rpcNonceTagLength + rpcNonceKeySelectorLength + rpcNonceCryptoAESLength + rpcNonceCryptoTSLength + rpcNonceLength ) var ( diff --git a/telegram/middle.go b/telegram/middle.go index 33d8326..81975ca 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -1,11 +1,13 @@ package telegram import ( + "io" "io/ioutil" "net" "net/http" "sync" + "github.com/juju/errors" "go.uber.org/zap" "github.com/9seconds/mtg/config" @@ -13,7 +15,6 @@ import ( "github.com/9seconds/mtg/mtproto/rpc" mtwrappers "github.com/9seconds/mtg/mtproto/wrappers" "github.com/9seconds/mtg/wrappers" - "github.com/juju/errors" ) type middleTelegram struct { @@ -70,7 +71,7 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return nil, nil } -func (t *middleTelegram) sendRPCNonceRequest(conn wrappers.ReadWriteCloserWithAddr) (*rpc.RPCNonceRequest, error) { +func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.RPCNonceRequest, error) { rpcNonceReq, err := rpc.NewRPCNonceRequest(t.proxySecret) if err != nil { return nil, errors.Annotate(err, "Cannot create RPC nonce request") @@ -82,7 +83,7 @@ func (t *middleTelegram) sendRPCNonceRequest(conn wrappers.ReadWriteCloserWithAd return rpcNonceReq, nil } -func (t *middleTelegram) receiveRPCNonceResponse(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest) (*rpc.RPCNonceResponse, error) { +func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.RPCNonceRequest) (*rpc.RPCNonceResponse, error) { ans, err := ioutil.ReadAll(conn) if err != nil { return nil, errors.Annotate(err, "Cannot read RPC nonce response") @@ -98,7 +99,7 @@ func (t *middleTelegram) receiveRPCNonceResponse(conn wrappers.ReadWriteCloserWi return rpcNonceResp, nil } -func (t *middleTelegram) sendRPCHandshakeRequest(conn wrappers.ReadWriteCloserWithAddr) (*rpc.RPCHandshakeRequest, error) { +func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.RPCHandshakeRequest, error) { req := rpc.NewRPCHandshakeRequest() if _, err := conn.Write(req.Bytes()); err != nil { return nil, errors.Annotate(err, "Cannot send RPC handshake request") @@ -107,7 +108,7 @@ func (t *middleTelegram) sendRPCHandshakeRequest(conn wrappers.ReadWriteCloserWi return req, nil } -func (t *middleTelegram) receiveRPCHandshakeResponse(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCHandshakeRequest) (*rpc.RPCHandshakeResponse, error) { +func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.RPCHandshakeRequest) (*rpc.RPCHandshakeResponse, error) { ans, err := ioutil.ReadAll(conn) if err != nil { return nil, errors.Annotate(err, "Cannot read RPC handshake response") From 4c4671ad3bed6166e6dd2e4a191a90974486f1ac Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 3 Jul 2018 14:08:01 +0300 Subject: [PATCH 21/71] Bugfix framerwc --- mtproto/wrappers/frame.go | 27 ++++++++++++++++++--------- telegram/middle.go | 13 ++++++++----- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index 9a77c73..784762f 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "hash/crc32" "io" + "io/ioutil" "net" "github.com/juju/errors" @@ -56,9 +57,13 @@ func (f *FrameRWC) Write(buf []byte) (int, error) { func (f *FrameRWC) Read(p []byte) (int, error) { return f.BufferedRead(p, func() error { buf := &bytes.Buffer{} + sum := crc32.NewIEEE() + writer := io.MultiWriter(buf, sum) + for { buf.Reset() - if _, err := io.CopyN(buf, f.conn, 4); err != nil { + sum.Reset() + if _, err := io.CopyN(writer, f.conn, 4); err != nil { return errors.Annotate(err, "Cannot read frame padding") } if !bytes.Equal(buf.Bytes(), frameRWCPadding[:]) { @@ -70,25 +75,29 @@ func (f *FrameRWC) Read(p []byte) (int, error) { if messageLength%4 != 0 || messageLength < frameRWCMinMessageLength || messageLength > frameRWCMaxMessageLength { return errors.Errorf("Incorrect frame message length %d", messageLength) } - sum := crc32.NewIEEE() - sum.Write(buf.Bytes()) buf.Reset() - buf.Grow(int(messageLength) - 4) // -4 because we already read the first number - if _, err := io.CopyN(buf, f.conn, int64(messageLength)-4); err != nil { + buf.Grow(int(messageLength) - 4 - 4) + if _, err := io.CopyN(writer, f.conn, int64(messageLength)-4-4); err != nil { return errors.Annotate(err, "Cannot read the message frame") } - sum.Write(buf.Bytes()) var seqNo int32 - binary.Read(buf, binary.LittleEndian, seqNo) + binary.Read(buf, binary.LittleEndian, &seqNo) if seqNo != f.readSeqNo { return errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, f.readSeqNo) } f.readSeqNo++ - data := buf.Bytes()[:int(messageLength)-4-4-4] - checksum := binary.LittleEndian.Uint32(buf.Bytes()[int(messageLength)-4-4-4:]) + data, _ := ioutil.ReadAll(buf) + buf.Reset() + // write to buf, not to writer. This is because we are going to fetch + // crc32 checksum. + if _, err := io.CopyN(buf, f.conn, 4); err != nil { + return errors.Annotate(err, "Cannot read checksum") + } + checksum := binary.LittleEndian.Uint32(buf.Bytes()) + if checksum != sum.Sum32() { return errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) diff --git a/telegram/middle.go b/telegram/middle.go index 81975ca..a747b78 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -2,7 +2,6 @@ package telegram import ( "io" - "io/ioutil" "net" "net/http" "sync" @@ -84,11 +83,13 @@ func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.RPCNonceReque } func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.RPCNonceRequest) (*rpc.RPCNonceResponse, error) { - ans, err := ioutil.ReadAll(conn) + var ans [128]byte + + n, err := conn.Read(ans[:]) if err != nil { return nil, errors.Annotate(err, "Cannot read RPC nonce response") } - rpcNonceResp, err := rpc.NewRPCNonceResponse(ans) + rpcNonceResp, err := rpc.NewRPCNonceResponse(ans[:n]) if err != nil { return nil, errors.Annotate(err, "Cannot initialize RPC nonce response") } @@ -109,11 +110,13 @@ func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.RPCHandsh } func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.RPCHandshakeRequest) (*rpc.RPCHandshakeResponse, error) { - ans, err := ioutil.ReadAll(conn) + var ans [128]byte + + n, err := conn.Read(ans[:]) if err != nil { return nil, errors.Annotate(err, "Cannot read RPC handshake response") } - rpcHandshakeResp, err := rpc.NewRPCHandshakeResponse(ans) + rpcHandshakeResp, err := rpc.NewRPCHandshakeResponse(ans[:n]) if err != nil { return nil, errors.Annotate(err, "Cannot initialize RPC handshake response") } From ffa7ef7a0a8042db2f7bdeeb9d33e64e64ec6ca2 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Tue, 3 Jul 2018 18:25:54 +0300 Subject: [PATCH 22/71] Make crypt testable --- mtproto/wrappers/crypt.go | 34 +++++++++++++------------ mtproto/wrappers/crypt_test.go | 45 ++++++++++++++++++++++++++++++++++ telegram/middle.go | 4 +-- 3 files changed, 66 insertions(+), 17 deletions(-) create mode 100644 mtproto/wrappers/crypt_test.go diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index 82ab6e4..b790348 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -25,14 +25,17 @@ var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, client *net.TCPAddr, secret []byte) wrappers.ReadWriteCloserWithAddr { remote := conn.Addr() - encryptor, _ := newCBCCipher(CipherPurposeClient, req, resp, client, remote, secret) - _, decryptor := newCBCCipher(CipherPurposeServer, req, resp, client, remote, secret) + encKey, encIV := makeKeys(CipherPurposeClient, req, resp, client, remote, secret) + decKey, decIV := makeKeys(CipherPurposeServer, req, resp, client, remote, secret) - return wrappers.NewBlockCipherRWC(conn, encryptor, decryptor) + enc, _ := makeEncrypterDecrypter(encKey, encIV) + _, dec := makeEncrypterDecrypter(decKey, decIV) + + return wrappers.NewBlockCipherRWC(conn, enc, dec) } -func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, - client *net.TCPAddr, remote *net.TCPAddr, secret []byte) (cipher.BlockMode, cipher.BlockMode) { +func makeKeys(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, + client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { message := bytes.Buffer{} message.Write(resp.Nonce[:]) message.Write(req.Nonce[:]) @@ -71,22 +74,23 @@ func newCBCCipher(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPC } message.Write(req.Nonce[:]) - return makeCipher(message.Bytes()) + data := message.Bytes() + md5sum := md5.Sum(data[1:]) + sha1sum := sha1.Sum(data) + + key := append(md5sum[:12], sha1sum[:]...) + iv := md5.Sum(data[2:]) + + return key, iv[:] } -func makeCipher(message []byte) (cipher.BlockMode, cipher.BlockMode) { - md5sum := md5.Sum(message[1:]) - sha1sum := sha1.Sum(message) - - key := append(md5sum[12:], sha1sum[:]...) - iv := md5.Sum(message[2:]) - +func makeEncrypterDecrypter(key, iv []byte) (cipher.BlockMode, cipher.BlockMode) { block, err := aes.NewCipher(key) if err != nil { - panic("Cannot create cipher from the given key") + panic(err) } - return cipher.NewCBCEncrypter(block, iv[:]), cipher.NewCBCDecrypter(block, iv[:]) + return cipher.NewCBCEncrypter(block, iv), cipher.NewCBCDecrypter(block, iv) } func reverseBytes(data []byte) []byte { diff --git a/mtproto/wrappers/crypt_test.go b/mtproto/wrappers/crypt_test.go new file mode 100644 index 0000000..62bf0b9 --- /dev/null +++ b/mtproto/wrappers/crypt_test.go @@ -0,0 +1,45 @@ +package wrappers + +import ( + "encoding/binary" + "net" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/9seconds/mtg/mtproto/rpc" +) + +var proxySecret = []byte{196, 249, 250, 202, 150, 120, 230, 187, 72, 173, + 108, 126, 44, 229, 192, 210, 68, 48, 100, 93, 85, 74, 221, 235, 85, 65, + 158, 3, 77, 166, 39, 33, 208, 70, 234, 171, 110, 82, 171, 20, 169, 90, 68, + 62, 207, 179, 70, 62, 121, 160, 90, 102, 97, 42, 223, 156, 174, 218, 139, + 233, 168, 13, 166, 152, 111, 176, 166, 255, 56, 122, 248, 77, 136, 239, + 58, 100, 19, 113, 62, 92, 51, 119, 246, 225, 163, 212, 125, 153, 245, 224, + 197, 110, 236, 232, 240, 92, 84, 196, 144, 176, 121, 227, 27, 239, 130, + 255, 14, 232, 242, 176, 163, 39, 86, 210, 73, 197, 242, 18, 105, 129, 108, + 183, 6, 27, 38, 93, 178, 18} + +func TestMakeKeys(t *testing.T) { + req, err := rpc.NewRPCNonceRequest(proxySecret) + assert.Nil(t, err) + + copy(req.Nonce[:], []byte{24, 49, 53, 111, 198, 10, 235, 180, 230, 112, 92, 78, 1, 201, 106, 105}) + binary.LittleEndian.PutUint32(req.CryptoTS[:], 1528396015) + + resp := &rpc.RPCNonceResponse{} + copy(resp.Nonce[:], []byte{247, 40, 210, 56, 65, 12, 101, 170, 216, 155, 14, 253, 250, 238, 219, 226}) + + cltAddr := &net.TCPAddr{ + IP: net.ParseIP("80.211.29.34"), + Port: 54208, + } + srvAddr := &net.TCPAddr{ + IP: net.ParseIP("149.154.162.38"), + Port: 80, + } + + key, iv := makeKeys(CipherPurposeClient, req, resp, cltAddr, srvAddr, proxySecret) + assert.Equal(t, key, []byte{165, 158, 127, 49, 41, 232, 187, 69, 38, 29, 163, 226, 183, 146, 28, 67, 225, 224, 134, 191, 207, 152, 255, 166, 152, 66, 169, 196, 54, 135, 50, 188}) + assert.Equal(t, iv, []byte{33, 110, 125, 221, 183, 121, 160, 116, 130, 180, 156, 249, 52, 111, 37, 178}) +} diff --git a/telegram/middle.go b/telegram/middle.go index a747b78..b3b4ed3 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -54,9 +54,9 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return nil, err } - secureConn := mtwrappers.NewFrameRWC(conn, rpc.RPCHandshakeSeqNo) - secureConn = mtwrappers.NewMiddleProxyCipherRWC(secureConn, rpcNonceReq, + secureConn := mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, rpcNonceResp, connOpts.ClientAddr, t.proxySecret) + secureConn = mtwrappers.NewFrameRWC(secureConn, rpc.RPCHandshakeSeqNo) rpcHandshakeReq, err := t.sendRPCHandshakeRequest(secureConn) if err != nil { From 90c10519ee601b7c46eff22b9a0fb7533dc65fc8 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 4 Jul 2018 11:17:05 +0300 Subject: [PATCH 23/71] Bugfix rpc handshake --- client/direct.go | 3 +-- config/config.go | 35 +++++++++++----------------- main.go | 34 ++++++++++++++++++++------- mtproto/connection_options.go | 4 ++-- mtproto/rpc/rpc_handshake_request.go | 4 +++- mtproto/wrappers/crypt.go | 11 +++++---- mtproto/wrappers/frame.go | 8 +++++-- telegram/dialer.go | 4 +++- telegram/direct.go | 5 +++- telegram/middle.go | 10 +++++--- wrappers/blockcipherrwc.go | 9 +++++-- wrappers/ctxrwc.go | 8 +++++-- wrappers/logrwc.go | 8 +++++-- wrappers/rwcaddr.go | 3 ++- wrappers/streamcipherrwc.go | 8 +++++-- wrappers/timeoutrwc.go | 29 +++++++++++++++++++---- wrappers/trafficrwc.go | 8 +++++-- 17 files changed, 129 insertions(+), 62 deletions(-) diff --git a/client/direct.go b/client/direct.go index 8ef2dc7..1e38e5e 100644 --- a/client/direct.go +++ b/client/direct.go @@ -32,9 +32,8 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wr return nil, nil, errors.Annotate(err, "Cannot parse obfuscated frame") } connOpts.ConnectionProto = mtproto.ConnectionProtocolAny - connOpts.ClientAddr = conn.RemoteAddr().(*net.TCPAddr) - socket := wrappers.NewTimeoutRWC(conn) + socket := wrappers.NewTimeoutRWC(conn, conf.PublicIPv4, conf.PublicIPv6) socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) return connOpts, socket, nil diff --git a/config/config.go b/config/config.go index 9935fa8..d988c38 100644 --- a/config/config.go +++ b/config/config.go @@ -59,16 +59,6 @@ func (c *Config) BindAddr() string { return getAddr(c.BindIP, c.BindPort) } -// IPv4Addr returns connection string to ipv6 for mtproto proxy. -func (c *Config) IPv4Addr() string { - return getAddr(c.PublicIPv4, c.PublicIPv4Port) -} - -// IPv6Addr returns connection string to ipv6 for mtproto proxy. -func (c *Config) IPv6Addr() string { - return getAddr(c.PublicIPv6, c.PublicIPv6Port) -} - // StatAddr returns connection string to the stats API. func (c *Config) StatAddr() string { return getAddr(c.StatsIP, c.StatsPort) @@ -76,10 +66,15 @@ func (c *Config) StatAddr() string { // GetURLs returns configured IPURLs instance with links to this server. func (c *Config) GetURLs() IPURLs { - return IPURLs{ - IPv4: getURLs(c.PublicIPv4, c.PublicIPv4Port, c.Secret), - IPv6: getURLs(c.PublicIPv6, c.PublicIPv6Port, c.Secret), + urls := IPURLs{} + if c.PublicIPv4 != nil { + urls.IPv4 = getURLs(c.PublicIPv4, c.PublicIPv4Port, c.Secret) } + if c.PublicIPv6 != nil { + urls.IPv6 = getURLs(c.PublicIPv6, c.PublicIPv6Port, c.Secret) + } + + return urls } func getAddr(host fmt.Stringer, port uint16) string { @@ -106,12 +101,11 @@ func NewConfig(debug, verbose bool, // nolint: gocyclo if publicIPv4 == nil { publicIPv4, err = getGlobalIPv4() if err != nil { - return nil, errors.Errorf("Cannot get public IP") + publicIPv4 = nil + } else if publicIPv4.To4() == nil { + return nil, errors.Errorf("IP %s is not IPv4", publicIPv4.String()) } } - if publicIPv4.To4() == nil { - return nil, errors.Errorf("IP %s is not IPv4", publicIPv4.String()) - } if PublicIPv4Port == 0 { PublicIPv4Port = bindPort } @@ -119,12 +113,11 @@ func NewConfig(debug, verbose bool, // nolint: gocyclo if publicIPv6 == nil { publicIPv6, err = getGlobalIPv6() if err != nil { - publicIPv6 = publicIPv4 + publicIPv6 = nil + } else if publicIPv6.To4() != nil { + return nil, errors.Errorf("IP %s is not IPv6", publicIPv6.String()) } } - if publicIPv6.To16() == nil { - return nil, errors.Errorf("IP %s is not IPv6", publicIPv6.String()) - } if publicIPv6Port == 0 { publicIPv6Port = bindPort } diff --git a/main.go b/main.go index fa63cfc..8154f6f 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "fmt" "io" "math/rand" "os" @@ -15,7 +16,8 @@ import ( kingpin "gopkg.in/alecthomas/kingpin.v2" "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/proxy" + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/telegram" "github.com/juju/errors" ) @@ -112,15 +114,29 @@ func main() { atom, )).Sugar() - stat := proxy.NewStats(conf) - go stat.Serve() - - srv := proxy.NewServer(conf, logger, stat) - printURLs(conf.GetURLs()) - - if err := srv.Serve(); err != nil { - logger.Fatal(err.Error()) + tg := telegram.NewMiddleTelegram(conf, logger) + connOpts := &mtproto.ConnectionOpts{ + DC: int16(1), + ConnectionType: mtproto.ConnectionTypeAbridged, + ConnectionProto: mtproto.ConnectionProtocolIPv4, } + + sock, err := tg.Dial(connOpts) + if err != nil { + panic(err) + } + _, err = tg.Init(connOpts, sock) + fmt.Println(err) + + // stat := proxy.NewStats(conf) + // go stat.Serve() + + // srv := proxy.NewServer(conf, logger, stat) + // printURLs(conf.GetURLs()) + + // if err := srv.Serve(); err != nil { + // logger.Fatal(err.Error()) + // } } func setRLimit() (err error) { diff --git a/mtproto/connection_options.go b/mtproto/connection_options.go index 0f82ec0..823b81b 100644 --- a/mtproto/connection_options.go +++ b/mtproto/connection_options.go @@ -2,7 +2,6 @@ package mtproto import ( "bytes" - "net" "github.com/juju/errors" ) @@ -19,7 +18,8 @@ type ConnectionOpts struct { DC int16 ConnectionType ConnectionType ConnectionProto ConnectionProtocol - ClientAddr *net.TCPAddr + QuickAck bool + SimpleAck bool } // Different connection types which user requests from Telegram. diff --git a/mtproto/rpc/rpc_handshake_request.go b/mtproto/rpc/rpc_handshake_request.go index c0a30a6..0625c67 100644 --- a/mtproto/rpc/rpc_handshake_request.go +++ b/mtproto/rpc/rpc_handshake_request.go @@ -1,6 +1,8 @@ package rpc -import "bytes" +import ( + "bytes" +) const ( rpcHandshakeTagLength = 4 diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index b790348..fc9bf63 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -22,11 +22,12 @@ const ( var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} -func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, - resp *rpc.RPCNonceResponse, client *net.TCPAddr, secret []byte) wrappers.ReadWriteCloserWithAddr { - remote := conn.Addr() - encKey, encIV := makeKeys(CipherPurposeClient, req, resp, client, remote, secret) - decKey, decIV := makeKeys(CipherPurposeServer, req, resp, client, remote, secret) +func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, secret []byte) wrappers.ReadWriteCloserWithAddr { + localAddr := conn.LocalAddr() + remoteAddr := conn.RemoteAddr() + + encKey, encIV := makeKeys(CipherPurposeClient, req, resp, localAddr, remoteAddr, secret) + decKey, decIV := makeKeys(CipherPurposeServer, req, resp, localAddr, remoteAddr, secret) enc, _ := makeEncrypterDecrypter(encKey, encIV) _, dec := makeEncrypterDecrypter(decKey, decIV) diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index 784762f..d008d37 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -112,8 +112,12 @@ func (f *FrameRWC) Close() error { return f.conn.Close() } -func (f *FrameRWC) Addr() *net.TCPAddr { - return f.conn.Addr() +func (f *FrameRWC) LocalAddr() *net.TCPAddr { + return f.conn.LocalAddr() +} + +func (f *FrameRWC) RemoteAddr() *net.TCPAddr { + return f.conn.RemoteAddr() } func NewFrameRWC(conn wrappers.ReadWriteCloserWithAddr, seqNo int32) wrappers.ReadWriteCloserWithAddr { diff --git a/telegram/dialer.go b/telegram/dialer.go index f0d9cb5..9cf048b 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -14,6 +14,8 @@ const telegramDialTimeout = 10 * time.Second type tgDialer struct { net.Dialer + + conf *config.Config } func (t *tgDialer) dial(addr string) (net.Conn, error) { @@ -34,5 +36,5 @@ func (t *tgDialer) dialRWC(addr string) (wrappers.ReadWriteCloserWithAddr, error return nil, err } - return wrappers.NewTimeoutRWC(conn), nil + return wrappers.NewTimeoutRWC(conn, t.conf.PublicIPv4, t.conf.PublicIPv6), nil } diff --git a/telegram/direct.go b/telegram/direct.go index 81fb2eb..8975f71 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -57,7 +57,10 @@ func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re // to Telegram bypassing middleproxies. func NewDirectTelegram(conf *config.Config) Telegram { return &directTelegram{baseTelegram{ - dialer: tgDialer{net.Dialer{Timeout: telegramDialTimeout}}, + dialer: tgDialer{ + Dialer: net.Dialer{Timeout: telegramDialTimeout}, + conf: conf, + }, v4Addresses: directV4Addresses, v6Addresses: directV6Addresses, }} diff --git a/telegram/middle.go b/telegram/middle.go index b3b4ed3..03d70cd 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -1,6 +1,7 @@ package telegram import ( + "fmt" "io" "net" "net/http" @@ -24,7 +25,10 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram tg := &middleTelegram{ middleTelegramCaller: middleTelegramCaller{ baseTelegram: baseTelegram{ - dialer: tgDialer{net.Dialer{Timeout: telegramDialTimeout}}, + dialer: tgDialer{ + Dialer: net.Dialer{Timeout: telegramDialTimeout}, + conf: conf, + }, }, logger: logger, httpClient: &http.Client{ @@ -54,8 +58,7 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return nil, err } - secureConn := mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, - rpcNonceResp, connOpts.ClientAddr, t.proxySecret) + secureConn := mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, rpcNonceResp, t.proxySecret) secureConn = mtwrappers.NewFrameRWC(secureConn, rpc.RPCHandshakeSeqNo) rpcHandshakeReq, err := t.sendRPCHandshakeRequest(secureConn) @@ -123,6 +126,7 @@ func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.RP if err = rpcHandshakeResp.Valid(req); err != nil { return nil, errors.Annotate(err, "Invalid RPC handshake response") } + fmt.Println("VICTORY") return rpcHandshakeResp, nil } diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go index 4d11d5d..9db024d 100644 --- a/wrappers/blockcipherrwc.go +++ b/wrappers/blockcipherrwc.go @@ -25,6 +25,7 @@ func (c *BlockCipherReadWriteCloserWithAddr) Read(p []byte) (int, error) { return errors.Annotate(err, "Cannot read from socket") } c.Buffer.Write(p[:n]) + bufferLength = c.Buffer.Len() } c.decryptor.CryptBlocks(c.Buffer.Bytes(), c.Buffer.Bytes()) @@ -47,8 +48,12 @@ func (c *BlockCipherReadWriteCloserWithAddr) Close() error { return c.conn.Close() } -func (c *BlockCipherReadWriteCloserWithAddr) Addr() *net.TCPAddr { - return c.conn.Addr() +func (c *BlockCipherReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return c.conn.LocalAddr() +} + +func (c *BlockCipherReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return c.conn.RemoteAddr() } func NewBlockCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.BlockMode) ReadWriteCloserWithAddr { diff --git a/wrappers/ctxrwc.go b/wrappers/ctxrwc.go index b378a96..64c1006 100644 --- a/wrappers/ctxrwc.go +++ b/wrappers/ctxrwc.go @@ -48,8 +48,12 @@ func (c *CtxReadWriteCloserWithAddr) Close() error { return c.conn.Close() } -func (c *CtxReadWriteCloserWithAddr) Addr() *net.TCPAddr { - return c.conn.Addr() +func (c *CtxReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return c.conn.LocalAddr() +} + +func (c *CtxReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return c.conn.RemoteAddr() } // NewCtxRWC returns ReadWriteCloser which respects given context, diff --git a/wrappers/logrwc.go b/wrappers/logrwc.go index bab7af7..fd80ad4 100644 --- a/wrappers/logrwc.go +++ b/wrappers/logrwc.go @@ -36,8 +36,12 @@ func (l *LogReadWriteCloserWithAddr) Close() error { return err } -func (l *LogReadWriteCloserWithAddr) Addr() *net.TCPAddr { - return l.conn.Addr() +func (l *LogReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return l.conn.LocalAddr() +} + +func (l *LogReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return l.conn.RemoteAddr() } // NewLogRWC wraps ReadWriteCloser with logger calls. diff --git a/wrappers/rwcaddr.go b/wrappers/rwcaddr.go index 804bc5a..6ca4299 100644 --- a/wrappers/rwcaddr.go +++ b/wrappers/rwcaddr.go @@ -8,5 +8,6 @@ import ( type ReadWriteCloserWithAddr interface { io.ReadWriteCloser - Addr() *net.TCPAddr + LocalAddr() *net.TCPAddr + RemoteAddr() *net.TCPAddr } diff --git a/wrappers/streamcipherrwc.go b/wrappers/streamcipherrwc.go index d847c72..a5f15d1 100644 --- a/wrappers/streamcipherrwc.go +++ b/wrappers/streamcipherrwc.go @@ -43,8 +43,12 @@ func (c *StreamCipherReadWriteCloserWithAddr) Close() error { return c.conn.Close() } -func (c *StreamCipherReadWriteCloserWithAddr) Addr() *net.TCPAddr { - return c.conn.Addr() +func (c *StreamCipherReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return c.conn.LocalAddr() +} + +func (c *StreamCipherReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return c.conn.RemoteAddr() } // NewStreamCipherRWC returns wrapper which transparently diff --git a/wrappers/timeoutrwc.go b/wrappers/timeoutrwc.go index e758600..952ade1 100644 --- a/wrappers/timeoutrwc.go +++ b/wrappers/timeoutrwc.go @@ -8,7 +8,9 @@ import ( ) type TimeoutReadWriteCloserWithAddr struct { - conn net.Conn + conn net.Conn + publicIPv4 net.IP + publicIPv6 net.IP } func (t *TimeoutReadWriteCloserWithAddr) Read(p []byte) (int, error) { @@ -25,10 +27,29 @@ func (t *TimeoutReadWriteCloserWithAddr) Close() error { return t.conn.Close() } -func (t *TimeoutReadWriteCloserWithAddr) Addr() *net.TCPAddr { +func (t *TimeoutReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return t.conn.RemoteAddr().(*net.TCPAddr) } -func NewTimeoutRWC(conn net.Conn) ReadWriteCloserWithAddr { - return &TimeoutReadWriteCloserWithAddr{conn} +func (t *TimeoutReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + addr := t.conn.LocalAddr().(*net.TCPAddr) + newAddr := *addr + + if t.RemoteAddr().IP.To4() != nil { + if t.publicIPv4 != nil { + newAddr.IP = t.publicIPv4 + } + } else if t.publicIPv6 != nil { + newAddr.IP = t.publicIPv6 + } + + return &newAddr +} + +func NewTimeoutRWC(conn net.Conn, ipv4, ipv6 net.IP) ReadWriteCloserWithAddr { + return &TimeoutReadWriteCloserWithAddr{ + conn: conn, + publicIPv4: ipv4, + publicIPv6: ipv6, + } } diff --git a/wrappers/trafficrwc.go b/wrappers/trafficrwc.go index 78518ad..60dd8c2 100644 --- a/wrappers/trafficrwc.go +++ b/wrappers/trafficrwc.go @@ -29,8 +29,12 @@ func (t *TrafficReadWriteCloserWithAddr) Close() error { return t.conn.Close() } -func (t *TrafficReadWriteCloserWithAddr) Addr() *net.TCPAddr { - return t.conn.Addr() +func (t *TrafficReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return t.conn.LocalAddr() +} + +func (t *TrafficReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return t.conn.RemoteAddr() } // NewTrafficRWC wraps ReadWriteCloser to have read/write callbacks. From a274493e3b453113cdfc4e995e1ca3ff8b964c43 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 4 Jul 2018 12:57:13 +0300 Subject: [PATCH 24/71] Fix rpc proxy request --- mtproto/rpc/rpc.go | 5 ---- mtproto/rpc/rpc_proxy_request.go | 39 +++++++++++++++----------------- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index 0ae4d7f..8f2ae99 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -4,8 +4,3 @@ const ( RPCNonceSeqNo = -2 RPCHandshakeSeqNo = -1 ) - -type Extras struct { - QuickAck bool - SimpleAck bool -} diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go index feaecb3..1f6a7d6 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/rpc_proxy_request.go @@ -25,17 +25,17 @@ var ( type RPCProxyRequest struct { Flags RPCProxyRequestFlags ConnectionID [rpcProxyRequestConnectionIDLength]byte - RemoteIPPort [rpcProxyRequestIPPortLength]byte - LocalIPPort [rpcProxyRequestIPPortLength]byte + OurIPPort [rpcProxyRequestIPPortLength]byte + ClientIPPort [rpcProxyRequestIPPortLength]byte ADTag []byte - Extras Extras + Options *mtproto.ConnectionOpts } func (r *RPCProxyRequest) Bytes(message []byte) []byte { buf := &bytes.Buffer{} flags := r.Flags - if r.Extras.QuickAck { + if r.Options.QuickAck { flags |= RPCProxyRequestFlagsQuickAck } @@ -46,25 +46,22 @@ func (r *RPCProxyRequest) Bytes(message []byte) []byte { buf.Write(rpcProxyRequestTag) buf.Write(flags.Bytes()) buf.Write(r.ConnectionID[:]) - buf.Write(r.RemoteIPPort[:]) - buf.Write(r.LocalIPPort[:]) + buf.Write(r.ClientIPPort[:]) + buf.Write(r.OurIPPort[:]) buf.Write(rpcProxyRequestExtraSize) buf.Write(rpcProxyRequestProxyTag) buf.WriteByte(byte(len(r.ADTag))) buf.Write(r.ADTag) - - for i := 0; i < (buf.Len() % 4); i++ { - buf.WriteByte(0x00) - } + buf.Write(bytes.Repeat([]byte{0x00}, buf.Len()%4)) buf.Write(message) return buf.Bytes() } -func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *net.TCPAddr, adTag []byte, extras Extras) (*RPCProxyRequest, error) { +func NewRPCProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.ConnectionOpts, adTag []byte) (*RPCProxyRequest, error) { flags := RPCProxyRequestFlagsHasAdTag | RPCProxyRequestFlagsMagic | RPCProxyRequestFlagsExtMode2 - switch connectionType { + switch opts.ConnectionType { case mtproto.ConnectionTypeAbridged: flags |= RPCProxyRequestFlagsAbdridged case mtproto.ConnectionTypeIntermediate: @@ -72,9 +69,9 @@ func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *ne } request := RPCProxyRequest{ - Flags: flags, - ADTag: adTag, - Extras: extras, + Flags: flags, + ADTag: adTag, + Options: opts, } if _, err := rand.Read(request.ConnectionID[:]); err != nil { @@ -82,13 +79,13 @@ func NewRPCProxyRequest(connectionType mtproto.ConnectionType, local, remote *ne } port := make([]byte, 4) - copy(request.LocalIPPort[:], local.IP.To16()) - binary.LittleEndian.PutUint32(port, uint32(local.Port)) - copy(request.LocalIPPort[16:], port) + copy(request.ClientIPPort[:16], clientAddr.IP.To16()) + binary.LittleEndian.PutUint32(port, uint32(clientAddr.Port)) + copy(request.ClientIPPort[16:], port) - copy(request.RemoteIPPort[:], remote.IP.To16()) - binary.LittleEndian.PutUint32(port, uint32(remote.Port)) - copy(request.RemoteIPPort[16:], port) + copy(request.OurIPPort[:16], ownAddr.IP.To16()) + binary.LittleEndian.PutUint32(port, uint32(ownAddr.Port)) + copy(request.OurIPPort[16:], port) return &request, nil } From 69c4997fd4657c829c3bf375812cee7fa878a1dc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 4 Jul 2018 13:28:23 +0300 Subject: [PATCH 25/71] Add proxy request wrapper --- client/direct.go | 1 + mtproto/connection_options.go | 2 + mtproto/wrappers/proxy_request.go | 102 ++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 mtproto/wrappers/proxy_request.go diff --git a/client/direct.go b/client/direct.go index 1e38e5e..98b518d 100644 --- a/client/direct.go +++ b/client/direct.go @@ -32,6 +32,7 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wr return nil, nil, errors.Annotate(err, "Cannot parse obfuscated frame") } connOpts.ConnectionProto = mtproto.ConnectionProtocolAny + connOpts.ClientAddr = conn.RemoteAddr().(*net.TCPAddr) socket := wrappers.NewTimeoutRWC(conn, conf.PublicIPv4, conf.PublicIPv6) socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) diff --git a/mtproto/connection_options.go b/mtproto/connection_options.go index 823b81b..585e0f7 100644 --- a/mtproto/connection_options.go +++ b/mtproto/connection_options.go @@ -2,6 +2,7 @@ package mtproto import ( "bytes" + "net" "github.com/juju/errors" ) @@ -20,6 +21,7 @@ type ConnectionOpts struct { ConnectionProto ConnectionProtocol QuickAck bool SimpleAck bool + ClientAddr *net.TCPAddr } // Different connection types which user requests from Telegram. diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go new file mode 100644 index 0000000..d048a85 --- /dev/null +++ b/mtproto/wrappers/proxy_request.go @@ -0,0 +1,102 @@ +package wrappers + +import ( + "bytes" + "io" + "io/ioutil" + "net" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/mtproto/rpc" + "github.com/9seconds/mtg/wrappers" +) + +var ( + rpcCloseExtTag = [4]byte{0xa2, 0x34, 0xb6, 0x5e} + rpcProxyAnsTag = [4]byte{0x0d, 0xda, 0x03, 0x44} + rpcSimpleAckTag = [4]byte{0x9b, 0x40, 0xac, 0x3b} +) + +type ProxyRequestReadWriteCloserWithAddr struct { + wrappers.BufferedReader + + conn wrappers.ReadWriteCloserWithAddr + req *rpc.RPCProxyRequest +} + +func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { + return p.BufferedRead(buf, func() error { + ansBuf := &bytes.Buffer{} + ansBuf.Grow(4) + + if _, err := io.CopyN(ansBuf, p.conn, 4); err != nil { + return errors.Annotate(err, "Cannot read RPC tag") + } + + if bytes.Equal(ansBuf.Bytes(), rpcCloseExtTag[:]) { + return errors.New("Connection has been closed remotely") + } else if bytes.Equal(ansBuf.Bytes(), rpcProxyAnsTag[:]) { + if _, err := io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { + return errors.Annotate(err, "Cannot skip flags and connid") + } + for { + n, err := p.conn.Read(buf) + if err != nil { + return errors.Annotate(err, "Cannot read proxy answer") + } + if n == 0 { + break + } + p.Buffer.Write(buf[:n]) + } + return nil + } else if bytes.Equal(ansBuf.Bytes(), rpcSimpleAckTag[:]) { + if _, err := io.CopyN(ioutil.Discard, p.conn, 8); err != nil { + return errors.Annotate(err, "Cannot skip connid") + } + if _, err := io.CopyN(p.Buffer, p.conn, 4); err != nil { + return errors.Annotate(err, "Cannot read simple ack") + } + p.req.Options.SimpleAck = true + return nil + } + + return nil + }) +} + +func (p *ProxyRequestReadWriteCloserWithAddr) Write(raw []byte) (int, error) { + if _, err := p.conn.Write(p.req.Bytes(raw)); err != nil { + return 0, err + } + p.req.Options.SimpleAck = false + p.req.Options.QuickAck = false + + return len(raw), nil +} + +func (p *ProxyRequestReadWriteCloserWithAddr) Close() error { + return p.conn.Close() +} + +func (p *ProxyRequestReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return p.conn.LocalAddr() +} + +func (p *ProxyRequestReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return p.conn.RemoteAddr() +} + +func NewProxyRequestRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts, adTag []byte) (wrappers.ReadWriteCloserWithAddr, error) { + req, err := rpc.NewRPCProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) + if err != nil { + return nil, errors.Annotate(err, "Cannot create new RPC proxy request") + } + + return &ProxyRequestReadWriteCloserWithAddr{ + conn: conn, + req: req, + }, nil +} From c35c482de7f060c49113e8a5884dbf4174373bdc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 4 Jul 2018 18:04:52 +0300 Subject: [PATCH 26/71] Propagate adtag --- config/config.go | 12 +++++++++++- main.go | 3 ++- telegram/middle.go | 5 ++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index d988c38..6235b8c 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,7 @@ type Config struct { StatsIP net.IP Secret []byte + AdTag []byte } // URLs contains links to the proxy (tg://, t.me) and their QR codes. @@ -89,7 +90,7 @@ func NewConfig(debug, verbose bool, // nolint: gocyclo publicIPv4 net.IP, PublicIPv4Port uint16, publicIPv6 net.IP, publicIPv6Port uint16, statsIP net.IP, statsPort uint16, - secret string) (*Config, error) { + secret, adtag string) (*Config, error) { if len(secret) != 32 { return nil, errors.New("Telegram demands secret of length 32") } @@ -98,6 +99,14 @@ func NewConfig(debug, verbose bool, // nolint: gocyclo return nil, errors.Annotate(err, "Cannot create config") } + var adTagBytes []byte + if len(adtag) != 0 { + adTagBytes, err = hex.DecodeString(adtag) + if err != nil { + return nil, errors.Annotate(err, "Cannot create config") + } + } + if publicIPv4 == nil { publicIPv4, err = getGlobalIPv4() if err != nil { @@ -138,6 +147,7 @@ func NewConfig(debug, verbose bool, // nolint: gocyclo StatsIP: statsIP, StatsPort: statsPort, Secret: secretBytes, + AdTag: adTagBytes, } return conf, nil diff --git a/main.go b/main.go index 8154f6f..fcb2599 100644 --- a/main.go +++ b/main.go @@ -72,6 +72,7 @@ var ( Uint16() secret = app.Arg("secret", "Secret of this proxy.").Required().String() + adtag = app.Arg("adtag", "ADTag of the proxy.").String() ) func init() { @@ -93,7 +94,7 @@ func main() { *publicIPv4, *publicIPv4Port, *publicIPv6, *publicIPv6Port, *statsIP, *statsPort, - *secret, + *secret, *adtag, ) if err != nil { usage(err.Error()) diff --git a/telegram/middle.go b/telegram/middle.go index 03d70cd..91f22d5 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -19,6 +19,8 @@ import ( type middleTelegram struct { middleTelegramCaller + + adtag []byte } func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram { @@ -36,6 +38,7 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram }, dialerMutex: &sync.RWMutex{}, }, + adtag: conf.AdTag, } if err := tg.update(); err != nil { @@ -70,7 +73,7 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return nil, err } - return nil, nil + return mtwrappers.NewProxyRequestRWC(secureConn, connOpts, t.adtag) } func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.RPCNonceRequest, error) { From 1894aa798916718c0e98b76844c28642576b5fbc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Wed, 4 Jul 2018 18:25:15 +0300 Subject: [PATCH 27/71] Refactorings of rpc --- mtproto/rpc/rpc.go | 6 +++ mtproto/wrappers/proxy_request.go | 73 +++++++++++++++++-------------- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index 8f2ae99..575b9b7 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -4,3 +4,9 @@ const ( RPCNonceSeqNo = -2 RPCHandshakeSeqNo = -1 ) + +var ( + RPCTagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} + RPCTagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} + RPCTagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} +) diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go index d048a85..5b1b269 100644 --- a/mtproto/wrappers/proxy_request.go +++ b/mtproto/wrappers/proxy_request.go @@ -13,12 +13,6 @@ import ( "github.com/9seconds/mtg/wrappers" ) -var ( - rpcCloseExtTag = [4]byte{0xa2, 0x34, 0xb6, 0x5e} - rpcProxyAnsTag = [4]byte{0x0d, 0xda, 0x03, 0x44} - rpcSimpleAckTag = [4]byte{0x9b, 0x40, 0xac, 0x3b} -) - type ProxyRequestReadWriteCloserWithAddr struct { wrappers.BufferedReader @@ -35,38 +29,53 @@ func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { return errors.Annotate(err, "Cannot read RPC tag") } - if bytes.Equal(ansBuf.Bytes(), rpcCloseExtTag[:]) { - return errors.New("Connection has been closed remotely") - } else if bytes.Equal(ansBuf.Bytes(), rpcProxyAnsTag[:]) { - if _, err := io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { - return errors.Annotate(err, "Cannot skip flags and connid") - } - for { - n, err := p.conn.Read(buf) - if err != nil { - return errors.Annotate(err, "Cannot read proxy answer") - } - if n == 0 { - break - } - p.Buffer.Write(buf[:n]) - } - return nil - } else if bytes.Equal(ansBuf.Bytes(), rpcSimpleAckTag[:]) { - if _, err := io.CopyN(ioutil.Discard, p.conn, 8); err != nil { - return errors.Annotate(err, "Cannot skip connid") - } - if _, err := io.CopyN(p.Buffer, p.conn, 4); err != nil { - return errors.Annotate(err, "Cannot read simple ack") - } - p.req.Options.SimpleAck = true - return nil + if bytes.Equal(ansBuf.Bytes(), rpc.RPCTagCloseExt) { + return p.readCloseExt() + } else if bytes.Equal(ansBuf.Bytes(), rpc.RPCTagProxyAns) { + return p.readProxyAns(buf) + } else if bytes.Equal(ansBuf.Bytes(), rpc.RPCTagSimpleAck) { + return p.readSimpleAck() } return nil }) } +func (p *ProxyRequestReadWriteCloserWithAddr) readCloseExt() error { + return errors.New("Connection has been closed remotely") +} + +func (p *ProxyRequestReadWriteCloserWithAddr) readProxyAns(buf []byte) error { + if _, err := io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { + return errors.Annotate(err, "Cannot skip flags and connid") + } + + for { + n, err := p.conn.Read(buf) + if err != nil { + return errors.Annotate(err, "Cannot read proxy answer") + } + if n == 0 { + break + } + p.Buffer.Write(buf[:n]) + } + + return nil +} + +func (p *ProxyRequestReadWriteCloserWithAddr) readSimpleAck() error { + if _, err := io.CopyN(ioutil.Discard, p.conn, 8); err != nil { + return errors.Annotate(err, "Cannot skip connid") + } + if _, err := io.CopyN(p.Buffer, p.conn, 4); err != nil { + return errors.Annotate(err, "Cannot read simple ack") + } + p.req.Options.SimpleAck = true + + return nil +} + func (p *ProxyRequestReadWriteCloserWithAddr) Write(raw []byte) (int, error) { if _, err := p.conn.Write(p.req.Bytes(raw)); err != nil { return 0, err From 4264fdadf84b1cc61a68caa2c37c6fa501918317 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 09:20:28 +0300 Subject: [PATCH 28/71] Add abridged rwc --- mtproto/wrappers/abridged.go | 120 +++++++++++++++++++++++++++++++++++ telegram/middle.go | 6 +- wrappers/buffered_reader.go | 16 ++--- 3 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 mtproto/wrappers/abridged.go diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go new file mode 100644 index 0000000..db5a99e --- /dev/null +++ b/mtproto/wrappers/abridged.go @@ -0,0 +1,120 @@ +package wrappers + +import ( + "bytes" + "encoding/binary" + "io" + "net" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" +) + +type uint24 [3]byte + +const ( + abridgedSmallPacketLength = 0x7f + abridgedQuickAckLength = 0x80 + abridgedLargePacketLength = 16777216 // 256 ^ 3 +) + +type AbridgedReadWriteCloserAddr struct { + wrappers.BufferedReader + + conn wrappers.ReadWriteCloserWithAddr + opts *mtproto.ConnectionOpts +} + +func (a *AbridgedReadWriteCloserAddr) Read(p []byte) (int, error) { + return a.BufferedRead(p, func() error { + var msgLength uint8 + if err := binary.Read(a.conn, binary.LittleEndian, &msgLength); err != nil { + return errors.Annotate(err, "Cannot read message length") + } + + a.opts.QuickAck = false + if msgLength >= abridgedQuickAckLength { + a.opts.QuickAck = true + msgLength -= 0x80 + } + + msgLength32 := uint32(msgLength) + if msgLength == abridgedSmallPacketLength { + buf := &bytes.Buffer{} + buf.Grow(3) + + if _, err := io.CopyN(buf, a.conn, 3); err != nil { + return errors.Annotate(err, "Cannot read correct message length") + } + number := uint24{} + copy(number[:], buf.Bytes()) + msgLength32 = fromUint24(number) + } + msgLength32 *= 4 + + if _, err := io.CopyN(a.Buffer, a.conn, int64(msgLength32)); err != nil { + return errors.Annotate(err, "Cannot read message") + } + + return nil + }) +} + +func (a *AbridgedReadWriteCloserAddr) Write(p []byte) (int, error) { + if len(p)%4 != 0 { + return 0, errors.Errorf("Incorrect packet length %d", len(p)) + } + if a.opts.SimpleAck { + return a.conn.Write(reverseBytes(p)) + } + + packetLength := len(p) / 4 + switch { + case packetLength < abridgedSmallPacketLength: + newData := append([]byte{byte(packetLength)}, p...) + return a.conn.Write(newData) + + case packetLength < abridgedLargePacketLength: + length24 := toUint24(uint32(packetLength)) + + buf := &bytes.Buffer{} + buf.Grow(1 + 3 + len(p)) + buf.WriteByte(byte(abridgedSmallPacketLength)) + buf.Write(length24[:]) + buf.Write(p) + + return a.conn.Write(buf.Bytes()) + + default: + return 0, errors.Errorf("Packet is too big %d", len(p)) + } +} + +func (a *AbridgedReadWriteCloserAddr) Close() error { + return a.conn.Close() +} + +func (a *AbridgedReadWriteCloserAddr) LocalAddr() *net.TCPAddr { + return a.conn.LocalAddr() +} + +func (a *AbridgedReadWriteCloserAddr) RemoteAddr() *net.TCPAddr { + return a.conn.RemoteAddr() +} + +func toUint24(number uint32) uint24 { + return uint24{byte(number), byte(number >> 8), byte(number >> 16)} +} + +func fromUint24(number uint24) uint32 { + return uint32(number[0]) + (uint32(number[1]) << 8) + (uint32(number[2]) << 16) +} + +func NewAbridgedRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts) wrappers.ReadWriteCloserWithAddr { + return &AbridgedReadWriteCloserAddr{ + conn: conn, + opts: connOpts, + } +} diff --git a/telegram/middle.go b/telegram/middle.go index 91f22d5..8e1bd7b 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -20,7 +20,7 @@ import ( type middleTelegram struct { middleTelegramCaller - adtag []byte + conf *config.Config } func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram { @@ -38,7 +38,7 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram }, dialerMutex: &sync.RWMutex{}, }, - adtag: conf.AdTag, + conf: conf, } if err := tg.update(); err != nil { @@ -73,7 +73,7 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return nil, err } - return mtwrappers.NewProxyRequestRWC(secureConn, connOpts, t.adtag) + return mtwrappers.NewProxyRequestRWC(secureConn, connOpts, t.conf.AdTag) } func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.RPCNonceRequest, error) { diff --git a/wrappers/buffered_reader.go b/wrappers/buffered_reader.go index a824805..cb48d56 100644 --- a/wrappers/buffered_reader.go +++ b/wrappers/buffered_reader.go @@ -17,20 +17,14 @@ func (b *BufferedReader) BufferedRead(p []byte, callback func() error) (int, err } func (b *BufferedReader) flush(p []byte) (int, error) { - sizeToRead := len(p) - if b.Buffer.Len() < sizeToRead { - sizeToRead = b.Buffer.Len() - } - - data := b.Buffer.Bytes() - copy(p, data[:sizeToRead]) - if sizeToRead == b.Buffer.Len() { + if b.Buffer.Len() < len(p) { + sizeToReturn := b.Buffer.Len() + copy(p, b.Buffer.Bytes()) b.Buffer.Reset() - } else { - b.Buffer = bytes.NewBuffer(data[sizeToRead:]) + return sizeToReturn, nil } - return sizeToRead, nil + return b.Buffer.Read(p) } func NewBufferedReader() BufferedReader { From d636010377261d962912038b967523ba3b0f7652 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 09:49:15 +0300 Subject: [PATCH 29/71] Add intermediate rwc --- mtproto/wrappers/abridged.go | 19 +++---- mtproto/wrappers/intermediate.go | 83 +++++++++++++++++++++++++++++++ mtproto/wrappers/proxy_request.go | 5 +- 3 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 mtproto/wrappers/intermediate.go diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go index db5a99e..66b440d 100644 --- a/mtproto/wrappers/abridged.go +++ b/mtproto/wrappers/abridged.go @@ -20,14 +20,14 @@ const ( abridgedLargePacketLength = 16777216 // 256 ^ 3 ) -type AbridgedReadWriteCloserAddr struct { +type AbridgedReadWriteCloserWithAddr struct { wrappers.BufferedReader conn wrappers.ReadWriteCloserWithAddr opts *mtproto.ConnectionOpts } -func (a *AbridgedReadWriteCloserAddr) Read(p []byte) (int, error) { +func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { return a.BufferedRead(p, func() error { var msgLength uint8 if err := binary.Read(a.conn, binary.LittleEndian, &msgLength); err != nil { @@ -62,7 +62,7 @@ func (a *AbridgedReadWriteCloserAddr) Read(p []byte) (int, error) { }) } -func (a *AbridgedReadWriteCloserAddr) Write(p []byte) (int, error) { +func (a *AbridgedReadWriteCloserWithAddr) Write(p []byte) (int, error) { if len(p)%4 != 0 { return 0, errors.Errorf("Incorrect packet length %d", len(p)) } @@ -92,15 +92,15 @@ func (a *AbridgedReadWriteCloserAddr) Write(p []byte) (int, error) { } } -func (a *AbridgedReadWriteCloserAddr) Close() error { +func (a *AbridgedReadWriteCloserWithAddr) Close() error { return a.conn.Close() } -func (a *AbridgedReadWriteCloserAddr) LocalAddr() *net.TCPAddr { +func (a *AbridgedReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { return a.conn.LocalAddr() } -func (a *AbridgedReadWriteCloserAddr) RemoteAddr() *net.TCPAddr { +func (a *AbridgedReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return a.conn.RemoteAddr() } @@ -113,8 +113,9 @@ func fromUint24(number uint24) uint32 { } func NewAbridgedRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts) wrappers.ReadWriteCloserWithAddr { - return &AbridgedReadWriteCloserAddr{ - conn: conn, - opts: connOpts, + return &AbridgedReadWriteCloserWithAddr{ + BufferedReader: wrappers.NewBufferedReader(), + conn: conn, + opts: connOpts, } } diff --git a/mtproto/wrappers/intermediate.go b/mtproto/wrappers/intermediate.go new file mode 100644 index 0000000..1c2a6ab --- /dev/null +++ b/mtproto/wrappers/intermediate.go @@ -0,0 +1,83 @@ +package wrappers + +import ( + "bytes" + "encoding/binary" + "io" + "net" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" +) + +const intermediateQuickAckLength = 0x80000000 + +type IntermediateReadWriteCloserWithAddr struct { + wrappers.BufferedReader + + conn wrappers.ReadWriteCloserWithAddr + opts *mtproto.ConnectionOpts +} + +func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { + return i.BufferedRead(p, func() error { + var length uint32 + if err := binary.Read(i.conn, binary.LittleEndian, &length); err != nil { + return errors.Annotate(err, "Cannot read message length") + } + + if length > intermediateQuickAckLength { + i.opts.QuickAck = true + length -= intermediateQuickAckLength + } + + buf := &bytes.Buffer{} + buf.Grow(int(length)) + if _, err := io.CopyN(buf, i.conn, int64(length)); err != nil { + return errors.Annotate(err, "Cannot read the message") + } + + if length%4 != 0 { + length -= length % 4 + i.Buffer.Write(buf.Bytes()[:length]) + return nil + } + + i.Buffer.Write(buf.Bytes()) + + return nil + }) +} + +func (i *IntermediateReadWriteCloserWithAddr) Write(p []byte) (int, error) { + if i.opts.SimpleAck { + return i.conn.Write(p) + } + + var length [4]byte + binary.LittleEndian.PutUint32(length[:], uint32(len(p))) + + return i.conn.Write(append(length[:], p...)) +} + +func (i *IntermediateReadWriteCloserWithAddr) Close() error { + return i.conn.Close() +} + +func (i *IntermediateReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { + return i.conn.LocalAddr() +} + +func (i *IntermediateReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { + return i.conn.RemoteAddr() +} + +func NewIntermediateRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts) wrappers.ReadWriteCloserWithAddr { + return &IntermediateReadWriteCloserWithAddr{ + BufferedReader: wrappers.NewBufferedReader(), + conn: conn, + opts: connOpts, + } +} diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go index 5b1b269..657f120 100644 --- a/mtproto/wrappers/proxy_request.go +++ b/mtproto/wrappers/proxy_request.go @@ -105,7 +105,8 @@ func NewProxyRequestRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto } return &ProxyRequestReadWriteCloserWithAddr{ - conn: conn, - req: req, + BufferedReader: wrappers.NewBufferedReader(), + conn: conn, + req: req, }, nil } From a7eb29cc94bc528b0926cf9c09e2abf4eb23a094 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 10:04:18 +0300 Subject: [PATCH 30/71] Propagate to middle client --- client/middle.go | 25 +++++++++++++++++++++++++ main.go | 34 +++++++++------------------------- proxy/server.go | 12 ++++++++++-- telegram/direct.go | 3 ++- 4 files changed, 46 insertions(+), 28 deletions(-) create mode 100644 client/middle.go diff --git a/client/middle.go b/client/middle.go new file mode 100644 index 0000000..d5dc4d3 --- /dev/null +++ b/client/middle.go @@ -0,0 +1,25 @@ +package client + +import ( + "net" + + "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/mtproto" + mtwrappers "github.com/9seconds/mtg/mtproto/wrappers" + "github.com/9seconds/mtg/wrappers" +) + +func MiddleInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr, error) { + opts, newConn, err := DirectInit(conn, conf) + if err != nil { + return nil, nil, err + } + + if opts.ConnectionType == mtproto.ConnectionTypeAbridged { + newConn = mtwrappers.NewAbridgedRWC(newConn, opts) + } else { + newConn = mtwrappers.NewIntermediateRWC(newConn, opts) + } + + return opts, newConn, nil +} diff --git a/main.go b/main.go index fcb2599..14cf85a 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ package main import ( "encoding/json" - "fmt" "io" "math/rand" "os" @@ -16,8 +15,7 @@ import ( kingpin "gopkg.in/alecthomas/kingpin.v2" "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/telegram" + "github.com/9seconds/mtg/proxy" "github.com/juju/errors" ) @@ -115,29 +113,15 @@ func main() { atom, )).Sugar() - tg := telegram.NewMiddleTelegram(conf, logger) - connOpts := &mtproto.ConnectionOpts{ - DC: int16(1), - ConnectionType: mtproto.ConnectionTypeAbridged, - ConnectionProto: mtproto.ConnectionProtocolIPv4, + stat := proxy.NewStats(conf) + go stat.Serve() + + srv := proxy.NewServer(conf, logger, stat) + printURLs(conf.GetURLs()) + + if err := srv.Serve(); err != nil { + logger.Fatal(err.Error()) } - - sock, err := tg.Dial(connOpts) - if err != nil { - panic(err) - } - _, err = tg.Init(connOpts, sock) - fmt.Println(err) - - // stat := proxy.NewStats(conf) - // go stat.Serve() - - // srv := proxy.NewServer(conf, logger, stat) - // printURLs(conf.GetURLs()) - - // if err := srv.Serve(); err != nil { - // logger.Fatal(err.Error()) - // } } func setRLimit() (err error) { diff --git a/proxy/server.go b/proxy/server.go index 1a6ea73..314b3d9 100644 --- a/proxy/server.go +++ b/proxy/server.go @@ -139,11 +139,19 @@ func (s *Server) pipe(dst io.Writer, src io.Reader, wait *sync.WaitGroup) { // NewServer creates new instance of MTPROTO proxy. func NewServer(conf *config.Config, logger *zap.SugaredLogger, stat *Stats) *Server { + clientInit := client.DirectInit + tg := telegram.NewDirectTelegram + + if len(conf.AdTag) > 0 { + clientInit = client.MiddleInit + tg = telegram.NewMiddleTelegram + } + return &Server{ conf: conf, logger: logger, stats: stat, - tg: telegram.NewDirectTelegram(conf), - clientInit: client.DirectInit, + tg: tg(conf, logger), + clientInit: clientInit, } } diff --git a/telegram/direct.go b/telegram/direct.go index 8975f71..ae89276 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -4,6 +4,7 @@ import ( "net" "github.com/juju/errors" + "go.uber.org/zap" "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" @@ -55,7 +56,7 @@ func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re // NewDirectTelegram returns Telegram instance which connects directly // to Telegram bypassing middleproxies. -func NewDirectTelegram(conf *config.Config) Telegram { +func NewDirectTelegram(conf *config.Config, _ *zap.SugaredLogger) Telegram { return &directTelegram{baseTelegram{ dialer: tgDialer{ Dialer: net.Dialer{Timeout: telegramDialTimeout}, From 0738631ef81ade2c43e4cd9b83e217e94e8a920c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 10:37:05 +0300 Subject: [PATCH 31/71] Small refactorings of rpc package --- mtproto/rpc/rpc.go | 24 +++++++++++++-- mtproto/rpc/rpc_handshake_request.go | 35 +++------------------ mtproto/rpc/rpc_handshake_response.go | 30 ++++++++---------- mtproto/rpc/rpc_nonce_request.go | 44 +++++++++------------------ mtproto/rpc/rpc_nonce_response.go | 42 ++++++++++++------------- mtproto/rpc/rpc_proxy_request.go | 44 +++++++++++---------------- 6 files changed, 91 insertions(+), 128 deletions(-) diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index 575b9b7..f4edf71 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -6,7 +6,25 @@ const ( ) var ( - RPCTagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} - RPCTagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} - RPCTagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} + RPCTagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} + RPCTagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} + RPCTagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} + RPCTagHandshake = []byte{0xf5, 0xee, 0x82, 0x76} + RPCTagNonce = []byte{0xaa, 0x87, 0xcb, 0x7a} + RPCTagProxyRequest = []byte{0xee, 0xf1, 0xce, 0x36} + + RPCNonceCryptoAES = []byte{0x01, 0x00, 0x00, 0x00} + + RPCHandshakeFlags = []byte{0x00, 0x00, 0x00, 0x00} + + RPCProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} + RPCProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} + + RPCHandshakeSenderPID = []byte{} + RPCHandshakePeerPID = []byte{} ) + +func init() { + RPCHandshakeSenderPID = []byte("IPIPPRPDTIME") + RPCHandshakePeerPID = []byte("IPIPPRPDTIME") +} diff --git a/mtproto/rpc/rpc_handshake_request.go b/mtproto/rpc/rpc_handshake_request.go index 0625c67..4a21fe4 100644 --- a/mtproto/rpc/rpc_handshake_request.go +++ b/mtproto/rpc/rpc_handshake_request.go @@ -1,46 +1,21 @@ package rpc -import ( - "bytes" -) - -const ( - rpcHandshakeTagLength = 4 - rpcHandshakeFlagsLength = 4 - rpcHandshakeSenderPIDLength = 12 - rpcHandshakePeerPIDLength = rpcHandshakeSenderPIDLength - - rpcHandshakeRequestLength = rpcHandshakeTagLength + rpcHandshakeFlagsLength + rpcHandshakeSenderPIDLength + rpcHandshakePeerPIDLength -) - -var ( - rpcHandshakeSenderPID [rpcHandshakeSenderPIDLength]byte - rpcHandshakePeerPID [rpcHandshakePeerPIDLength]byte - - rpcHandshakeTag = [rpcHandshakeTagLength]byte{0xf5, 0xee, 0x82, 0x76} - rpcHandshakeFlags = [rpcHandshakeFlagsLength]byte{0x00, 0x00, 0x00, 0x00} -) +import "bytes" type RPCHandshakeRequest struct { } func (r *RPCHandshakeRequest) Bytes() []byte { buf := &bytes.Buffer{} - buf.Grow(rpcHandshakeRequestLength) - buf.Write(rpcHandshakeTag[:]) - buf.Write(rpcHandshakeFlags[:]) - buf.Write(rpcHandshakeSenderPID[:]) - buf.Write(rpcHandshakePeerPID[:]) + buf.Write(RPCTagHandshake) + buf.Write(RPCHandshakeFlags) + buf.Write(RPCHandshakeSenderPID) + buf.Write(RPCHandshakePeerPID) return buf.Bytes() } -func init() { - copy(rpcHandshakeSenderPID[:], "IPIPPRPDTIME") - copy(rpcHandshakePeerPID[:], "IPIPPRPDTIME") -} - func NewRPCHandshakeRequest() *RPCHandshakeRequest { return &RPCHandshakeRequest{} } diff --git a/mtproto/rpc/rpc_handshake_response.go b/mtproto/rpc/rpc_handshake_response.go index f415840..af74469 100644 --- a/mtproto/rpc/rpc_handshake_response.go +++ b/mtproto/rpc/rpc_handshake_response.go @@ -6,18 +6,15 @@ import ( "github.com/juju/errors" ) -const rpcHandshakeResponseLength = rpcHandshakeRequestLength - type RPCHandshakeResponse struct { - Type [rpcHandshakeTagLength]byte - Flags [rpcHandshakeFlagsLength]byte - SenderPID [rpcHandshakeSenderPIDLength]byte - PeerPID [rpcHandshakePeerPIDLength]byte + Type []byte + Flags []byte + SenderPID []byte + PeerPID []byte } func (r *RPCHandshakeResponse) Bytes() []byte { buf := &bytes.Buffer{} - buf.Grow(rpcHandshakeResponseLength) buf.Write(r.Type[:]) buf.Write(r.Flags[:]) @@ -28,10 +25,10 @@ func (r *RPCHandshakeResponse) Bytes() []byte { } func (r *RPCHandshakeResponse) Valid(req *RPCHandshakeRequest) error { - if r.Type != rpcHandshakeTag { + if !bytes.Equal(r.Type, RPCTagHandshake) { return errors.New("Unexpected handshake tag") } - if r.PeerPID != rpcHandshakeSenderPID { + if !bytes.Equal(r.PeerPID, RPCHandshakeSenderPID) { return errors.New("Incorrect sender PID") } @@ -39,15 +36,14 @@ func (r *RPCHandshakeResponse) Valid(req *RPCHandshakeRequest) error { } func NewRPCHandshakeResponse(data []byte) (*RPCHandshakeResponse, error) { - if len(data) != rpcHandshakeResponseLength { + if len(data) != 32 { return nil, errors.New("Incorrect handshake response length") } - resp := RPCHandshakeResponse{} - copy(resp.Type[:], data[:4]) - copy(resp.Flags[:], data[4:8]) - copy(resp.SenderPID[:], data[8:20]) - copy(resp.PeerPID[:], data[20:]) - - return &resp, nil + return &RPCHandshakeResponse{ + Type: data[:4], + Flags: data[4:8], + SenderPID: data[8:20], + PeerPID: data[20:], + }, nil } diff --git a/mtproto/rpc/rpc_nonce_request.go b/mtproto/rpc/rpc_nonce_request.go index 65a6947..55f5543 100644 --- a/mtproto/rpc/rpc_nonce_request.go +++ b/mtproto/rpc/rpc_nonce_request.go @@ -9,52 +9,36 @@ import ( "github.com/juju/errors" ) -const ( - rpcNonceLength = 16 - rpcNonceKeySelectorLength = 4 - rpcNonceCryptoTSLength = 4 - rpcNonceTagLength = 4 - rpcNonceCryptoAESLength = 4 - - rpcNonceRequestLength = rpcNonceTagLength + rpcNonceKeySelectorLength + rpcNonceCryptoAESLength + rpcNonceCryptoTSLength + rpcNonceLength -) - -var ( - rpcNonceTag = [rpcNonceTagLength]byte{0xaa, 0x87, 0xcb, 0x7a} - rpcNonceCryptoAESTag = [rpcNonceCryptoAESLength]byte{0x01, 0x00, 0x00, 0x00} -) - type RPCNonceRequest struct { - KeySelector [rpcNonceKeySelectorLength]byte - CryptoTS [rpcNonceCryptoTSLength]byte - Nonce [rpcNonceLength]byte + KeySelector []byte + CryptoTS []byte + Nonce []byte } func (r *RPCNonceRequest) Bytes() []byte { buf := &bytes.Buffer{} - buf.Grow(rpcNonceRequestLength) - buf.Write(rpcNonceTag[:]) - buf.Write(r.KeySelector[:]) - buf.Write(rpcNonceCryptoAESTag[:]) - buf.Write(r.CryptoTS[:]) - buf.Write(r.Nonce[:]) + buf.Write(RPCTagNonce) + buf.Write(r.KeySelector) + buf.Write(RPCNonceCryptoAES) + buf.Write(r.CryptoTS) + buf.Write(r.Nonce) return buf.Bytes() } func NewRPCNonceRequest(proxySecret []byte) (*RPCNonceRequest, error) { - var nonce [rpcNonceLength]byte - var keySelector [rpcNonceKeySelectorLength]byte - var cryptoTS [rpcNonceCryptoTSLength]byte + nonce := make([]byte, 16) + keySelector := make([]byte, 4) + cryptoTS := make([]byte, 4) - if _, err := rand.Read(nonce[:]); err != nil { + if _, err := rand.Read(nonce); err != nil { return nil, errors.Annotate(err, "Cannot generate nonce") } - copy(keySelector[:], proxySecret) + copy(keySelector, proxySecret) timestamp := time.Now().Truncate(time.Second).Unix() % 4294967296 // 256 ^ 4 - do not know how to name - binary.LittleEndian.PutUint32(cryptoTS[:], uint32(timestamp)) + binary.LittleEndian.PutUint32(cryptoTS, uint32(timestamp)) return &RPCNonceRequest{ KeySelector: keySelector, diff --git a/mtproto/rpc/rpc_nonce_response.go b/mtproto/rpc/rpc_nonce_response.go index 7cf90fc..4f753b6 100644 --- a/mtproto/rpc/rpc_nonce_response.go +++ b/mtproto/rpc/rpc_nonce_response.go @@ -6,36 +6,33 @@ import ( "github.com/juju/errors" ) -const rpcNonceResponseLength = rpcNonceRequestLength - type RPCNonceResponse struct { RPCNonceRequest - RPCType [rpcNonceTagLength]byte - Crypto [rpcNonceCryptoAESLength]byte + RPCType []byte + Crypto []byte } func (r *RPCNonceResponse) Bytes() []byte { buf := &bytes.Buffer{} - buf.Grow(rpcNonceResponseLength) - buf.Write(r.RPCType[:]) - buf.Write(r.KeySelector[:]) - buf.Write(r.Crypto[:]) - buf.Write(r.CryptoTS[:]) - buf.Write(r.Nonce[:]) + buf.Write(r.RPCType) + buf.Write(r.KeySelector) + buf.Write(r.Crypto) + buf.Write(r.CryptoTS) + buf.Write(r.Nonce) return buf.Bytes() } func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { - if r.RPCType != rpcNonceTag { + if !bytes.Equal(r.RPCType, RPCTagNonce) { return errors.New("Unexpected RPC type") } - if r.Crypto != rpcNonceCryptoAESTag { + if !bytes.Equal(r.Crypto, RPCNonceCryptoAES) { return errors.New("Unexpected crypto type") } - if r.KeySelector != req.KeySelector { + if !bytes.Equal(r.KeySelector, req.KeySelector) { return errors.New("Unexpected key selector") } @@ -43,16 +40,17 @@ func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { } func NewRPCNonceResponse(data []byte) (*RPCNonceResponse, error) { - if len(data) != rpcNonceResponseLength { + if len(data) != 32 { return nil, errors.New("Unexpected message length") } - resp := RPCNonceResponse{} - copy(resp.RPCType[:], data[:4]) - copy(resp.KeySelector[:], data[4:8]) - copy(resp.Crypto[:], data[8:12]) - copy(resp.CryptoTS[:], data[12:16]) - copy(resp.Nonce[:], data[16:]) - - return &resp, nil + return &RPCNonceResponse{ + RPCNonceRequest: RPCNonceRequest{ + KeySelector: data[4:8], + CryptoTS: data[12:16], + Nonce: data[16:], + }, + RPCType: data[:4], + Crypto: data[8:12], + }, nil } diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/rpc_proxy_request.go index 1f6a7d6..d2b8392 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/rpc_proxy_request.go @@ -11,22 +11,11 @@ import ( "github.com/9seconds/mtg/mtproto" ) -const ( - rpcProxyRequestConnectionIDLength = 8 - rpcProxyRequestIPPortLength = 16 + 4 -) - -var ( - rpcProxyRequestTag = []byte{0xee, 0xf1, 0xce, 0x36} - rpcProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} - rpcProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} -) - type RPCProxyRequest struct { Flags RPCProxyRequestFlags - ConnectionID [rpcProxyRequestConnectionIDLength]byte - OurIPPort [rpcProxyRequestIPPortLength]byte - ClientIPPort [rpcProxyRequestIPPortLength]byte + ConnectionID []byte + OurIPPort []byte + ClientIPPort []byte ADTag []byte Options *mtproto.ConnectionOpts } @@ -43,13 +32,13 @@ func (r *RPCProxyRequest) Bytes(message []byte) []byte { flags |= RPCProxyRequestFlagsEncrypted } - buf.Write(rpcProxyRequestTag) + buf.Write(RPCTagProxyRequest) buf.Write(flags.Bytes()) buf.Write(r.ConnectionID[:]) buf.Write(r.ClientIPPort[:]) buf.Write(r.OurIPPort[:]) - buf.Write(rpcProxyRequestExtraSize) - buf.Write(rpcProxyRequestProxyTag) + buf.Write(RPCProxyRequestExtraSize) + buf.Write(RPCProxyRequestProxyTag) buf.WriteByte(byte(len(r.ADTag))) buf.Write(r.ADTag) buf.Write(bytes.Repeat([]byte{0x00}, buf.Len()%4)) @@ -69,23 +58,26 @@ func NewRPCProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.Connecti } request := RPCProxyRequest{ - Flags: flags, - ADTag: adTag, - Options: opts, + Flags: flags, + ADTag: adTag, + Options: opts, + ConnectionID: make([]byte, 8), + ClientIPPort: make([]byte, 16+4), + OurIPPort: make([]byte, 16+4), } - if _, err := rand.Read(request.ConnectionID[:]); err != nil { + if _, err := rand.Read(request.ConnectionID); err != nil { return nil, errors.Annotate(err, "Cannot generate connection ID") } - port := make([]byte, 4) + port := [4]byte{} copy(request.ClientIPPort[:16], clientAddr.IP.To16()) - binary.LittleEndian.PutUint32(port, uint32(clientAddr.Port)) - copy(request.ClientIPPort[16:], port) + binary.LittleEndian.PutUint32(port[:], uint32(clientAddr.Port)) + copy(request.ClientIPPort[16:], port[:]) copy(request.OurIPPort[:16], ownAddr.IP.To16()) - binary.LittleEndian.PutUint32(port, uint32(ownAddr.Port)) - copy(request.OurIPPort[16:], port) + binary.LittleEndian.PutUint32(port[:], uint32(ownAddr.Port)) + copy(request.OurIPPort[16:], port[:]) return &request, nil } From 8ce97f96c5f0955bbd86fa00e67124f5a4c54495 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 10:37:58 +0300 Subject: [PATCH 32/71] Remove redundant keepalive --- config/config.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/config/config.go b/config/config.go index 6235b8c..b5cfe9e 100644 --- a/config/config.go +++ b/config/config.go @@ -18,8 +18,6 @@ const ( TimeoutRead = time.Minute TimeoutWrite = time.Minute - - keepAlivePeriod = 20 * time.Second ) // Config represents common configuration of mtg. @@ -163,12 +161,6 @@ func SetSocketOptions(conn net.Conn) error { if err := socket.SetWriteBuffer(BufferWriteSize); err != nil { return errors.Annotate(err, "Cannot set write buffer size") } - if err := socket.SetKeepAlive(true); err != nil { - return errors.Annotate(err, "Cannot make socket keepalive") - } - if err := socket.SetKeepAlivePeriod(keepAlivePeriod); err != nil { - return errors.Annotate(err, "Cannot set keepalive period") - } if err := socket.SetNoDelay(true); err != nil { return errors.Annotate(err, "Cannot activate nodelay for the socket") } From 7b43766a081d478f53c89bc5d87803675c13acaa Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 10:58:58 +0300 Subject: [PATCH 33/71] More optimal byte reversing --- mtproto/wrappers/crypt.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index fc9bf63..e28ad8e 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -95,9 +95,13 @@ func makeEncrypterDecrypter(key, iv []byte) (cipher.BlockMode, cipher.BlockMode) } func reverseBytes(data []byte) []byte { - rv := make([]byte, len(data)) - for k, v := range data { - rv[len(data)-1-k] = v + dataLen := len(data) + rv := make([]byte, dataLen) + + rv[dataLen/2] = data[dataLen/2] + for i := dataLen/2 - 1; i >= 0; i-- { + opp := dataLen - i - 1 + rv[i], rv[opp] = data[opp], data[i] } return rv From d3e685de97853684eced30d91cdff8f83897d75b Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 11:16:53 +0300 Subject: [PATCH 34/71] Remove borsh borsh new borsch --- mtproto/rpc/handshake_request.go | 21 ++++++++++++ ...hake_response.go => handshake_response.go} | 14 ++++---- ...{rpc_nonce_request.go => nonce_request.go} | 12 +++---- ...pc_nonce_response.go => nonce_response.go} | 28 ++++++++-------- mtproto/rpc/proxy_flags.go | 24 ++++++++++++++ ...{rpc_proxy_request.go => proxy_request.go} | 30 ++++++++--------- mtproto/rpc/rpc.go | 32 +++++++++---------- mtproto/rpc/rpc_handshake_request.go | 21 ------------ mtproto/rpc/rpc_proxy_flags.go | 24 -------------- mtproto/wrappers/crypt.go | 5 ++- mtproto/wrappers/crypt_test.go | 4 +-- mtproto/wrappers/proxy_request.go | 10 +++--- telegram/middle.go | 22 +++++++------ 13 files changed, 124 insertions(+), 123 deletions(-) create mode 100644 mtproto/rpc/handshake_request.go rename mtproto/rpc/{rpc_handshake_response.go => handshake_response.go} (62%) rename mtproto/rpc/{rpc_nonce_request.go => nonce_request.go} (77%) rename mtproto/rpc/{rpc_nonce_response.go => nonce_response.go} (55%) create mode 100644 mtproto/rpc/proxy_flags.go rename mtproto/rpc/{rpc_proxy_request.go => proxy_request.go} (64%) delete mode 100644 mtproto/rpc/rpc_handshake_request.go delete mode 100644 mtproto/rpc/rpc_proxy_flags.go diff --git a/mtproto/rpc/handshake_request.go b/mtproto/rpc/handshake_request.go new file mode 100644 index 0000000..262656f --- /dev/null +++ b/mtproto/rpc/handshake_request.go @@ -0,0 +1,21 @@ +package rpc + +import "bytes" + +type HandshakeRequest struct { +} + +func (r *HandshakeRequest) Bytes() []byte { + buf := &bytes.Buffer{} + + buf.Write(TagHandshake) + buf.Write(HandshakeFlags) + buf.Write(HandshakeSenderPID) + buf.Write(HandshakePeerPID) + + return buf.Bytes() +} + +func NewHandshakeRequest() *HandshakeRequest { + return &HandshakeRequest{} +} diff --git a/mtproto/rpc/rpc_handshake_response.go b/mtproto/rpc/handshake_response.go similarity index 62% rename from mtproto/rpc/rpc_handshake_response.go rename to mtproto/rpc/handshake_response.go index af74469..40534d4 100644 --- a/mtproto/rpc/rpc_handshake_response.go +++ b/mtproto/rpc/handshake_response.go @@ -6,14 +6,14 @@ import ( "github.com/juju/errors" ) -type RPCHandshakeResponse struct { +type HandshakeResponse struct { Type []byte Flags []byte SenderPID []byte PeerPID []byte } -func (r *RPCHandshakeResponse) Bytes() []byte { +func (r *HandshakeResponse) Bytes() []byte { buf := &bytes.Buffer{} buf.Write(r.Type[:]) @@ -24,23 +24,23 @@ func (r *RPCHandshakeResponse) Bytes() []byte { return buf.Bytes() } -func (r *RPCHandshakeResponse) Valid(req *RPCHandshakeRequest) error { - if !bytes.Equal(r.Type, RPCTagHandshake) { +func (r *HandshakeResponse) Valid(req *HandshakeRequest) error { + if !bytes.Equal(r.Type, TagHandshake) { return errors.New("Unexpected handshake tag") } - if !bytes.Equal(r.PeerPID, RPCHandshakeSenderPID) { + if !bytes.Equal(r.PeerPID, HandshakeSenderPID) { return errors.New("Incorrect sender PID") } return nil } -func NewRPCHandshakeResponse(data []byte) (*RPCHandshakeResponse, error) { +func NewHandshakeResponse(data []byte) (*HandshakeResponse, error) { if len(data) != 32 { return nil, errors.New("Incorrect handshake response length") } - return &RPCHandshakeResponse{ + return &HandshakeResponse{ Type: data[:4], Flags: data[4:8], SenderPID: data[8:20], diff --git a/mtproto/rpc/rpc_nonce_request.go b/mtproto/rpc/nonce_request.go similarity index 77% rename from mtproto/rpc/rpc_nonce_request.go rename to mtproto/rpc/nonce_request.go index 55f5543..e4fb8a0 100644 --- a/mtproto/rpc/rpc_nonce_request.go +++ b/mtproto/rpc/nonce_request.go @@ -9,25 +9,25 @@ import ( "github.com/juju/errors" ) -type RPCNonceRequest struct { +type NonceRequest struct { KeySelector []byte CryptoTS []byte Nonce []byte } -func (r *RPCNonceRequest) Bytes() []byte { +func (r *NonceRequest) Bytes() []byte { buf := &bytes.Buffer{} - buf.Write(RPCTagNonce) + buf.Write(TagNonce) buf.Write(r.KeySelector) - buf.Write(RPCNonceCryptoAES) + buf.Write(NonceCryptoAES) buf.Write(r.CryptoTS) buf.Write(r.Nonce) return buf.Bytes() } -func NewRPCNonceRequest(proxySecret []byte) (*RPCNonceRequest, error) { +func NewNonceRequest(proxySecret []byte) (*NonceRequest, error) { nonce := make([]byte, 16) keySelector := make([]byte, 4) cryptoTS := make([]byte, 4) @@ -40,7 +40,7 @@ func NewRPCNonceRequest(proxySecret []byte) (*RPCNonceRequest, error) { timestamp := time.Now().Truncate(time.Second).Unix() % 4294967296 // 256 ^ 4 - do not know how to name binary.LittleEndian.PutUint32(cryptoTS, uint32(timestamp)) - return &RPCNonceRequest{ + return &NonceRequest{ KeySelector: keySelector, CryptoTS: cryptoTS, Nonce: nonce, diff --git a/mtproto/rpc/rpc_nonce_response.go b/mtproto/rpc/nonce_response.go similarity index 55% rename from mtproto/rpc/rpc_nonce_response.go rename to mtproto/rpc/nonce_response.go index 4f753b6..85c9383 100644 --- a/mtproto/rpc/rpc_nonce_response.go +++ b/mtproto/rpc/nonce_response.go @@ -6,17 +6,17 @@ import ( "github.com/juju/errors" ) -type RPCNonceResponse struct { - RPCNonceRequest +type NonceResponse struct { + NonceRequest - RPCType []byte - Crypto []byte + Type []byte + Crypto []byte } -func (r *RPCNonceResponse) Bytes() []byte { +func (r *NonceResponse) Bytes() []byte { buf := &bytes.Buffer{} - buf.Write(r.RPCType) + buf.Write(r.Type) buf.Write(r.KeySelector) buf.Write(r.Crypto) buf.Write(r.CryptoTS) @@ -25,11 +25,11 @@ func (r *RPCNonceResponse) Bytes() []byte { return buf.Bytes() } -func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { - if !bytes.Equal(r.RPCType, RPCTagNonce) { +func (r *NonceResponse) Valid(req *NonceRequest) error { + if !bytes.Equal(r.Type, TagNonce) { return errors.New("Unexpected RPC type") } - if !bytes.Equal(r.Crypto, RPCNonceCryptoAES) { + if !bytes.Equal(r.Crypto, NonceCryptoAES) { return errors.New("Unexpected crypto type") } if !bytes.Equal(r.KeySelector, req.KeySelector) { @@ -39,18 +39,18 @@ func (r *RPCNonceResponse) Valid(req *RPCNonceRequest) error { return nil } -func NewRPCNonceResponse(data []byte) (*RPCNonceResponse, error) { +func NewNonceResponse(data []byte) (*NonceResponse, error) { if len(data) != 32 { return nil, errors.New("Unexpected message length") } - return &RPCNonceResponse{ - RPCNonceRequest: RPCNonceRequest{ + return &NonceResponse{ + NonceRequest: NonceRequest{ KeySelector: data[4:8], CryptoTS: data[12:16], Nonce: data[16:], }, - RPCType: data[:4], - Crypto: data[8:12], + Type: data[:4], + Crypto: data[8:12], }, nil } diff --git a/mtproto/rpc/proxy_flags.go b/mtproto/rpc/proxy_flags.go new file mode 100644 index 0000000..0ac3c30 --- /dev/null +++ b/mtproto/rpc/proxy_flags.go @@ -0,0 +1,24 @@ +package rpc + +import "encoding/binary" + +type proxyRequestFlags uint32 + +const ( + proxyRequestFlagsHasAdTag proxyRequestFlags = 0x8 + proxyRequestFlagsEncrypted = 0x2 + proxyRequestFlagsMagic = 0x1000 + proxyRequestFlagsExtMode2 = 0x20000 + proxyRequestFlagsIntermediate = 0x20000000 + proxyRequestFlagsAbdridged = 0x40000000 + proxyRequestFlagsQuickAck = 0x80000000 +) + +var proxyRequestFlagsEncryptedPrefix [8]byte + +func (r proxyRequestFlags) Bytes() []byte { + converted := make([]byte, 4) + binary.LittleEndian.PutUint32(converted, uint32(r)) + + return converted +} diff --git a/mtproto/rpc/rpc_proxy_request.go b/mtproto/rpc/proxy_request.go similarity index 64% rename from mtproto/rpc/rpc_proxy_request.go rename to mtproto/rpc/proxy_request.go index d2b8392..6870273 100644 --- a/mtproto/rpc/rpc_proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -11,8 +11,8 @@ import ( "github.com/9seconds/mtg/mtproto" ) -type RPCProxyRequest struct { - Flags RPCProxyRequestFlags +type ProxyRequest struct { + Flags proxyRequestFlags ConnectionID []byte OurIPPort []byte ClientIPPort []byte @@ -20,25 +20,25 @@ type RPCProxyRequest struct { Options *mtproto.ConnectionOpts } -func (r *RPCProxyRequest) Bytes(message []byte) []byte { +func (r *ProxyRequest) Bytes(message []byte) []byte { buf := &bytes.Buffer{} flags := r.Flags if r.Options.QuickAck { - flags |= RPCProxyRequestFlagsQuickAck + flags |= proxyRequestFlagsQuickAck } - if bytes.HasPrefix(message, rpcProxyRequestFlagsEncryptedPrefix[:]) { - flags |= RPCProxyRequestFlagsEncrypted + if bytes.HasPrefix(message, proxyRequestFlagsEncryptedPrefix[:]) { + flags |= proxyRequestFlagsEncrypted } - buf.Write(RPCTagProxyRequest) + buf.Write(TagProxyRequest) buf.Write(flags.Bytes()) buf.Write(r.ConnectionID[:]) buf.Write(r.ClientIPPort[:]) buf.Write(r.OurIPPort[:]) - buf.Write(RPCProxyRequestExtraSize) - buf.Write(RPCProxyRequestProxyTag) + buf.Write(ProxyRequestExtraSize) + buf.Write(ProxyRequestProxyTag) buf.WriteByte(byte(len(r.ADTag))) buf.Write(r.ADTag) buf.Write(bytes.Repeat([]byte{0x00}, buf.Len()%4)) @@ -47,17 +47,17 @@ func (r *RPCProxyRequest) Bytes(message []byte) []byte { return buf.Bytes() } -func NewRPCProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.ConnectionOpts, adTag []byte) (*RPCProxyRequest, error) { - flags := RPCProxyRequestFlagsHasAdTag | RPCProxyRequestFlagsMagic | RPCProxyRequestFlagsExtMode2 +func NewProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.ConnectionOpts, adTag []byte) (*ProxyRequest, error) { + flags := proxyRequestFlagsHasAdTag | proxyRequestFlagsMagic | proxyRequestFlagsExtMode2 switch opts.ConnectionType { case mtproto.ConnectionTypeAbridged: - flags |= RPCProxyRequestFlagsAbdridged + flags |= proxyRequestFlagsAbdridged case mtproto.ConnectionTypeIntermediate: - flags |= RPCProxyRequestFlagsIntermediate + flags |= proxyRequestFlagsIntermediate } - request := RPCProxyRequest{ + request := &ProxyRequest{ Flags: flags, ADTag: adTag, Options: opts, @@ -79,5 +79,5 @@ func NewRPCProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.Connecti binary.LittleEndian.PutUint32(port[:], uint32(ownAddr.Port)) copy(request.OurIPPort[16:], port[:]) - return &request, nil + return request, nil } diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index f4edf71..bfb0555 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -1,30 +1,30 @@ package rpc const ( - RPCNonceSeqNo = -2 - RPCHandshakeSeqNo = -1 + SeqNoNonce = -2 + SeqNoHandshake = -1 ) var ( - RPCTagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} - RPCTagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} - RPCTagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} - RPCTagHandshake = []byte{0xf5, 0xee, 0x82, 0x76} - RPCTagNonce = []byte{0xaa, 0x87, 0xcb, 0x7a} - RPCTagProxyRequest = []byte{0xee, 0xf1, 0xce, 0x36} + TagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} + TagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} + TagSimpleAck = []byte{0x9b, 0x40, 0xac, 0x3b} + TagHandshake = []byte{0xf5, 0xee, 0x82, 0x76} + TagNonce = []byte{0xaa, 0x87, 0xcb, 0x7a} + TagProxyRequest = []byte{0xee, 0xf1, 0xce, 0x36} - RPCNonceCryptoAES = []byte{0x01, 0x00, 0x00, 0x00} + NonceCryptoAES = []byte{0x01, 0x00, 0x00, 0x00} - RPCHandshakeFlags = []byte{0x00, 0x00, 0x00, 0x00} + HandshakeFlags = []byte{0x00, 0x00, 0x00, 0x00} - RPCProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} - RPCProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} + ProxyRequestExtraSize = []byte{0x18, 0x00, 0x00, 0x00} + ProxyRequestProxyTag = []byte{0xae, 0x26, 0x1e, 0xdb} - RPCHandshakeSenderPID = []byte{} - RPCHandshakePeerPID = []byte{} + HandshakeSenderPID []byte + HandshakePeerPID []byte ) func init() { - RPCHandshakeSenderPID = []byte("IPIPPRPDTIME") - RPCHandshakePeerPID = []byte("IPIPPRPDTIME") + HandshakeSenderPID = []byte("IPIPPRPDTIME") + HandshakePeerPID = []byte("IPIPPRPDTIME") } diff --git a/mtproto/rpc/rpc_handshake_request.go b/mtproto/rpc/rpc_handshake_request.go deleted file mode 100644 index 4a21fe4..0000000 --- a/mtproto/rpc/rpc_handshake_request.go +++ /dev/null @@ -1,21 +0,0 @@ -package rpc - -import "bytes" - -type RPCHandshakeRequest struct { -} - -func (r *RPCHandshakeRequest) Bytes() []byte { - buf := &bytes.Buffer{} - - buf.Write(RPCTagHandshake) - buf.Write(RPCHandshakeFlags) - buf.Write(RPCHandshakeSenderPID) - buf.Write(RPCHandshakePeerPID) - - return buf.Bytes() -} - -func NewRPCHandshakeRequest() *RPCHandshakeRequest { - return &RPCHandshakeRequest{} -} diff --git a/mtproto/rpc/rpc_proxy_flags.go b/mtproto/rpc/rpc_proxy_flags.go deleted file mode 100644 index 4edd74d..0000000 --- a/mtproto/rpc/rpc_proxy_flags.go +++ /dev/null @@ -1,24 +0,0 @@ -package rpc - -import "encoding/binary" - -type RPCProxyRequestFlags uint32 - -const ( - RPCProxyRequestFlagsHasAdTag RPCProxyRequestFlags = 0x8 - RPCProxyRequestFlagsEncrypted = 0x2 - RPCProxyRequestFlagsMagic = 0x1000 - RPCProxyRequestFlagsExtMode2 = 0x20000 - RPCProxyRequestFlagsIntermediate = 0x20000000 - RPCProxyRequestFlagsAbdridged = 0x40000000 - RPCProxyRequestFlagsQuickAck = 0x80000000 -) - -var rpcProxyRequestFlagsEncryptedPrefix [8]byte - -func (r RPCProxyRequestFlags) Bytes() []byte { - converted := make([]byte, 4) - binary.LittleEndian.PutUint32(converted, uint32(r)) - - return converted -} diff --git a/mtproto/wrappers/crypt.go b/mtproto/wrappers/crypt.go index e28ad8e..f47b466 100644 --- a/mtproto/wrappers/crypt.go +++ b/mtproto/wrappers/crypt.go @@ -22,7 +22,7 @@ const ( var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} -func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, secret []byte) wrappers.ReadWriteCloserWithAddr { +func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.NonceRequest, resp *rpc.NonceResponse, secret []byte) wrappers.ReadWriteCloserWithAddr { localAddr := conn.LocalAddr() remoteAddr := conn.RemoteAddr() @@ -35,8 +35,7 @@ func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.RPC return wrappers.NewBlockCipherRWC(conn, enc, dec) } -func makeKeys(purpose CipherPurpose, req *rpc.RPCNonceRequest, resp *rpc.RPCNonceResponse, - client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { +func makeKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { message := bytes.Buffer{} message.Write(resp.Nonce[:]) message.Write(req.Nonce[:]) diff --git a/mtproto/wrappers/crypt_test.go b/mtproto/wrappers/crypt_test.go index 62bf0b9..921ad49 100644 --- a/mtproto/wrappers/crypt_test.go +++ b/mtproto/wrappers/crypt_test.go @@ -21,13 +21,13 @@ var proxySecret = []byte{196, 249, 250, 202, 150, 120, 230, 187, 72, 173, 183, 6, 27, 38, 93, 178, 18} func TestMakeKeys(t *testing.T) { - req, err := rpc.NewRPCNonceRequest(proxySecret) + req, err := rpc.NewNonceRequest(proxySecret) assert.Nil(t, err) copy(req.Nonce[:], []byte{24, 49, 53, 111, 198, 10, 235, 180, 230, 112, 92, 78, 1, 201, 106, 105}) binary.LittleEndian.PutUint32(req.CryptoTS[:], 1528396015) - resp := &rpc.RPCNonceResponse{} + resp := &rpc.NonceResponse{} copy(resp.Nonce[:], []byte{247, 40, 210, 56, 65, 12, 101, 170, 216, 155, 14, 253, 250, 238, 219, 226}) cltAddr := &net.TCPAddr{ diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go index 657f120..1d2ba67 100644 --- a/mtproto/wrappers/proxy_request.go +++ b/mtproto/wrappers/proxy_request.go @@ -17,7 +17,7 @@ type ProxyRequestReadWriteCloserWithAddr struct { wrappers.BufferedReader conn wrappers.ReadWriteCloserWithAddr - req *rpc.RPCProxyRequest + req *rpc.ProxyRequest } func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { @@ -29,11 +29,11 @@ func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { return errors.Annotate(err, "Cannot read RPC tag") } - if bytes.Equal(ansBuf.Bytes(), rpc.RPCTagCloseExt) { + if bytes.Equal(ansBuf.Bytes(), rpc.TagCloseExt) { return p.readCloseExt() - } else if bytes.Equal(ansBuf.Bytes(), rpc.RPCTagProxyAns) { + } else if bytes.Equal(ansBuf.Bytes(), rpc.TagProxyAns) { return p.readProxyAns(buf) - } else if bytes.Equal(ansBuf.Bytes(), rpc.RPCTagSimpleAck) { + } else if bytes.Equal(ansBuf.Bytes(), rpc.TagSimpleAck) { return p.readSimpleAck() } @@ -99,7 +99,7 @@ func (p *ProxyRequestReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { } func NewProxyRequestRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts, adTag []byte) (wrappers.ReadWriteCloserWithAddr, error) { - req, err := rpc.NewRPCProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) + req, err := rpc.NewProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) if err != nil { return nil, errors.Annotate(err, "Cannot create new RPC proxy request") } diff --git a/telegram/middle.go b/telegram/middle.go index 8e1bd7b..326c095 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -50,7 +50,7 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram } func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { - rpcNonceConn := mtwrappers.NewFrameRWC(conn, rpc.RPCNonceSeqNo) + rpcNonceConn := mtwrappers.NewFrameRWC(conn, rpc.SeqNoNonce) rpcNonceReq, err := t.sendRPCNonceRequest(rpcNonceConn) if err != nil { @@ -62,7 +62,7 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re } secureConn := mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, rpcNonceResp, t.proxySecret) - secureConn = mtwrappers.NewFrameRWC(secureConn, rpc.RPCHandshakeSeqNo) + secureConn = mtwrappers.NewFrameRWC(secureConn, rpc.SeqNoHandshake) rpcHandshakeReq, err := t.sendRPCHandshakeRequest(secureConn) if err != nil { @@ -76,8 +76,8 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return mtwrappers.NewProxyRequestRWC(secureConn, connOpts, t.conf.AdTag) } -func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.RPCNonceRequest, error) { - rpcNonceReq, err := rpc.NewRPCNonceRequest(t.proxySecret) +func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.NonceRequest, error) { + rpcNonceReq, err := rpc.NewNonceRequest(t.proxySecret) if err != nil { return nil, errors.Annotate(err, "Cannot create RPC nonce request") } @@ -88,14 +88,15 @@ func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.RPCNonceReque return rpcNonceReq, nil } -func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.RPCNonceRequest) (*rpc.RPCNonceResponse, error) { +func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { var ans [128]byte n, err := conn.Read(ans[:]) if err != nil { return nil, errors.Annotate(err, "Cannot read RPC nonce response") } - rpcNonceResp, err := rpc.NewRPCNonceResponse(ans[:n]) + + rpcNonceResp, err := rpc.NewNonceResponse(ans[:n]) if err != nil { return nil, errors.Annotate(err, "Cannot initialize RPC nonce response") } @@ -106,8 +107,8 @@ func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.RPCNon return rpcNonceResp, nil } -func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.RPCHandshakeRequest, error) { - req := rpc.NewRPCHandshakeRequest() +func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.HandshakeRequest, error) { + req := rpc.NewHandshakeRequest() if _, err := conn.Write(req.Bytes()); err != nil { return nil, errors.Annotate(err, "Cannot send RPC handshake request") } @@ -115,14 +116,15 @@ func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.RPCHandsh return req, nil } -func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.RPCHandshakeRequest) (*rpc.RPCHandshakeResponse, error) { +func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { var ans [128]byte n, err := conn.Read(ans[:]) if err != nil { return nil, errors.Annotate(err, "Cannot read RPC handshake response") } - rpcHandshakeResp, err := rpc.NewRPCHandshakeResponse(ans[:n]) + + rpcHandshakeResp, err := rpc.NewHandshakeResponse(ans[:n]) if err != nil { return nil, errors.Annotate(err, "Cannot initialize RPC handshake response") } From 4ed877efbc21aaf8b5e58b3508489f7f0dae7531 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 11:25:43 +0300 Subject: [PATCH 35/71] Fix cyrpt test --- mtproto/wrappers/crypt_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mtproto/wrappers/crypt_test.go b/mtproto/wrappers/crypt_test.go index 921ad49..c9914f2 100644 --- a/mtproto/wrappers/crypt_test.go +++ b/mtproto/wrappers/crypt_test.go @@ -27,8 +27,11 @@ func TestMakeKeys(t *testing.T) { copy(req.Nonce[:], []byte{24, 49, 53, 111, 198, 10, 235, 180, 230, 112, 92, 78, 1, 201, 106, 105}) binary.LittleEndian.PutUint32(req.CryptoTS[:], 1528396015) - resp := &rpc.NonceResponse{} - copy(resp.Nonce[:], []byte{247, 40, 210, 56, 65, 12, 101, 170, 216, 155, 14, 253, 250, 238, 219, 226}) + resp := &rpc.NonceResponse{ + NonceRequest: rpc.NonceRequest{ + Nonce: []byte{247, 40, 210, 56, 65, 12, 101, 170, 216, 155, 14, 253, 250, 238, 219, 226}, + }, + } cltAddr := &net.TCPAddr{ IP: net.ParseIP("80.211.29.34"), From 4210cf014eec3e3d6a3b61f20a5b4df830c1af33 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 14:59:33 +0300 Subject: [PATCH 36/71] Bugfix proxy request handler --- mtproto/rpc/proxy_request.go | 8 ++++---- mtproto/wrappers/frame.go | 5 +++-- telegram/middle.go | 2 -- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/mtproto/rpc/proxy_request.go b/mtproto/rpc/proxy_request.go index 6870273..994af28 100644 --- a/mtproto/rpc/proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -34,14 +34,14 @@ func (r *ProxyRequest) Bytes(message []byte) []byte { buf.Write(TagProxyRequest) buf.Write(flags.Bytes()) - buf.Write(r.ConnectionID[:]) - buf.Write(r.ClientIPPort[:]) - buf.Write(r.OurIPPort[:]) + buf.Write(r.ConnectionID) + buf.Write(r.ClientIPPort) + buf.Write(r.OurIPPort) buf.Write(ProxyRequestExtraSize) buf.Write(ProxyRequestProxyTag) buf.WriteByte(byte(len(r.ADTag))) buf.Write(r.ADTag) - buf.Write(bytes.Repeat([]byte{0x00}, buf.Len()%4)) + buf.Write(make([]byte, (4-buf.Len()%4)%4)) buf.Write(message) return buf.Bytes() diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index d008d37..62406e7 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/aes" "encoding/binary" + "fmt" "hash/crc32" "io" "io/ioutil" @@ -20,7 +21,7 @@ const ( frameRWCMaxMessageLength = 16777216 ) -var frameRWCPadding = [4]byte{0x04, 0x00, 0x00, 0x00} +var frameRWCPadding = []byte{0x04, 0x00, 0x00, 0x00} type FrameRWC struct { wrappers.BufferedReader @@ -48,7 +49,7 @@ func (f *FrameRWC) Write(buf []byte) (int, error) { checksum := crc32.ChecksumIEEE(writeBuf.Bytes()) binary.Write(writeBuf, binary.LittleEndian, checksum) - writeBuf.Write(bytes.Repeat(frameRWCPadding[:], paddingLength/4)) + writeBuf.Write(bytes.Repeat(frameRWCPadding, paddingLength/4)) _, err := f.conn.Write(writeBuf.Bytes()) return len(buf), err diff --git a/telegram/middle.go b/telegram/middle.go index 326c095..62a9c65 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -1,7 +1,6 @@ package telegram import ( - "fmt" "io" "net" "net/http" @@ -131,7 +130,6 @@ func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.Ha if err = rpcHandshakeResp.Valid(req); err != nil { return nil, errors.Annotate(err, "Invalid RPC handshake response") } - fmt.Println("VICTORY") return rpcHandshakeResp, nil } From 154ff70bf72e541b771c0ccedf9a2b63c049662a Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 16:37:25 +0300 Subject: [PATCH 37/71] Small changes for abridged/intermediate --- mtproto/wrappers/abridged.go | 14 +++++++------- mtproto/wrappers/frame.go | 1 - mtproto/wrappers/intermediate.go | 11 +++++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go index 66b440d..8d38a97 100644 --- a/mtproto/wrappers/abridged.go +++ b/mtproto/wrappers/abridged.go @@ -2,7 +2,6 @@ package wrappers import ( "bytes" - "encoding/binary" "io" "net" @@ -29,10 +28,14 @@ type AbridgedReadWriteCloserWithAddr struct { func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { return a.BufferedRead(p, func() error { - var msgLength uint8 - if err := binary.Read(a.conn, binary.LittleEndian, &msgLength); err != nil { + buf := &bytes.Buffer{} + buf.Grow(3) + + if _, err := io.CopyN(buf, a.conn, 1); err != nil { return errors.Annotate(err, "Cannot read message length") } + msgLength := uint8(buf.Bytes()[0]) + buf.Reset() a.opts.QuickAck = false if msgLength >= abridgedQuickAckLength { @@ -42,11 +45,8 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { msgLength32 := uint32(msgLength) if msgLength == abridgedSmallPacketLength { - buf := &bytes.Buffer{} - buf.Grow(3) - if _, err := io.CopyN(buf, a.conn, 3); err != nil { - return errors.Annotate(err, "Cannot read correct message length") + return errors.Annotate(err, "Cannot read the correct message length") } number := uint24{} copy(number[:], buf.Bytes()) diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index 62406e7..907dbde 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -4,7 +4,6 @@ import ( "bytes" "crypto/aes" "encoding/binary" - "fmt" "hash/crc32" "io" "io/ioutil" diff --git a/mtproto/wrappers/intermediate.go b/mtproto/wrappers/intermediate.go index 1c2a6ab..4e35223 100644 --- a/mtproto/wrappers/intermediate.go +++ b/mtproto/wrappers/intermediate.go @@ -23,18 +23,21 @@ type IntermediateReadWriteCloserWithAddr struct { func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { return i.BufferedRead(p, func() error { - var length uint32 - if err := binary.Read(i.conn, binary.LittleEndian, &length); err != nil { + buf := &bytes.Buffer{} + buf.Grow(4) + + if _, err := io.CopyN(buf, i.conn, 4); err != nil { return errors.Annotate(err, "Cannot read message length") } + length := binary.LittleEndian.Uint32(buf.Bytes()) + buf.Reset() + buf.Grow(int(length)) if length > intermediateQuickAckLength { i.opts.QuickAck = true length -= intermediateQuickAckLength } - buf := &bytes.Buffer{} - buf.Grow(int(length)) if _, err := io.CopyN(buf, i.conn, int64(length)); err != nil { return errors.Annotate(err, "Cannot read the message") } From 53ef81a4fc52a63614202367913c8c1acd534b24 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 17:10:49 +0300 Subject: [PATCH 38/71] Use the same protocol as client --- client/middle.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/middle.go b/client/middle.go index d5dc4d3..2197cec 100644 --- a/client/middle.go +++ b/client/middle.go @@ -21,5 +21,10 @@ func MiddleInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wr newConn = mtwrappers.NewIntermediateRWC(newConn, opts) } + opts.ConnectionProto = mtproto.ConnectionProtocolIPv4 + if conn.LocalAddr().(*net.TCPAddr).IP.To4() == nil { + opts.ConnectionProto = mtproto.ConnectionProtocolIPv6 + } + return opts, newConn, nil } From 5413a5b0904ed032a6722f33ed54293943194a29 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Thu, 5 Jul 2018 20:46:44 +0300 Subject: [PATCH 39/71] Fix infinite hangs --- mtproto/wrappers/abridged.go | 6 +++--- mtproto/wrappers/frame.go | 2 +- mtproto/wrappers/intermediate.go | 3 +++ mtproto/wrappers/proxy_request.go | 11 +++++------ wrappers/blockcipherrwc.go | 11 ++++++----- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go index 8d38a97..9c22785 100644 --- a/mtproto/wrappers/abridged.go +++ b/mtproto/wrappers/abridged.go @@ -28,6 +28,9 @@ type AbridgedReadWriteCloserWithAddr struct { func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { return a.BufferedRead(p, func() error { + a.opts.QuickAck = false + a.opts.SimpleAck = false + buf := &bytes.Buffer{} buf.Grow(3) @@ -37,7 +40,6 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { msgLength := uint8(buf.Bytes()[0]) buf.Reset() - a.opts.QuickAck = false if msgLength >= abridgedQuickAckLength { a.opts.QuickAck = true msgLength -= 0x80 @@ -78,13 +80,11 @@ func (a *AbridgedReadWriteCloserWithAddr) Write(p []byte) (int, error) { case packetLength < abridgedLargePacketLength: length24 := toUint24(uint32(packetLength)) - buf := &bytes.Buffer{} buf.Grow(1 + 3 + len(p)) buf.WriteByte(byte(abridgedSmallPacketLength)) buf.Write(length24[:]) buf.Write(p) - return a.conn.Write(buf.Bytes()) default: diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index 907dbde..2c5cf8a 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -66,7 +66,7 @@ func (f *FrameRWC) Read(p []byte) (int, error) { if _, err := io.CopyN(writer, f.conn, 4); err != nil { return errors.Annotate(err, "Cannot read frame padding") } - if !bytes.Equal(buf.Bytes(), frameRWCPadding[:]) { + if !bytes.Equal(buf.Bytes(), frameRWCPadding) { break } } diff --git a/mtproto/wrappers/intermediate.go b/mtproto/wrappers/intermediate.go index 4e35223..d80eabe 100644 --- a/mtproto/wrappers/intermediate.go +++ b/mtproto/wrappers/intermediate.go @@ -23,6 +23,9 @@ type IntermediateReadWriteCloserWithAddr struct { func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { return i.BufferedRead(p, func() error { + i.opts.QuickAck = false + i.opts.SimpleAck = false + buf := &bytes.Buffer{} buf.Grow(4) diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go index 1d2ba67..7d0adbe 100644 --- a/mtproto/wrappers/proxy_request.go +++ b/mtproto/wrappers/proxy_request.go @@ -29,15 +29,16 @@ func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { return errors.Annotate(err, "Cannot read RPC tag") } - if bytes.Equal(ansBuf.Bytes(), rpc.TagCloseExt) { + switch { + case bytes.Equal(ansBuf.Bytes(), rpc.TagCloseExt): return p.readCloseExt() - } else if bytes.Equal(ansBuf.Bytes(), rpc.TagProxyAns) { + case bytes.Equal(ansBuf.Bytes(), rpc.TagProxyAns): return p.readProxyAns(buf) - } else if bytes.Equal(ansBuf.Bytes(), rpc.TagSimpleAck) { + case bytes.Equal(ansBuf.Bytes(), rpc.TagSimpleAck): return p.readSimpleAck() } - return nil + return errors.Errorf("Unknown RPC answer %s", ansBuf.Bytes()) }) } @@ -80,8 +81,6 @@ func (p *ProxyRequestReadWriteCloserWithAddr) Write(raw []byte) (int, error) { if _, err := p.conn.Write(p.req.Bytes(raw)); err != nil { return 0, err } - p.req.Options.SimpleAck = false - p.req.Options.QuickAck = false return len(raw), nil } diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go index 9db024d..8d2a06c 100644 --- a/wrappers/blockcipherrwc.go +++ b/wrappers/blockcipherrwc.go @@ -1,6 +1,7 @@ package wrappers import ( + "bytes" "crypto/aes" "crypto/cipher" "net" @@ -18,16 +19,16 @@ type BlockCipherReadWriteCloserWithAddr struct { func (c *BlockCipherReadWriteCloserWithAddr) Read(p []byte) (int, error) { return c.BufferedRead(p, func() error { - bufferLength := c.Buffer.Len() - for bufferLength%aes.BlockSize != 0 || bufferLength == 0 { + buf := &bytes.Buffer{} + for buf.Len()%aes.BlockSize != 0 || buf.Len() == 0 { n, err := c.conn.Read(p) if err != nil { return errors.Annotate(err, "Cannot read from socket") } - c.Buffer.Write(p[:n]) - bufferLength = c.Buffer.Len() + buf.Write(p[:n]) } - c.decryptor.CryptBlocks(c.Buffer.Bytes(), c.Buffer.Bytes()) + c.decryptor.CryptBlocks(buf.Bytes(), buf.Bytes()) + c.Buffer.Write(buf.Bytes()) return nil }) From 88faeb7195354cf7c64c5bfd452b08b1e7ce7a8d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 6 Jul 2018 10:21:04 +0300 Subject: [PATCH 40/71] Proxy is working but unstable --- client/client.go | 2 +- client/direct.go | 4 +-- client/middle.go | 6 ++-- mtproto/connection_options.go | 9 ++++-- mtproto/rpc/proxy_request.go | 2 +- mtproto/wrappers/abridged.go | 13 ++++---- mtproto/wrappers/intermediate.go | 7 ++--- mtproto/wrappers/proxy_request.go | 38 ++++++++++++------------ proxy/copy_pool.go | 18 ------------ proxy/server.go | 49 ++++++++++++++++++++++++++----- 10 files changed, 83 insertions(+), 65 deletions(-) delete mode 100644 proxy/copy_pool.go diff --git a/client/client.go b/client/client.go index cb6b291..32e23a9 100644 --- a/client/client.go +++ b/client/client.go @@ -9,4 +9,4 @@ import ( ) // Init has to initialize client connection based on given config. -type Init func(net.Conn, *config.Config) (*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr, error) +type Init func(net.Conn, *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) diff --git a/client/direct.go b/client/direct.go index 98b518d..3ce1eed 100644 --- a/client/direct.go +++ b/client/direct.go @@ -15,7 +15,7 @@ import ( const handshakeTimeout = 10 * time.Second // DirectInit initializes client to access Telegram bypassing middleproxies. -func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr, error) { +func DirectInit(conn net.Conn, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { if err := config.SetSocketOptions(conn); err != nil { return nil, nil, errors.Annotate(err, "Cannot set socket options") } @@ -37,5 +37,5 @@ func DirectInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wr socket := wrappers.NewTimeoutRWC(conn, conf.PublicIPv4, conf.PublicIPv6) socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) - return connOpts, socket, nil + return socket, connOpts, nil } diff --git a/client/middle.go b/client/middle.go index 2197cec..f8877fc 100644 --- a/client/middle.go +++ b/client/middle.go @@ -9,8 +9,8 @@ import ( "github.com/9seconds/mtg/wrappers" ) -func MiddleInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr, error) { - opts, newConn, err := DirectInit(conn, conf) +func MiddleInit(conn net.Conn, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { + newConn, opts, err := DirectInit(conn, conf) if err != nil { return nil, nil, err } @@ -26,5 +26,5 @@ func MiddleInit(conn net.Conn, conf *config.Config) (*mtproto.ConnectionOpts, wr opts.ConnectionProto = mtproto.ConnectionProtocolIPv6 } - return opts, newConn, nil + return newConn, opts, nil } diff --git a/mtproto/connection_options.go b/mtproto/connection_options.go index 585e0f7..0285488 100644 --- a/mtproto/connection_options.go +++ b/mtproto/connection_options.go @@ -13,14 +13,19 @@ type ConnectionType uint8 type ConnectionProtocol uint8 +type Hacks struct { + SimpleAck bool + QuickAck bool +} + // ConnectionOpts presents an options, metadata on connection requested // by the user on handshake. type ConnectionOpts struct { DC int16 ConnectionType ConnectionType ConnectionProto ConnectionProtocol - QuickAck bool - SimpleAck bool + ReadHacks Hacks + WriteHacks Hacks ClientAddr *net.TCPAddr } diff --git a/mtproto/rpc/proxy_request.go b/mtproto/rpc/proxy_request.go index 994af28..e36a2ec 100644 --- a/mtproto/rpc/proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -24,7 +24,7 @@ func (r *ProxyRequest) Bytes(message []byte) []byte { buf := &bytes.Buffer{} flags := r.Flags - if r.Options.QuickAck { + if r.Options.ReadHacks.QuickAck { flags |= proxyRequestFlagsQuickAck } diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go index 9c22785..1850220 100644 --- a/mtproto/wrappers/abridged.go +++ b/mtproto/wrappers/abridged.go @@ -28,9 +28,6 @@ type AbridgedReadWriteCloserWithAddr struct { func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { return a.BufferedRead(p, func() error { - a.opts.QuickAck = false - a.opts.SimpleAck = false - buf := &bytes.Buffer{} buf.Grow(3) @@ -41,7 +38,7 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { buf.Reset() if msgLength >= abridgedQuickAckLength { - a.opts.QuickAck = true + a.opts.ReadHacks.QuickAck = true msgLength -= 0x80 } @@ -56,9 +53,13 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { } msgLength32 *= 4 - if _, err := io.CopyN(a.Buffer, a.conn, int64(msgLength32)); err != nil { + buf.Reset() + buf.Grow(int(msgLength32)) + + if _, err := io.CopyN(buf, a.conn, int64(msgLength32)); err != nil { return errors.Annotate(err, "Cannot read message") } + a.Buffer.Write(buf.Bytes()) return nil }) @@ -68,7 +69,7 @@ func (a *AbridgedReadWriteCloserWithAddr) Write(p []byte) (int, error) { if len(p)%4 != 0 { return 0, errors.Errorf("Incorrect packet length %d", len(p)) } - if a.opts.SimpleAck { + if a.opts.WriteHacks.SimpleAck { return a.conn.Write(reverseBytes(p)) } diff --git a/mtproto/wrappers/intermediate.go b/mtproto/wrappers/intermediate.go index d80eabe..a22e56d 100644 --- a/mtproto/wrappers/intermediate.go +++ b/mtproto/wrappers/intermediate.go @@ -23,9 +23,6 @@ type IntermediateReadWriteCloserWithAddr struct { func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { return i.BufferedRead(p, func() error { - i.opts.QuickAck = false - i.opts.SimpleAck = false - buf := &bytes.Buffer{} buf.Grow(4) @@ -37,7 +34,7 @@ func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { buf.Grow(int(length)) if length > intermediateQuickAckLength { - i.opts.QuickAck = true + i.opts.ReadHacks.QuickAck = true length -= intermediateQuickAckLength } @@ -58,7 +55,7 @@ func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { } func (i *IntermediateReadWriteCloserWithAddr) Write(p []byte) (int, error) { - if i.opts.SimpleAck { + if i.opts.WriteHacks.SimpleAck { return i.conn.Write(p) } diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go index 7d0adbe..2641715 100644 --- a/mtproto/wrappers/proxy_request.go +++ b/mtproto/wrappers/proxy_request.go @@ -22,23 +22,21 @@ type ProxyRequestReadWriteCloserWithAddr struct { func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { return p.BufferedRead(buf, func() error { - ansBuf := &bytes.Buffer{} - ansBuf.Grow(4) - - if _, err := io.CopyN(ansBuf, p.conn, 4); err != nil { + ans := make([]byte, 4) + if _, err := io.ReadFull(p.conn, ans); err != nil { return errors.Annotate(err, "Cannot read RPC tag") } switch { - case bytes.Equal(ansBuf.Bytes(), rpc.TagCloseExt): - return p.readCloseExt() - case bytes.Equal(ansBuf.Bytes(), rpc.TagProxyAns): + case bytes.Equal(ans, rpc.TagProxyAns): return p.readProxyAns(buf) - case bytes.Equal(ansBuf.Bytes(), rpc.TagSimpleAck): + case bytes.Equal(ans, rpc.TagSimpleAck): return p.readSimpleAck() + case bytes.Equal(ans, rpc.TagCloseExt): + return p.readCloseExt() } - return errors.Errorf("Unknown RPC answer %s", ansBuf.Bytes()) + return errors.Errorf("Unknown RPC answer %v", ans) }) } @@ -46,21 +44,21 @@ func (p *ProxyRequestReadWriteCloserWithAddr) readCloseExt() error { return errors.New("Connection has been closed remotely") } -func (p *ProxyRequestReadWriteCloserWithAddr) readProxyAns(buf []byte) error { - if _, err := io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { +func (p *ProxyRequestReadWriteCloserWithAddr) readProxyAns(buf []byte) (err error) { + if _, err = io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { return errors.Annotate(err, "Cannot skip flags and connid") } - for { - n, err := p.conn.Read(buf) + n := len(buf) + preBuffer := &bytes.Buffer{} + for n == len(buf) { + n, err = p.conn.Read(buf) if err != nil { return errors.Annotate(err, "Cannot read proxy answer") } - if n == 0 { - break - } - p.Buffer.Write(buf[:n]) + preBuffer.Write(buf[:n]) } + p.Buffer.Write(preBuffer.Bytes()) return nil } @@ -69,10 +67,12 @@ func (p *ProxyRequestReadWriteCloserWithAddr) readSimpleAck() error { if _, err := io.CopyN(ioutil.Discard, p.conn, 8); err != nil { return errors.Annotate(err, "Cannot skip connid") } - if _, err := io.CopyN(p.Buffer, p.conn, 4); err != nil { + + ackData := make([]byte, 4) + if _, err := io.ReadFull(p.conn, ackData); err != nil { return errors.Annotate(err, "Cannot read simple ack") } - p.req.Options.SimpleAck = true + p.Buffer.Write(ackData) return nil } diff --git a/proxy/copy_pool.go b/proxy/copy_pool.go deleted file mode 100644 index 23477aa..0000000 --- a/proxy/copy_pool.go +++ /dev/null @@ -1,18 +0,0 @@ -package proxy - -import ( - "sync" - - "github.com/9seconds/mtg/config" -) - -var copyPool sync.Pool - -func init() { - copyPool = sync.Pool{ - New: func() interface{} { - data := make([]byte, config.BufferSizeCopy) - return &data - }, - } -} diff --git a/proxy/server.go b/proxy/server.go index 314b3d9..9b43613 100644 --- a/proxy/server.go +++ b/proxy/server.go @@ -85,8 +85,32 @@ func (s *Server) accept(conn net.Conn) { wait := &sync.WaitGroup{} wait.Add(2) - go s.pipe(clientConn, tgConn, wait) - go s.pipe(tgConn, clientConn, wait) + go func() { + defer wait.Done() + + for { + connOpts.ReadHacks.QuickAck = false + connOpts.ReadHacks.SimpleAck = false + if err := s.pump(clientConn, tgConn, socketID, "client"); err != nil { + s.logger.Infow("Client stream is aborted", + "socketid", socketID, "error", err) + return + } + } + }() + go func() { + defer wait.Done() + + for { + connOpts.WriteHacks.QuickAck = false + connOpts.WriteHacks.SimpleAck = false + if err := s.pump(tgConn, clientConn, socketID, "telegram"); err != nil { + s.logger.Infow("Telegram stream is aborted", + "socketid", socketID, "error", err) + return + } + } + }() <-ctx.Done() wait.Wait() @@ -98,7 +122,7 @@ func (s *Server) accept(conn net.Conn) { } func (s *Server) getClientStream(ctx context.Context, cancel context.CancelFunc, conn net.Conn, socketID string) (*mtproto.ConnectionOpts, io.ReadWriteCloser, error) { - connOpts, socket, err := s.clientInit(conn, s.conf) + socket, connOpts, err := s.clientInit(conn, s.conf) if err != nil { return nil, nil, errors.Annotate(err, "Cannot init client connection") } @@ -128,13 +152,22 @@ func (s *Server) getTelegramStream(ctx context.Context, cancel context.CancelFun return conn, nil } -func (s *Server) pipe(dst io.Writer, src io.Reader, wait *sync.WaitGroup) { - defer wait.Done() +func (s *Server) pump(src io.Reader, dst io.Writer, socketID, name string) (err error) { + copyBuf := make([]byte, 1024*1024*2) - buf := copyPool.Get().(*[]byte) - defer copyPool.Put(buf) + n := config.BufferSizeCopy + for n == config.BufferSizeCopy { + n, err = src.Read(copyBuf) + if err != nil { + break + } + _, err = dst.Write(copyBuf[:n]) + if err != nil { + break + } + } - io.CopyBuffer(dst, src, *buf) // nolint: errcheck + return } // NewServer creates new instance of MTPROTO proxy. From a3933e6ede78eb40ea347b699b7a7637d7bca18d Mon Sep 17 00:00:00 2001 From: 9seconds Date: Fri, 6 Jul 2018 17:29:24 +0300 Subject: [PATCH 41/71] Preliminary debug state --- client/client.go | 2 +- client/direct.go | 4 ++-- client/middle.go | 4 ++-- mtproto/wrappers/abridged.go | 16 ++++++++++++---- mtproto/wrappers/frame.go | 4 ++++ mtproto/wrappers/intermediate.go | 4 ++++ mtproto/wrappers/proxy_request.go | 22 +++++++++++----------- proxy/server.go | 26 ++++++++++---------------- telegram/dialer.go | 4 ++-- telegram/direct.go | 4 ++-- telegram/middle_caller.go | 4 ++-- telegram/telegram.go | 6 +++--- utils/read_current_data.go | 20 ++++++++++++++++++++ wrappers/blockcipherrwc.go | 23 ++++++++++++++++------- wrappers/buffered_reader.go | 12 ++++++++++-- wrappers/ctxrwc.go | 4 ++++ wrappers/logrwc.go | 8 ++++++-- wrappers/rwcaddr.go | 1 + wrappers/streamcipherrwc.go | 4 ++++ wrappers/timeoutrwc.go | 8 +++++++- wrappers/trafficrwc.go | 4 ++++ 21 files changed, 127 insertions(+), 57 deletions(-) create mode 100644 utils/read_current_data.go diff --git a/client/client.go b/client/client.go index 32e23a9..318c476 100644 --- a/client/client.go +++ b/client/client.go @@ -9,4 +9,4 @@ import ( ) // Init has to initialize client connection based on given config. -type Init func(net.Conn, *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) +type Init func(net.Conn, string, *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) diff --git a/client/direct.go b/client/direct.go index 3ce1eed..7ee7e75 100644 --- a/client/direct.go +++ b/client/direct.go @@ -15,7 +15,7 @@ import ( const handshakeTimeout = 10 * time.Second // DirectInit initializes client to access Telegram bypassing middleproxies. -func DirectInit(conn net.Conn, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { +func DirectInit(conn net.Conn, socketID string, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { if err := config.SetSocketOptions(conn); err != nil { return nil, nil, errors.Annotate(err, "Cannot set socket options") } @@ -34,7 +34,7 @@ func DirectInit(conn net.Conn, conf *config.Config) (wrappers.ReadWriteCloserWit connOpts.ConnectionProto = mtproto.ConnectionProtocolAny connOpts.ClientAddr = conn.RemoteAddr().(*net.TCPAddr) - socket := wrappers.NewTimeoutRWC(conn, conf.PublicIPv4, conf.PublicIPv6) + socket := wrappers.NewTimeoutRWC(conn, socketID, conf.PublicIPv4, conf.PublicIPv6) socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) return socket, connOpts, nil diff --git a/client/middle.go b/client/middle.go index f8877fc..2c978ec 100644 --- a/client/middle.go +++ b/client/middle.go @@ -9,8 +9,8 @@ import ( "github.com/9seconds/mtg/wrappers" ) -func MiddleInit(conn net.Conn, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { - newConn, opts, err := DirectInit(conn, conf) +func MiddleInit(conn net.Conn, socketID string, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { + newConn, opts, err := DirectInit(conn, socketID, conf) if err != nil { return nil, nil, err } diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go index 1850220..e4bfc13 100644 --- a/mtproto/wrappers/abridged.go +++ b/mtproto/wrappers/abridged.go @@ -31,15 +31,18 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { buf := &bytes.Buffer{} buf.Grow(3) + q := make([]byte, 1) + if _, err := io.CopyN(buf, a.conn, 1); err != nil { return errors.Annotate(err, "Cannot read message length") } msgLength := uint8(buf.Bytes()[0]) + q[0] = msgLength buf.Reset() if msgLength >= abridgedQuickAckLength { a.opts.ReadHacks.QuickAck = true - msgLength -= 0x80 + msgLength -= abridgedQuickAckLength } msgLength32 := uint32(msgLength) @@ -49,6 +52,7 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { } number := uint24{} copy(number[:], buf.Bytes()) + q = append(q, buf.Bytes()...) msgLength32 = fromUint24(number) } msgLength32 *= 4 @@ -59,6 +63,7 @@ func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { if _, err := io.CopyN(buf, a.conn, int64(msgLength32)); err != nil { return errors.Annotate(err, "Cannot read message") } + q = append(q, buf.Bytes()...) a.Buffer.Write(buf.Bytes()) return nil @@ -87,10 +92,9 @@ func (a *AbridgedReadWriteCloserWithAddr) Write(p []byte) (int, error) { buf.Write(length24[:]) buf.Write(p) return a.conn.Write(buf.Bytes()) - - default: - return 0, errors.Errorf("Packet is too big %d", len(p)) } + + return 0, errors.Errorf("Packet is too big %d", len(p)) } func (a *AbridgedReadWriteCloserWithAddr) Close() error { @@ -105,6 +109,10 @@ func (a *AbridgedReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return a.conn.RemoteAddr() } +func (a *AbridgedReadWriteCloserWithAddr) SocketID() string { + return a.conn.SocketID() +} + func toUint24(number uint32) uint24 { return uint24{byte(number), byte(number >> 8), byte(number >> 16)} } diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go index 2c5cf8a..633c4ad 100644 --- a/mtproto/wrappers/frame.go +++ b/mtproto/wrappers/frame.go @@ -120,6 +120,10 @@ func (f *FrameRWC) RemoteAddr() *net.TCPAddr { return f.conn.RemoteAddr() } +func (f *FrameRWC) SocketID() string { + return f.conn.SocketID() +} + func NewFrameRWC(conn wrappers.ReadWriteCloserWithAddr, seqNo int32) wrappers.ReadWriteCloserWithAddr { return &FrameRWC{ BufferedReader: wrappers.NewBufferedReader(), diff --git a/mtproto/wrappers/intermediate.go b/mtproto/wrappers/intermediate.go index a22e56d..0468d47 100644 --- a/mtproto/wrappers/intermediate.go +++ b/mtproto/wrappers/intermediate.go @@ -77,6 +77,10 @@ func (i *IntermediateReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return i.conn.RemoteAddr() } +func (i *IntermediateReadWriteCloserWithAddr) SocketID() string { + return i.conn.SocketID() +} + func NewIntermediateRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts) wrappers.ReadWriteCloserWithAddr { return &IntermediateReadWriteCloserWithAddr{ BufferedReader: wrappers.NewBufferedReader(), diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go index 2641715..088565d 100644 --- a/mtproto/wrappers/proxy_request.go +++ b/mtproto/wrappers/proxy_request.go @@ -10,6 +10,7 @@ import ( "github.com/9seconds/mtg/mtproto" "github.com/9seconds/mtg/mtproto/rpc" + "github.com/9seconds/mtg/utils" "github.com/9seconds/mtg/wrappers" ) @@ -29,7 +30,7 @@ func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { switch { case bytes.Equal(ans, rpc.TagProxyAns): - return p.readProxyAns(buf) + return p.readProxyAns() case bytes.Equal(ans, rpc.TagSimpleAck): return p.readSimpleAck() case bytes.Equal(ans, rpc.TagCloseExt): @@ -44,21 +45,16 @@ func (p *ProxyRequestReadWriteCloserWithAddr) readCloseExt() error { return errors.New("Connection has been closed remotely") } -func (p *ProxyRequestReadWriteCloserWithAddr) readProxyAns(buf []byte) (err error) { +func (p *ProxyRequestReadWriteCloserWithAddr) readProxyAns() (err error) { if _, err = io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { return errors.Annotate(err, "Cannot skip flags and connid") } - n := len(buf) - preBuffer := &bytes.Buffer{} - for n == len(buf) { - n, err = p.conn.Read(buf) - if err != nil { - return errors.Annotate(err, "Cannot read proxy answer") - } - preBuffer.Write(buf[:n]) + buf, err := utils.ReadCurrentData(p.conn) + if err != nil { + return errors.Annotate(err, "Cannot read proxy answer") } - p.Buffer.Write(preBuffer.Bytes()) + p.Buffer.Write(buf) return nil } @@ -97,6 +93,10 @@ func (p *ProxyRequestReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return p.conn.RemoteAddr() } +func (p *ProxyRequestReadWriteCloserWithAddr) SocketID() string { + return p.conn.SocketID() +} + func NewProxyRequestRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts, adTag []byte) (wrappers.ReadWriteCloserWithAddr, error) { req, err := rpc.NewProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) if err != nil { diff --git a/proxy/server.go b/proxy/server.go index 9b43613..18bb98e 100644 --- a/proxy/server.go +++ b/proxy/server.go @@ -14,6 +14,7 @@ import ( "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" "github.com/9seconds/mtg/telegram" + "github.com/9seconds/mtg/utils" "github.com/9seconds/mtg/wrappers" ) @@ -122,7 +123,7 @@ func (s *Server) accept(conn net.Conn) { } func (s *Server) getClientStream(ctx context.Context, cancel context.CancelFunc, conn net.Conn, socketID string) (*mtproto.ConnectionOpts, io.ReadWriteCloser, error) { - socket, connOpts, err := s.clientInit(conn, s.conf) + socket, connOpts, err := s.clientInit(conn, socketID, s.conf) if err != nil { return nil, nil, errors.Annotate(err, "Cannot init client connection") } @@ -135,7 +136,7 @@ func (s *Server) getClientStream(ctx context.Context, cancel context.CancelFunc, } func (s *Server) getTelegramStream(ctx context.Context, cancel context.CancelFunc, connOpts *mtproto.ConnectionOpts, socketID string) (io.ReadWriteCloser, error) { - conn, err := s.tg.Dial(connOpts) + conn, err := s.tg.Dial(socketID, connOpts) if err != nil { return nil, errors.Annotate(err, "Cannot connect to Telegram") } @@ -152,22 +153,15 @@ func (s *Server) getTelegramStream(ctx context.Context, cancel context.CancelFun return conn, nil } -func (s *Server) pump(src io.Reader, dst io.Writer, socketID, name string) (err error) { - copyBuf := make([]byte, 1024*1024*2) - - n := config.BufferSizeCopy - for n == config.BufferSizeCopy { - n, err = src.Read(copyBuf) - if err != nil { - break - } - _, err = dst.Write(copyBuf[:n]) - if err != nil { - break - } +func (s *Server) pump(src io.Reader, dst io.Writer, socketID, name string) error { + buf, err := utils.ReadCurrentData(src) + if err != nil { + return errors.Annotate(err, "Cannot pump the socket") } - return + _, err = dst.Write(buf) + + return err } // NewServer creates new instance of MTPROTO proxy. diff --git a/telegram/dialer.go b/telegram/dialer.go index 9cf048b..5f3e20c 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -30,11 +30,11 @@ func (t *tgDialer) dial(addr string) (net.Conn, error) { return conn, nil } -func (t *tgDialer) dialRWC(addr string) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *tgDialer) dialRWC(addr, sock string) (wrappers.ReadWriteCloserWithAddr, error) { conn, err := t.dial(addr) if err != nil { return nil, err } - return wrappers.NewTimeoutRWC(conn, t.conf.PublicIPv4, t.conf.PublicIPv6), nil + return wrappers.NewTimeoutRWC(conn, sock, t.conf.PublicIPv4, t.conf.PublicIPv6), nil } diff --git a/telegram/direct.go b/telegram/direct.go index ae89276..601fa52 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -33,7 +33,7 @@ type directTelegram struct { baseTelegram } -func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *directTelegram) Dial(sock string, connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { dc := connOpts.DC if dc < 0 { dc = -dc @@ -41,7 +41,7 @@ func (t *directTelegram) Dial(connOpts *mtproto.ConnectionOpts) (wrappers.ReadWr dc = 1 } - return t.baseTelegram.dial(dc-1, connOpts.ConnectionProto) + return t.baseTelegram.dial(dc-1, sock, connOpts.ConnectionProto) } func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go index acc1b27..490ed26 100644 --- a/telegram/middle_caller.go +++ b/telegram/middle_caller.go @@ -39,7 +39,7 @@ type middleTelegramCaller struct { httpClient *http.Client } -func (t *middleTelegramCaller) Dial(connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *middleTelegramCaller) Dial(sock string, connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { dc := connOpts.DC if dc == 0 { dc = 1 @@ -47,7 +47,7 @@ func (t *middleTelegramCaller) Dial(connOpts *mtproto.ConnectionOpts) (wrappers. t.dialerMutex.RLock() defer t.dialerMutex.RUnlock() - return t.baseTelegram.dial(dc, connOpts.ConnectionProto) + return t.baseTelegram.dial(dc, sock, connOpts.ConnectionProto) } func (t *middleTelegramCaller) autoUpdate() { diff --git a/telegram/telegram.go b/telegram/telegram.go index 5ce16ad..9413882 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -13,7 +13,7 @@ import ( // encapsulates logic of working with middleproxies or direct // connections. type Telegram interface { - Dial(*mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) + Dial(string, *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) Init(*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) } @@ -24,7 +24,7 @@ type baseTelegram struct { v6Addresses map[int16][]string } -func (b *baseTelegram) dial(dcIdx int16, proto mtproto.ConnectionProtocol) (wrappers.ReadWriteCloserWithAddr, error) { +func (b *baseTelegram) dial(dcIdx int16, sock string, proto mtproto.ConnectionProtocol) (wrappers.ReadWriteCloserWithAddr, error) { addrs := make([]string, 2) if proto&mtproto.ConnectionProtocolIPv6 != 0 { @@ -39,7 +39,7 @@ func (b *baseTelegram) dial(dcIdx int16, proto mtproto.ConnectionProtocol) (wrap } for _, addr := range addrs { - if conn, err := b.dialer.dialRWC(addr); err == nil { + if conn, err := b.dialer.dialRWC(addr, sock); err == nil { return conn, err } } diff --git a/utils/read_current_data.go b/utils/read_current_data.go new file mode 100644 index 0000000..c8669e2 --- /dev/null +++ b/utils/read_current_data.go @@ -0,0 +1,20 @@ +package utils + +import "io" + +const readCurrentDataBufferSize = 1024 + 1 + +func ReadCurrentData(src io.Reader) (rv []byte, err error) { + buf := make([]byte, readCurrentDataBufferSize) + n := readCurrentDataBufferSize + + for n == len(buf) { + n, err = src.Read(buf) + if err != nil { + return nil, err + } + rv = append(rv, buf[:n]...) + } + + return rv, nil +} diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go index 8d2a06c..af5ccf0 100644 --- a/wrappers/blockcipherrwc.go +++ b/wrappers/blockcipherrwc.go @@ -1,12 +1,14 @@ package wrappers import ( - "bytes" "crypto/aes" "crypto/cipher" + "fmt" "net" "github.com/juju/errors" + + "github.com/9seconds/mtg/utils" ) type BlockCipherReadWriteCloserWithAddr struct { @@ -19,16 +21,18 @@ type BlockCipherReadWriteCloserWithAddr struct { func (c *BlockCipherReadWriteCloserWithAddr) Read(p []byte) (int, error) { return c.BufferedRead(p, func() error { - buf := &bytes.Buffer{} - for buf.Len()%aes.BlockSize != 0 || buf.Len() == 0 { - n, err := c.conn.Read(p) + var buf []byte + + for len(buf) == 0 || len(buf)%aes.BlockSize != 0 { + rv, err := utils.ReadCurrentData(c.conn) if err != nil { return errors.Annotate(err, "Cannot read from socket") } - buf.Write(p[:n]) + buf = append(buf, rv...) } - c.decryptor.CryptBlocks(buf.Bytes(), buf.Bytes()) - c.Buffer.Write(buf.Bytes()) + + c.decryptor.CryptBlocks(buf, buf) + c.Buffer.Write(buf) return nil }) @@ -46,6 +50,7 @@ func (c *BlockCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { } func (c *BlockCipherReadWriteCloserWithAddr) Close() error { + fmt.Println("BlockCipherReadWriteCloserWithAddr closes", "sockid", c.SocketID(), "bufsize", c.Buffer.Len()) return c.conn.Close() } @@ -57,6 +62,10 @@ func (c *BlockCipherReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return c.conn.RemoteAddr() } +func (c *BlockCipherReadWriteCloserWithAddr) SocketID() string { + return c.conn.SocketID() +} + func NewBlockCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.BlockMode) ReadWriteCloserWithAddr { return &BlockCipherReadWriteCloserWithAddr{ BufferedReader: NewBufferedReader(), diff --git a/wrappers/buffered_reader.go b/wrappers/buffered_reader.go index cb48d56..09d3bca 100644 --- a/wrappers/buffered_reader.go +++ b/wrappers/buffered_reader.go @@ -1,11 +1,19 @@ package wrappers -import "bytes" +import ( + "bytes" + + "github.com/juju/errors" +) type BufferedReader struct { Buffer *bytes.Buffer } +var ( + BufferedReaderContinue = errors.New("Please continue reading") +) + func (b *BufferedReader) BufferedRead(p []byte, callback func() error) (int, error) { if b.Buffer.Len() > 0 { return b.flush(p) @@ -17,7 +25,7 @@ func (b *BufferedReader) BufferedRead(p []byte, callback func() error) (int, err } func (b *BufferedReader) flush(p []byte) (int, error) { - if b.Buffer.Len() < len(p) { + if b.Buffer.Len() <= len(p) { sizeToReturn := b.Buffer.Len() copy(p, b.Buffer.Bytes()) b.Buffer.Reset() diff --git a/wrappers/ctxrwc.go b/wrappers/ctxrwc.go index 64c1006..fe6f515 100644 --- a/wrappers/ctxrwc.go +++ b/wrappers/ctxrwc.go @@ -56,6 +56,10 @@ func (c *CtxReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return c.conn.RemoteAddr() } +func (c *CtxReadWriteCloserWithAddr) SocketID() string { + return c.conn.SocketID() +} + // NewCtxRWC returns ReadWriteCloser which respects given context, // cancellation etc. func NewCtxRWC(ctx context.Context, cancel context.CancelFunc, conn ReadWriteCloserWithAddr) ReadWriteCloserWithAddr { diff --git a/wrappers/logrwc.go b/wrappers/logrwc.go index fd80ad4..74d3cbf 100644 --- a/wrappers/logrwc.go +++ b/wrappers/logrwc.go @@ -18,14 +18,14 @@ type LogReadWriteCloserWithAddr struct { // Read reads from connection func (l *LogReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { n, err = l.conn.Read(p) - l.logger.Debugw("Finish reading", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err) + l.logger.Debugw("Finish reading", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err, "localAddr", l.LocalAddr()) return } // Write writes into connection. func (l *LogReadWriteCloserWithAddr) Write(p []byte) (n int, err error) { n, err = l.conn.Write(p) - l.logger.Debugw("Finish writing", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err) + l.logger.Debugw("Finish writing", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err, "localAddr", l.LocalAddr()) return } @@ -44,6 +44,10 @@ func (l *LogReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return l.conn.RemoteAddr() } +func (l *LogReadWriteCloserWithAddr) SocketID() string { + return l.sockid +} + // NewLogRWC wraps ReadWriteCloser with logger calls. func NewLogRWC(conn ReadWriteCloserWithAddr, logger *zap.SugaredLogger, sockid string, name string) ReadWriteCloserWithAddr { return &LogReadWriteCloserWithAddr{ diff --git a/wrappers/rwcaddr.go b/wrappers/rwcaddr.go index 6ca4299..fee186c 100644 --- a/wrappers/rwcaddr.go +++ b/wrappers/rwcaddr.go @@ -10,4 +10,5 @@ type ReadWriteCloserWithAddr interface { LocalAddr() *net.TCPAddr RemoteAddr() *net.TCPAddr + SocketID() string } diff --git a/wrappers/streamcipherrwc.go b/wrappers/streamcipherrwc.go index a5f15d1..f9ad598 100644 --- a/wrappers/streamcipherrwc.go +++ b/wrappers/streamcipherrwc.go @@ -51,6 +51,10 @@ func (c *StreamCipherReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return c.conn.RemoteAddr() } +func (c *StreamCipherReadWriteCloserWithAddr) SocketID() string { + return c.conn.SocketID() +} + // NewStreamCipherRWC returns wrapper which transparently // encrypts/decrypts traffic with obfuscated2 protocol. func NewStreamCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.Stream) ReadWriteCloserWithAddr { diff --git a/wrappers/timeoutrwc.go b/wrappers/timeoutrwc.go index 952ade1..3c48546 100644 --- a/wrappers/timeoutrwc.go +++ b/wrappers/timeoutrwc.go @@ -9,6 +9,7 @@ import ( type TimeoutReadWriteCloserWithAddr struct { conn net.Conn + sock string publicIPv4 net.IP publicIPv6 net.IP } @@ -46,10 +47,15 @@ func (t *TimeoutReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { return &newAddr } -func NewTimeoutRWC(conn net.Conn, ipv4, ipv6 net.IP) ReadWriteCloserWithAddr { +func (t *TimeoutReadWriteCloserWithAddr) SocketID() string { + return t.sock +} + +func NewTimeoutRWC(conn net.Conn, sock string, ipv4, ipv6 net.IP) ReadWriteCloserWithAddr { return &TimeoutReadWriteCloserWithAddr{ conn: conn, publicIPv4: ipv4, publicIPv6: ipv6, + sock: sock, } } diff --git a/wrappers/trafficrwc.go b/wrappers/trafficrwc.go index 60dd8c2..344a451 100644 --- a/wrappers/trafficrwc.go +++ b/wrappers/trafficrwc.go @@ -37,6 +37,10 @@ func (t *TrafficReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { return t.conn.RemoteAddr() } +func (t *TrafficReadWriteCloserWithAddr) SocketID() string { + return t.conn.SocketID() +} + // NewTrafficRWC wraps ReadWriteCloser to have read/write callbacks. func NewTrafficRWC(conn ReadWriteCloserWithAddr, readCallback, writeCallback func(int)) ReadWriteCloserWithAddr { return &TrafficReadWriteCloserWithAddr{ From 41b2b1c819662bd95ec4c15175a824c5f41edd7c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 7 Jul 2018 16:50:29 +0300 Subject: [PATCH 42/71] Start to rewrite --- mtproto/wrappers/abridged.go | 130 ----------------------------- utils/read_current_data.go | 2 +- utils/reverse_bytes.go | 14 ++++ utils/uint24.go | 11 +++ wrappers/blockcipher.go | 85 +++++++++++++++++++ wrappers/blockcipherrwc.go | 76 ----------------- wrappers/buffer_pool.go | 27 ------ wrappers/buffered_reader.go | 10 +-- wrappers/conn.go | 115 ++++++++++++++++++++++++++ wrappers/ctx.go | 76 +++++++++++++++++ wrappers/ctxrwc.go | 71 ---------------- wrappers/logrwc.go | 59 ------------- wrappers/mtproto_abridged.go | 155 +++++++++++++++++++++++++++++++++++ wrappers/rwcaddr.go | 14 ---- wrappers/streamcipher.go | 67 +++++++++++++++ wrappers/streamcipherrwc.go | 66 --------------- wrappers/timeoutrwc.go | 61 -------------- wrappers/trafficrwc.go | 51 ------------ wrappers/wrap.go | 78 ++++++++++++++++++ 19 files changed, 603 insertions(+), 565 deletions(-) delete mode 100644 mtproto/wrappers/abridged.go create mode 100644 utils/reverse_bytes.go create mode 100644 utils/uint24.go create mode 100644 wrappers/blockcipher.go delete mode 100644 wrappers/blockcipherrwc.go delete mode 100644 wrappers/buffer_pool.go create mode 100644 wrappers/conn.go create mode 100644 wrappers/ctx.go delete mode 100644 wrappers/ctxrwc.go delete mode 100644 wrappers/logrwc.go create mode 100644 wrappers/mtproto_abridged.go delete mode 100644 wrappers/rwcaddr.go create mode 100644 wrappers/streamcipher.go delete mode 100644 wrappers/streamcipherrwc.go delete mode 100644 wrappers/timeoutrwc.go delete mode 100644 wrappers/trafficrwc.go create mode 100644 wrappers/wrap.go diff --git a/mtproto/wrappers/abridged.go b/mtproto/wrappers/abridged.go deleted file mode 100644 index e4bfc13..0000000 --- a/mtproto/wrappers/abridged.go +++ /dev/null @@ -1,130 +0,0 @@ -package wrappers - -import ( - "bytes" - "io" - "net" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/wrappers" -) - -type uint24 [3]byte - -const ( - abridgedSmallPacketLength = 0x7f - abridgedQuickAckLength = 0x80 - abridgedLargePacketLength = 16777216 // 256 ^ 3 -) - -type AbridgedReadWriteCloserWithAddr struct { - wrappers.BufferedReader - - conn wrappers.ReadWriteCloserWithAddr - opts *mtproto.ConnectionOpts -} - -func (a *AbridgedReadWriteCloserWithAddr) Read(p []byte) (int, error) { - return a.BufferedRead(p, func() error { - buf := &bytes.Buffer{} - buf.Grow(3) - - q := make([]byte, 1) - - if _, err := io.CopyN(buf, a.conn, 1); err != nil { - return errors.Annotate(err, "Cannot read message length") - } - msgLength := uint8(buf.Bytes()[0]) - q[0] = msgLength - buf.Reset() - - if msgLength >= abridgedQuickAckLength { - a.opts.ReadHacks.QuickAck = true - msgLength -= abridgedQuickAckLength - } - - msgLength32 := uint32(msgLength) - if msgLength == abridgedSmallPacketLength { - if _, err := io.CopyN(buf, a.conn, 3); err != nil { - return errors.Annotate(err, "Cannot read the correct message length") - } - number := uint24{} - copy(number[:], buf.Bytes()) - q = append(q, buf.Bytes()...) - msgLength32 = fromUint24(number) - } - msgLength32 *= 4 - - buf.Reset() - buf.Grow(int(msgLength32)) - - if _, err := io.CopyN(buf, a.conn, int64(msgLength32)); err != nil { - return errors.Annotate(err, "Cannot read message") - } - q = append(q, buf.Bytes()...) - a.Buffer.Write(buf.Bytes()) - - return nil - }) -} - -func (a *AbridgedReadWriteCloserWithAddr) Write(p []byte) (int, error) { - if len(p)%4 != 0 { - return 0, errors.Errorf("Incorrect packet length %d", len(p)) - } - if a.opts.WriteHacks.SimpleAck { - return a.conn.Write(reverseBytes(p)) - } - - packetLength := len(p) / 4 - switch { - case packetLength < abridgedSmallPacketLength: - newData := append([]byte{byte(packetLength)}, p...) - return a.conn.Write(newData) - - case packetLength < abridgedLargePacketLength: - length24 := toUint24(uint32(packetLength)) - buf := &bytes.Buffer{} - buf.Grow(1 + 3 + len(p)) - buf.WriteByte(byte(abridgedSmallPacketLength)) - buf.Write(length24[:]) - buf.Write(p) - return a.conn.Write(buf.Bytes()) - } - - return 0, errors.Errorf("Packet is too big %d", len(p)) -} - -func (a *AbridgedReadWriteCloserWithAddr) Close() error { - return a.conn.Close() -} - -func (a *AbridgedReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return a.conn.LocalAddr() -} - -func (a *AbridgedReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return a.conn.RemoteAddr() -} - -func (a *AbridgedReadWriteCloserWithAddr) SocketID() string { - return a.conn.SocketID() -} - -func toUint24(number uint32) uint24 { - return uint24{byte(number), byte(number >> 8), byte(number >> 16)} -} - -func fromUint24(number uint24) uint32 { - return uint32(number[0]) + (uint32(number[1]) << 8) + (uint32(number[2]) << 16) -} - -func NewAbridgedRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts) wrappers.ReadWriteCloserWithAddr { - return &AbridgedReadWriteCloserWithAddr{ - BufferedReader: wrappers.NewBufferedReader(), - conn: conn, - opts: connOpts, - } -} diff --git a/utils/read_current_data.go b/utils/read_current_data.go index c8669e2..d66802b 100644 --- a/utils/read_current_data.go +++ b/utils/read_current_data.go @@ -2,7 +2,7 @@ package utils import "io" -const readCurrentDataBufferSize = 1024 + 1 +const readCurrentDataBufferSize = 1024 + 1 // + 1 because telegram operates with blocks mod 4 func ReadCurrentData(src io.Reader) (rv []byte, err error) { buf := make([]byte, readCurrentDataBufferSize) diff --git a/utils/reverse_bytes.go b/utils/reverse_bytes.go new file mode 100644 index 0000000..447adbd --- /dev/null +++ b/utils/reverse_bytes.go @@ -0,0 +1,14 @@ +package utils + +func ReverseBytes(data []byte) []byte { + dataLen := len(data) + rv := make([]byte, dataLen) + + rv[dataLen/2] = data[dataLen/2] + for i := dataLen/2 - 1; i >= 0; i-- { + opp := dataLen - i - 1 + rv[i], rv[opp] = data[opp], data[i] + } + + return rv +} diff --git a/utils/uint24.go b/utils/uint24.go new file mode 100644 index 0000000..350f3d5 --- /dev/null +++ b/utils/uint24.go @@ -0,0 +1,11 @@ +package utils + +type Uint24 [3]byte + +func ToUint24(number uint32) Uint24 { + return Uint24{byte(number), byte(number >> 8), byte(number >> 16)} +} + +func FromUint24(number Uint24) uint32 { + return uint32(number[0]) + (uint32(number[1]) << 8) + (uint32(number[2]) << 16) +} diff --git a/wrappers/blockcipher.go b/wrappers/blockcipher.go new file mode 100644 index 0000000..55b90e1 --- /dev/null +++ b/wrappers/blockcipher.go @@ -0,0 +1,85 @@ +package wrappers + +import ( + "crypto/aes" + "crypto/cipher" + "net" + + "github.com/9seconds/mtg/utils" + "github.com/juju/errors" +) + +type WrapBlockCipher struct { + BufferedReader + + conn WrapStreamReadWriteCloser + encryptor cipher.BlockMode + decryptor cipher.BlockMode +} + +func (w *WrapBlockCipher) Read(p []byte) (int, error) { + return w.BufferedRead(p, func() error { + var buf []byte + + for len(buf) == 0 || len(buf)%aes.BlockSize != 0 { + rv, err := utils.ReadCurrentData(w.conn) + if err != nil { + return errors.Annotate(err, "Cannot read from socket") + } + buf = append(buf, rv...) + } + + w.decryptor.CryptBlocks(buf, buf) + w.Buffer.Write(buf) + + return nil + }) +} + +func (w *WrapBlockCipher) Write(p []byte) (int, error) { + if len(p)%aes.BlockSize > 0 { + return 0, errors.Errorf("Incorrect block size %d", len(p)) + } + + encrypted := make([]byte, len(p)) + w.encryptor.CryptBlocks(encrypted, p) + + return w.conn.Write(encrypted) +} + +func (w *WrapBlockCipher) LogDebug(msg string, data ...interface{}) { + w.conn.LogDebug(msg, data...) +} + +func (w *WrapBlockCipher) LogInfo(msg string, data ...interface{}) { + w.conn.LogInfo(msg, data...) +} + +func (w *WrapBlockCipher) LogWarn(msg string, data ...interface{}) { + w.conn.LogWarn(msg, data...) +} + +func (w *WrapBlockCipher) LogError(msg string, data ...interface{}) { + w.conn.LogError(msg, data...) +} + +func (w *WrapBlockCipher) LocalAddr() *net.TCPAddr { + return w.conn.LocalAddr() +} + +func (w *WrapBlockCipher) RemoteAddr() *net.TCPAddr { + return w.conn.RemoteAddr() +} + +func (w *WrapBlockCipher) Close() error { + return w.conn.Close() +} + +func NewWrapBlockCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.BlockMode) WrapStreamReadWriteCloser { + return &WrapBlockCipher{ + BufferedReader: NewBufferedReader(), + conn: conn, + encryptor: encryptor, + decryptor: decryptor, + } +} diff --git a/wrappers/blockcipherrwc.go b/wrappers/blockcipherrwc.go deleted file mode 100644 index af5ccf0..0000000 --- a/wrappers/blockcipherrwc.go +++ /dev/null @@ -1,76 +0,0 @@ -package wrappers - -import ( - "crypto/aes" - "crypto/cipher" - "fmt" - "net" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/utils" -) - -type BlockCipherReadWriteCloserWithAddr struct { - BufferedReader - - conn ReadWriteCloserWithAddr - encryptor cipher.BlockMode - decryptor cipher.BlockMode -} - -func (c *BlockCipherReadWriteCloserWithAddr) Read(p []byte) (int, error) { - return c.BufferedRead(p, func() error { - var buf []byte - - for len(buf) == 0 || len(buf)%aes.BlockSize != 0 { - rv, err := utils.ReadCurrentData(c.conn) - if err != nil { - return errors.Annotate(err, "Cannot read from socket") - } - buf = append(buf, rv...) - } - - c.decryptor.CryptBlocks(buf, buf) - c.Buffer.Write(buf) - - return nil - }) -} - -func (c *BlockCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { - if len(p)%aes.BlockSize > 0 { - return 0, errors.Errorf("Incorrect block size %d", len(p)) - } - - encrypted := make([]byte, len(p)) - c.encryptor.CryptBlocks(encrypted, p) - - return c.conn.Write(encrypted) -} - -func (c *BlockCipherReadWriteCloserWithAddr) Close() error { - fmt.Println("BlockCipherReadWriteCloserWithAddr closes", "sockid", c.SocketID(), "bufsize", c.Buffer.Len()) - return c.conn.Close() -} - -func (c *BlockCipherReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return c.conn.LocalAddr() -} - -func (c *BlockCipherReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return c.conn.RemoteAddr() -} - -func (c *BlockCipherReadWriteCloserWithAddr) SocketID() string { - return c.conn.SocketID() -} - -func NewBlockCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.BlockMode) ReadWriteCloserWithAddr { - return &BlockCipherReadWriteCloserWithAddr{ - BufferedReader: NewBufferedReader(), - conn: conn, - encryptor: encryptor, - decryptor: decryptor, - } -} diff --git a/wrappers/buffer_pool.go b/wrappers/buffer_pool.go deleted file mode 100644 index ead700d..0000000 --- a/wrappers/buffer_pool.go +++ /dev/null @@ -1,27 +0,0 @@ -package wrappers - -import ( - "bytes" - "sync" -) - -var bufPool sync.Pool - -func getBuffer() *bytes.Buffer { - buf := bufPool.Get().(*bytes.Buffer) - buf.Reset() - - return buf -} - -func putBuffer(buf *bytes.Buffer) { - bufPool.Put(buf) -} - -func init() { - bufPool = sync.Pool{ - New: func() interface{} { - return &bytes.Buffer{} - }, - } -} diff --git a/wrappers/buffered_reader.go b/wrappers/buffered_reader.go index 09d3bca..379534a 100644 --- a/wrappers/buffered_reader.go +++ b/wrappers/buffered_reader.go @@ -1,19 +1,11 @@ package wrappers -import ( - "bytes" - - "github.com/juju/errors" -) +import "bytes" type BufferedReader struct { Buffer *bytes.Buffer } -var ( - BufferedReaderContinue = errors.New("Please continue reading") -) - func (b *BufferedReader) BufferedRead(p []byte, callback func() error) (int, error) { if b.Buffer.Len() > 0 { return b.flush(p) diff --git a/wrappers/conn.go b/wrappers/conn.go new file mode 100644 index 0000000..468820c --- /dev/null +++ b/wrappers/conn.go @@ -0,0 +1,115 @@ +package wrappers + +import ( + "net" + "time" + + "go.uber.org/zap" +) + +type ConnPurpose uint8 + +func (c ConnPurpose) String() string { + switch c { + case ConnPurposeClient: + return "client" + case ConnPurposeTelegram: + return "telegram" + } + + return "" +} + +const ( + ConnPurposeClient = iota + ConnPurposeTelegram +) + +const ( + connTimeoutRead = 5 * time.Minute + connTimeoutWrite = 5 * time.Minute +) + +type WrapConn struct { + purpose ConnPurpose + connID string + conn net.Conn + logger *zap.SugaredLogger + publicIPv4 net.IP + publicIPv6 net.IP +} + +func (w *WrapConn) Write(p []byte) (int, error) { + w.conn.SetWriteDeadline(time.Now().Add(connTimeoutWrite)) + n, err := w.conn.Write(p) + + w.logger.Debugw("Write to stream", "bytes", n, "error", err) + + return n, err +} + +func (w *WrapConn) Read(p []byte) (int, error) { + w.conn.SetReadDeadline(time.Now().Add(connTimeoutRead)) + n, err := w.conn.Read(p) + + w.logger.Debugw("Read from stream", "bytes", n, "error", err) + + return n, err +} + +func (w *WrapConn) Close() error { + defer w.LogDebug("Closed connection") + return w.conn.Close() +} + +func (w *WrapConn) LocalAddr() *net.TCPAddr { + addr := w.conn.LocalAddr().(*net.TCPAddr) + newAddr := *addr + + if w.RemoteAddr().IP.To4() != nil { + if w.publicIPv4 != nil { + newAddr.IP = w.publicIPv4 + } + } else if w.publicIPv6 != nil { + newAddr.IP = w.publicIPv6 + } + + return &newAddr +} + +func (w *WrapConn) RemoteAddr() *net.TCPAddr { + return w.conn.RemoteAddr().(*net.TCPAddr) +} + +func (w *WrapConn) LogDebug(msg string, data ...interface{}) { + w.logger.Debugw(msg, data...) +} + +func (w *WrapConn) LogInfo(msg string, data ...interface{}) { + w.logger.Infow(msg, data...) +} + +func (w *WrapConn) LogWarn(msg string, data ...interface{}) { + w.logger.Warnw(msg, data...) +} + +func (w *WrapConn) LogError(msg string, data ...interface{}) { + w.logger.Errorw(msg, data...) +} + +func NewConn(connID string, purpose ConnPurpose, conn net.Conn, publicIPv4, publicIPv6 net.IP) WrapStreamReadWriteCloser { + logger := zap.S().With( + "connection_id", connID, + "local_address", conn.LocalAddr(), + "remote_address", conn.RemoteAddr(), + ) + + return &WrapConn{ + logger: logger, + purpose: purpose, + connID: connID, + conn: conn, + publicIPv4: publicIPv4, + publicIPv6: publicIPv6, + } +} diff --git a/wrappers/ctx.go b/wrappers/ctx.go new file mode 100644 index 0000000..ee766f8 --- /dev/null +++ b/wrappers/ctx.go @@ -0,0 +1,76 @@ +package wrappers + +import ( + "context" + "net" + + "github.com/juju/errors" +) + +type WrapCtx struct { + cancel context.CancelFunc + conn WrapStreamReadWriteCloser + ctx context.Context +} + +func (w *WrapCtx) Read(p []byte) (int, error) { + select { + case <-w.ctx.Done(): + return 0, errors.Annotate(w.ctx.Err(), "Read is failed because of closed context") + default: + n, err := w.conn.Read(p) + if err != nil { + w.cancel() + } + return n, err + } +} + +func (w *WrapCtx) Write(p []byte) (int, error) { + select { + case <-w.ctx.Done(): + return 0, errors.Annotate(w.ctx.Err(), "Write is failed because of closed context") + default: + n, err := w.conn.Write(p) + if err != nil { + w.cancel() + } + return n, err + } +} + +func (w *WrapCtx) LogDebug(msg string, data ...interface{}) { + w.conn.LogDebug(msg, data...) +} + +func (w *WrapCtx) LogInfo(msg string, data ...interface{}) { + w.conn.LogInfo(msg, data...) +} + +func (w *WrapCtx) LogWarn(msg string, data ...interface{}) { + w.conn.LogWarn(msg, data...) +} + +func (w *WrapCtx) LogError(msg string, data ...interface{}) { + w.conn.LogError(msg, data...) +} + +func (w *WrapCtx) LocalAddr() *net.TCPAddr { + return w.conn.LocalAddr() +} + +func (w *WrapCtx) RemoteAddr() *net.TCPAddr { + return w.conn.RemoteAddr() +} + +func (w *WrapCtx) Close() error { + return w.conn.Close() +} + +func NewCtx(ctx context.Context, cancel context.CancelFunc, conn WrapStreamReadWriteCloser) WrapStreamReadWriteCloser { + return &WrapCtx{ + ctx: ctx, + cancel: cancel, + conn: conn, + } +} diff --git a/wrappers/ctxrwc.go b/wrappers/ctxrwc.go deleted file mode 100644 index fe6f515..0000000 --- a/wrappers/ctxrwc.go +++ /dev/null @@ -1,71 +0,0 @@ -package wrappers - -import ( - "context" - "net" - - "github.com/juju/errors" -) - -// CtxReadWriteCloser wraps underlying connection and does management of the -// context and its cancel function. -type CtxReadWriteCloserWithAddr struct { - ctx context.Context - conn ReadWriteCloserWithAddr - cancel context.CancelFunc -} - -// Read reads from connection -func (c *CtxReadWriteCloserWithAddr) Read(p []byte) (int, error) { - select { - case <-c.ctx.Done(): - return 0, errors.Annotate(c.ctx.Err(), "Read is failed because of closed context") - default: - n, err := c.conn.Read(p) - if err != nil { - c.cancel() - } - return n, err - } -} - -// Write writes into connection. -func (c *CtxReadWriteCloserWithAddr) Write(p []byte) (int, error) { - select { - case <-c.ctx.Done(): - return 0, errors.Annotate(c.ctx.Err(), "Write is failed because of closed context") - default: - n, err := c.conn.Write(p) - if err != nil { - c.cancel() - } - return n, err - } -} - -// Close closes underlying connection. -func (c *CtxReadWriteCloserWithAddr) Close() error { - return c.conn.Close() -} - -func (c *CtxReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return c.conn.LocalAddr() -} - -func (c *CtxReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return c.conn.RemoteAddr() -} - -func (c *CtxReadWriteCloserWithAddr) SocketID() string { - return c.conn.SocketID() -} - -// NewCtxRWC returns ReadWriteCloser which respects given context, -// cancellation etc. -func NewCtxRWC(ctx context.Context, cancel context.CancelFunc, conn ReadWriteCloserWithAddr) ReadWriteCloserWithAddr { - return &CtxReadWriteCloserWithAddr{ - conn: conn, - ctx: ctx, - cancel: cancel, - } -} diff --git a/wrappers/logrwc.go b/wrappers/logrwc.go deleted file mode 100644 index 74d3cbf..0000000 --- a/wrappers/logrwc.go +++ /dev/null @@ -1,59 +0,0 @@ -package wrappers - -import ( - "net" - - "go.uber.org/zap" -) - -// LogReadWriteCloser adds additional logging for reading/writing. All -// logging is performed for debug mode only. -type LogReadWriteCloserWithAddr struct { - conn ReadWriteCloserWithAddr - logger *zap.SugaredLogger - sockid string - name string -} - -// Read reads from connection -func (l *LogReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { - n, err = l.conn.Read(p) - l.logger.Debugw("Finish reading", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err, "localAddr", l.LocalAddr()) - return -} - -// Write writes into connection. -func (l *LogReadWriteCloserWithAddr) Write(p []byte) (n int, err error) { - n, err = l.conn.Write(p) - l.logger.Debugw("Finish writing", "name", l.name, "socketid", l.sockid, "nbytes", n, "error", err, "localAddr", l.LocalAddr()) - return -} - -// Close closes underlying connection. -func (l *LogReadWriteCloserWithAddr) Close() error { - err := l.conn.Close() - l.logger.Debugw("Finish closing socket", "name", l.name, "socketid", l.sockid, "error", err) - return err -} - -func (l *LogReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return l.conn.LocalAddr() -} - -func (l *LogReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return l.conn.RemoteAddr() -} - -func (l *LogReadWriteCloserWithAddr) SocketID() string { - return l.sockid -} - -// NewLogRWC wraps ReadWriteCloser with logger calls. -func NewLogRWC(conn ReadWriteCloserWithAddr, logger *zap.SugaredLogger, sockid string, name string) ReadWriteCloserWithAddr { - return &LogReadWriteCloserWithAddr{ - conn: conn, - logger: logger, - sockid: sockid, - name: name, - } -} diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go new file mode 100644 index 0000000..b9950c3 --- /dev/null +++ b/wrappers/mtproto_abridged.go @@ -0,0 +1,155 @@ +package wrappers + +import ( + "bytes" + "io" + "net" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/utils" +) + +const ( + abridgedSmallPacketLength = 0x7f + abridgedQuickAckLength = 0x80 + abridgedLargePacketLength = 16777216 // 256 ^ 3 +) + +type MTProtoAbridged struct { + conn WrapStreamReadWriteCloser + opts *mtproto.ConnectionOpts + + readCounter uint32 + writeCounter uint32 +} + +func (m *MTProtoAbridged) Read() ([]byte, error) { + m.LogDebug("Read abridged packet", + "simple_ack", m.opts.WriteHacks.SimpleAck, + "quick_ack", m.opts.WriteHacks.QuickAck, + "counter", m.readCounter, + ) + + buf := &bytes.Buffer{} + buf.Grow(3) + + if _, err := io.CopyN(buf, m.conn, 1); err != nil { + return nil, errors.Annotate(err, "Cannot read message length") + } + msgLength := uint8(buf.Bytes()[0]) + buf.Reset() + + m.LogDebug("Abridged packet first byte", + "byte", msgLength, + "counter", m.readCounter, + ) + + if msgLength >= abridgedQuickAckLength { + m.opts.ReadHacks.QuickAck = true + msgLength -= abridgedQuickAckLength + } + + msgLength32 := uint32(msgLength) + if msgLength == abridgedSmallPacketLength { + if _, err := io.CopyN(buf, m.conn, 3); err != nil { + return nil, errors.Annotate(err, "Cannot read the correct message length") + } + number := utils.Uint24{} + copy(number[:], buf.Bytes()) + msgLength32 = utils.FromUint24(number) + } + msgLength32 *= 4 + + m.LogDebug("Abridged packet length", + "length", msgLength32, + "counter", m.readCounter, + ) + + buf.Reset() + buf.Grow(int(msgLength32)) + + if _, err := io.CopyN(buf, m.conn, int64(msgLength32)); err != nil { + return nil, errors.Annotate(err, "Cannot read message") + } + m.readCounter++ + + return buf.Bytes(), nil +} + +func (m *MTProtoAbridged) Write(p []byte) (int, error) { + m.LogDebug("Write abridged packet", + "length", len(p), + "simple_ack", m.opts.WriteHacks.SimpleAck, + "quick_ack", m.opts.WriteHacks.QuickAck, + "counter", m.writeCounter, + ) + + if len(p)%4 == 0 { + return 0, errors.Errorf("Incorrect packet length %d", len(p)) + } + + if m.opts.WriteHacks.SimpleAck { + return m.conn.Write(utils.ReverseBytes(p)) + } + + packetLength := len(p) / 4 + switch { + case packetLength < abridgedSmallPacketLength: + newData := append([]byte{byte(packetLength)}, p...) + + m.writeCounter++ + return m.conn.Write(newData) + + case packetLength < abridgedLargePacketLength: + length24 := utils.ToUint24(uint32(packetLength)) + + buf := &bytes.Buffer{} + buf.Grow(1 + 3 + len(p)) + + buf.WriteByte(byte(abridgedSmallPacketLength)) + buf.Write(length24[:]) + buf.Write(p) + + m.writeCounter++ + return m.conn.Write(buf.Bytes()) + } + + return 0, errors.Errorf("Packet is too big %d", len(p)) +} + +func (m *MTProtoAbridged) LogDebug(msg string, data ...interface{}) { + m.conn.LogDebug(msg, data...) +} + +func (m *MTProtoAbridged) LogInfo(msg string, data ...interface{}) { + m.conn.LogInfo(msg, data...) +} + +func (m *MTProtoAbridged) LogWarn(msg string, data ...interface{}) { + m.conn.LogWarn(msg, data...) +} + +func (m *MTProtoAbridged) LogError(msg string, data ...interface{}) { + m.conn.LogError(msg, data...) +} + +func (m *MTProtoAbridged) LocalAddr() *net.TCPAddr { + return m.conn.LocalAddr() +} + +func (m *MTProtoAbridged) RemoteAddr() *net.TCPAddr { + return m.conn.RemoteAddr() +} + +func (m *MTProtoAbridged) Close() error { + return m.conn.Close() +} + +func NewMTProtoAbridged(conn WrapStreamReadWriteCloser, opts *mtproto.ConnectionOpts) WrapPacketReadWriteCloser { + return &MTProtoAbridged{ + conn: conn, + opts: opts, + } +} diff --git a/wrappers/rwcaddr.go b/wrappers/rwcaddr.go deleted file mode 100644 index fee186c..0000000 --- a/wrappers/rwcaddr.go +++ /dev/null @@ -1,14 +0,0 @@ -package wrappers - -import ( - "io" - "net" -) - -type ReadWriteCloserWithAddr interface { - io.ReadWriteCloser - - LocalAddr() *net.TCPAddr - RemoteAddr() *net.TCPAddr - SocketID() string -} diff --git a/wrappers/streamcipher.go b/wrappers/streamcipher.go new file mode 100644 index 0000000..1ad764f --- /dev/null +++ b/wrappers/streamcipher.go @@ -0,0 +1,67 @@ +package wrappers + +import ( + "crypto/cipher" + "net" + + "github.com/juju/errors" +) + +type WrapStreamCipher struct { + encryptor cipher.Stream + decryptor cipher.Stream + conn WrapStreamReadWriteCloser +} + +func (w *WrapStreamCipher) Read(p []byte) (int, error) { + n, err := w.conn.Read(p) + if err != nil { + return 0, errors.Annotate(err, "Cannot read stream ciphered data") + } + w.decryptor.XORKeyStream(p, p[:n]) + + return n, nil +} + +func (w *WrapStreamCipher) Write(p []byte) (int, error) { + encrypted := make([]byte, len(p)) + w.encryptor.XORKeyStream(encrypted, p) + + return w.conn.Write(encrypted) +} + +func (w *WrapStreamCipher) LogDebug(msg string, data ...interface{}) { + w.conn.LogDebug(msg, data...) +} + +func (w *WrapStreamCipher) LogInfo(msg string, data ...interface{}) { + w.conn.LogInfo(msg, data...) +} + +func (w *WrapStreamCipher) LogWarn(msg string, data ...interface{}) { + w.conn.LogWarn(msg, data...) +} + +func (w *WrapStreamCipher) LogError(msg string, data ...interface{}) { + w.conn.LogError(msg, data...) +} + +func (w *WrapStreamCipher) LocalAddr() *net.TCPAddr { + return w.conn.LocalAddr() +} + +func (w *WrapStreamCipher) RemoteAddr() *net.TCPAddr { + return w.conn.RemoteAddr() +} + +func (w *WrapStreamCipher) Close() error { + return w.conn.Close() +} + +func NewStreamCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.Stream) WrapStreamReadWriteCloser { + return &WrapStreamCipher{ + conn: conn, + encryptor: encryptor, + decryptor: decryptor, + } +} diff --git a/wrappers/streamcipherrwc.go b/wrappers/streamcipherrwc.go deleted file mode 100644 index f9ad598..0000000 --- a/wrappers/streamcipherrwc.go +++ /dev/null @@ -1,66 +0,0 @@ -package wrappers - -import ( - "crypto/cipher" - "net" -) - -// StreamCipherReadWriteCloser is a ReadWriteCloser which ciphers -// incoming and outgoing data with givem cipher.Stream instances. -type StreamCipherReadWriteCloserWithAddr struct { - encryptor cipher.Stream - decryptor cipher.Stream - conn ReadWriteCloserWithAddr -} - -// Read reads from connection -func (c *StreamCipherReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { - n, err = c.conn.Read(p) - c.decryptor.XORKeyStream(p, p[:n]) - return -} - -// Write writes into connection. -func (c *StreamCipherReadWriteCloserWithAddr) Write(p []byte) (int, error) { - // This is to decrease an amount of allocations. Unfortunately, escape - // analysis in (at least Golang 1.10) is absolutely not perfect. For - // example, it understands that we want to have a slice locally, right? - // But since slice is effectively 2 ints + uintptr to [number]byte, the - // most heavyweight part is placed in heap. - buf := getBuffer() - defer putBuffer(buf) - buf.Grow(len(p)) - buf.Write(p) - - encrypted := buf.Bytes() - c.encryptor.XORKeyStream(encrypted, p) - - return c.conn.Write(encrypted) -} - -// Close closes underlying connection. -func (c *StreamCipherReadWriteCloserWithAddr) Close() error { - return c.conn.Close() -} - -func (c *StreamCipherReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return c.conn.LocalAddr() -} - -func (c *StreamCipherReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return c.conn.RemoteAddr() -} - -func (c *StreamCipherReadWriteCloserWithAddr) SocketID() string { - return c.conn.SocketID() -} - -// NewStreamCipherRWC returns wrapper which transparently -// encrypts/decrypts traffic with obfuscated2 protocol. -func NewStreamCipherRWC(conn ReadWriteCloserWithAddr, encryptor, decryptor cipher.Stream) ReadWriteCloserWithAddr { - return &StreamCipherReadWriteCloserWithAddr{ - conn: conn, - encryptor: encryptor, - decryptor: decryptor, - } -} diff --git a/wrappers/timeoutrwc.go b/wrappers/timeoutrwc.go deleted file mode 100644 index 3c48546..0000000 --- a/wrappers/timeoutrwc.go +++ /dev/null @@ -1,61 +0,0 @@ -package wrappers - -import ( - "net" - "time" - - "github.com/9seconds/mtg/config" -) - -type TimeoutReadWriteCloserWithAddr struct { - conn net.Conn - sock string - publicIPv4 net.IP - publicIPv6 net.IP -} - -func (t *TimeoutReadWriteCloserWithAddr) Read(p []byte) (int, error) { - t.conn.SetReadDeadline(time.Now().Add(config.TimeoutRead)) - return t.conn.Read(p) -} - -func (t *TimeoutReadWriteCloserWithAddr) Write(p []byte) (int, error) { - t.conn.SetWriteDeadline(time.Now().Add(config.TimeoutWrite)) - return t.conn.Write(p) -} - -func (t *TimeoutReadWriteCloserWithAddr) Close() error { - return t.conn.Close() -} - -func (t *TimeoutReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return t.conn.RemoteAddr().(*net.TCPAddr) -} - -func (t *TimeoutReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - addr := t.conn.LocalAddr().(*net.TCPAddr) - newAddr := *addr - - if t.RemoteAddr().IP.To4() != nil { - if t.publicIPv4 != nil { - newAddr.IP = t.publicIPv4 - } - } else if t.publicIPv6 != nil { - newAddr.IP = t.publicIPv6 - } - - return &newAddr -} - -func (t *TimeoutReadWriteCloserWithAddr) SocketID() string { - return t.sock -} - -func NewTimeoutRWC(conn net.Conn, sock string, ipv4, ipv6 net.IP) ReadWriteCloserWithAddr { - return &TimeoutReadWriteCloserWithAddr{ - conn: conn, - publicIPv4: ipv4, - publicIPv6: ipv6, - sock: sock, - } -} diff --git a/wrappers/trafficrwc.go b/wrappers/trafficrwc.go deleted file mode 100644 index 344a451..0000000 --- a/wrappers/trafficrwc.go +++ /dev/null @@ -1,51 +0,0 @@ -package wrappers - -import "net" - -// TrafficReadWriteCloser counts an amount of ingress/egress traffic by -// calling given callbacks. -type TrafficReadWriteCloserWithAddr struct { - conn ReadWriteCloserWithAddr - readCallback func(int) - writeCallback func(int) -} - -// Read reads from connection -func (t *TrafficReadWriteCloserWithAddr) Read(p []byte) (n int, err error) { - n, err = t.conn.Read(p) - t.readCallback(n) - return -} - -// Write writes into connection. -func (t *TrafficReadWriteCloserWithAddr) Write(p []byte) (n int, err error) { - n, err = t.conn.Write(p) - t.writeCallback(n) - return -} - -// Close closes underlying connection. -func (t *TrafficReadWriteCloserWithAddr) Close() error { - return t.conn.Close() -} - -func (t *TrafficReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return t.conn.LocalAddr() -} - -func (t *TrafficReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return t.conn.RemoteAddr() -} - -func (t *TrafficReadWriteCloserWithAddr) SocketID() string { - return t.conn.SocketID() -} - -// NewTrafficRWC wraps ReadWriteCloser to have read/write callbacks. -func NewTrafficRWC(conn ReadWriteCloserWithAddr, readCallback, writeCallback func(int)) ReadWriteCloserWithAddr { - return &TrafficReadWriteCloserWithAddr{ - conn: conn, - readCallback: readCallback, - writeCallback: writeCallback, - } -} diff --git a/wrappers/wrap.go b/wrappers/wrap.go new file mode 100644 index 0000000..d4bc7d4 --- /dev/null +++ b/wrappers/wrap.go @@ -0,0 +1,78 @@ +package wrappers + +import ( + "io" + "net" +) + +type Wrap interface { + LogDebug(msg string, data ...interface{}) + LogInfo(msg string, data ...interface{}) + LogWarn(msg string, data ...interface{}) + LogError(msg string, data ...interface{}) + + LocalAddr() *net.TCPAddr + RemoteAddr() *net.TCPAddr +} + +type WrapWriter interface { + io.Writer + Wrap +} + +type WrapWriteCloser interface { + io.Closer + WrapWriter +} + +type WrapStreamReader interface { + io.Reader + Wrap +} + +type WrapStreamReadCloser interface { + io.Closer + WrapStreamReader +} + +type WrapStreamReadWriter interface { + io.Writer + WrapStreamReader +} + +type WrapStreamWriteCloser interface { + io.Closer + io.Writer + Wrap +} + +type WrapStreamReadWriteCloser interface { + io.Closer + WrapStreamReadWriter +} + +type WrapPacketReader interface { + Read() ([]byte, error) + Wrap +} + +type WrapPacketReadWriter interface { + io.Writer + WrapPacketReader +} + +type WrapBlockReadCloser interface { + io.Closer + WrapPacketReader +} + +type WrapPacketWriteCloser interface { + io.Writer + io.Closer + Wrap +} + +type WrapPacketReadWriteCloser interface { + io.Closer + WrapPacketReadWriter +} From 1d93bef9130d81c6ea808ab3eb80aa04ff00f0db Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 7 Jul 2018 17:18:42 +0300 Subject: [PATCH 43/71] Move frame wrapper --- mtproto/wrappers/frame.go | 134 ------------------------------- wrappers/mtproto_abridged.go | 18 ++--- wrappers/mtproto_frame.go | 151 +++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 143 deletions(-) delete mode 100644 mtproto/wrappers/frame.go create mode 100644 wrappers/mtproto_frame.go diff --git a/mtproto/wrappers/frame.go b/mtproto/wrappers/frame.go deleted file mode 100644 index 633c4ad..0000000 --- a/mtproto/wrappers/frame.go +++ /dev/null @@ -1,134 +0,0 @@ -package wrappers - -import ( - "bytes" - "crypto/aes" - "encoding/binary" - "hash/crc32" - "io" - "io/ioutil" - "net" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/wrappers" -) - -// Frame: { MessageLength(4) | SequenceNumber(4) | Message(???) | CRC32(4) [| padding(4), ...] } -const ( - frameRWCMinMessageLength = 12 - frameRWCMaxMessageLength = 16777216 -) - -var frameRWCPadding = []byte{0x04, 0x00, 0x00, 0x00} - -type FrameRWC struct { - wrappers.BufferedReader - - conn wrappers.ReadWriteCloserWithAddr - readSeqNo int32 - writeSeqNo int32 -} - -func (f *FrameRWC) Write(buf []byte) (int, error) { - writeBuf := &bytes.Buffer{} - - // 4 - len bytes - // 4 - seq bytes - // . - message - // 4 - crc32 - messageLength := 4 + 4 + len(buf) + 4 - paddingLength := (aes.BlockSize - messageLength%aes.BlockSize) % aes.BlockSize - writeBuf.Grow(messageLength + paddingLength) - - binary.Write(writeBuf, binary.LittleEndian, uint32(messageLength)) - binary.Write(writeBuf, binary.LittleEndian, f.writeSeqNo) - writeBuf.Write(buf) - f.writeSeqNo++ - - checksum := crc32.ChecksumIEEE(writeBuf.Bytes()) - binary.Write(writeBuf, binary.LittleEndian, checksum) - writeBuf.Write(bytes.Repeat(frameRWCPadding, paddingLength/4)) - - _, err := f.conn.Write(writeBuf.Bytes()) - return len(buf), err -} - -func (f *FrameRWC) Read(p []byte) (int, error) { - return f.BufferedRead(p, func() error { - buf := &bytes.Buffer{} - sum := crc32.NewIEEE() - writer := io.MultiWriter(buf, sum) - - for { - buf.Reset() - sum.Reset() - if _, err := io.CopyN(writer, f.conn, 4); err != nil { - return errors.Annotate(err, "Cannot read frame padding") - } - if !bytes.Equal(buf.Bytes(), frameRWCPadding) { - break - } - } - - messageLength := binary.LittleEndian.Uint32(buf.Bytes()) - if messageLength%4 != 0 || messageLength < frameRWCMinMessageLength || messageLength > frameRWCMaxMessageLength { - return errors.Errorf("Incorrect frame message length %d", messageLength) - } - - buf.Reset() - buf.Grow(int(messageLength) - 4 - 4) - if _, err := io.CopyN(writer, f.conn, int64(messageLength)-4-4); err != nil { - return errors.Annotate(err, "Cannot read the message frame") - } - - var seqNo int32 - binary.Read(buf, binary.LittleEndian, &seqNo) - if seqNo != f.readSeqNo { - return errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, f.readSeqNo) - } - f.readSeqNo++ - - data, _ := ioutil.ReadAll(buf) - buf.Reset() - // write to buf, not to writer. This is because we are going to fetch - // crc32 checksum. - if _, err := io.CopyN(buf, f.conn, 4); err != nil { - return errors.Annotate(err, "Cannot read checksum") - } - checksum := binary.LittleEndian.Uint32(buf.Bytes()) - - if checksum != sum.Sum32() { - return errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) - - } - f.Buffer.Write(data) - - return nil - }) -} - -func (f *FrameRWC) Close() error { - return f.conn.Close() -} - -func (f *FrameRWC) LocalAddr() *net.TCPAddr { - return f.conn.LocalAddr() -} - -func (f *FrameRWC) RemoteAddr() *net.TCPAddr { - return f.conn.RemoteAddr() -} - -func (f *FrameRWC) SocketID() string { - return f.conn.SocketID() -} - -func NewFrameRWC(conn wrappers.ReadWriteCloserWithAddr, seqNo int32) wrappers.ReadWriteCloserWithAddr { - return &FrameRWC{ - BufferedReader: wrappers.NewBufferedReader(), - conn: conn, - readSeqNo: seqNo, - writeSeqNo: seqNo, - } -} diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index b9950c3..1aa7f8c 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -12,9 +12,9 @@ import ( ) const ( - abridgedSmallPacketLength = 0x7f - abridgedQuickAckLength = 0x80 - abridgedLargePacketLength = 16777216 // 256 ^ 3 + mtprotoAbridgedSmallPacketLength = 0x7f + mtprotoAbridgedQuickAckLength = 0x80 + mtprotoAbridgedLargePacketLength = 16777216 // 256 ^ 3 ) type MTProtoAbridged struct { @@ -46,13 +46,13 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { "counter", m.readCounter, ) - if msgLength >= abridgedQuickAckLength { + if msgLength >= mtprotoAbridgedQuickAckLength { m.opts.ReadHacks.QuickAck = true - msgLength -= abridgedQuickAckLength + msgLength -= mtprotoAbridgedQuickAckLength } msgLength32 := uint32(msgLength) - if msgLength == abridgedSmallPacketLength { + if msgLength == mtprotoAbridgedSmallPacketLength { if _, err := io.CopyN(buf, m.conn, 3); err != nil { return nil, errors.Annotate(err, "Cannot read the correct message length") } @@ -96,19 +96,19 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { packetLength := len(p) / 4 switch { - case packetLength < abridgedSmallPacketLength: + case packetLength < mtprotoAbridgedSmallPacketLength: newData := append([]byte{byte(packetLength)}, p...) m.writeCounter++ return m.conn.Write(newData) - case packetLength < abridgedLargePacketLength: + case packetLength < mtprotoAbridgedLargePacketLength: length24 := utils.ToUint24(uint32(packetLength)) buf := &bytes.Buffer{} buf.Grow(1 + 3 + len(p)) - buf.WriteByte(byte(abridgedSmallPacketLength)) + buf.WriteByte(byte(mtprotoAbridgedSmallPacketLength)) buf.Write(length24[:]) buf.Write(p) diff --git a/wrappers/mtproto_frame.go b/wrappers/mtproto_frame.go new file mode 100644 index 0000000..d3eff6f --- /dev/null +++ b/wrappers/mtproto_frame.go @@ -0,0 +1,151 @@ +package wrappers + +import ( + "bytes" + "crypto/aes" + "encoding/binary" + "hash/crc32" + "io" + "io/ioutil" + "net" + + "github.com/juju/errors" +) + +const ( + mtprotoFrameMinMessageLength = 12 + mtprotoFrameMaxMessageLength = 16777216 +) + +var mtprotoFramePadding = []byte{0x04, 0x00, 0x00, 0x00} + +type MTProtoFrame struct { + conn WrapStreamReadWriteCloser + readSeqNo int32 + writeSeqNo int32 +} + +func (m *MTProtoFrame) Read() ([]byte, error) { + buf := &bytes.Buffer{} + sum := crc32.NewIEEE() + writer := io.MultiWriter(buf, sum) + + for { + buf.Reset() + sum.Reset() + if _, err := io.CopyN(writer, m.conn, 4); err != nil { + return nil, errors.Annotate(err, "Cannot read frame padding") + } + if !bytes.Equal(buf.Bytes(), mtprotoFramePadding) { + break + } + } + + messageLength := binary.LittleEndian.Uint32(buf.Bytes()) + m.LogDebug("Read MTProto frame", + "messageLength", messageLength, + "sequence_number", m.readSeqNo, + ) + if messageLength%4 != 0 || messageLength < mtprotoFrameMinMessageLength || messageLength > mtprotoFrameMaxMessageLength { + return nil, errors.Errorf("Incorrect frame message length %d", messageLength) + } + + buf.Reset() + buf.Grow(int(messageLength) - 4 - 4) + if _, err := io.CopyN(writer, m.conn, int64(messageLength)-4-4); err != nil { + return nil, errors.Annotate(err, "Cannot read the message frame") + } + + var seqNo int32 + binary.Read(buf, binary.LittleEndian, &seqNo) + if seqNo != m.readSeqNo { + return nil, errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, m.readSeqNo) + } + + data, _ := ioutil.ReadAll(buf) + buf.Reset() + // write to buf, not to writer. This is because we are going to fetch + // crc32 checksum. + if _, err := io.CopyN(buf, m.conn, 4); err != nil { + return nil, errors.Annotate(err, "Cannot read checksum") + } + + checksum := binary.LittleEndian.Uint32(buf.Bytes()) + if checksum != sum.Sum32() { + return nil, errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) + } + + m.LogDebug("Read MTProto frame", + "messageLength", messageLength, + "sequence_number", m.readSeqNo, + "dataLength", len(data), + "checksum", checksum, + ) + m.readSeqNo++ + + return data, nil +} + +func (m *MTProtoFrame) Write(p []byte) (int, error) { + messageLength := 4 + 4 + len(p) + 4 + paddingLength := (aes.BlockSize - messageLength%aes.BlockSize) % aes.BlockSize + + buf := &bytes.Buffer{} + buf.Grow(messageLength + paddingLength) + + binary.Write(buf, binary.LittleEndian, uint32(messageLength)) + binary.Write(buf, binary.LittleEndian, m.writeSeqNo) + buf.Write(p) + + checksum := crc32.ChecksumIEEE(buf.Bytes()) + binary.Write(buf, binary.LittleEndian, checksum) + buf.Write(bytes.Repeat(mtprotoFramePadding, paddingLength/4)) + + m.LogDebug("Write MTProto frame", + "length", len(p), + "sequence_number", m.writeSeqNo, + "crc32", checksum, + "frame_length", buf.Len(), + ) + m.writeSeqNo++ + + _, err := m.conn.Write(buf.Bytes()) + + return len(p), err +} + +func (m *MTProtoFrame) LogDebug(msg string, data ...interface{}) { + m.conn.LogDebug(msg, data...) +} + +func (m *MTProtoFrame) LogInfo(msg string, data ...interface{}) { + m.conn.LogInfo(msg, data...) +} + +func (m *MTProtoFrame) LogWarn(msg string, data ...interface{}) { + m.conn.LogWarn(msg, data...) +} + +func (m *MTProtoFrame) LogError(msg string, data ...interface{}) { + m.conn.LogError(msg, data...) +} + +func (m *MTProtoFrame) LocalAddr() *net.TCPAddr { + return m.conn.LocalAddr() +} + +func (m *MTProtoFrame) RemoteAddr() *net.TCPAddr { + return m.conn.RemoteAddr() +} + +func (m *MTProtoFrame) Close() error { + return m.conn.Close() +} + +func NewMTProtoFrame(conn WrapStreamReadWriteCloser, seqNo int32) WrapPacketReadWriteCloser { + return &MTProtoFrame{ + conn: conn, + readSeqNo: seqNo, + writeSeqNo: seqNo, + } +} From a7d46727baf50d3aa9bca708ad05473fc6f7eedb Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 7 Jul 2018 17:40:21 +0300 Subject: [PATCH 44/71] Move intermediate --- mtproto/wrappers/intermediate.go | 90 ----------------------- wrappers/mtproto_abridged.go | 21 ++++-- wrappers/mtproto_intermediate.go | 119 +++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 96 deletions(-) delete mode 100644 mtproto/wrappers/intermediate.go create mode 100644 wrappers/mtproto_intermediate.go diff --git a/mtproto/wrappers/intermediate.go b/mtproto/wrappers/intermediate.go deleted file mode 100644 index 0468d47..0000000 --- a/mtproto/wrappers/intermediate.go +++ /dev/null @@ -1,90 +0,0 @@ -package wrappers - -import ( - "bytes" - "encoding/binary" - "io" - "net" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/wrappers" -) - -const intermediateQuickAckLength = 0x80000000 - -type IntermediateReadWriteCloserWithAddr struct { - wrappers.BufferedReader - - conn wrappers.ReadWriteCloserWithAddr - opts *mtproto.ConnectionOpts -} - -func (i *IntermediateReadWriteCloserWithAddr) Read(p []byte) (int, error) { - return i.BufferedRead(p, func() error { - buf := &bytes.Buffer{} - buf.Grow(4) - - if _, err := io.CopyN(buf, i.conn, 4); err != nil { - return errors.Annotate(err, "Cannot read message length") - } - length := binary.LittleEndian.Uint32(buf.Bytes()) - buf.Reset() - buf.Grow(int(length)) - - if length > intermediateQuickAckLength { - i.opts.ReadHacks.QuickAck = true - length -= intermediateQuickAckLength - } - - if _, err := io.CopyN(buf, i.conn, int64(length)); err != nil { - return errors.Annotate(err, "Cannot read the message") - } - - if length%4 != 0 { - length -= length % 4 - i.Buffer.Write(buf.Bytes()[:length]) - return nil - } - - i.Buffer.Write(buf.Bytes()) - - return nil - }) -} - -func (i *IntermediateReadWriteCloserWithAddr) Write(p []byte) (int, error) { - if i.opts.WriteHacks.SimpleAck { - return i.conn.Write(p) - } - - var length [4]byte - binary.LittleEndian.PutUint32(length[:], uint32(len(p))) - - return i.conn.Write(append(length[:], p...)) -} - -func (i *IntermediateReadWriteCloserWithAddr) Close() error { - return i.conn.Close() -} - -func (i *IntermediateReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return i.conn.LocalAddr() -} - -func (i *IntermediateReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return i.conn.RemoteAddr() -} - -func (i *IntermediateReadWriteCloserWithAddr) SocketID() string { - return i.conn.SocketID() -} - -func NewIntermediateRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts) wrappers.ReadWriteCloserWithAddr { - return &IntermediateReadWriteCloserWithAddr{ - BufferedReader: wrappers.NewBufferedReader(), - conn: conn, - opts: connOpts, - } -} diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index 1aa7f8c..f1c1a5c 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -26,9 +26,9 @@ type MTProtoAbridged struct { } func (m *MTProtoAbridged) Read() ([]byte, error) { - m.LogDebug("Read abridged packet", - "simple_ack", m.opts.WriteHacks.SimpleAck, - "quick_ack", m.opts.WriteHacks.QuickAck, + m.LogDebug("Read packet", + "simple_ack", m.opts.ReadHacks.SimpleAck, + "quick_ack", m.opts.ReadHacks.QuickAck, "counter", m.readCounter, ) @@ -41,9 +41,11 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { msgLength := uint8(buf.Bytes()[0]) buf.Reset() - m.LogDebug("Abridged packet first byte", + m.LogDebug("Packet first byte", "byte", msgLength, "counter", m.readCounter, + "simple_ack", m.opts.ReadHacks.SimpleAck, + "quick_ack", m.opts.ReadHacks.QuickAck, ) if msgLength >= mtprotoAbridgedQuickAckLength { @@ -62,8 +64,10 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { } msgLength32 *= 4 - m.LogDebug("Abridged packet length", + m.LogDebug("Packet length", "length", msgLength32, + "simple_ack", m.opts.ReadHacks.SimpleAck, + "quick_ack", m.opts.ReadHacks.QuickAck, "counter", m.readCounter, ) @@ -79,7 +83,7 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { } func (m *MTProtoAbridged) Write(p []byte) (int, error) { - m.LogDebug("Write abridged packet", + m.LogDebug("Write packet", "length", len(p), "simple_ack", m.opts.WriteHacks.SimpleAck, "quick_ack", m.opts.WriteHacks.QuickAck, @@ -91,6 +95,7 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { } if m.opts.WriteHacks.SimpleAck { + m.writeCounter++ return m.conn.Write(utils.ReverseBytes(p)) } @@ -120,18 +125,22 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { } func (m *MTProtoAbridged) LogDebug(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "abridged"}) m.conn.LogDebug(msg, data...) } func (m *MTProtoAbridged) LogInfo(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "abridged"}) m.conn.LogInfo(msg, data...) } func (m *MTProtoAbridged) LogWarn(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "abridged"}) m.conn.LogWarn(msg, data...) } func (m *MTProtoAbridged) LogError(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "abridged"}) m.conn.LogError(msg, data...) } diff --git a/wrappers/mtproto_intermediate.go b/wrappers/mtproto_intermediate.go new file mode 100644 index 0000000..2605a71 --- /dev/null +++ b/wrappers/mtproto_intermediate.go @@ -0,0 +1,119 @@ +package wrappers + +import ( + "bytes" + "encoding/binary" + "io" + "net" + + "github.com/9seconds/mtg/mtproto" + "github.com/juju/errors" +) + +const mtprotoIntermediateQuickAckLength = 0x80000000 + +type MTProtoIntermediate struct { + conn WrapStreamReadWriteCloser + opts *mtproto.ConnectionOpts + + readCounter uint32 + writeCounter uint32 +} + +func (m *MTProtoIntermediate) Read() ([]byte, error) { + m.LogDebug("Read packet", + "simple_ack", m.opts.ReadHacks.SimpleAck, + "quick_ack", m.opts.ReadHacks.QuickAck, + "counter", m.readCounter, + ) + + buf := &bytes.Buffer{} + buf.Grow(4) + + if _, err := io.CopyN(buf, m.conn, 4); err != nil { + return nil, errors.Annotate(err, "Cannot read message length") + } + length := binary.LittleEndian.Uint32(buf.Bytes()) + + m.LogDebug("Packet message length", + "simple_ack", m.opts.ReadHacks.SimpleAck, + "quick_ack", m.opts.ReadHacks.QuickAck, + "counter", m.readCounter, + "length", length, + ) + + if length > mtprotoIntermediateQuickAckLength { + m.opts.ReadHacks.QuickAck = true + length -= mtprotoIntermediateQuickAckLength + } + + buf.Reset() + buf.Grow(int(length)) + if _, err := io.CopyN(buf, m.conn, int64(length)); err != nil { + return nil, errors.Annotate(err, "Cannot read the message") + } + + if length%4 != 0 { + length -= length % 4 + } + m.readCounter++ + + return buf.Bytes()[:length], nil +} + +func (m *MTProtoIntermediate) Write(p []byte) (int, error) { + m.LogDebug("Write packet", + "simple_ack", m.opts.WriteHacks.SimpleAck, + "quick_ack", m.opts.WriteHacks.QuickAck, + "counter", m.writeCounter, + ) + m.writeCounter++ + + if m.opts.ReadHacks.SimpleAck { + return m.conn.Write(p) + } + + var length [4]byte + binary.LittleEndian.PutUint32(length[:], uint32(len(p))) + + return m.conn.Write(append(length[:], p...)) +} + +func (m *MTProtoIntermediate) LogDebug(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "intermediate"}) + m.conn.LogDebug(msg, data...) +} + +func (m *MTProtoIntermediate) LogInfo(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "intermediate"}) + m.conn.LogInfo(msg, data...) +} + +func (m *MTProtoIntermediate) LogWarn(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "intermediate"}) + m.conn.LogWarn(msg, data...) +} + +func (m *MTProtoIntermediate) LogError(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "intermediate"}) + m.conn.LogError(msg, data...) +} + +func (m *MTProtoIntermediate) LocalAddr() *net.TCPAddr { + return m.conn.LocalAddr() +} + +func (m *MTProtoIntermediate) RemoteAddr() *net.TCPAddr { + return m.conn.RemoteAddr() +} + +func (m *MTProtoIntermediate) Close() error { + return m.conn.Close() +} + +func NewMTProtoIntermediate(conn WrapStreamReadWriteCloser, opts *mtproto.ConnectionOpts) WrapPacketReadWriteCloser { + return &MTProtoIntermediate{ + conn: conn, + opts: opts, + } +} From 4262e5f5de711a91276491770361154a06cafff0 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 7 Jul 2018 18:05:24 +0300 Subject: [PATCH 45/71] Move all wrappers --- mtproto/wrappers/crypt_test.go | 48 ------- mtproto/wrappers/proxy_request.go | 111 -------------- .../crypt.go => wrappers/mtproto_cipher.go | 29 ++-- wrappers/mtproto_proxy.go | 135 ++++++++++++++++++ 4 files changed, 143 insertions(+), 180 deletions(-) delete mode 100644 mtproto/wrappers/crypt_test.go delete mode 100644 mtproto/wrappers/proxy_request.go rename mtproto/wrappers/crypt.go => wrappers/mtproto_cipher.go (64%) create mode 100644 wrappers/mtproto_proxy.go diff --git a/mtproto/wrappers/crypt_test.go b/mtproto/wrappers/crypt_test.go deleted file mode 100644 index c9914f2..0000000 --- a/mtproto/wrappers/crypt_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package wrappers - -import ( - "encoding/binary" - "net" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/9seconds/mtg/mtproto/rpc" -) - -var proxySecret = []byte{196, 249, 250, 202, 150, 120, 230, 187, 72, 173, - 108, 126, 44, 229, 192, 210, 68, 48, 100, 93, 85, 74, 221, 235, 85, 65, - 158, 3, 77, 166, 39, 33, 208, 70, 234, 171, 110, 82, 171, 20, 169, 90, 68, - 62, 207, 179, 70, 62, 121, 160, 90, 102, 97, 42, 223, 156, 174, 218, 139, - 233, 168, 13, 166, 152, 111, 176, 166, 255, 56, 122, 248, 77, 136, 239, - 58, 100, 19, 113, 62, 92, 51, 119, 246, 225, 163, 212, 125, 153, 245, 224, - 197, 110, 236, 232, 240, 92, 84, 196, 144, 176, 121, 227, 27, 239, 130, - 255, 14, 232, 242, 176, 163, 39, 86, 210, 73, 197, 242, 18, 105, 129, 108, - 183, 6, 27, 38, 93, 178, 18} - -func TestMakeKeys(t *testing.T) { - req, err := rpc.NewNonceRequest(proxySecret) - assert.Nil(t, err) - - copy(req.Nonce[:], []byte{24, 49, 53, 111, 198, 10, 235, 180, 230, 112, 92, 78, 1, 201, 106, 105}) - binary.LittleEndian.PutUint32(req.CryptoTS[:], 1528396015) - - resp := &rpc.NonceResponse{ - NonceRequest: rpc.NonceRequest{ - Nonce: []byte{247, 40, 210, 56, 65, 12, 101, 170, 216, 155, 14, 253, 250, 238, 219, 226}, - }, - } - - cltAddr := &net.TCPAddr{ - IP: net.ParseIP("80.211.29.34"), - Port: 54208, - } - srvAddr := &net.TCPAddr{ - IP: net.ParseIP("149.154.162.38"), - Port: 80, - } - - key, iv := makeKeys(CipherPurposeClient, req, resp, cltAddr, srvAddr, proxySecret) - assert.Equal(t, key, []byte{165, 158, 127, 49, 41, 232, 187, 69, 38, 29, 163, 226, 183, 146, 28, 67, 225, 224, 134, 191, 207, 152, 255, 166, 152, 66, 169, 196, 54, 135, 50, 188}) - assert.Equal(t, iv, []byte{33, 110, 125, 221, 183, 121, 160, 116, 130, 180, 156, 249, 52, 111, 37, 178}) -} diff --git a/mtproto/wrappers/proxy_request.go b/mtproto/wrappers/proxy_request.go deleted file mode 100644 index 088565d..0000000 --- a/mtproto/wrappers/proxy_request.go +++ /dev/null @@ -1,111 +0,0 @@ -package wrappers - -import ( - "bytes" - "io" - "io/ioutil" - "net" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers" -) - -type ProxyRequestReadWriteCloserWithAddr struct { - wrappers.BufferedReader - - conn wrappers.ReadWriteCloserWithAddr - req *rpc.ProxyRequest -} - -func (p *ProxyRequestReadWriteCloserWithAddr) Read(buf []byte) (int, error) { - return p.BufferedRead(buf, func() error { - ans := make([]byte, 4) - if _, err := io.ReadFull(p.conn, ans); err != nil { - return errors.Annotate(err, "Cannot read RPC tag") - } - - switch { - case bytes.Equal(ans, rpc.TagProxyAns): - return p.readProxyAns() - case bytes.Equal(ans, rpc.TagSimpleAck): - return p.readSimpleAck() - case bytes.Equal(ans, rpc.TagCloseExt): - return p.readCloseExt() - } - - return errors.Errorf("Unknown RPC answer %v", ans) - }) -} - -func (p *ProxyRequestReadWriteCloserWithAddr) readCloseExt() error { - return errors.New("Connection has been closed remotely") -} - -func (p *ProxyRequestReadWriteCloserWithAddr) readProxyAns() (err error) { - if _, err = io.CopyN(ioutil.Discard, p.conn, 8+4); err != nil { - return errors.Annotate(err, "Cannot skip flags and connid") - } - - buf, err := utils.ReadCurrentData(p.conn) - if err != nil { - return errors.Annotate(err, "Cannot read proxy answer") - } - p.Buffer.Write(buf) - - return nil -} - -func (p *ProxyRequestReadWriteCloserWithAddr) readSimpleAck() error { - if _, err := io.CopyN(ioutil.Discard, p.conn, 8); err != nil { - return errors.Annotate(err, "Cannot skip connid") - } - - ackData := make([]byte, 4) - if _, err := io.ReadFull(p.conn, ackData); err != nil { - return errors.Annotate(err, "Cannot read simple ack") - } - p.Buffer.Write(ackData) - - return nil -} - -func (p *ProxyRequestReadWriteCloserWithAddr) Write(raw []byte) (int, error) { - if _, err := p.conn.Write(p.req.Bytes(raw)); err != nil { - return 0, err - } - - return len(raw), nil -} - -func (p *ProxyRequestReadWriteCloserWithAddr) Close() error { - return p.conn.Close() -} - -func (p *ProxyRequestReadWriteCloserWithAddr) LocalAddr() *net.TCPAddr { - return p.conn.LocalAddr() -} - -func (p *ProxyRequestReadWriteCloserWithAddr) RemoteAddr() *net.TCPAddr { - return p.conn.RemoteAddr() -} - -func (p *ProxyRequestReadWriteCloserWithAddr) SocketID() string { - return p.conn.SocketID() -} - -func NewProxyRequestRWC(conn wrappers.ReadWriteCloserWithAddr, connOpts *mtproto.ConnectionOpts, adTag []byte) (wrappers.ReadWriteCloserWithAddr, error) { - req, err := rpc.NewProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) - if err != nil { - return nil, errors.Annotate(err, "Cannot create new RPC proxy request") - } - - return &ProxyRequestReadWriteCloserWithAddr{ - BufferedReader: wrappers.NewBufferedReader(), - conn: conn, - req: req, - }, nil -} diff --git a/mtproto/wrappers/crypt.go b/wrappers/mtproto_cipher.go similarity index 64% rename from mtproto/wrappers/crypt.go rename to wrappers/mtproto_cipher.go index f47b466..146016e 100644 --- a/mtproto/wrappers/crypt.go +++ b/wrappers/mtproto_cipher.go @@ -10,7 +10,7 @@ import ( "net" "github.com/9seconds/mtg/mtproto/rpc" - "github.com/9seconds/mtg/wrappers" + "github.com/9seconds/mtg/utils" ) type CipherPurpose uint8 @@ -22,20 +22,20 @@ const ( var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} -func NewMiddleProxyCipherRWC(conn wrappers.ReadWriteCloserWithAddr, req *rpc.NonceRequest, resp *rpc.NonceResponse, secret []byte) wrappers.ReadWriteCloserWithAddr { +func NewMiddleProxyCipher(conn WrapStreamReadWriteCloser, req *rpc.NonceRequest, resp *rpc.NonceResponse, secret []byte) WrapStreamReadWriteCloser { localAddr := conn.LocalAddr() remoteAddr := conn.RemoteAddr() - encKey, encIV := makeKeys(CipherPurposeClient, req, resp, localAddr, remoteAddr, secret) - decKey, decIV := makeKeys(CipherPurposeServer, req, resp, localAddr, remoteAddr, secret) + encKey, encIV := deriveKeys(CipherPurposeClient, req, resp, localAddr, remoteAddr, secret) + decKey, decIV := deriveKeys(CipherPurposeServer, req, resp, localAddr, remoteAddr, secret) enc, _ := makeEncrypterDecrypter(encKey, encIV) _, dec := makeEncrypterDecrypter(decKey, decIV) - return wrappers.NewBlockCipherRWC(conn, enc, dec) + return NewWrapBlockCipher(conn, enc, dec) } -func makeKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { +func deriveKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { message := bytes.Buffer{} message.Write(resp.Nonce[:]) message.Write(req.Nonce[:]) @@ -44,8 +44,8 @@ func makeKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceRespo clientIPv4 := emptyIP[:] serverIPv4 := emptyIP[:] if client.IP.To4() != nil { - clientIPv4 = reverseBytes(client.IP.To4()) - serverIPv4 = reverseBytes(remote.IP.To4()) + clientIPv4 = utils.ReverseBytes(client.IP.To4()) + serverIPv4 = utils.ReverseBytes(remote.IP.To4()) } message.Write(serverIPv4) @@ -92,16 +92,3 @@ func makeEncrypterDecrypter(key, iv []byte) (cipher.BlockMode, cipher.BlockMode) return cipher.NewCBCEncrypter(block, iv), cipher.NewCBCDecrypter(block, iv) } - -func reverseBytes(data []byte) []byte { - dataLen := len(data) - rv := make([]byte, dataLen) - - rv[dataLen/2] = data[dataLen/2] - for i := dataLen/2 - 1; i >= 0; i-- { - opp := dataLen - i - 1 - rv[i], rv[opp] = data[opp], data[i] - } - - return rv -} diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go new file mode 100644 index 0000000..0a6d1a8 --- /dev/null +++ b/wrappers/mtproto_proxy.go @@ -0,0 +1,135 @@ +package wrappers + +import ( + "bytes" + "net" + + "github.com/9seconds/mtg/mtproto/rpc" + "github.com/juju/errors" +) + +type MTProtoProxy struct { + conn WrapPacketReadWriteCloser + req *rpc.ProxyRequest + + readCounter uint32 + writeCounter uint32 +} + +func (m *MTProtoProxy) Read() ([]byte, error) { + m.LogDebug("Read packet", + "counter", m.readCounter, + "simple_ack", m.req.Options.WriteHacks.SimpleAck, + "quick_ack", m.req.Options.WriteHacks.QuickAck, + ) + + packet, err := m.conn.Read() + if err != nil { + return nil, errors.Annotate(err, "Cannot read packet") + } + m.LogDebug("Read packet length", + "counter", m.readCounter, + "simple_ack", m.req.Options.WriteHacks.SimpleAck, + "quick_ack", m.req.Options.WriteHacks.QuickAck, + "length", len(packet), + ) + + if len(packet) < 4 { + return nil, errors.Annotate(err, "Incorrect packet length") + } + + tag, packet := packet[:4], packet[4:] + m.LogDebug("Read RPC tag", + "counter", m.readCounter, + "simple_ack", m.req.Options.WriteHacks.SimpleAck, + "quick_ack", m.req.Options.WriteHacks.QuickAck, + "tag", tag, + ) + + m.readCounter++ + switch { + case bytes.Equal(tag, rpc.TagProxyAns): + return m.readProxyAns(packet) + case bytes.Equal(tag, rpc.TagSimpleAck): + return m.readSimpleAck(packet) + case bytes.Equal(tag, rpc.TagCloseExt): + return m.readCloseExt(packet) + } + + return nil, errors.Errorf("Unknown RPC answer %v", tag) +} + +func (m *MTProtoProxy) readProxyAns(data []byte) ([]byte, error) { + if len(data) < 12 { + return nil, errors.Errorf("Incorrect data of proxy answer: %d", len(data)) + } + + return data[12:], nil +} + +func (m *MTProtoProxy) readSimpleAck(data []byte) ([]byte, error) { + if len(data) != 12 { + return nil, errors.Errorf("Incorrect data of simple ack: %d", len(data)) + } + + return data[8:12], nil // 0:8 - connection id +} + +func (m *MTProtoProxy) readCloseExt(data []byte) ([]byte, error) { + return nil, errors.New("Connection has been closed remotely by RPC call") +} + +func (m *MTProtoProxy) Write(p []byte) (int, error) { + m.LogDebug("Write packet", + "length", len(p), + "counter", m.writeCounter, + "simple_ack", m.req.Options.ReadHacks.SimpleAck, + "quick_ack", m.req.Options.ReadHacks.QuickAck, + ) + m.writeCounter++ + + if _, err := m.conn.Write(p); err != nil { + return 0, err + } + + return len(p), nil +} + +func (m *MTProtoProxy) LogDebug(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "proxy"}) + m.conn.LogDebug(msg, data...) +} + +func (m *MTProtoProxy) LogInfo(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "proxy"}) + m.conn.LogInfo(msg, data...) +} + +func (m *MTProtoProxy) LogWarn(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "proxy"}) + m.conn.LogWarn(msg, data...) +} + +func (m *MTProtoProxy) LogError(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "proxy"}) + m.conn.LogError(msg, data...) +} + +func (m *MTProtoProxy) LocalAddr() *net.TCPAddr { + return m.conn.LocalAddr() +} + +func (m *MTProtoProxy) RemoteAddr() *net.TCPAddr { + return m.conn.RemoteAddr() +} + +func (m *MTProtoProxy) Close() error { + return m.conn.Close() +} + +func NewMTProtoProxy(conn WrapPacketReadWriteCloser, req *rpc.ProxyRequest) WrapPacketReadWriteCloser { + return &MTProtoProxy{ + conn: conn, + req: req, + } +} From f82ff1f6fe6e24b60b04f4a3dd5fd111ead7933e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 7 Jul 2018 20:03:25 +0300 Subject: [PATCH 46/71] Refactored all the things! --- client/client.go | 12 --- client/direct.go | 22 +++-- client/middle.go | 18 ++-- config/config.go | 5 - main.go | 17 ++-- obfuscated2/frame.go | 10 +- proxy/direct.go | 66 +++++++++++++ proxy/middle.go | 77 +++++++++++++++ proxy/proxy.go | 63 ++++++++++++ proxy/server.go | 184 ------------------------------------ proxy/stats.go | 74 --------------- telegram/dialer.go | 7 +- telegram/direct.go | 17 ++-- telegram/middle.go | 44 ++++----- telegram/middle_caller.go | 9 +- telegram/telegram.go | 12 +-- wrappers/blockcipher.go | 89 +++++++++-------- wrappers/buffered_reader.go | 32 ------- wrappers/conn.go | 68 ++++++------- wrappers/ctx.go | 52 +++++----- wrappers/mtproto_cipher.go | 2 +- wrappers/mtproto_proxy.go | 13 ++- wrappers/streamcipher.go | 44 ++++----- wrappers/wrap.go | 5 + 24 files changed, 427 insertions(+), 515 deletions(-) delete mode 100644 client/client.go create mode 100644 proxy/direct.go create mode 100644 proxy/middle.go create mode 100644 proxy/proxy.go delete mode 100644 proxy/server.go delete mode 100644 proxy/stats.go delete mode 100644 wrappers/buffered_reader.go diff --git a/client/client.go b/client/client.go deleted file mode 100644 index 318c476..0000000 --- a/client/client.go +++ /dev/null @@ -1,12 +0,0 @@ -package client - -import ( - "net" - - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/wrappers" -) - -// Init has to initialize client connection based on given config. -type Init func(net.Conn, string, *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) diff --git a/client/direct.go b/client/direct.go index 7ee7e75..5abe56c 100644 --- a/client/direct.go +++ b/client/direct.go @@ -1,6 +1,7 @@ package client import ( + "context" "net" "time" @@ -14,28 +15,29 @@ import ( const handshakeTimeout = 10 * time.Second -// DirectInit initializes client to access Telegram bypassing middleproxies. -func DirectInit(conn net.Conn, socketID string, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { - if err := config.SetSocketOptions(conn); err != nil { +func DirectInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connID string, + conf *config.Config) (wrappers.WrapStreamReadWriteCloser, *mtproto.ConnectionOpts, error) { + if err := config.SetSocketOptions(socket); err != nil { return nil, nil, errors.Annotate(err, "Cannot set socket options") } - conn.SetReadDeadline(time.Now().Add(handshakeTimeout)) // nolint: errcheck - frame, err := obfuscated2.ExtractFrame(conn) - conn.SetReadDeadline(time.Time{}) // nolint: errcheck + socket.SetReadDeadline(time.Now().Add(handshakeTimeout)) + frame, err := obfuscated2.ExtractFrame(socket) if err != nil { return nil, nil, errors.Annotate(err, "Cannot extract frame") } + socket.SetReadDeadline(time.Time{}) + conn := wrappers.NewConn(socket, connID, wrappers.ConnPurposeClient, conf.PublicIPv4, conf.PublicIPv6) obfs2, connOpts, err := obfuscated2.ParseObfuscated2ClientFrame(conf.Secret, frame) if err != nil { return nil, nil, errors.Annotate(err, "Cannot parse obfuscated frame") } connOpts.ConnectionProto = mtproto.ConnectionProtocolAny - connOpts.ClientAddr = conn.RemoteAddr().(*net.TCPAddr) + connOpts.ClientAddr = conn.RemoteAddr() - socket := wrappers.NewTimeoutRWC(conn, socketID, conf.PublicIPv4, conf.PublicIPv6) - socket = wrappers.NewStreamCipherRWC(socket, obfs2.Encryptor, obfs2.Decryptor) + conn = wrappers.NewCtx(ctx, cancel, conn) + conn = wrappers.NewStreamCipher(conn, obfs2.Encryptor, obfs2.Decryptor) - return socket, connOpts, nil + return conn, connOpts, nil } diff --git a/client/middle.go b/client/middle.go index 2c978ec..635277d 100644 --- a/client/middle.go +++ b/client/middle.go @@ -1,30 +1,30 @@ package client import ( + "context" "net" "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" - mtwrappers "github.com/9seconds/mtg/mtproto/wrappers" "github.com/9seconds/mtg/wrappers" ) -func MiddleInit(conn net.Conn, socketID string, conf *config.Config) (wrappers.ReadWriteCloserWithAddr, *mtproto.ConnectionOpts, error) { - newConn, opts, err := DirectInit(conn, socketID, conf) +func MiddleInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connID string, + conf *config.Config) (wrappers.WrapPacketReadWriteCloser, *mtproto.ConnectionOpts, error) { + conn, opts, err := DirectInit(ctx, cancel, socket, connID, conf) if err != nil { return nil, nil, err } - if opts.ConnectionType == mtproto.ConnectionTypeAbridged { - newConn = mtwrappers.NewAbridgedRWC(newConn, opts) - } else { - newConn = mtwrappers.NewIntermediateRWC(newConn, opts) + newConn := wrappers.NewMTProtoAbridged(conn, opts) + if opts.ConnectionType != mtproto.ConnectionTypeAbridged { + newConn = wrappers.NewMTProtoIntermediate(conn, opts) } opts.ConnectionProto = mtproto.ConnectionProtocolIPv4 - if conn.LocalAddr().(*net.TCPAddr).IP.To4() == nil { + if socket.LocalAddr().(*net.TCPAddr).IP.To4() == nil { opts.ConnectionProto = mtproto.ConnectionProtocolIPv6 } - return newConn, opts, nil + return newConn, opts, err } diff --git a/config/config.go b/config/config.go index b5cfe9e..1a8f95e 100644 --- a/config/config.go +++ b/config/config.go @@ -5,7 +5,6 @@ import ( "fmt" "net" "strconv" - "time" "github.com/juju/errors" ) @@ -14,10 +13,6 @@ import ( const ( BufferWriteSize = 32 * 1024 BufferReadSize = 32 * 1024 - BufferSizeCopy = 32 * 1024 - - TimeoutRead = time.Minute - TimeoutWrite = time.Minute ) // Config represents common configuration of mtg. diff --git a/main.go b/main.go index 14cf85a..5cc71be 100644 --- a/main.go +++ b/main.go @@ -111,16 +111,21 @@ func main() { zapcore.NewJSONEncoder(encoderCfg), zapcore.Lock(os.Stderr), atom, - )).Sugar() + )) + zap.ReplaceGlobals(logger) + defer logger.Sync() - stat := proxy.NewStats(conf) - go stat.Serve() + var server *proxy.Proxy + if len(conf.AdTag) == 0 { + server = proxy.NewProxyDirect(conf) + } else { + server = proxy.NewProxyMiddle(conf) + } - srv := proxy.NewServer(conf, logger, stat) printURLs(conf.GetURLs()) - if err := srv.Serve(); err != nil { - logger.Fatal(err.Error()) + if err := server.Serve(); err != nil { + zap.S().Fatalw("Server stopped", "error", err) } } diff --git a/obfuscated2/frame.go b/obfuscated2/frame.go index 6c6f6ab..edcb8b4 100644 --- a/obfuscated2/frame.go +++ b/obfuscated2/frame.go @@ -9,6 +9,7 @@ import ( "github.com/juju/errors" "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/utils" ) // [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd] @@ -67,14 +68,7 @@ func (f Frame) ConnectionType() (mtproto.ConnectionType, error) { // Invert inverts frame for extracting encryption keys. Pkease check that link: // https://blog.susanka.eu/how-telegram-obfuscates-its-mtproto-traffic/ func (f Frame) Invert() Frame { - reversed := make(Frame, FrameLen) - copy(reversed, f) - - for i := 0; i < frameLenKey+frameLenIV; i++ { - reversed[frameOffsetFirst+i] = f[frameOffsetIV-1-i] - } - - return reversed + return Frame(utils.ReverseBytes([]byte(f))) } // ExtractFrame extracts exact obfuscated2 handshake frame from given reader. diff --git a/proxy/direct.go b/proxy/direct.go new file mode 100644 index 0000000..56b6ce2 --- /dev/null +++ b/proxy/direct.go @@ -0,0 +1,66 @@ +package proxy + +import ( + "context" + "io" + "net" + "sync" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/client" + "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/telegram" + "github.com/9seconds/mtg/wrappers" +) + +func NewProxyDirect(conf *config.Config) *Proxy { + tg := telegram.NewDirectTelegram(conf) + + return &Proxy{ + conf: conf, + acceptCallback: func(ctx context.Context, cancel context.CancelFunc, clientSocket net.Conn, + connID string, wait *sync.WaitGroup, conf *config.Config) error { + client, opts, err := client.DirectInit(ctx, cancel, clientSocket, connID, conf) + if err != nil { + return errors.Annotate(err, "Cannot initialize client connection") + } + defer client.Close() + + server, err := directTelegramStream(ctx, cancel, opts, connID, tg) + if err != nil { + return errors.Annotate(err, "Cannot initialize telegram connection") + } + defer server.Close() + + wait.Add(2) + + go directPipe(client, server, wait) + go directPipe(server, client, wait) + + return nil + }, + } +} + +func directTelegramStream(ctx context.Context, cancel context.CancelFunc, opts *mtproto.ConnectionOpts, + connID string, tg *telegram.DirectTelegram) (wrappers.WrapStreamReadWriteCloser, error) { + streamConn, err := tg.Dial(connID, opts) + if err != nil { + return nil, errors.Annotate(err, "Cannot dial to Telegram") + } + streamConn = wrappers.NewCtx(ctx, cancel, streamConn) + + packetConn, err := tg.Init(opts, streamConn) + if err != nil { + return nil, errors.Annotate(err, "Cannot handshake telegram") + } + + return packetConn, nil +} + +func directPipe(src io.Reader, dst io.Writer, wait *sync.WaitGroup) { + defer wait.Done() + io.Copy(dst, src) +} diff --git a/proxy/middle.go b/proxy/middle.go new file mode 100644 index 0000000..4987a13 --- /dev/null +++ b/proxy/middle.go @@ -0,0 +1,77 @@ +package proxy + +import ( + "context" + "net" + "sync" + + "github.com/juju/errors" + + "github.com/9seconds/mtg/client" + "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/telegram" + "github.com/9seconds/mtg/wrappers" +) + +func NewProxyMiddle(conf *config.Config) *Proxy { + tg := telegram.NewMiddleTelegram(conf) + + return &Proxy{ + conf: conf, + acceptCallback: func(ctx context.Context, cancel context.CancelFunc, clientSocket net.Conn, + connID string, wait *sync.WaitGroup, conf *config.Config) error { + client, opts, err := client.MiddleInit(ctx, cancel, clientSocket, connID, conf) + if err != nil { + return errors.Annotate(err, "Cannot initialize client connection") + } + defer client.Close() + + server, err := middleTelegramStream(ctx, cancel, opts, connID, tg) + if err != nil { + return errors.Annotate(err, "Cannot initialize telegram connection") + } + defer server.Close() + + wait.Add(2) + + go middlePipe(client, server, wait, &opts.ReadHacks) + go middlePipe(server, client, wait, &opts.WriteHacks) + + return nil + }, + } +} + +func middleTelegramStream(ctx context.Context, cancel context.CancelFunc, opts *mtproto.ConnectionOpts, + connID string, tg *telegram.MiddleTelegram) (wrappers.WrapPacketReadWriteCloser, error) { + streamConn, err := tg.Dial(connID, opts) + if err != nil { + return nil, errors.Annotate(err, "Cannot dial to Telegram") + } + streamConn = wrappers.NewCtx(ctx, cancel, streamConn) + + packetConn, err := tg.Init(opts, streamConn) + if err != nil { + return nil, errors.Annotate(err, "Cannot handshake telegram") + } + + return packetConn, nil +} + +func middlePipe(src wrappers.WrapPacketReader, dst wrappers.WrapPacketWriter, wait *sync.WaitGroup, hacks *mtproto.Hacks) { + defer wait.Done() + + for { + hacks.SimpleAck = false + hacks.QuickAck = false + + packet, err := src.Read() + if err != nil { + return + } + if _, err = dst.Write(packet); err != nil { + return + } + } +} diff --git a/proxy/proxy.go b/proxy/proxy.go new file mode 100644 index 0000000..d1baf93 --- /dev/null +++ b/proxy/proxy.go @@ -0,0 +1,63 @@ +package proxy + +import ( + "context" + "net" + "sync" + + "github.com/juju/errors" + uuid "github.com/satori/go.uuid" + "go.uber.org/zap" + + "github.com/9seconds/mtg/config" +) + +type proxyAcceptCallback func(context.Context, context.CancelFunc, net.Conn, string, *sync.WaitGroup, *config.Config) error + +type Proxy struct { + conf *config.Config + acceptCallback proxyAcceptCallback +} + +func (p *Proxy) Serve() error { + lsock, err := net.Listen("tcp", p.conf.BindAddr()) + if err != nil { + return errors.Annotate(err, "Cannot create listen socket") + } + + for { + if conn, err := lsock.Accept(); err != nil { + zap.S().Errorw("Cannot allocate incoming connection", "error", err) + } else { + go p.accept(conn) + } + } +} + +func (p *Proxy) accept(conn net.Conn) { + connID := uuid.NewV4().String() + log := zap.S().With("connection_id", connID) + + defer func() { + conn.Close() + + if err := recover(); err != nil { + log.Errorw("Crash of accept handler", "error", err) + } + }() + + log.Infow("Client connected", "addr", conn.RemoteAddr()) + + ctx, cancel := context.WithCancel(context.Background()) + wait := &sync.WaitGroup{} + + if err := p.acceptCallback(ctx, cancel, conn, connID, wait, p.conf); err != nil { + log.Errorw("Cannot initialize connection", "error", err) + cancel() + } + + <-ctx.Done() + wait.Wait() + + log.Infow("Client disconnected", "addr", conn.RemoteAddr()) +} diff --git a/proxy/server.go b/proxy/server.go deleted file mode 100644 index 18bb98e..0000000 --- a/proxy/server.go +++ /dev/null @@ -1,184 +0,0 @@ -package proxy - -import ( - "context" - "io" - "net" - "sync" - - "github.com/juju/errors" - uuid "github.com/satori/go.uuid" - "go.uber.org/zap" - - "github.com/9seconds/mtg/client" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/utils" - "github.com/9seconds/mtg/wrappers" -) - -// Server is an insgtance of MTPROTO proxy. -type Server struct { - conf *config.Config - logger *zap.SugaredLogger - stats *Stats - tg telegram.Telegram - clientInit client.Init -} - -// Serve does MTPROTO proxying. -func (s *Server) Serve() error { - lsock, err := net.Listen("tcp", s.conf.BindAddr()) - if err != nil { - return errors.Annotate(err, "Cannot create listen socket") - } - - for { - if conn, err := lsock.Accept(); err != nil { - s.logger.Warn("Cannot allocate incoming connection", "error", err) - } else { - go s.accept(conn) - } - } -} - -func (s *Server) accept(conn net.Conn) { - defer func() { - s.stats.closeConnection() - conn.Close() // nolint: errcheck - - if r := recover(); r != nil { - s.logger.Errorw("Crash of accept handler", "error", r) - } - }() - - s.stats.newConnection() - ctx, cancel := context.WithCancel(context.Background()) - socketID := uuid.NewV4().String() - - s.logger.Debugw("Client connected", - "addr", conn.RemoteAddr().String(), - "socketid", socketID, - ) - - connOpts, clientConn, err := s.getClientStream(ctx, cancel, conn, socketID) - if err != nil { - s.logger.Warnw("Cannot initialize client connection", - "addr", conn.RemoteAddr().String(), - "socketid", socketID, - "error", err, - ) - return - } - defer clientConn.Close() // nolint: errcheck - - tgConn, err := s.getTelegramStream(ctx, cancel, connOpts, socketID) - if err != nil { - s.logger.Warnw("Cannot initialize Telegram connection", - "socketid", socketID, - "error", err, - ) - return - } - defer tgConn.Close() // nolint: errcheck - - wait := &sync.WaitGroup{} - wait.Add(2) - - go func() { - defer wait.Done() - - for { - connOpts.ReadHacks.QuickAck = false - connOpts.ReadHacks.SimpleAck = false - if err := s.pump(clientConn, tgConn, socketID, "client"); err != nil { - s.logger.Infow("Client stream is aborted", - "socketid", socketID, "error", err) - return - } - } - }() - go func() { - defer wait.Done() - - for { - connOpts.WriteHacks.QuickAck = false - connOpts.WriteHacks.SimpleAck = false - if err := s.pump(tgConn, clientConn, socketID, "telegram"); err != nil { - s.logger.Infow("Telegram stream is aborted", - "socketid", socketID, "error", err) - return - } - } - }() - - <-ctx.Done() - wait.Wait() - - s.logger.Debugw("Client disconnected", - "addr", conn.RemoteAddr().String(), - "socketid", socketID, - ) -} - -func (s *Server) getClientStream(ctx context.Context, cancel context.CancelFunc, conn net.Conn, socketID string) (*mtproto.ConnectionOpts, io.ReadWriteCloser, error) { - socket, connOpts, err := s.clientInit(conn, socketID, s.conf) - if err != nil { - return nil, nil, errors.Annotate(err, "Cannot init client connection") - } - - socket = wrappers.NewTrafficRWC(socket, s.stats.addIncomingTraffic, s.stats.addOutgoingTraffic) - socket = wrappers.NewLogRWC(socket, s.logger, socketID, "client") - socket = wrappers.NewCtxRWC(ctx, cancel, socket) - - return connOpts, socket, nil -} - -func (s *Server) getTelegramStream(ctx context.Context, cancel context.CancelFunc, connOpts *mtproto.ConnectionOpts, socketID string) (io.ReadWriteCloser, error) { - conn, err := s.tg.Dial(socketID, connOpts) - if err != nil { - return nil, errors.Annotate(err, "Cannot connect to Telegram") - } - - conn = wrappers.NewTrafficRWC(conn, s.stats.addIncomingTraffic, s.stats.addOutgoingTraffic) - conn, err = s.tg.Init(connOpts, conn) - if err != nil { - return nil, errors.Annotate(err, "Cannot handshake Telegram") - } - - conn = wrappers.NewLogRWC(conn, s.logger, socketID, "telegram") - conn = wrappers.NewCtxRWC(ctx, cancel, conn) - - return conn, nil -} - -func (s *Server) pump(src io.Reader, dst io.Writer, socketID, name string) error { - buf, err := utils.ReadCurrentData(src) - if err != nil { - return errors.Annotate(err, "Cannot pump the socket") - } - - _, err = dst.Write(buf) - - return err -} - -// NewServer creates new instance of MTPROTO proxy. -func NewServer(conf *config.Config, logger *zap.SugaredLogger, stat *Stats) *Server { - clientInit := client.DirectInit - tg := telegram.NewDirectTelegram - - if len(conf.AdTag) > 0 { - clientInit = client.MiddleInit - tg = telegram.NewMiddleTelegram - } - - return &Server{ - conf: conf, - logger: logger, - stats: stat, - tg: tg(conf, logger), - clientInit: clientInit, - } -} diff --git a/proxy/stats.go b/proxy/stats.go deleted file mode 100644 index 9469c2f..0000000 --- a/proxy/stats.go +++ /dev/null @@ -1,74 +0,0 @@ -package proxy - -import ( - "encoding/json" - "net/http" - "strconv" - "sync/atomic" - "time" - - "github.com/9seconds/mtg/config" -) - -type statsUptime time.Time - -func (s statsUptime) MarshalJSON() ([]byte, error) { - uptime := int(time.Since(time.Time(s)).Seconds()) - return []byte(strconv.Itoa(uptime)), nil -} - -// Stats is a datastructure for statistics on work of this proxy. -type Stats struct { - AllConnections uint64 `json:"all_connections"` - ActiveConnections uint32 `json:"active_connections"` - Traffic struct { - Incoming uint64 `json:"incoming"` - Outgoing uint64 `json:"outgoing"` - } `json:"traffic"` - URLs config.IPURLs `json:"urls"` - Uptime statsUptime `json:"uptime"` - - conf *config.Config -} - -func (s *Stats) newConnection() { - atomic.AddUint64(&s.AllConnections, 1) - atomic.AddUint32(&s.ActiveConnections, 1) -} - -func (s *Stats) closeConnection() { - atomic.AddUint32(&s.ActiveConnections, ^uint32(0)) -} - -func (s *Stats) addIncomingTraffic(n int) { - atomic.AddUint64(&s.Traffic.Incoming, uint64(n)) -} - -func (s *Stats) addOutgoingTraffic(n int) { - atomic.AddUint64(&s.Traffic.Outgoing, uint64(n)) -} - -// Serve runs statistics HTTP server. -func (s *Stats) Serve() { - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - - encoder := json.NewEncoder(w) - encoder.SetEscapeHTML(false) - encoder.SetIndent("", " ") - encoder.Encode(s) // nolint: errcheck, gas - }) - - http.ListenAndServe(s.conf.StatAddr(), nil) // nolint: errcheck, gas -} - -// NewStats returns new instance of statistics datastructure. -func NewStats(conf *config.Config) *Stats { - stat := &Stats{ - Uptime: statsUptime(time.Now()), - conf: conf, - } - stat.URLs = conf.GetURLs() - - return stat -} diff --git a/telegram/dialer.go b/telegram/dialer.go index 5f3e20c..d710f38 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -30,11 +30,14 @@ func (t *tgDialer) dial(addr string) (net.Conn, error) { return conn, nil } -func (t *tgDialer) dialRWC(addr, sock string) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *tgDialer) dialRWC(addr, connID string) (wrappers.WrapStreamReadWriteCloser, error) { conn, err := t.dial(addr) if err != nil { return nil, err } - return wrappers.NewTimeoutRWC(conn, sock, t.conf.PublicIPv4, t.conf.PublicIPv6), nil + tgConn := wrappers.NewConn(conn, connID, wrappers.ConnPurposeTelegram, + t.conf.PublicIPv4, t.conf.PublicIPv6) + + return tgConn, nil } diff --git a/telegram/direct.go b/telegram/direct.go index 601fa52..0d1dd28 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -4,7 +4,6 @@ import ( "net" "github.com/juju/errors" - "go.uber.org/zap" "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" @@ -29,11 +28,11 @@ var ( } ) -type directTelegram struct { +type DirectTelegram struct { baseTelegram } -func (t *directTelegram) Dial(sock string, connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.WrapStreamReadWriteCloser, error) { dc := connOpts.DC if dc < 0 { dc = -dc @@ -41,23 +40,23 @@ func (t *directTelegram) Dial(sock string, connOpts *mtproto.ConnectionOpts) (wr dc = 1 } - return t.baseTelegram.dial(dc-1, sock, connOpts.ConnectionProto) + return t.baseTelegram.dial(dc-1, connID, connOpts.ConnectionProto) } -func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.WrapStreamReadWriteCloser, error) { obfs2, frame := obfuscated2.MakeTelegramObfuscated2Frame(connOpts) - if n, err := conn.Write(frame); err != nil || n != obfuscated2.FrameLen { + if _, err := conn.Write(frame); err != nil { return nil, errors.Annotate(err, "Cannot write hadnshake frame") } - return wrappers.NewStreamCipherRWC(conn, obfs2.Encryptor, obfs2.Decryptor), nil + return wrappers.NewStreamCipher(conn, obfs2.Encryptor, obfs2.Decryptor), nil } // NewDirectTelegram returns Telegram instance which connects directly // to Telegram bypassing middleproxies. -func NewDirectTelegram(conf *config.Config, _ *zap.SugaredLogger) Telegram { - return &directTelegram{baseTelegram{ +func NewDirectTelegram(conf *config.Config) *DirectTelegram { + return &DirectTelegram{baseTelegram{ dialer: tgDialer{ Dialer: net.Dialer{Timeout: telegramDialTimeout}, conf: conf, diff --git a/telegram/middle.go b/telegram/middle.go index 62a9c65..8a25996 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -1,29 +1,26 @@ package telegram import ( - "io" "net" "net/http" "sync" "github.com/juju/errors" - "go.uber.org/zap" "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" "github.com/9seconds/mtg/mtproto/rpc" - mtwrappers "github.com/9seconds/mtg/mtproto/wrappers" "github.com/9seconds/mtg/wrappers" ) -type middleTelegram struct { +type MiddleTelegram struct { middleTelegramCaller conf *config.Config } -func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram { - tg := &middleTelegram{ +func NewMiddleTelegram(conf *config.Config) *MiddleTelegram { + tg := &MiddleTelegram{ middleTelegramCaller: middleTelegramCaller{ baseTelegram: baseTelegram{ dialer: tgDialer{ @@ -31,7 +28,6 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram conf: conf, }, }, - logger: logger, httpClient: &http.Client{ Timeout: middleTelegramHTTPClientTimeout, }, @@ -48,8 +44,8 @@ func NewMiddleTelegram(conf *config.Config, logger *zap.SugaredLogger) Telegram return tg } -func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) { - rpcNonceConn := mtwrappers.NewFrameRWC(conn, rpc.SeqNoNonce) +func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.WrapPacketReadWriteCloser, error) { + rpcNonceConn := wrappers.NewMTProtoFrame(conn, rpc.SeqNoNonce) rpcNonceReq, err := t.sendRPCNonceRequest(rpcNonceConn) if err != nil { @@ -60,22 +56,22 @@ func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Re return nil, err } - secureConn := mtwrappers.NewMiddleProxyCipherRWC(conn, rpcNonceReq, rpcNonceResp, t.proxySecret) - secureConn = mtwrappers.NewFrameRWC(secureConn, rpc.SeqNoHandshake) + secureConn := wrappers.NewMiddleProxyCipher(conn, rpcNonceReq, rpcNonceResp, t.proxySecret) + frameConn := wrappers.NewMTProtoFrame(secureConn, rpc.SeqNoHandshake) - rpcHandshakeReq, err := t.sendRPCHandshakeRequest(secureConn) + rpcHandshakeReq, err := t.sendRPCHandshakeRequest(frameConn) if err != nil { return nil, err } - _, err = t.receiveRPCHandshakeResponse(secureConn, rpcHandshakeReq) + _, err = t.receiveRPCHandshakeResponse(frameConn, rpcHandshakeReq) if err != nil { return nil, err } - return mtwrappers.NewProxyRequestRWC(secureConn, connOpts, t.conf.AdTag) + return wrappers.NewMTProtoProxy(frameConn, connOpts, t.conf.AdTag) } -func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.NonceRequest, error) { +func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.WrapPacketWriter) (*rpc.NonceRequest, error) { rpcNonceReq, err := rpc.NewNonceRequest(t.proxySecret) if err != nil { return nil, errors.Annotate(err, "Cannot create RPC nonce request") @@ -87,15 +83,13 @@ func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.NonceRequest, return rpcNonceReq, nil } -func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { - var ans [128]byte - - n, err := conn.Read(ans[:]) +func (t *MiddleTelegram) receiveRPCNonceResponse(conn wrappers.WrapPacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { + packet, err := conn.Read() if err != nil { return nil, errors.Annotate(err, "Cannot read RPC nonce response") } - rpcNonceResp, err := rpc.NewNonceResponse(ans[:n]) + rpcNonceResp, err := rpc.NewNonceResponse(packet) if err != nil { return nil, errors.Annotate(err, "Cannot initialize RPC nonce response") } @@ -106,7 +100,7 @@ func (t *middleTelegram) receiveRPCNonceResponse(conn io.Reader, req *rpc.NonceR return rpcNonceResp, nil } -func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.HandshakeRequest, error) { +func (t *MiddleTelegram) sendRPCHandshakeRequest(conn wrappers.WrapPacketWriter) (*rpc.HandshakeRequest, error) { req := rpc.NewHandshakeRequest() if _, err := conn.Write(req.Bytes()); err != nil { return nil, errors.Annotate(err, "Cannot send RPC handshake request") @@ -115,15 +109,13 @@ func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.Handshake return req, nil } -func (t *middleTelegram) receiveRPCHandshakeResponse(conn io.Reader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { - var ans [128]byte - - n, err := conn.Read(ans[:]) +func (t *MiddleTelegram) receiveRPCHandshakeResponse(conn wrappers.WrapPacketReader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { + packet, err := conn.Read() if err != nil { return nil, errors.Annotate(err, "Cannot read RPC handshake response") } - rpcHandshakeResp, err := rpc.NewHandshakeResponse(ans[:n]) + rpcHandshakeResp, err := rpc.NewHandshakeResponse(packet) if err != nil { return nil, errors.Annotate(err, "Cannot initialize RPC handshake response") } diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go index 490ed26..c0d40a2 100644 --- a/telegram/middle_caller.go +++ b/telegram/middle_caller.go @@ -35,11 +35,10 @@ type middleTelegramCaller struct { proxySecret []byte dialerMutex *sync.RWMutex - logger *zap.SugaredLogger httpClient *http.Client } -func (t *middleTelegramCaller) Dial(sock string, connOpts *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) { +func (t *middleTelegramCaller) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.WrapStreamReadWriteCloser, error) { dc := connOpts.DC if dc == 0 { dc = 1 @@ -47,13 +46,13 @@ func (t *middleTelegramCaller) Dial(sock string, connOpts *mtproto.ConnectionOpt t.dialerMutex.RLock() defer t.dialerMutex.RUnlock() - return t.baseTelegram.dial(dc, sock, connOpts.ConnectionProto) + return t.baseTelegram.dial(dc, connID, connOpts.ConnectionProto) } func (t *middleTelegramCaller) autoUpdate() { for range time.Tick(middleTelegramAutoUpdateInterval) { if err := t.update(); err != nil { - t.logger.Warnw("Cannot update from Telegram", "error", err) + zap.S().Warnw("Cannot update from Telegram", "error", err) } } } @@ -80,7 +79,7 @@ func (t *middleTelegramCaller) update() error { t.v6Addresses = v6Addresses t.dialerMutex.Unlock() - t.logger.Infow("Telegram middle proxy data has been updated") + zap.S().Infow("Telegram middle proxy data has been updated") return nil } diff --git a/telegram/telegram.go b/telegram/telegram.go index 9413882..da8e27a 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -9,14 +9,6 @@ import ( "github.com/9seconds/mtg/wrappers" ) -// Telegram defines an interface to connect to Telegram. This -// encapsulates logic of working with middleproxies or direct -// connections. -type Telegram interface { - Dial(string, *mtproto.ConnectionOpts) (wrappers.ReadWriteCloserWithAddr, error) - Init(*mtproto.ConnectionOpts, wrappers.ReadWriteCloserWithAddr) (wrappers.ReadWriteCloserWithAddr, error) -} - type baseTelegram struct { dialer tgDialer @@ -24,7 +16,7 @@ type baseTelegram struct { v6Addresses map[int16][]string } -func (b *baseTelegram) dial(dcIdx int16, sock string, proto mtproto.ConnectionProtocol) (wrappers.ReadWriteCloserWithAddr, error) { +func (b *baseTelegram) dial(dcIdx int16, connID string, proto mtproto.ConnectionProtocol) (wrappers.WrapStreamReadWriteCloser, error) { addrs := make([]string, 2) if proto&mtproto.ConnectionProtocolIPv6 != 0 { @@ -39,7 +31,7 @@ func (b *baseTelegram) dial(dcIdx int16, sock string, proto mtproto.ConnectionPr } for _, addr := range addrs { - if conn, err := b.dialer.dialRWC(addr, sock); err == nil { + if conn, err := b.dialer.dialRWC(addr, connID); err == nil { return conn, err } } diff --git a/wrappers/blockcipher.go b/wrappers/blockcipher.go index 55b90e1..5283f90 100644 --- a/wrappers/blockcipher.go +++ b/wrappers/blockcipher.go @@ -1,6 +1,7 @@ package wrappers import ( + "bytes" "crypto/aes" "crypto/cipher" "net" @@ -9,77 +10,89 @@ import ( "github.com/juju/errors" ) -type WrapBlockCipher struct { - BufferedReader +type BlockCipher struct { + buf *bytes.Buffer conn WrapStreamReadWriteCloser encryptor cipher.BlockMode decryptor cipher.BlockMode } -func (w *WrapBlockCipher) Read(p []byte) (int, error) { - return w.BufferedRead(p, func() error { - var buf []byte +func (b *BlockCipher) Read(p []byte) (int, error) { + if b.buf.Len() > 0 { + return b.flush(p) + } - for len(buf) == 0 || len(buf)%aes.BlockSize != 0 { - rv, err := utils.ReadCurrentData(w.conn) - if err != nil { - return errors.Annotate(err, "Cannot read from socket") - } - buf = append(buf, rv...) + buf := []byte{} + for len(buf) == 0 || len(buf)%aes.BlockSize != 0 { + rv, err := utils.ReadCurrentData(b.conn) + if err != nil { + return 0, errors.Annotate(err, "Cannot read from socket") } + buf = append(buf, rv...) + } - w.decryptor.CryptBlocks(buf, buf) - w.Buffer.Write(buf) + b.decryptor.CryptBlocks(buf, buf) + b.buf.Write(buf) - return nil - }) + return b.flush(p) } -func (w *WrapBlockCipher) Write(p []byte) (int, error) { +func (b *BlockCipher) flush(p []byte) (int, error) { + if b.buf.Len() <= len(p) { + sizeToReturn := b.buf.Len() + copy(p, b.buf.Bytes()) + b.buf.Reset() + return sizeToReturn, nil + } + + return b.buf.Read(p) +} + +func (b *BlockCipher) Write(p []byte) (int, error) { if len(p)%aes.BlockSize > 0 { return 0, errors.Errorf("Incorrect block size %d", len(p)) } encrypted := make([]byte, len(p)) - w.encryptor.CryptBlocks(encrypted, p) + b.encryptor.CryptBlocks(encrypted, p) - return w.conn.Write(encrypted) + return b.conn.Write(encrypted) } -func (w *WrapBlockCipher) LogDebug(msg string, data ...interface{}) { - w.conn.LogDebug(msg, data...) +func (b *BlockCipher) LogDebug(msg string, data ...interface{}) { + b.conn.LogDebug(msg, data...) } -func (w *WrapBlockCipher) LogInfo(msg string, data ...interface{}) { - w.conn.LogInfo(msg, data...) +func (b *BlockCipher) LogInfo(msg string, data ...interface{}) { + b.conn.LogInfo(msg, data...) } -func (w *WrapBlockCipher) LogWarn(msg string, data ...interface{}) { - w.conn.LogWarn(msg, data...) +func (b *BlockCipher) LogWarn(msg string, data ...interface{}) { + b.conn.LogWarn(msg, data...) } -func (w *WrapBlockCipher) LogError(msg string, data ...interface{}) { - w.conn.LogError(msg, data...) +func (b *BlockCipher) LogError(msg string, data ...interface{}) { + b.conn.LogError(msg, data...) } -func (w *WrapBlockCipher) LocalAddr() *net.TCPAddr { - return w.conn.LocalAddr() +func (b *BlockCipher) LocalAddr() *net.TCPAddr { + return b.conn.LocalAddr() } -func (w *WrapBlockCipher) RemoteAddr() *net.TCPAddr { - return w.conn.RemoteAddr() +func (b *BlockCipher) RemoteAddr() *net.TCPAddr { + return b.conn.RemoteAddr() } -func (w *WrapBlockCipher) Close() error { - return w.conn.Close() +func (b *BlockCipher) Close() error { + return b.conn.Close() } -func NewWrapBlockCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.BlockMode) WrapStreamReadWriteCloser { - return &WrapBlockCipher{ - BufferedReader: NewBufferedReader(), - conn: conn, - encryptor: encryptor, - decryptor: decryptor, +func NewBlockCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.BlockMode) WrapStreamReadWriteCloser { + return &BlockCipher{ + buf: &bytes.Buffer{}, + conn: conn, + encryptor: encryptor, + decryptor: decryptor, } } diff --git a/wrappers/buffered_reader.go b/wrappers/buffered_reader.go deleted file mode 100644 index 379534a..0000000 --- a/wrappers/buffered_reader.go +++ /dev/null @@ -1,32 +0,0 @@ -package wrappers - -import "bytes" - -type BufferedReader struct { - Buffer *bytes.Buffer -} - -func (b *BufferedReader) BufferedRead(p []byte, callback func() error) (int, error) { - if b.Buffer.Len() > 0 { - return b.flush(p) - } - if err := callback(); err != nil { - return 0, err - } - return b.flush(p) -} - -func (b *BufferedReader) flush(p []byte) (int, error) { - if b.Buffer.Len() <= len(p) { - sizeToReturn := b.Buffer.Len() - copy(p, b.Buffer.Bytes()) - b.Buffer.Reset() - return sizeToReturn, nil - } - - return b.Buffer.Read(p) -} - -func NewBufferedReader() BufferedReader { - return BufferedReader{Buffer: &bytes.Buffer{}} -} diff --git a/wrappers/conn.go b/wrappers/conn.go index 468820c..2a84941 100644 --- a/wrappers/conn.go +++ b/wrappers/conn.go @@ -30,8 +30,7 @@ const ( connTimeoutWrite = 5 * time.Minute ) -type WrapConn struct { - purpose ConnPurpose +type Conn struct { connID string conn net.Conn logger *zap.SugaredLogger @@ -39,77 +38,80 @@ type WrapConn struct { publicIPv6 net.IP } -func (w *WrapConn) Write(p []byte) (int, error) { - w.conn.SetWriteDeadline(time.Now().Add(connTimeoutWrite)) - n, err := w.conn.Write(p) +func (c *Conn) Write(p []byte) (int, error) { + c.conn.SetWriteDeadline(time.Now().Add(connTimeoutWrite)) + n, err := c.conn.Write(p) - w.logger.Debugw("Write to stream", "bytes", n, "error", err) + c.logger.Debugw("Write to stream", "bytes", n, "error", err) return n, err } -func (w *WrapConn) Read(p []byte) (int, error) { - w.conn.SetReadDeadline(time.Now().Add(connTimeoutRead)) - n, err := w.conn.Read(p) +func (c *Conn) Read(p []byte) (int, error) { + c.conn.SetReadDeadline(time.Now().Add(connTimeoutRead)) + n, err := c.conn.Read(p) - w.logger.Debugw("Read from stream", "bytes", n, "error", err) + c.logger.Debugw("Read from stream", "bytes", n, "error", err) return n, err } -func (w *WrapConn) Close() error { - defer w.LogDebug("Closed connection") - return w.conn.Close() +func (c *Conn) Close() error { + defer c.LogDebug("Closed connection") + return c.conn.Close() } -func (w *WrapConn) LocalAddr() *net.TCPAddr { - addr := w.conn.LocalAddr().(*net.TCPAddr) +func (c *Conn) LocalAddr() *net.TCPAddr { + addr := c.conn.LocalAddr().(*net.TCPAddr) newAddr := *addr - if w.RemoteAddr().IP.To4() != nil { - if w.publicIPv4 != nil { - newAddr.IP = w.publicIPv4 + if c.RemoteAddr().IP.To4() != nil { + if c.publicIPv4 != nil { + newAddr.IP = c.publicIPv4 } - } else if w.publicIPv6 != nil { - newAddr.IP = w.publicIPv6 + } else if c.publicIPv6 != nil { + newAddr.IP = c.publicIPv6 } return &newAddr } -func (w *WrapConn) RemoteAddr() *net.TCPAddr { - return w.conn.RemoteAddr().(*net.TCPAddr) +func (c *Conn) RemoteAddr() *net.TCPAddr { + return c.conn.RemoteAddr().(*net.TCPAddr) } -func (w *WrapConn) LogDebug(msg string, data ...interface{}) { - w.logger.Debugw(msg, data...) +func (c *Conn) LogDebug(msg string, data ...interface{}) { + c.logger.Debugw(msg, data...) } -func (w *WrapConn) LogInfo(msg string, data ...interface{}) { - w.logger.Infow(msg, data...) +func (c *Conn) LogInfo(msg string, data ...interface{}) { + c.logger.Infow(msg, data...) } -func (w *WrapConn) LogWarn(msg string, data ...interface{}) { - w.logger.Warnw(msg, data...) +func (c *Conn) LogWarn(msg string, data ...interface{}) { + c.logger.Warnw(msg, data...) } -func (w *WrapConn) LogError(msg string, data ...interface{}) { - w.logger.Errorw(msg, data...) +func (c *Conn) LogError(msg string, data ...interface{}) { + c.logger.Errorw(msg, data...) } -func NewConn(connID string, purpose ConnPurpose, conn net.Conn, publicIPv4, publicIPv6 net.IP) WrapStreamReadWriteCloser { +func NewConn(conn net.Conn, connID string, purpose ConnPurpose, publicIPv4, publicIPv6 net.IP) WrapStreamReadWriteCloser { logger := zap.S().With( "connection_id", connID, "local_address", conn.LocalAddr(), "remote_address", conn.RemoteAddr(), + "purpose", purpose, ) - return &WrapConn{ + wrapper := Conn{ logger: logger, - purpose: purpose, connID: connID, conn: conn, publicIPv4: publicIPv4, publicIPv6: publicIPv6, } + wrapper.logger = logger.With("faked_local_addr", wrapper.LocalAddr()) + + return &wrapper } diff --git a/wrappers/ctx.go b/wrappers/ctx.go index ee766f8..bc38211 100644 --- a/wrappers/ctx.go +++ b/wrappers/ctx.go @@ -7,68 +7,68 @@ import ( "github.com/juju/errors" ) -type WrapCtx struct { +type Ctx struct { cancel context.CancelFunc conn WrapStreamReadWriteCloser ctx context.Context } -func (w *WrapCtx) Read(p []byte) (int, error) { +func (c *Ctx) Read(p []byte) (int, error) { select { - case <-w.ctx.Done(): - return 0, errors.Annotate(w.ctx.Err(), "Read is failed because of closed context") + case <-c.ctx.Done(): + return 0, errors.Annotate(c.ctx.Err(), "Read is failed because of closed context") default: - n, err := w.conn.Read(p) + n, err := c.conn.Read(p) if err != nil { - w.cancel() + c.cancel() } return n, err } } -func (w *WrapCtx) Write(p []byte) (int, error) { +func (c *Ctx) Write(p []byte) (int, error) { select { - case <-w.ctx.Done(): - return 0, errors.Annotate(w.ctx.Err(), "Write is failed because of closed context") + case <-c.ctx.Done(): + return 0, errors.Annotate(c.ctx.Err(), "Write is failed because of closed context") default: - n, err := w.conn.Write(p) + n, err := c.conn.Write(p) if err != nil { - w.cancel() + c.cancel() } return n, err } } -func (w *WrapCtx) LogDebug(msg string, data ...interface{}) { - w.conn.LogDebug(msg, data...) +func (c *Ctx) LogDebug(msg string, data ...interface{}) { + c.conn.LogDebug(msg, data...) } -func (w *WrapCtx) LogInfo(msg string, data ...interface{}) { - w.conn.LogInfo(msg, data...) +func (c *Ctx) LogInfo(msg string, data ...interface{}) { + c.conn.LogInfo(msg, data...) } -func (w *WrapCtx) LogWarn(msg string, data ...interface{}) { - w.conn.LogWarn(msg, data...) +func (c *Ctx) LogWarn(msg string, data ...interface{}) { + c.conn.LogWarn(msg, data...) } -func (w *WrapCtx) LogError(msg string, data ...interface{}) { - w.conn.LogError(msg, data...) +func (c *Ctx) LogError(msg string, data ...interface{}) { + c.conn.LogError(msg, data...) } -func (w *WrapCtx) LocalAddr() *net.TCPAddr { - return w.conn.LocalAddr() +func (c *Ctx) LocalAddr() *net.TCPAddr { + return c.conn.LocalAddr() } -func (w *WrapCtx) RemoteAddr() *net.TCPAddr { - return w.conn.RemoteAddr() +func (c *Ctx) RemoteAddr() *net.TCPAddr { + return c.conn.RemoteAddr() } -func (w *WrapCtx) Close() error { - return w.conn.Close() +func (c *Ctx) Close() error { + return c.conn.Close() } func NewCtx(ctx context.Context, cancel context.CancelFunc, conn WrapStreamReadWriteCloser) WrapStreamReadWriteCloser { - return &WrapCtx{ + return &Ctx{ ctx: ctx, cancel: cancel, conn: conn, diff --git a/wrappers/mtproto_cipher.go b/wrappers/mtproto_cipher.go index 146016e..71170dc 100644 --- a/wrappers/mtproto_cipher.go +++ b/wrappers/mtproto_cipher.go @@ -32,7 +32,7 @@ func NewMiddleProxyCipher(conn WrapStreamReadWriteCloser, req *rpc.NonceRequest, enc, _ := makeEncrypterDecrypter(encKey, encIV) _, dec := makeEncrypterDecrypter(decKey, decIV) - return NewWrapBlockCipher(conn, enc, dec) + return NewBlockCipher(conn, enc, dec) } func deriveKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index 0a6d1a8..c8f1796 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -4,8 +4,10 @@ import ( "bytes" "net" - "github.com/9seconds/mtg/mtproto/rpc" "github.com/juju/errors" + + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/mtproto/rpc" ) type MTProtoProxy struct { @@ -127,9 +129,14 @@ func (m *MTProtoProxy) Close() error { return m.conn.Close() } -func NewMTProtoProxy(conn WrapPacketReadWriteCloser, req *rpc.ProxyRequest) WrapPacketReadWriteCloser { +func NewMTProtoProxy(conn WrapPacketReadWriteCloser, connOpts *mtproto.ConnectionOpts, adTag []byte) (WrapPacketReadWriteCloser, error) { + req, err := rpc.NewProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) + if err != nil { + return nil, errors.Annotate(err, "Cannot create new RPC proxy request") + } + return &MTProtoProxy{ conn: conn, req: req, - } + }, nil } diff --git a/wrappers/streamcipher.go b/wrappers/streamcipher.go index 1ad764f..1057d33 100644 --- a/wrappers/streamcipher.go +++ b/wrappers/streamcipher.go @@ -7,59 +7,59 @@ import ( "github.com/juju/errors" ) -type WrapStreamCipher struct { +type StreamCipher struct { encryptor cipher.Stream decryptor cipher.Stream conn WrapStreamReadWriteCloser } -func (w *WrapStreamCipher) Read(p []byte) (int, error) { - n, err := w.conn.Read(p) +func (s *StreamCipher) Read(p []byte) (int, error) { + n, err := s.conn.Read(p) if err != nil { return 0, errors.Annotate(err, "Cannot read stream ciphered data") } - w.decryptor.XORKeyStream(p, p[:n]) + s.decryptor.XORKeyStream(p, p[:n]) return n, nil } -func (w *WrapStreamCipher) Write(p []byte) (int, error) { +func (s *StreamCipher) Write(p []byte) (int, error) { encrypted := make([]byte, len(p)) - w.encryptor.XORKeyStream(encrypted, p) + s.encryptor.XORKeyStream(encrypted, p) - return w.conn.Write(encrypted) + return s.conn.Write(encrypted) } -func (w *WrapStreamCipher) LogDebug(msg string, data ...interface{}) { - w.conn.LogDebug(msg, data...) +func (s *StreamCipher) LogDebug(msg string, data ...interface{}) { + s.conn.LogDebug(msg, data...) } -func (w *WrapStreamCipher) LogInfo(msg string, data ...interface{}) { - w.conn.LogInfo(msg, data...) +func (s *StreamCipher) LogInfo(msg string, data ...interface{}) { + s.conn.LogInfo(msg, data...) } -func (w *WrapStreamCipher) LogWarn(msg string, data ...interface{}) { - w.conn.LogWarn(msg, data...) +func (s *StreamCipher) LogWarn(msg string, data ...interface{}) { + s.conn.LogWarn(msg, data...) } -func (w *WrapStreamCipher) LogError(msg string, data ...interface{}) { - w.conn.LogError(msg, data...) +func (s *StreamCipher) LogError(msg string, data ...interface{}) { + s.conn.LogError(msg, data...) } -func (w *WrapStreamCipher) LocalAddr() *net.TCPAddr { - return w.conn.LocalAddr() +func (s *StreamCipher) LocalAddr() *net.TCPAddr { + return s.conn.LocalAddr() } -func (w *WrapStreamCipher) RemoteAddr() *net.TCPAddr { - return w.conn.RemoteAddr() +func (s *StreamCipher) RemoteAddr() *net.TCPAddr { + return s.conn.RemoteAddr() } -func (w *WrapStreamCipher) Close() error { - return w.conn.Close() +func (s *StreamCipher) Close() error { + return s.conn.Close() } func NewStreamCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.Stream) WrapStreamReadWriteCloser { - return &WrapStreamCipher{ + return &StreamCipher{ conn: conn, encryptor: encryptor, decryptor: decryptor, diff --git a/wrappers/wrap.go b/wrappers/wrap.go index d4bc7d4..3d54bc1 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -56,6 +56,11 @@ type WrapPacketReader interface { Wrap } +type WrapPacketWriter interface { + io.Writer + Wrap +} + type WrapPacketReadWriter interface { io.Writer WrapPacketReader From cd63483503cd024759d48bb011a1f06421598ea6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sat, 7 Jul 2018 20:45:10 +0300 Subject: [PATCH 47/71] Debug direct connection --- client/direct.go | 2 +- main.go | 2 ++ obfuscated2/frame.go | 10 ++++++++-- proxy/direct.go | 10 ++++------ proxy/middle.go | 11 +++++------ proxy/proxy.go | 14 ++++++++++++-- 6 files changed, 32 insertions(+), 17 deletions(-) diff --git a/client/direct.go b/client/direct.go index 5abe56c..40e0696 100644 --- a/client/direct.go +++ b/client/direct.go @@ -36,8 +36,8 @@ func DirectInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connOpts.ConnectionProto = mtproto.ConnectionProtocolAny connOpts.ClientAddr = conn.RemoteAddr() - conn = wrappers.NewCtx(ctx, cancel, conn) conn = wrappers.NewStreamCipher(conn, obfs2.Encryptor, obfs2.Decryptor) + conn = wrappers.NewCtx(ctx, cancel, conn) return conn, connOpts, nil } diff --git a/main.go b/main.go index 5cc71be..a023f76 100644 --- a/main.go +++ b/main.go @@ -117,8 +117,10 @@ func main() { var server *proxy.Proxy if len(conf.AdTag) == 0 { + zap.S().Infow("Use direct connection to Telegram") server = proxy.NewProxyDirect(conf) } else { + zap.S().Infow("Use middle proxy connection to Telegram") server = proxy.NewProxyMiddle(conf) } diff --git a/obfuscated2/frame.go b/obfuscated2/frame.go index edcb8b4..6c6f6ab 100644 --- a/obfuscated2/frame.go +++ b/obfuscated2/frame.go @@ -9,7 +9,6 @@ import ( "github.com/juju/errors" "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/utils" ) // [frameOffsetFirst:frameOffsetKey:frameOffsetIV:frameOffsetMagic:frameOffsetDC:frameOffsetEnd] @@ -68,7 +67,14 @@ func (f Frame) ConnectionType() (mtproto.ConnectionType, error) { // Invert inverts frame for extracting encryption keys. Pkease check that link: // https://blog.susanka.eu/how-telegram-obfuscates-its-mtproto-traffic/ func (f Frame) Invert() Frame { - return Frame(utils.ReverseBytes([]byte(f))) + reversed := make(Frame, FrameLen) + copy(reversed, f) + + for i := 0; i < frameLenKey+frameLenIV; i++ { + reversed[frameOffsetFirst+i] = f[frameOffsetIV-1-i] + } + + return reversed } // ExtractFrame extracts exact obfuscated2 handshake frame from given reader. diff --git a/proxy/direct.go b/proxy/direct.go index 56b6ce2..663f93b 100644 --- a/proxy/direct.go +++ b/proxy/direct.go @@ -21,25 +21,23 @@ func NewProxyDirect(conf *config.Config) *Proxy { return &Proxy{ conf: conf, acceptCallback: func(ctx context.Context, cancel context.CancelFunc, clientSocket net.Conn, - connID string, wait *sync.WaitGroup, conf *config.Config) error { + connID string, wait *sync.WaitGroup, conf *config.Config) (io.Closer, io.Closer, error) { client, opts, err := client.DirectInit(ctx, cancel, clientSocket, connID, conf) if err != nil { - return errors.Annotate(err, "Cannot initialize client connection") + return nil, nil, errors.Annotate(err, "Cannot initialize client connection") } - defer client.Close() server, err := directTelegramStream(ctx, cancel, opts, connID, tg) if err != nil { - return errors.Annotate(err, "Cannot initialize telegram connection") + return client, nil, errors.Annotate(err, "Cannot initialize telegram connection") } - defer server.Close() wait.Add(2) go directPipe(client, server, wait) go directPipe(server, client, wait) - return nil + return client, server, nil }, } } diff --git a/proxy/middle.go b/proxy/middle.go index 4987a13..6ab300b 100644 --- a/proxy/middle.go +++ b/proxy/middle.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "io" "net" "sync" @@ -20,25 +21,23 @@ func NewProxyMiddle(conf *config.Config) *Proxy { return &Proxy{ conf: conf, acceptCallback: func(ctx context.Context, cancel context.CancelFunc, clientSocket net.Conn, - connID string, wait *sync.WaitGroup, conf *config.Config) error { + connID string, wait *sync.WaitGroup, conf *config.Config) (io.Closer, io.Closer, error) { client, opts, err := client.MiddleInit(ctx, cancel, clientSocket, connID, conf) if err != nil { - return errors.Annotate(err, "Cannot initialize client connection") + return nil, nil, errors.Annotate(err, "Cannot initialize client connection") } - defer client.Close() server, err := middleTelegramStream(ctx, cancel, opts, connID, tg) if err != nil { - return errors.Annotate(err, "Cannot initialize telegram connection") + return client, nil, errors.Annotate(err, "Cannot initialize telegram connection") } - defer server.Close() wait.Add(2) go middlePipe(client, server, wait, &opts.ReadHacks) go middlePipe(server, client, wait, &opts.WriteHacks) - return nil + return client, server, nil }, } } diff --git a/proxy/proxy.go b/proxy/proxy.go index d1baf93..0debedd 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "io" "net" "sync" @@ -12,7 +13,7 @@ import ( "github.com/9seconds/mtg/config" ) -type proxyAcceptCallback func(context.Context, context.CancelFunc, net.Conn, string, *sync.WaitGroup, *config.Config) error +type proxyAcceptCallback func(context.Context, context.CancelFunc, net.Conn, string, *sync.WaitGroup, *config.Config) (io.Closer, io.Closer, error) type Proxy struct { conf *config.Config @@ -51,7 +52,16 @@ func (p *Proxy) accept(conn net.Conn) { ctx, cancel := context.WithCancel(context.Background()) wait := &sync.WaitGroup{} - if err := p.acceptCallback(ctx, cancel, conn, connID, wait, p.conf); err != nil { + client, server, err := p.acceptCallback(ctx, cancel, conn, connID, wait, p.conf) + defer func() { + if client != nil { + client.Close() + } + if server != nil { + server.Close() + } + }() + if err != nil { log.Errorw("Cannot initialize connection", "error", err) cancel() } From a0ba89b105ac06616ae50ba86d1a4ea7d5128831 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 08:04:29 +0300 Subject: [PATCH 48/71] Rework for simplier proxy --- client/client.go | 12 +++++ client/direct.go | 2 +- client/middle.go | 7 +-- config/config.go | 4 ++ main.go | 10 ++-- proxy/direct.go | 64 ------------------------- proxy/middle.go | 76 ------------------------------ proxy/proxy.go | 108 ++++++++++++++++++++++++++++++++++++------- telegram/direct.go | 4 +- telegram/middle.go | 52 ++++++++++----------- telegram/telegram.go | 5 ++ wrappers/wrap.go | 5 ++ 12 files changed, 154 insertions(+), 195 deletions(-) create mode 100644 client/client.go delete mode 100644 proxy/direct.go delete mode 100644 proxy/middle.go diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..379ead8 --- /dev/null +++ b/client/client.go @@ -0,0 +1,12 @@ +package client + +import ( + "context" + "net" + + "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/wrappers" +) + +type Init func(context.Context, context.CancelFunc, net.Conn, string, *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) diff --git a/client/direct.go b/client/direct.go index 40e0696..09c0178 100644 --- a/client/direct.go +++ b/client/direct.go @@ -16,7 +16,7 @@ import ( const handshakeTimeout = 10 * time.Second func DirectInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connID string, - conf *config.Config) (wrappers.WrapStreamReadWriteCloser, *mtproto.ConnectionOpts, error) { + conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { if err := config.SetSocketOptions(socket); err != nil { return nil, nil, errors.Annotate(err, "Cannot set socket options") } diff --git a/client/middle.go b/client/middle.go index 635277d..bbc7347 100644 --- a/client/middle.go +++ b/client/middle.go @@ -10,15 +10,16 @@ import ( ) func MiddleInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connID string, - conf *config.Config) (wrappers.WrapPacketReadWriteCloser, *mtproto.ConnectionOpts, error) { + conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { conn, opts, err := DirectInit(ctx, cancel, socket, connID, conf) if err != nil { return nil, nil, err } + connStream := conn.(wrappers.WrapStreamReadWriteCloser) - newConn := wrappers.NewMTProtoAbridged(conn, opts) + newConn := wrappers.NewMTProtoAbridged(connStream, opts) if opts.ConnectionType != mtproto.ConnectionTypeAbridged { - newConn = wrappers.NewMTProtoIntermediate(conn, opts) + newConn = wrappers.NewMTProtoIntermediate(connStream, opts) } opts.ConnectionProto = mtproto.ConnectionProtocolIPv4 diff --git a/config/config.go b/config/config.go index 1a8f95e..59d06cd 100644 --- a/config/config.go +++ b/config/config.go @@ -58,6 +58,10 @@ func (c *Config) StatAddr() string { return getAddr(c.StatsIP, c.StatsPort) } +func (c *Config) UseMiddleProxy() bool { + return len(c.AdTag) > 0 +} + // GetURLs returns configured IPURLs instance with links to this server. func (c *Config) GetURLs() IPURLs { urls := IPURLs{} diff --git a/main.go b/main.go index a023f76..c0ffba1 100644 --- a/main.go +++ b/main.go @@ -115,17 +115,15 @@ func main() { zap.ReplaceGlobals(logger) defer logger.Sync() - var server *proxy.Proxy - if len(conf.AdTag) == 0 { - zap.S().Infow("Use direct connection to Telegram") - server = proxy.NewProxyDirect(conf) - } else { + if conf.UseMiddleProxy() { zap.S().Infow("Use middle proxy connection to Telegram") - server = proxy.NewProxyMiddle(conf) + } else { + zap.S().Infow("Use direct connection to Telegram") } printURLs(conf.GetURLs()) + server := proxy.NewProxy(conf) if err := server.Serve(); err != nil { zap.S().Fatalw("Server stopped", "error", err) } diff --git a/proxy/direct.go b/proxy/direct.go deleted file mode 100644 index 663f93b..0000000 --- a/proxy/direct.go +++ /dev/null @@ -1,64 +0,0 @@ -package proxy - -import ( - "context" - "io" - "net" - "sync" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/client" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/wrappers" -) - -func NewProxyDirect(conf *config.Config) *Proxy { - tg := telegram.NewDirectTelegram(conf) - - return &Proxy{ - conf: conf, - acceptCallback: func(ctx context.Context, cancel context.CancelFunc, clientSocket net.Conn, - connID string, wait *sync.WaitGroup, conf *config.Config) (io.Closer, io.Closer, error) { - client, opts, err := client.DirectInit(ctx, cancel, clientSocket, connID, conf) - if err != nil { - return nil, nil, errors.Annotate(err, "Cannot initialize client connection") - } - - server, err := directTelegramStream(ctx, cancel, opts, connID, tg) - if err != nil { - return client, nil, errors.Annotate(err, "Cannot initialize telegram connection") - } - - wait.Add(2) - - go directPipe(client, server, wait) - go directPipe(server, client, wait) - - return client, server, nil - }, - } -} - -func directTelegramStream(ctx context.Context, cancel context.CancelFunc, opts *mtproto.ConnectionOpts, - connID string, tg *telegram.DirectTelegram) (wrappers.WrapStreamReadWriteCloser, error) { - streamConn, err := tg.Dial(connID, opts) - if err != nil { - return nil, errors.Annotate(err, "Cannot dial to Telegram") - } - streamConn = wrappers.NewCtx(ctx, cancel, streamConn) - - packetConn, err := tg.Init(opts, streamConn) - if err != nil { - return nil, errors.Annotate(err, "Cannot handshake telegram") - } - - return packetConn, nil -} - -func directPipe(src io.Reader, dst io.Writer, wait *sync.WaitGroup) { - defer wait.Done() - io.Copy(dst, src) -} diff --git a/proxy/middle.go b/proxy/middle.go deleted file mode 100644 index 6ab300b..0000000 --- a/proxy/middle.go +++ /dev/null @@ -1,76 +0,0 @@ -package proxy - -import ( - "context" - "io" - "net" - "sync" - - "github.com/juju/errors" - - "github.com/9seconds/mtg/client" - "github.com/9seconds/mtg/config" - "github.com/9seconds/mtg/mtproto" - "github.com/9seconds/mtg/telegram" - "github.com/9seconds/mtg/wrappers" -) - -func NewProxyMiddle(conf *config.Config) *Proxy { - tg := telegram.NewMiddleTelegram(conf) - - return &Proxy{ - conf: conf, - acceptCallback: func(ctx context.Context, cancel context.CancelFunc, clientSocket net.Conn, - connID string, wait *sync.WaitGroup, conf *config.Config) (io.Closer, io.Closer, error) { - client, opts, err := client.MiddleInit(ctx, cancel, clientSocket, connID, conf) - if err != nil { - return nil, nil, errors.Annotate(err, "Cannot initialize client connection") - } - - server, err := middleTelegramStream(ctx, cancel, opts, connID, tg) - if err != nil { - return client, nil, errors.Annotate(err, "Cannot initialize telegram connection") - } - - wait.Add(2) - - go middlePipe(client, server, wait, &opts.ReadHacks) - go middlePipe(server, client, wait, &opts.WriteHacks) - - return client, server, nil - }, - } -} - -func middleTelegramStream(ctx context.Context, cancel context.CancelFunc, opts *mtproto.ConnectionOpts, - connID string, tg *telegram.MiddleTelegram) (wrappers.WrapPacketReadWriteCloser, error) { - streamConn, err := tg.Dial(connID, opts) - if err != nil { - return nil, errors.Annotate(err, "Cannot dial to Telegram") - } - streamConn = wrappers.NewCtx(ctx, cancel, streamConn) - - packetConn, err := tg.Init(opts, streamConn) - if err != nil { - return nil, errors.Annotate(err, "Cannot handshake telegram") - } - - return packetConn, nil -} - -func middlePipe(src wrappers.WrapPacketReader, dst wrappers.WrapPacketWriter, wait *sync.WaitGroup, hacks *mtproto.Hacks) { - defer wait.Done() - - for { - hacks.SimpleAck = false - hacks.QuickAck = false - - packet, err := src.Read() - if err != nil { - return - } - if _, err = dst.Write(packet); err != nil { - return - } - } -} diff --git a/proxy/proxy.go b/proxy/proxy.go index 0debedd..182b980 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -10,14 +10,17 @@ import ( uuid "github.com/satori/go.uuid" "go.uber.org/zap" + "github.com/9seconds/mtg/client" "github.com/9seconds/mtg/config" + "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/telegram" + "github.com/9seconds/mtg/wrappers" ) -type proxyAcceptCallback func(context.Context, context.CancelFunc, net.Conn, string, *sync.WaitGroup, *config.Config) (io.Closer, io.Closer, error) - type Proxy struct { - conf *config.Config - acceptCallback proxyAcceptCallback + clientInit client.Init + tg telegram.Telegram + conf *config.Config } func (p *Proxy) Serve() error { @@ -50,20 +53,33 @@ func (p *Proxy) accept(conn net.Conn) { log.Infow("Client connected", "addr", conn.RemoteAddr()) ctx, cancel := context.WithCancel(context.Background()) - wait := &sync.WaitGroup{} - - client, server, err := p.acceptCallback(ctx, cancel, conn, connID, wait, p.conf) - defer func() { - if client != nil { - client.Close() - } - if server != nil { - server.Close() - } - }() + client, opts, err := p.clientInit(ctx, cancel, conn, connID, p.conf) if err != nil { - log.Errorw("Cannot initialize connection", "error", err) - cancel() + log.Errorw("Cannot initialize client connection", "error", err) + return + } + defer client.(wrappers.WrapCloser).Close() + + server, err := p.getTelegramConn(ctx, cancel, opts, connID) + if err != nil { + log.Errorw("Cannot initialize server connection", "error", err) + return + } + defer server.(wrappers.WrapCloser).Close() + + wait := &sync.WaitGroup{} + wait.Add(2) + + if p.conf.UseMiddleProxy() { + clientPacket := client.(wrappers.WrapPacketReadWriteCloser) + serverPacket := server.(wrappers.WrapPacketReadWriteCloser) + go p.middlePipe(clientPacket, serverPacket, wait, &opts.ReadHacks) + go p.middlePipe(serverPacket, clientPacket, wait, &opts.WriteHacks) + } else { + clientStream := client.(wrappers.WrapStreamReadWriteCloser) + serverStream := server.(wrappers.WrapStreamReadWriteCloser) + go p.directPipe(clientStream, serverStream, wait) + go p.directPipe(serverStream, clientStream, wait) } <-ctx.Done() @@ -71,3 +87,61 @@ func (p *Proxy) accept(conn net.Conn) { log.Infow("Client disconnected", "addr", conn.RemoteAddr()) } + +func (p *Proxy) getTelegramConn(ctx context.Context, cancel context.CancelFunc, opts *mtproto.ConnectionOpts, + connID string) (wrappers.Wrap, error) { + streamConn, err := p.tg.Dial(connID, opts) + if err != nil { + return nil, errors.Annotate(err, "Cannot dial to Telegram") + } + streamConn = wrappers.NewCtx(ctx, cancel, streamConn) + + packetConn, err := p.tg.Init(opts, streamConn) + if err != nil { + return nil, errors.Annotate(err, "Cannot handshake telegram") + } + + return packetConn, nil +} + +func (p *Proxy) middlePipe(src wrappers.WrapPacketReader, dst wrappers.WrapPacketWriter, wait *sync.WaitGroup, hacks *mtproto.Hacks) { + defer wait.Done() + + for { + hacks.SimpleAck = false + hacks.QuickAck = false + + packet, err := src.Read() + if err != nil { + return + } + if _, err = dst.Write(packet); err != nil { + return + } + } +} + +func (p *Proxy) directPipe(src io.Reader, dst io.Writer, wait *sync.WaitGroup) { + defer wait.Done() + io.Copy(dst, src) + +} + +func NewProxy(conf *config.Config) *Proxy { + var clientInit client.Init + var tg telegram.Telegram + + if conf.UseMiddleProxy() { + clientInit = client.MiddleInit + tg = telegram.NewMiddleTelegram(conf) + } else { + clientInit = client.DirectInit + tg = telegram.NewDirectTelegram(conf) + } + + return &Proxy{ + conf: conf, + clientInit: clientInit, + tg: tg, + } +} diff --git a/telegram/direct.go b/telegram/direct.go index 0d1dd28..23ca66c 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -43,7 +43,7 @@ func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) ( return t.baseTelegram.dial(dc-1, connID, connOpts.ConnectionProto) } -func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.WrapStreamReadWriteCloser, error) { +func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.Wrap, error) { obfs2, frame := obfuscated2.MakeTelegramObfuscated2Frame(connOpts) if _, err := conn.Write(frame); err != nil { @@ -55,7 +55,7 @@ func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Wr // NewDirectTelegram returns Telegram instance which connects directly // to Telegram bypassing middleproxies. -func NewDirectTelegram(conf *config.Config) *DirectTelegram { +func NewDirectTelegram(conf *config.Config) Telegram { return &DirectTelegram{baseTelegram{ dialer: tgDialer{ Dialer: net.Dialer{Timeout: telegramDialTimeout}, diff --git a/telegram/middle.go b/telegram/middle.go index 8a25996..5069e9a 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -19,32 +19,7 @@ type MiddleTelegram struct { conf *config.Config } -func NewMiddleTelegram(conf *config.Config) *MiddleTelegram { - tg := &MiddleTelegram{ - middleTelegramCaller: middleTelegramCaller{ - baseTelegram: baseTelegram{ - dialer: tgDialer{ - Dialer: net.Dialer{Timeout: telegramDialTimeout}, - conf: conf, - }, - }, - httpClient: &http.Client{ - Timeout: middleTelegramHTTPClientTimeout, - }, - dialerMutex: &sync.RWMutex{}, - }, - conf: conf, - } - - if err := tg.update(); err != nil { - panic(err) - } - go tg.autoUpdate() - - return tg -} - -func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.WrapPacketReadWriteCloser, error) { +func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.Wrap, error) { rpcNonceConn := wrappers.NewMTProtoFrame(conn, rpc.SeqNoNonce) rpcNonceReq, err := t.sendRPCNonceRequest(rpcNonceConn) @@ -125,3 +100,28 @@ func (t *MiddleTelegram) receiveRPCHandshakeResponse(conn wrappers.WrapPacketRea return rpcHandshakeResp, nil } + +func NewMiddleTelegram(conf *config.Config) Telegram { + tg := &MiddleTelegram{ + middleTelegramCaller: middleTelegramCaller{ + baseTelegram: baseTelegram{ + dialer: tgDialer{ + Dialer: net.Dialer{Timeout: telegramDialTimeout}, + conf: conf, + }, + }, + httpClient: &http.Client{ + Timeout: middleTelegramHTTPClientTimeout, + }, + dialerMutex: &sync.RWMutex{}, + }, + conf: conf, + } + + if err := tg.update(); err != nil { + panic(err) + } + go tg.autoUpdate() + + return tg +} diff --git a/telegram/telegram.go b/telegram/telegram.go index da8e27a..a24531e 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -9,6 +9,11 @@ import ( "github.com/9seconds/mtg/wrappers" ) +type Telegram interface { + Dial(string, *mtproto.ConnectionOpts) (wrappers.WrapStreamReadWriteCloser, error) + Init(*mtproto.ConnectionOpts, wrappers.WrapStreamReadWriteCloser) (wrappers.Wrap, error) +} + type baseTelegram struct { dialer tgDialer diff --git a/wrappers/wrap.go b/wrappers/wrap.go index 3d54bc1..7ebe04c 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -20,6 +20,11 @@ type WrapWriter interface { Wrap } +type WrapCloser interface { + io.Closer + Wrap +} + type WrapWriteCloser interface { io.Closer WrapWriter From 388f7a23d3c5e7bcb938e04889e0d8f3f0b775d0 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 08:14:43 +0300 Subject: [PATCH 49/71] Remove wrap prefix --- client/middle.go | 2 +- proxy/proxy.go | 15 ++++++------ telegram/dialer.go | 6 ++--- telegram/direct.go | 4 +-- telegram/middle.go | 10 ++++---- telegram/middle_caller.go | 8 ++---- telegram/telegram.go | 6 ++--- wrappers/blockcipher.go | 4 +-- wrappers/conn.go | 2 +- wrappers/ctx.go | 4 +-- wrappers/mtproto_abridged.go | 4 +-- wrappers/mtproto_cipher.go | 2 +- wrappers/mtproto_frame.go | 4 +-- wrappers/mtproto_intermediate.go | 4 +-- wrappers/mtproto_proxy.go | 4 +-- wrappers/streamcipher.go | 4 +-- wrappers/wrap.go | 42 ++++++++++++++++---------------- 17 files changed, 59 insertions(+), 66 deletions(-) diff --git a/client/middle.go b/client/middle.go index bbc7347..4f0be19 100644 --- a/client/middle.go +++ b/client/middle.go @@ -15,7 +15,7 @@ func MiddleInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, if err != nil { return nil, nil, err } - connStream := conn.(wrappers.WrapStreamReadWriteCloser) + connStream := conn.(wrappers.StreamReadWriteCloser) newConn := wrappers.NewMTProtoAbridged(connStream, opts) if opts.ConnectionType != mtproto.ConnectionTypeAbridged { diff --git a/proxy/proxy.go b/proxy/proxy.go index 182b980..f559f68 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -58,26 +58,26 @@ func (p *Proxy) accept(conn net.Conn) { log.Errorw("Cannot initialize client connection", "error", err) return } - defer client.(wrappers.WrapCloser).Close() + defer client.(io.Closer).Close() server, err := p.getTelegramConn(ctx, cancel, opts, connID) if err != nil { log.Errorw("Cannot initialize server connection", "error", err) return } - defer server.(wrappers.WrapCloser).Close() + defer server.(io.Closer).Close() wait := &sync.WaitGroup{} wait.Add(2) if p.conf.UseMiddleProxy() { - clientPacket := client.(wrappers.WrapPacketReadWriteCloser) - serverPacket := server.(wrappers.WrapPacketReadWriteCloser) + clientPacket := client.(wrappers.PacketReadWriteCloser) + serverPacket := server.(wrappers.PacketReadWriteCloser) go p.middlePipe(clientPacket, serverPacket, wait, &opts.ReadHacks) go p.middlePipe(serverPacket, clientPacket, wait, &opts.WriteHacks) } else { - clientStream := client.(wrappers.WrapStreamReadWriteCloser) - serverStream := server.(wrappers.WrapStreamReadWriteCloser) + clientStream := client.(wrappers.StreamReadWriteCloser) + serverStream := server.(wrappers.StreamReadWriteCloser) go p.directPipe(clientStream, serverStream, wait) go p.directPipe(serverStream, clientStream, wait) } @@ -104,9 +104,8 @@ func (p *Proxy) getTelegramConn(ctx context.Context, cancel context.CancelFunc, return packetConn, nil } -func (p *Proxy) middlePipe(src wrappers.WrapPacketReader, dst wrappers.WrapPacketWriter, wait *sync.WaitGroup, hacks *mtproto.Hacks) { +func (p *Proxy) middlePipe(src wrappers.PacketReader, dst wrappers.PacketWriter, wait *sync.WaitGroup, hacks *mtproto.Hacks) { defer wait.Done() - for { hacks.SimpleAck = false hacks.QuickAck = false diff --git a/telegram/dialer.go b/telegram/dialer.go index d710f38..fc70d64 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -30,14 +30,12 @@ func (t *tgDialer) dial(addr string) (net.Conn, error) { return conn, nil } -func (t *tgDialer) dialRWC(addr, connID string) (wrappers.WrapStreamReadWriteCloser, error) { +func (t *tgDialer) dialRWC(addr, connID string) (wrappers.StreamReadWriteCloser, error) { conn, err := t.dial(addr) if err != nil { return nil, err } - - tgConn := wrappers.NewConn(conn, connID, wrappers.ConnPurposeTelegram, - t.conf.PublicIPv4, t.conf.PublicIPv6) + tgConn := wrappers.NewConn(conn, connID, wrappers.ConnPurposeTelegram, t.conf.PublicIPv4, t.conf.PublicIPv6) return tgConn, nil } diff --git a/telegram/direct.go b/telegram/direct.go index 23ca66c..749aa07 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -32,7 +32,7 @@ type DirectTelegram struct { baseTelegram } -func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.WrapStreamReadWriteCloser, error) { +func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.StreamReadWriteCloser, error) { dc := connOpts.DC if dc < 0 { dc = -dc @@ -43,7 +43,7 @@ func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) ( return t.baseTelegram.dial(dc-1, connID, connOpts.ConnectionProto) } -func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.Wrap, error) { +func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) { obfs2, frame := obfuscated2.MakeTelegramObfuscated2Frame(connOpts) if _, err := conn.Write(frame); err != nil { diff --git a/telegram/middle.go b/telegram/middle.go index 5069e9a..b800860 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -19,7 +19,7 @@ type MiddleTelegram struct { conf *config.Config } -func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.WrapStreamReadWriteCloser) (wrappers.Wrap, error) { +func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) { rpcNonceConn := wrappers.NewMTProtoFrame(conn, rpc.SeqNoNonce) rpcNonceReq, err := t.sendRPCNonceRequest(rpcNonceConn) @@ -46,7 +46,7 @@ func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.Wr return wrappers.NewMTProtoProxy(frameConn, connOpts, t.conf.AdTag) } -func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.WrapPacketWriter) (*rpc.NonceRequest, error) { +func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.PacketWriter) (*rpc.NonceRequest, error) { rpcNonceReq, err := rpc.NewNonceRequest(t.proxySecret) if err != nil { return nil, errors.Annotate(err, "Cannot create RPC nonce request") @@ -58,7 +58,7 @@ func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.WrapPacketWriter) (*r return rpcNonceReq, nil } -func (t *MiddleTelegram) receiveRPCNonceResponse(conn wrappers.WrapPacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { +func (t *MiddleTelegram) receiveRPCNonceResponse(conn wrappers.PacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { packet, err := conn.Read() if err != nil { return nil, errors.Annotate(err, "Cannot read RPC nonce response") @@ -75,7 +75,7 @@ func (t *MiddleTelegram) receiveRPCNonceResponse(conn wrappers.WrapPacketReader, return rpcNonceResp, nil } -func (t *MiddleTelegram) sendRPCHandshakeRequest(conn wrappers.WrapPacketWriter) (*rpc.HandshakeRequest, error) { +func (t *MiddleTelegram) sendRPCHandshakeRequest(conn wrappers.PacketWriter) (*rpc.HandshakeRequest, error) { req := rpc.NewHandshakeRequest() if _, err := conn.Write(req.Bytes()); err != nil { return nil, errors.Annotate(err, "Cannot send RPC handshake request") @@ -84,7 +84,7 @@ func (t *MiddleTelegram) sendRPCHandshakeRequest(conn wrappers.WrapPacketWriter) return req, nil } -func (t *MiddleTelegram) receiveRPCHandshakeResponse(conn wrappers.WrapPacketReader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { +func (t *MiddleTelegram) receiveRPCHandshakeResponse(conn wrappers.PacketReader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { packet, err := conn.Read() if err != nil { return nil, errors.Annotate(err, "Cannot read RPC handshake response") diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go index c0d40a2..832430a 100644 --- a/telegram/middle_caller.go +++ b/telegram/middle_caller.go @@ -28,7 +28,7 @@ const ( tgUserAgent = "mtg" ) -var middleTelegramProxyConfigSplitter *regexp.Regexp +var middleTelegramProxyConfigSplitter = regexp.MustCompile(`\s+`) type middleTelegramCaller struct { baseTelegram @@ -38,7 +38,7 @@ type middleTelegramCaller struct { httpClient *http.Client } -func (t *middleTelegramCaller) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.WrapStreamReadWriteCloser, error) { +func (t *middleTelegramCaller) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.StreamReadWriteCloser, error) { dc := connOpts.DC if dc == 0 { dc = 1 @@ -150,7 +150,3 @@ func (t *middleTelegramCaller) call(url string) (*http.Response, error) { return t.httpClient.Do(req) } - -func init() { - middleTelegramProxyConfigSplitter = regexp.MustCompile(`\s+`) -} diff --git a/telegram/telegram.go b/telegram/telegram.go index a24531e..43f436d 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -10,8 +10,8 @@ import ( ) type Telegram interface { - Dial(string, *mtproto.ConnectionOpts) (wrappers.WrapStreamReadWriteCloser, error) - Init(*mtproto.ConnectionOpts, wrappers.WrapStreamReadWriteCloser) (wrappers.Wrap, error) + Dial(string, *mtproto.ConnectionOpts) (wrappers.StreamReadWriteCloser, error) + Init(*mtproto.ConnectionOpts, wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) } type baseTelegram struct { @@ -21,7 +21,7 @@ type baseTelegram struct { v6Addresses map[int16][]string } -func (b *baseTelegram) dial(dcIdx int16, connID string, proto mtproto.ConnectionProtocol) (wrappers.WrapStreamReadWriteCloser, error) { +func (b *baseTelegram) dial(dcIdx int16, connID string, proto mtproto.ConnectionProtocol) (wrappers.StreamReadWriteCloser, error) { addrs := make([]string, 2) if proto&mtproto.ConnectionProtocolIPv6 != 0 { diff --git a/wrappers/blockcipher.go b/wrappers/blockcipher.go index 5283f90..b4348c2 100644 --- a/wrappers/blockcipher.go +++ b/wrappers/blockcipher.go @@ -13,7 +13,7 @@ import ( type BlockCipher struct { buf *bytes.Buffer - conn WrapStreamReadWriteCloser + conn StreamReadWriteCloser encryptor cipher.BlockMode decryptor cipher.BlockMode } @@ -88,7 +88,7 @@ func (b *BlockCipher) Close() error { return b.conn.Close() } -func NewBlockCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.BlockMode) WrapStreamReadWriteCloser { +func NewBlockCipher(conn StreamReadWriteCloser, encryptor, decryptor cipher.BlockMode) StreamReadWriteCloser { return &BlockCipher{ buf: &bytes.Buffer{}, conn: conn, diff --git a/wrappers/conn.go b/wrappers/conn.go index 2a84941..f5484c6 100644 --- a/wrappers/conn.go +++ b/wrappers/conn.go @@ -96,7 +96,7 @@ func (c *Conn) LogError(msg string, data ...interface{}) { c.logger.Errorw(msg, data...) } -func NewConn(conn net.Conn, connID string, purpose ConnPurpose, publicIPv4, publicIPv6 net.IP) WrapStreamReadWriteCloser { +func NewConn(conn net.Conn, connID string, purpose ConnPurpose, publicIPv4, publicIPv6 net.IP) StreamReadWriteCloser { logger := zap.S().With( "connection_id", connID, "local_address", conn.LocalAddr(), diff --git a/wrappers/ctx.go b/wrappers/ctx.go index bc38211..0ea4af4 100644 --- a/wrappers/ctx.go +++ b/wrappers/ctx.go @@ -9,7 +9,7 @@ import ( type Ctx struct { cancel context.CancelFunc - conn WrapStreamReadWriteCloser + conn StreamReadWriteCloser ctx context.Context } @@ -67,7 +67,7 @@ func (c *Ctx) Close() error { return c.conn.Close() } -func NewCtx(ctx context.Context, cancel context.CancelFunc, conn WrapStreamReadWriteCloser) WrapStreamReadWriteCloser { +func NewCtx(ctx context.Context, cancel context.CancelFunc, conn StreamReadWriteCloser) StreamReadWriteCloser { return &Ctx{ ctx: ctx, cancel: cancel, diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index f1c1a5c..a844452 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -18,7 +18,7 @@ const ( ) type MTProtoAbridged struct { - conn WrapStreamReadWriteCloser + conn StreamReadWriteCloser opts *mtproto.ConnectionOpts readCounter uint32 @@ -156,7 +156,7 @@ func (m *MTProtoAbridged) Close() error { return m.conn.Close() } -func NewMTProtoAbridged(conn WrapStreamReadWriteCloser, opts *mtproto.ConnectionOpts) WrapPacketReadWriteCloser { +func NewMTProtoAbridged(conn StreamReadWriteCloser, opts *mtproto.ConnectionOpts) PacketReadWriteCloser { return &MTProtoAbridged{ conn: conn, opts: opts, diff --git a/wrappers/mtproto_cipher.go b/wrappers/mtproto_cipher.go index 71170dc..72dad3f 100644 --- a/wrappers/mtproto_cipher.go +++ b/wrappers/mtproto_cipher.go @@ -22,7 +22,7 @@ const ( var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} -func NewMiddleProxyCipher(conn WrapStreamReadWriteCloser, req *rpc.NonceRequest, resp *rpc.NonceResponse, secret []byte) WrapStreamReadWriteCloser { +func NewMiddleProxyCipher(conn StreamReadWriteCloser, req *rpc.NonceRequest, resp *rpc.NonceResponse, secret []byte) StreamReadWriteCloser { localAddr := conn.LocalAddr() remoteAddr := conn.RemoteAddr() diff --git a/wrappers/mtproto_frame.go b/wrappers/mtproto_frame.go index d3eff6f..7ace6f8 100644 --- a/wrappers/mtproto_frame.go +++ b/wrappers/mtproto_frame.go @@ -20,7 +20,7 @@ const ( var mtprotoFramePadding = []byte{0x04, 0x00, 0x00, 0x00} type MTProtoFrame struct { - conn WrapStreamReadWriteCloser + conn StreamReadWriteCloser readSeqNo int32 writeSeqNo int32 } @@ -142,7 +142,7 @@ func (m *MTProtoFrame) Close() error { return m.conn.Close() } -func NewMTProtoFrame(conn WrapStreamReadWriteCloser, seqNo int32) WrapPacketReadWriteCloser { +func NewMTProtoFrame(conn StreamReadWriteCloser, seqNo int32) PacketReadWriteCloser { return &MTProtoFrame{ conn: conn, readSeqNo: seqNo, diff --git a/wrappers/mtproto_intermediate.go b/wrappers/mtproto_intermediate.go index 2605a71..d9ed753 100644 --- a/wrappers/mtproto_intermediate.go +++ b/wrappers/mtproto_intermediate.go @@ -13,7 +13,7 @@ import ( const mtprotoIntermediateQuickAckLength = 0x80000000 type MTProtoIntermediate struct { - conn WrapStreamReadWriteCloser + conn StreamReadWriteCloser opts *mtproto.ConnectionOpts readCounter uint32 @@ -111,7 +111,7 @@ func (m *MTProtoIntermediate) Close() error { return m.conn.Close() } -func NewMTProtoIntermediate(conn WrapStreamReadWriteCloser, opts *mtproto.ConnectionOpts) WrapPacketReadWriteCloser { +func NewMTProtoIntermediate(conn StreamReadWriteCloser, opts *mtproto.ConnectionOpts) PacketReadWriteCloser { return &MTProtoIntermediate{ conn: conn, opts: opts, diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index c8f1796..ea2552d 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -11,7 +11,7 @@ import ( ) type MTProtoProxy struct { - conn WrapPacketReadWriteCloser + conn PacketReadWriteCloser req *rpc.ProxyRequest readCounter uint32 @@ -129,7 +129,7 @@ func (m *MTProtoProxy) Close() error { return m.conn.Close() } -func NewMTProtoProxy(conn WrapPacketReadWriteCloser, connOpts *mtproto.ConnectionOpts, adTag []byte) (WrapPacketReadWriteCloser, error) { +func NewMTProtoProxy(conn PacketReadWriteCloser, connOpts *mtproto.ConnectionOpts, adTag []byte) (PacketReadWriteCloser, error) { req, err := rpc.NewProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) if err != nil { return nil, errors.Annotate(err, "Cannot create new RPC proxy request") diff --git a/wrappers/streamcipher.go b/wrappers/streamcipher.go index 1057d33..f7c6376 100644 --- a/wrappers/streamcipher.go +++ b/wrappers/streamcipher.go @@ -10,7 +10,7 @@ import ( type StreamCipher struct { encryptor cipher.Stream decryptor cipher.Stream - conn WrapStreamReadWriteCloser + conn StreamReadWriteCloser } func (s *StreamCipher) Read(p []byte) (int, error) { @@ -58,7 +58,7 @@ func (s *StreamCipher) Close() error { return s.conn.Close() } -func NewStreamCipher(conn WrapStreamReadWriteCloser, encryptor, decryptor cipher.Stream) WrapStreamReadWriteCloser { +func NewStreamCipher(conn StreamReadWriteCloser, encryptor, decryptor cipher.Stream) StreamReadWriteCloser { return &StreamCipher{ conn: conn, encryptor: encryptor, diff --git a/wrappers/wrap.go b/wrappers/wrap.go index 7ebe04c..05ba8ea 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -15,74 +15,74 @@ type Wrap interface { RemoteAddr() *net.TCPAddr } -type WrapWriter interface { +type Writer interface { io.Writer Wrap } -type WrapCloser interface { +type Closer interface { io.Closer Wrap } -type WrapWriteCloser interface { +type WriteCloser interface { io.Closer - WrapWriter + Writer } -type WrapStreamReader interface { +type StreamReader interface { io.Reader Wrap } -type WrapStreamReadCloser interface { +type StreamReadCloser interface { io.Closer - WrapStreamReader + StreamReader } -type WrapStreamReadWriter interface { +type StreamReadWriter interface { io.Writer - WrapStreamReader + StreamReader } -type WrapStreamWriteCloser interface { +type StreamWriteCloser interface { io.Closer io.Writer Wrap } -type WrapStreamReadWriteCloser interface { +type StreamReadWriteCloser interface { io.Closer - WrapStreamReadWriter + StreamReadWriter } -type WrapPacketReader interface { +type PacketReader interface { Read() ([]byte, error) Wrap } -type WrapPacketWriter interface { +type PacketWriter interface { io.Writer Wrap } -type WrapPacketReadWriter interface { +type PacketReadWriter interface { io.Writer - WrapPacketReader + PacketReader } -type WrapBlockReadCloser interface { +type BlockReadCloser interface { io.Closer - WrapPacketReader + PacketReader } -type WrapPacketWriteCloser interface { +type PacketWriteCloser interface { io.Writer io.Closer Wrap } -type WrapPacketReadWriteCloser interface { +type PacketReadWriteCloser interface { io.Closer - WrapPacketReadWriter + PacketReadWriter } From 4fc6b074480dd35cebfe844e66cd79940c9f1360 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 08:30:08 +0300 Subject: [PATCH 50/71] Fix logging --- wrappers/mtproto_abridged.go | 8 ++++---- wrappers/mtproto_frame.go | 4 ++++ wrappers/mtproto_intermediate.go | 8 ++++---- wrappers/mtproto_proxy.go | 8 ++++---- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index a844452..5a3803d 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -125,22 +125,22 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { } func (m *MTProtoAbridged) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}) + data = append(data, []interface{}{"type", "abridged"}...) m.conn.LogDebug(msg, data...) } func (m *MTProtoAbridged) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}) + data = append(data, []interface{}{"type", "abridged"}...) m.conn.LogInfo(msg, data...) } func (m *MTProtoAbridged) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}) + data = append(data, []interface{}{"type", "abridged"}...) m.conn.LogWarn(msg, data...) } func (m *MTProtoAbridged) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}) + data = append(data, []interface{}{"type", "abridged"}...) m.conn.LogError(msg, data...) } diff --git a/wrappers/mtproto_frame.go b/wrappers/mtproto_frame.go index 7ace6f8..ad54941 100644 --- a/wrappers/mtproto_frame.go +++ b/wrappers/mtproto_frame.go @@ -115,18 +115,22 @@ func (m *MTProtoFrame) Write(p []byte) (int, error) { } func (m *MTProtoFrame) LogDebug(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "frame"}...) m.conn.LogDebug(msg, data...) } func (m *MTProtoFrame) LogInfo(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "frame"}...) m.conn.LogInfo(msg, data...) } func (m *MTProtoFrame) LogWarn(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "frame"}...) m.conn.LogWarn(msg, data...) } func (m *MTProtoFrame) LogError(msg string, data ...interface{}) { + data = append(data, []interface{}{"type", "frame"}...) m.conn.LogError(msg, data...) } diff --git a/wrappers/mtproto_intermediate.go b/wrappers/mtproto_intermediate.go index d9ed753..9ef9673 100644 --- a/wrappers/mtproto_intermediate.go +++ b/wrappers/mtproto_intermediate.go @@ -80,22 +80,22 @@ func (m *MTProtoIntermediate) Write(p []byte) (int, error) { } func (m *MTProtoIntermediate) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}) + data = append(data, []interface{}{"type", "intermediate"}...) m.conn.LogDebug(msg, data...) } func (m *MTProtoIntermediate) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}) + data = append(data, []interface{}{"type", "intermediate"}...) m.conn.LogInfo(msg, data...) } func (m *MTProtoIntermediate) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}) + data = append(data, []interface{}{"type", "intermediate"}...) m.conn.LogWarn(msg, data...) } func (m *MTProtoIntermediate) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}) + data = append(data, []interface{}{"type", "intermediate"}...) m.conn.LogError(msg, data...) } diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index ea2552d..b12e00e 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -98,22 +98,22 @@ func (m *MTProtoProxy) Write(p []byte) (int, error) { } func (m *MTProtoProxy) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}) + data = append(data, []interface{}{"type", "proxy"}...) m.conn.LogDebug(msg, data...) } func (m *MTProtoProxy) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}) + data = append(data, []interface{}{"type", "proxy"}...) m.conn.LogInfo(msg, data...) } func (m *MTProtoProxy) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}) + data = append(data, []interface{}{"type", "proxy"}...) m.conn.LogWarn(msg, data...) } func (m *MTProtoProxy) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}) + data = append(data, []interface{}{"type", "proxy"}...) m.conn.LogError(msg, data...) } From 1fe3319988ca88ebfa47be8562db8469b3930ebc Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 12:10:27 +0300 Subject: [PATCH 51/71] Remove contexts --- client/client.go | 3 +- client/direct.go | 5 +--- client/middle.go | 6 ++-- proxy/proxy.go | 30 ++++++++++--------- wrappers/ctx.go | 76 ------------------------------------------------ wrappers/wrap.go | 2 +- 6 files changed, 22 insertions(+), 100 deletions(-) delete mode 100644 wrappers/ctx.go diff --git a/client/client.go b/client/client.go index 379ead8..648fec2 100644 --- a/client/client.go +++ b/client/client.go @@ -1,7 +1,6 @@ package client import ( - "context" "net" "github.com/9seconds/mtg/config" @@ -9,4 +8,4 @@ import ( "github.com/9seconds/mtg/wrappers" ) -type Init func(context.Context, context.CancelFunc, net.Conn, string, *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) +type Init func(net.Conn, string, *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) diff --git a/client/direct.go b/client/direct.go index 09c0178..2c16436 100644 --- a/client/direct.go +++ b/client/direct.go @@ -1,7 +1,6 @@ package client import ( - "context" "net" "time" @@ -15,8 +14,7 @@ import ( const handshakeTimeout = 10 * time.Second -func DirectInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connID string, - conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { +func DirectInit(socket net.Conn, connID string, conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { if err := config.SetSocketOptions(socket); err != nil { return nil, nil, errors.Annotate(err, "Cannot set socket options") } @@ -37,7 +35,6 @@ func DirectInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connOpts.ClientAddr = conn.RemoteAddr() conn = wrappers.NewStreamCipher(conn, obfs2.Encryptor, obfs2.Decryptor) - conn = wrappers.NewCtx(ctx, cancel, conn) return conn, connOpts, nil } diff --git a/client/middle.go b/client/middle.go index 4f0be19..b85deef 100644 --- a/client/middle.go +++ b/client/middle.go @@ -1,7 +1,6 @@ package client import ( - "context" "net" "github.com/9seconds/mtg/config" @@ -9,9 +8,8 @@ import ( "github.com/9seconds/mtg/wrappers" ) -func MiddleInit(ctx context.Context, cancel context.CancelFunc, socket net.Conn, connID string, - conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { - conn, opts, err := DirectInit(ctx, cancel, socket, connID, conf) +func MiddleInit(socket net.Conn, connID string, conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { + conn, opts, err := DirectInit(socket, connID, conf) if err != nil { return nil, nil, err } diff --git a/proxy/proxy.go b/proxy/proxy.go index f559f68..d11fc02 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -1,7 +1,6 @@ package proxy import ( - "context" "io" "net" "sync" @@ -52,15 +51,14 @@ func (p *Proxy) accept(conn net.Conn) { log.Infow("Client connected", "addr", conn.RemoteAddr()) - ctx, cancel := context.WithCancel(context.Background()) - client, opts, err := p.clientInit(ctx, cancel, conn, connID, p.conf) + client, opts, err := p.clientInit(conn, connID, p.conf) if err != nil { log.Errorw("Cannot initialize client connection", "error", err) return } defer client.(io.Closer).Close() - server, err := p.getTelegramConn(ctx, cancel, opts, connID) + server, err := p.getTelegramConn(opts, connID) if err != nil { log.Errorw("Cannot initialize server connection", "error", err) return @@ -82,19 +80,16 @@ func (p *Proxy) accept(conn net.Conn) { go p.directPipe(serverStream, clientStream, wait) } - <-ctx.Done() wait.Wait() log.Infow("Client disconnected", "addr", conn.RemoteAddr()) } -func (p *Proxy) getTelegramConn(ctx context.Context, cancel context.CancelFunc, opts *mtproto.ConnectionOpts, - connID string) (wrappers.Wrap, error) { +func (p *Proxy) getTelegramConn(opts *mtproto.ConnectionOpts, connID string) (wrappers.Wrap, error) { streamConn, err := p.tg.Dial(connID, opts) if err != nil { return nil, errors.Annotate(err, "Cannot dial to Telegram") } - streamConn = wrappers.NewCtx(ctx, cancel, streamConn) packetConn, err := p.tg.Init(opts, streamConn) if err != nil { @@ -104,8 +99,13 @@ func (p *Proxy) getTelegramConn(ctx context.Context, cancel context.CancelFunc, return packetConn, nil } -func (p *Proxy) middlePipe(src wrappers.PacketReader, dst wrappers.PacketWriter, wait *sync.WaitGroup, hacks *mtproto.Hacks) { - defer wait.Done() +func (p *Proxy) middlePipe(src wrappers.PacketReadCloser, dst wrappers.PacketWriteCloser, wait *sync.WaitGroup, hacks *mtproto.Hacks) { + defer func() { + src.Close() + dst.Close() + wait.Done() + }() + for { hacks.SimpleAck = false hacks.QuickAck = false @@ -120,10 +120,14 @@ func (p *Proxy) middlePipe(src wrappers.PacketReader, dst wrappers.PacketWriter, } } -func (p *Proxy) directPipe(src io.Reader, dst io.Writer, wait *sync.WaitGroup) { - defer wait.Done() - io.Copy(dst, src) +func (p *Proxy) directPipe(src io.ReadCloser, dst io.WriteCloser, wait *sync.WaitGroup) { + defer func() { + src.Close() + dst.Close() + wait.Done() + }() + io.Copy(dst, src) } func NewProxy(conf *config.Config) *Proxy { diff --git a/wrappers/ctx.go b/wrappers/ctx.go deleted file mode 100644 index 0ea4af4..0000000 --- a/wrappers/ctx.go +++ /dev/null @@ -1,76 +0,0 @@ -package wrappers - -import ( - "context" - "net" - - "github.com/juju/errors" -) - -type Ctx struct { - cancel context.CancelFunc - conn StreamReadWriteCloser - ctx context.Context -} - -func (c *Ctx) Read(p []byte) (int, error) { - select { - case <-c.ctx.Done(): - return 0, errors.Annotate(c.ctx.Err(), "Read is failed because of closed context") - default: - n, err := c.conn.Read(p) - if err != nil { - c.cancel() - } - return n, err - } -} - -func (c *Ctx) Write(p []byte) (int, error) { - select { - case <-c.ctx.Done(): - return 0, errors.Annotate(c.ctx.Err(), "Write is failed because of closed context") - default: - n, err := c.conn.Write(p) - if err != nil { - c.cancel() - } - return n, err - } -} - -func (c *Ctx) LogDebug(msg string, data ...interface{}) { - c.conn.LogDebug(msg, data...) -} - -func (c *Ctx) LogInfo(msg string, data ...interface{}) { - c.conn.LogInfo(msg, data...) -} - -func (c *Ctx) LogWarn(msg string, data ...interface{}) { - c.conn.LogWarn(msg, data...) -} - -func (c *Ctx) LogError(msg string, data ...interface{}) { - c.conn.LogError(msg, data...) -} - -func (c *Ctx) LocalAddr() *net.TCPAddr { - return c.conn.LocalAddr() -} - -func (c *Ctx) RemoteAddr() *net.TCPAddr { - return c.conn.RemoteAddr() -} - -func (c *Ctx) Close() error { - return c.conn.Close() -} - -func NewCtx(ctx context.Context, cancel context.CancelFunc, conn StreamReadWriteCloser) StreamReadWriteCloser { - return &Ctx{ - ctx: ctx, - cancel: cancel, - conn: conn, - } -} diff --git a/wrappers/wrap.go b/wrappers/wrap.go index 05ba8ea..20cf7a0 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -71,7 +71,7 @@ type PacketReadWriter interface { PacketReader } -type BlockReadCloser interface { +type PacketReadCloser interface { io.Closer PacketReader } From 67151746b0f004ac031e4b0559f94467aa2f24c6 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 12:29:04 +0300 Subject: [PATCH 52/71] Rework logging --- wrappers/blockcipher.go | 20 +++++----------- wrappers/conn.go | 20 ++++------------ wrappers/mtproto_abridged.go | 39 +++++++++++-------------------- wrappers/mtproto_frame.go | 32 ++++++++----------------- wrappers/mtproto_intermediate.go | 40 +++++++++++--------------------- wrappers/mtproto_proxy.go | 39 +++++++++++-------------------- wrappers/streamcipher.go | 19 ++++----------- wrappers/wrap.go | 8 +++---- 8 files changed, 68 insertions(+), 149 deletions(-) diff --git a/wrappers/blockcipher.go b/wrappers/blockcipher.go index b4348c2..a4a7149 100644 --- a/wrappers/blockcipher.go +++ b/wrappers/blockcipher.go @@ -6,6 +6,8 @@ import ( "crypto/cipher" "net" + "go.uber.org/zap" + "github.com/9seconds/mtg/utils" "github.com/juju/errors" ) @@ -13,6 +15,7 @@ import ( type BlockCipher struct { buf *bytes.Buffer + logger *zap.SugaredLogger conn StreamReadWriteCloser encryptor cipher.BlockMode decryptor cipher.BlockMode @@ -60,20 +63,8 @@ func (b *BlockCipher) Write(p []byte) (int, error) { return b.conn.Write(encrypted) } -func (b *BlockCipher) LogDebug(msg string, data ...interface{}) { - b.conn.LogDebug(msg, data...) -} - -func (b *BlockCipher) LogInfo(msg string, data ...interface{}) { - b.conn.LogInfo(msg, data...) -} - -func (b *BlockCipher) LogWarn(msg string, data ...interface{}) { - b.conn.LogWarn(msg, data...) -} - -func (b *BlockCipher) LogError(msg string, data ...interface{}) { - b.conn.LogError(msg, data...) +func (b *BlockCipher) Logger() *zap.SugaredLogger { + return b.logger } func (b *BlockCipher) LocalAddr() *net.TCPAddr { @@ -92,6 +83,7 @@ func NewBlockCipher(conn StreamReadWriteCloser, encryptor, decryptor cipher.Bloc return &BlockCipher{ buf: &bytes.Buffer{}, conn: conn, + logger: conn.Logger().Named("block-cipher"), encryptor: encryptor, decryptor: decryptor, } diff --git a/wrappers/conn.go b/wrappers/conn.go index f5484c6..ad853af 100644 --- a/wrappers/conn.go +++ b/wrappers/conn.go @@ -57,7 +57,7 @@ func (c *Conn) Read(p []byte) (int, error) { } func (c *Conn) Close() error { - defer c.LogDebug("Closed connection") + defer c.logger.Debugw("Closed connection") return c.conn.Close() } @@ -80,20 +80,8 @@ func (c *Conn) RemoteAddr() *net.TCPAddr { return c.conn.RemoteAddr().(*net.TCPAddr) } -func (c *Conn) LogDebug(msg string, data ...interface{}) { - c.logger.Debugw(msg, data...) -} - -func (c *Conn) LogInfo(msg string, data ...interface{}) { - c.logger.Infow(msg, data...) -} - -func (c *Conn) LogWarn(msg string, data ...interface{}) { - c.logger.Warnw(msg, data...) -} - -func (c *Conn) LogError(msg string, data ...interface{}) { - c.logger.Errorw(msg, data...) +func (c *Conn) Logger() *zap.SugaredLogger { + return c.logger } func NewConn(conn net.Conn, connID string, purpose ConnPurpose, publicIPv4, publicIPv6 net.IP) StreamReadWriteCloser { @@ -102,7 +90,7 @@ func NewConn(conn net.Conn, connID string, purpose ConnPurpose, publicIPv4, publ "local_address", conn.LocalAddr(), "remote_address", conn.RemoteAddr(), "purpose", purpose, - ) + ).Named("conn") wrapper := Conn{ logger: logger, diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index 5a3803d..41f4e62 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -6,6 +6,7 @@ import ( "net" "github.com/juju/errors" + "go.uber.org/zap" "github.com/9seconds/mtg/mtproto" "github.com/9seconds/mtg/utils" @@ -18,15 +19,16 @@ const ( ) type MTProtoAbridged struct { - conn StreamReadWriteCloser - opts *mtproto.ConnectionOpts + conn StreamReadWriteCloser + opts *mtproto.ConnectionOpts + logger *zap.SugaredLogger readCounter uint32 writeCounter uint32 } func (m *MTProtoAbridged) Read() ([]byte, error) { - m.LogDebug("Read packet", + m.logger.Debugw("Read packet", "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, "counter", m.readCounter, @@ -41,7 +43,7 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { msgLength := uint8(buf.Bytes()[0]) buf.Reset() - m.LogDebug("Packet first byte", + m.logger.Debugw("Packet first byte", "byte", msgLength, "counter", m.readCounter, "simple_ack", m.opts.ReadHacks.SimpleAck, @@ -64,7 +66,7 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { } msgLength32 *= 4 - m.LogDebug("Packet length", + m.logger.Debugw("Packet length", "length", msgLength32, "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, @@ -83,7 +85,7 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { } func (m *MTProtoAbridged) Write(p []byte) (int, error) { - m.LogDebug("Write packet", + m.logger.Debugw("Write packet", "length", len(p), "simple_ack", m.opts.WriteHacks.SimpleAck, "quick_ack", m.opts.WriteHacks.QuickAck, @@ -124,24 +126,8 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { return 0, errors.Errorf("Packet is too big %d", len(p)) } -func (m *MTProtoAbridged) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}...) - m.conn.LogDebug(msg, data...) -} - -func (m *MTProtoAbridged) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}...) - m.conn.LogInfo(msg, data...) -} - -func (m *MTProtoAbridged) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}...) - m.conn.LogWarn(msg, data...) -} - -func (m *MTProtoAbridged) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "abridged"}...) - m.conn.LogError(msg, data...) +func (m *MTProtoAbridged) Logger() *zap.SugaredLogger { + return m.logger } func (m *MTProtoAbridged) LocalAddr() *net.TCPAddr { @@ -158,7 +144,8 @@ func (m *MTProtoAbridged) Close() error { func NewMTProtoAbridged(conn StreamReadWriteCloser, opts *mtproto.ConnectionOpts) PacketReadWriteCloser { return &MTProtoAbridged{ - conn: conn, - opts: opts, + conn: conn, + opts: opts, + logger: conn.Logger().Named("mtproto-abridged"), } } diff --git a/wrappers/mtproto_frame.go b/wrappers/mtproto_frame.go index ad54941..71ee084 100644 --- a/wrappers/mtproto_frame.go +++ b/wrappers/mtproto_frame.go @@ -10,6 +10,7 @@ import ( "net" "github.com/juju/errors" + "go.uber.org/zap" ) const ( @@ -20,7 +21,9 @@ const ( var mtprotoFramePadding = []byte{0x04, 0x00, 0x00, 0x00} type MTProtoFrame struct { - conn StreamReadWriteCloser + conn StreamReadWriteCloser + logger *zap.SugaredLogger + readSeqNo int32 writeSeqNo int32 } @@ -42,7 +45,7 @@ func (m *MTProtoFrame) Read() ([]byte, error) { } messageLength := binary.LittleEndian.Uint32(buf.Bytes()) - m.LogDebug("Read MTProto frame", + m.logger.Debugw("Read MTProto frame", "messageLength", messageLength, "sequence_number", m.readSeqNo, ) @@ -75,7 +78,7 @@ func (m *MTProtoFrame) Read() ([]byte, error) { return nil, errors.Errorf("CRC32 checksum mismatch. Wait for %d, got %d", sum.Sum32(), checksum) } - m.LogDebug("Read MTProto frame", + m.logger.Debugw("Read MTProto frame", "messageLength", messageLength, "sequence_number", m.readSeqNo, "dataLength", len(data), @@ -101,7 +104,7 @@ func (m *MTProtoFrame) Write(p []byte) (int, error) { binary.Write(buf, binary.LittleEndian, checksum) buf.Write(bytes.Repeat(mtprotoFramePadding, paddingLength/4)) - m.LogDebug("Write MTProto frame", + m.logger.Debugw("Write MTProto frame", "length", len(p), "sequence_number", m.writeSeqNo, "crc32", checksum, @@ -114,24 +117,8 @@ func (m *MTProtoFrame) Write(p []byte) (int, error) { return len(p), err } -func (m *MTProtoFrame) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "frame"}...) - m.conn.LogDebug(msg, data...) -} - -func (m *MTProtoFrame) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "frame"}...) - m.conn.LogInfo(msg, data...) -} - -func (m *MTProtoFrame) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "frame"}...) - m.conn.LogWarn(msg, data...) -} - -func (m *MTProtoFrame) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "frame"}...) - m.conn.LogError(msg, data...) +func (m *MTProtoFrame) Logger() *zap.SugaredLogger { + return m.logger } func (m *MTProtoFrame) LocalAddr() *net.TCPAddr { @@ -149,6 +136,7 @@ func (m *MTProtoFrame) Close() error { func NewMTProtoFrame(conn StreamReadWriteCloser, seqNo int32) PacketReadWriteCloser { return &MTProtoFrame{ conn: conn, + logger: conn.Logger().Named("mtproto-frame"), readSeqNo: seqNo, writeSeqNo: seqNo, } diff --git a/wrappers/mtproto_intermediate.go b/wrappers/mtproto_intermediate.go index 9ef9673..5ed8e00 100644 --- a/wrappers/mtproto_intermediate.go +++ b/wrappers/mtproto_intermediate.go @@ -6,22 +6,25 @@ import ( "io" "net" - "github.com/9seconds/mtg/mtproto" "github.com/juju/errors" + "go.uber.org/zap" + + "github.com/9seconds/mtg/mtproto" ) const mtprotoIntermediateQuickAckLength = 0x80000000 type MTProtoIntermediate struct { - conn StreamReadWriteCloser - opts *mtproto.ConnectionOpts + conn StreamReadWriteCloser + opts *mtproto.ConnectionOpts + logger *zap.SugaredLogger readCounter uint32 writeCounter uint32 } func (m *MTProtoIntermediate) Read() ([]byte, error) { - m.LogDebug("Read packet", + m.logger.Debugw("Read packet", "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, "counter", m.readCounter, @@ -35,7 +38,7 @@ func (m *MTProtoIntermediate) Read() ([]byte, error) { } length := binary.LittleEndian.Uint32(buf.Bytes()) - m.LogDebug("Packet message length", + m.logger.Debugw("Packet message length", "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, "counter", m.readCounter, @@ -62,7 +65,7 @@ func (m *MTProtoIntermediate) Read() ([]byte, error) { } func (m *MTProtoIntermediate) Write(p []byte) (int, error) { - m.LogDebug("Write packet", + m.logger.Debugw("Write packet", "simple_ack", m.opts.WriteHacks.SimpleAck, "quick_ack", m.opts.WriteHacks.QuickAck, "counter", m.writeCounter, @@ -79,24 +82,8 @@ func (m *MTProtoIntermediate) Write(p []byte) (int, error) { return m.conn.Write(append(length[:], p...)) } -func (m *MTProtoIntermediate) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}...) - m.conn.LogDebug(msg, data...) -} - -func (m *MTProtoIntermediate) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}...) - m.conn.LogInfo(msg, data...) -} - -func (m *MTProtoIntermediate) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}...) - m.conn.LogWarn(msg, data...) -} - -func (m *MTProtoIntermediate) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "intermediate"}...) - m.conn.LogError(msg, data...) +func (m *MTProtoIntermediate) Logger() *zap.SugaredLogger { + return m.logger } func (m *MTProtoIntermediate) LocalAddr() *net.TCPAddr { @@ -113,7 +100,8 @@ func (m *MTProtoIntermediate) Close() error { func NewMTProtoIntermediate(conn StreamReadWriteCloser, opts *mtproto.ConnectionOpts) PacketReadWriteCloser { return &MTProtoIntermediate{ - conn: conn, - opts: opts, + conn: conn, + logger: conn.Logger().Named("mtproto-intermediate"), + opts: opts, } } diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index b12e00e..d6ffcef 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -5,21 +5,23 @@ import ( "net" "github.com/juju/errors" + "go.uber.org/zap" "github.com/9seconds/mtg/mtproto" "github.com/9seconds/mtg/mtproto/rpc" ) type MTProtoProxy struct { - conn PacketReadWriteCloser - req *rpc.ProxyRequest + conn PacketReadWriteCloser + req *rpc.ProxyRequest + logger *zap.SugaredLogger readCounter uint32 writeCounter uint32 } func (m *MTProtoProxy) Read() ([]byte, error) { - m.LogDebug("Read packet", + m.logger.Debugw("Read packet", "counter", m.readCounter, "simple_ack", m.req.Options.WriteHacks.SimpleAck, "quick_ack", m.req.Options.WriteHacks.QuickAck, @@ -29,7 +31,7 @@ func (m *MTProtoProxy) Read() ([]byte, error) { if err != nil { return nil, errors.Annotate(err, "Cannot read packet") } - m.LogDebug("Read packet length", + m.logger.Debugw("Read packet length", "counter", m.readCounter, "simple_ack", m.req.Options.WriteHacks.SimpleAck, "quick_ack", m.req.Options.WriteHacks.QuickAck, @@ -41,7 +43,7 @@ func (m *MTProtoProxy) Read() ([]byte, error) { } tag, packet := packet[:4], packet[4:] - m.LogDebug("Read RPC tag", + m.logger.Debugw("Read RPC tag", "counter", m.readCounter, "simple_ack", m.req.Options.WriteHacks.SimpleAck, "quick_ack", m.req.Options.WriteHacks.QuickAck, @@ -82,7 +84,7 @@ func (m *MTProtoProxy) readCloseExt(data []byte) ([]byte, error) { } func (m *MTProtoProxy) Write(p []byte) (int, error) { - m.LogDebug("Write packet", + m.logger.Debugw("Write packet", "length", len(p), "counter", m.writeCounter, "simple_ack", m.req.Options.ReadHacks.SimpleAck, @@ -97,24 +99,8 @@ func (m *MTProtoProxy) Write(p []byte) (int, error) { return len(p), nil } -func (m *MTProtoProxy) LogDebug(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}...) - m.conn.LogDebug(msg, data...) -} - -func (m *MTProtoProxy) LogInfo(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}...) - m.conn.LogInfo(msg, data...) -} - -func (m *MTProtoProxy) LogWarn(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}...) - m.conn.LogWarn(msg, data...) -} - -func (m *MTProtoProxy) LogError(msg string, data ...interface{}) { - data = append(data, []interface{}{"type", "proxy"}...) - m.conn.LogError(msg, data...) +func (m *MTProtoProxy) Logger() *zap.SugaredLogger { + return m.logger } func (m *MTProtoProxy) LocalAddr() *net.TCPAddr { @@ -136,7 +122,8 @@ func NewMTProtoProxy(conn PacketReadWriteCloser, connOpts *mtproto.ConnectionOpt } return &MTProtoProxy{ - conn: conn, - req: req, + conn: conn, + logger: conn.Logger().Named("mtproto-proxy"), + req: req, }, nil } diff --git a/wrappers/streamcipher.go b/wrappers/streamcipher.go index f7c6376..da89535 100644 --- a/wrappers/streamcipher.go +++ b/wrappers/streamcipher.go @@ -5,12 +5,14 @@ import ( "net" "github.com/juju/errors" + "go.uber.org/zap" ) type StreamCipher struct { encryptor cipher.Stream decryptor cipher.Stream conn StreamReadWriteCloser + logger *zap.SugaredLogger } func (s *StreamCipher) Read(p []byte) (int, error) { @@ -30,20 +32,8 @@ func (s *StreamCipher) Write(p []byte) (int, error) { return s.conn.Write(encrypted) } -func (s *StreamCipher) LogDebug(msg string, data ...interface{}) { - s.conn.LogDebug(msg, data...) -} - -func (s *StreamCipher) LogInfo(msg string, data ...interface{}) { - s.conn.LogInfo(msg, data...) -} - -func (s *StreamCipher) LogWarn(msg string, data ...interface{}) { - s.conn.LogWarn(msg, data...) -} - -func (s *StreamCipher) LogError(msg string, data ...interface{}) { - s.conn.LogError(msg, data...) +func (s *StreamCipher) Logger() *zap.SugaredLogger { + return s.logger } func (s *StreamCipher) LocalAddr() *net.TCPAddr { @@ -61,6 +51,7 @@ func (s *StreamCipher) Close() error { func NewStreamCipher(conn StreamReadWriteCloser, encryptor, decryptor cipher.Stream) StreamReadWriteCloser { return &StreamCipher{ conn: conn, + logger: conn.Logger().Named("stream-cipher"), encryptor: encryptor, decryptor: decryptor, } diff --git a/wrappers/wrap.go b/wrappers/wrap.go index 20cf7a0..923bb20 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -3,14 +3,12 @@ package wrappers import ( "io" "net" + + "go.uber.org/zap" ) type Wrap interface { - LogDebug(msg string, data ...interface{}) - LogInfo(msg string, data ...interface{}) - LogWarn(msg string, data ...interface{}) - LogError(msg string, data ...interface{}) - + Logger() *zap.SugaredLogger LocalAddr() *net.TCPAddr RemoteAddr() *net.TCPAddr } From 6adc59249044bd8376857a90fb7d0fcbb35565de Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 12:37:43 +0300 Subject: [PATCH 53/71] Add logging for proxy tags --- wrappers/mtproto_proxy.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index d6ffcef..0a808a3 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -67,19 +67,27 @@ func (m *MTProtoProxy) readProxyAns(data []byte) ([]byte, error) { if len(data) < 12 { return nil, errors.Errorf("Incorrect data of proxy answer: %d", len(data)) } + data = data[12:] - return data[12:], nil + m.logger.Debugw("Read RPC_PROXY_ANS", "length", len(data)) + + return data, nil } func (m *MTProtoProxy) readSimpleAck(data []byte) ([]byte, error) { if len(data) != 12 { return nil, errors.Errorf("Incorrect data of simple ack: %d", len(data)) } + data = data[8:12] - return data[8:12], nil // 0:8 - connection id + m.logger.Debugw("Read RPC_SIMPLE_ACK", "length", len(data)) + + return data, nil } func (m *MTProtoProxy) readCloseExt(data []byte) ([]byte, error) { + m.logger.Debugw("Read RPC_CLOSE_EXT") + return nil, errors.New("Connection has been closed remotely by RPC call") } From 76e88edac342ffdfe41b53b5f11400c7b9308f5e Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 13:19:16 +0300 Subject: [PATCH 54/71] Send correct proxy header --- client/direct.go | 2 ++ mtproto/rpc/proxy_request.go | 17 ++++++++++++++--- telegram/middle.go | 8 +++++++- wrappers/mtproto_proxy.go | 15 ++++++++++++++- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/client/direct.go b/client/direct.go index 2c16436..9f0df10 100644 --- a/client/direct.go +++ b/client/direct.go @@ -36,5 +36,7 @@ func DirectInit(socket net.Conn, connID string, conf *config.Config) (wrappers.W conn = wrappers.NewStreamCipher(conn, obfs2.Encryptor, obfs2.Decryptor) + conn.Logger().Infow("Client connection initialized") + return conn, connOpts, nil } diff --git a/mtproto/rpc/proxy_request.go b/mtproto/rpc/proxy_request.go index e36a2ec..0895f1a 100644 --- a/mtproto/rpc/proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -20,8 +20,20 @@ type ProxyRequest struct { Options *mtproto.ConnectionOpts } -func (r *ProxyRequest) Bytes(message []byte) []byte { +func (r *ProxyRequest) MakeHeader(message []byte) *bytes.Buffer { + bufferLength := len(TagProxyRequest) + + 4 + // len(flags) + len(r.ConnectionID) + + len(r.ClientIPPort) + + len(r.OurIPPort) + + len(ProxyRequestExtraSize) + + len(ProxyRequestProxyTag) + + 1 + // len(AdTag) + len(r.ADTag) + bufferLength += bufferLength % 4 + buf := &bytes.Buffer{} + buf.Grow(bufferLength) flags := r.Flags if r.Options.ReadHacks.QuickAck { @@ -42,9 +54,8 @@ func (r *ProxyRequest) Bytes(message []byte) []byte { buf.WriteByte(byte(len(r.ADTag))) buf.Write(r.ADTag) buf.Write(make([]byte, (4-buf.Len()%4)%4)) - buf.Write(message) - return buf.Bytes() + return buf } func NewProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.ConnectionOpts, adTag []byte) (*ProxyRequest, error) { diff --git a/telegram/middle.go b/telegram/middle.go index b800860..7404e3d 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -43,7 +43,13 @@ func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.St return nil, err } - return wrappers.NewMTProtoProxy(frameConn, connOpts, t.conf.AdTag) + proxyConn, err := wrappers.NewMTProtoProxy(frameConn, connOpts, t.conf.AdTag) + if err != nil { + return nil, err + } + proxyConn.Logger().Infow("Telegram connection initialized") + + return proxyConn, nil } func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.PacketWriter) (*rpc.NonceRequest, error) { diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index 0a808a3..1a82eb4 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -2,6 +2,7 @@ package wrappers import ( "bytes" + "fmt" "net" "github.com/juju/errors" @@ -100,7 +101,19 @@ func (m *MTProtoProxy) Write(p []byte) (int, error) { ) m.writeCounter++ - if _, err := m.conn.Write(p); err != nil { + header := m.req.MakeHeader(p) + if ce := m.logger.Desugar().Check(zap.DebugLevel, "RPC_PROXY_REQ header"); ce != nil { + ce.Write( + zap.Int("length", len(p)), + zap.Uint32("counter", m.writeCounter), + zap.Bool("simple_ack", m.req.Options.ReadHacks.QuickAck), + zap.Bool("quick_ack", m.req.Options.ReadHacks.SimpleAck), + zap.String("header", fmt.Sprintf("%v", header.Bytes())), + ) + } + header.Write(p) + + if _, err := m.conn.Write(header.Bytes()); err != nil { return 0, err } From 474fb8a294e615108a4b98eb77983b40f96884d7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 13:23:46 +0300 Subject: [PATCH 55/71] More logging --- mtproto/rpc/handshake_request.go | 1 + wrappers/mtproto_proxy.go | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/mtproto/rpc/handshake_request.go b/mtproto/rpc/handshake_request.go index 262656f..924859d 100644 --- a/mtproto/rpc/handshake_request.go +++ b/mtproto/rpc/handshake_request.go @@ -7,6 +7,7 @@ type HandshakeRequest struct { func (r *HandshakeRequest) Bytes() []byte { buf := &bytes.Buffer{} + buf.Grow(len(TagHandshake) + len(HandshakeFlags) + len(HandshakeSenderPID) + len(HandshakePeerPID)) buf.Write(TagHandshake) buf.Write(HandshakeFlags) diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index 1a82eb4..d226cc8 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -32,6 +32,10 @@ func (m *MTProtoProxy) Read() ([]byte, error) { if err != nil { return nil, errors.Annotate(err, "Cannot read packet") } + defer func() { + m.readCounter++ + }() + m.logger.Debugw("Read packet length", "counter", m.readCounter, "simple_ack", m.req.Options.WriteHacks.SimpleAck, @@ -44,14 +48,6 @@ func (m *MTProtoProxy) Read() ([]byte, error) { } tag, packet := packet[:4], packet[4:] - m.logger.Debugw("Read RPC tag", - "counter", m.readCounter, - "simple_ack", m.req.Options.WriteHacks.SimpleAck, - "quick_ack", m.req.Options.WriteHacks.QuickAck, - "tag", tag, - ) - - m.readCounter++ switch { case bytes.Equal(tag, rpc.TagProxyAns): return m.readProxyAns(packet) @@ -70,7 +66,10 @@ func (m *MTProtoProxy) readProxyAns(data []byte) ([]byte, error) { } data = data[12:] - m.logger.Debugw("Read RPC_PROXY_ANS", "length", len(data)) + m.logger.Debugw("Read RPC_PROXY_ANS", + "counter", m.readCounter, + "length", len(data), + ) return data, nil } @@ -81,13 +80,16 @@ func (m *MTProtoProxy) readSimpleAck(data []byte) ([]byte, error) { } data = data[8:12] - m.logger.Debugw("Read RPC_SIMPLE_ACK", "length", len(data)) + m.logger.Debugw("Read RPC_SIMPLE_ACK", + "counter", m.readCounter, + "length", len(data), + ) return data, nil } func (m *MTProtoProxy) readCloseExt(data []byte) ([]byte, error) { - m.logger.Debugw("Read RPC_CLOSE_EXT") + m.logger.Debugw("Read RPC_CLOSE_EXT", "counter", m.readCounter) return nil, errors.New("Connection has been closed remotely by RPC call") } From a99c5b244295453b1806c4d2d1ebecb3e16375d3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 17:56:44 +0300 Subject: [PATCH 56/71] Defer counters --- wrappers/mtproto_abridged.go | 14 ++++++++------ wrappers/mtproto_intermediate.go | 10 ++++++++-- wrappers/mtproto_proxy.go | 12 ++++++++---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index 41f4e62..5ed40d6 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -28,6 +28,10 @@ type MTProtoAbridged struct { } func (m *MTProtoAbridged) Read() ([]byte, error) { + defer func() { + m.readCounter++ + }() + m.logger.Debugw("Read packet", "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, @@ -75,16 +79,18 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { buf.Reset() buf.Grow(int(msgLength32)) - if _, err := io.CopyN(buf, m.conn, int64(msgLength32)); err != nil { return nil, errors.Annotate(err, "Cannot read message") } - m.readCounter++ return buf.Bytes(), nil } func (m *MTProtoAbridged) Write(p []byte) (int, error) { + defer func() { + m.writeCounter++ + }() + m.logger.Debugw("Write packet", "length", len(p), "simple_ack", m.opts.WriteHacks.SimpleAck, @@ -97,7 +103,6 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { } if m.opts.WriteHacks.SimpleAck { - m.writeCounter++ return m.conn.Write(utils.ReverseBytes(p)) } @@ -105,8 +110,6 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { switch { case packetLength < mtprotoAbridgedSmallPacketLength: newData := append([]byte{byte(packetLength)}, p...) - - m.writeCounter++ return m.conn.Write(newData) case packetLength < mtprotoAbridgedLargePacketLength: @@ -119,7 +122,6 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { buf.Write(length24[:]) buf.Write(p) - m.writeCounter++ return m.conn.Write(buf.Bytes()) } diff --git a/wrappers/mtproto_intermediate.go b/wrappers/mtproto_intermediate.go index 5ed8e00..5c29c79 100644 --- a/wrappers/mtproto_intermediate.go +++ b/wrappers/mtproto_intermediate.go @@ -24,6 +24,10 @@ type MTProtoIntermediate struct { } func (m *MTProtoIntermediate) Read() ([]byte, error) { + defer func() { + m.readCounter++ + }() + m.logger.Debugw("Read packet", "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, @@ -59,18 +63,20 @@ func (m *MTProtoIntermediate) Read() ([]byte, error) { if length%4 != 0 { length -= length % 4 } - m.readCounter++ return buf.Bytes()[:length], nil } func (m *MTProtoIntermediate) Write(p []byte) (int, error) { + defer func() { + m.writeCounter++ + }() + m.logger.Debugw("Write packet", "simple_ack", m.opts.WriteHacks.SimpleAck, "quick_ack", m.opts.WriteHacks.QuickAck, "counter", m.writeCounter, ) - m.writeCounter++ if m.opts.ReadHacks.SimpleAck { return m.conn.Write(p) diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index d226cc8..c56e9a2 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -22,6 +22,10 @@ type MTProtoProxy struct { } func (m *MTProtoProxy) Read() ([]byte, error) { + defer func() { + m.readCounter++ + }() + m.logger.Debugw("Read packet", "counter", m.readCounter, "simple_ack", m.req.Options.WriteHacks.SimpleAck, @@ -32,9 +36,6 @@ func (m *MTProtoProxy) Read() ([]byte, error) { if err != nil { return nil, errors.Annotate(err, "Cannot read packet") } - defer func() { - m.readCounter++ - }() m.logger.Debugw("Read packet length", "counter", m.readCounter, @@ -95,13 +96,16 @@ func (m *MTProtoProxy) readCloseExt(data []byte) ([]byte, error) { } func (m *MTProtoProxy) Write(p []byte) (int, error) { + defer func() { + m.writeCounter++ + }() + m.logger.Debugw("Write packet", "length", len(p), "counter", m.writeCounter, "simple_ack", m.req.Options.ReadHacks.SimpleAck, "quick_ack", m.req.Options.ReadHacks.QuickAck, ) - m.writeCounter++ header := m.req.MakeHeader(p) if ce := m.logger.Desugar().Check(zap.DebugLevel, "RPC_PROXY_REQ header"); ce != nil { From 35b38fe55c9fe19e8cef03d56121c380cd63b23c Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 18:11:23 +0300 Subject: [PATCH 57/71] Debug ping --- proxy/proxy.go | 8 ++++++-- wrappers/mtproto_abridged.go | 2 +- wrappers/wrap.go | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index d11fc02..a8399eb 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -112,22 +112,26 @@ func (p *Proxy) middlePipe(src wrappers.PacketReadCloser, dst wrappers.PacketWri packet, err := src.Read() if err != nil { + src.Logger().Warnw("Cannot read packet", "error", err) return } if _, err = dst.Write(packet); err != nil { + src.Logger().Warnw("Cannot write packet", "error", err) return } } } -func (p *Proxy) directPipe(src io.ReadCloser, dst io.WriteCloser, wait *sync.WaitGroup) { +func (p *Proxy) directPipe(src wrappers.StreamReadCloser, dst wrappers.StreamWriteCloser, wait *sync.WaitGroup) { defer func() { src.Close() dst.Close() wait.Done() }() - io.Copy(dst, src) + if _, err := io.Copy(dst, src); err != nil { + src.Logger().Warnw("Cannot pump sockets", "error", err) + } } func NewProxy(conf *config.Config) *Proxy { diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index 5ed40d6..e90233c 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -98,7 +98,7 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { "counter", m.writeCounter, ) - if len(p)%4 == 0 { + if len(p)%4 != 0 { return 0, errors.Errorf("Incorrect packet length %d", len(p)) } diff --git a/wrappers/wrap.go b/wrappers/wrap.go index 923bb20..2023694 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -44,8 +44,7 @@ type StreamReadWriter interface { } type StreamWriteCloser interface { - io.Closer - io.Writer + io.WriteCloser Wrap } From 8b178e2340eec48ebf6aebb050bfb5b0670ca358 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 18:20:21 +0300 Subject: [PATCH 58/71] Grow correct buffer size --- mtproto/rpc/proxy_request.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mtproto/rpc/proxy_request.go b/mtproto/rpc/proxy_request.go index 0895f1a..c8666be 100644 --- a/mtproto/rpc/proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -33,7 +33,7 @@ func (r *ProxyRequest) MakeHeader(message []byte) *bytes.Buffer { bufferLength += bufferLength % 4 buf := &bytes.Buffer{} - buf.Grow(bufferLength) + buf.Grow(bufferLength + len(message)) flags := r.Flags if r.Options.ReadHacks.QuickAck { From 6ea4f3dbc2b6de1f5fe1c89439ec724d7c049329 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Sun, 8 Jul 2018 18:43:27 +0300 Subject: [PATCH 59/71] Log proxt req header flags --- mtproto/rpc/proxy_flags.go | 33 ++++++++++++++++++++++++++++++++- mtproto/rpc/proxy_request.go | 5 +++-- wrappers/mtproto_proxy.go | 3 ++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/mtproto/rpc/proxy_flags.go b/mtproto/rpc/proxy_flags.go index 0ac3c30..5599537 100644 --- a/mtproto/rpc/proxy_flags.go +++ b/mtproto/rpc/proxy_flags.go @@ -1,6 +1,9 @@ package rpc -import "encoding/binary" +import ( + "encoding/binary" + "strings" +) type proxyRequestFlags uint32 @@ -22,3 +25,31 @@ func (r proxyRequestFlags) Bytes() []byte { return converted } + +func (r proxyRequestFlags) String() string { + flags := make([]string, 0, 7) + + if r&proxyRequestFlagsHasAdTag != 0 { + flags = append(flags, "HAS_AD_TAG") + } + if r&proxyRequestFlagsEncrypted != 0 { + flags = append(flags, "ENCRYPTED") + } + if r&proxyRequestFlagsMagic != 0 { + flags = append(flags, "MAGIC") + } + if r&proxyRequestFlagsExtMode2 != 0 { + flags = append(flags, "EXT_MODE_2") + } + if r&proxyRequestFlagsIntermediate != 0 { + flags = append(flags, "INTERMEDIATE") + } + if r&proxyRequestFlagsAbdridged != 0 { + flags = append(flags, "ABRIDGED") + } + if r&proxyRequestFlagsQuickAck != 0 { + flags = append(flags, "QUICK_ACK") + } + + return strings.Join(flags, " | ") +} diff --git a/mtproto/rpc/proxy_request.go b/mtproto/rpc/proxy_request.go index c8666be..40ab3cf 100644 --- a/mtproto/rpc/proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/rand" "encoding/binary" + "fmt" "net" "github.com/juju/errors" @@ -20,7 +21,7 @@ type ProxyRequest struct { Options *mtproto.ConnectionOpts } -func (r *ProxyRequest) MakeHeader(message []byte) *bytes.Buffer { +func (r *ProxyRequest) MakeHeader(message []byte) (*bytes.Buffer, fmt.Stringer) { bufferLength := len(TagProxyRequest) + 4 + // len(flags) len(r.ConnectionID) + @@ -55,7 +56,7 @@ func (r *ProxyRequest) MakeHeader(message []byte) *bytes.Buffer { buf.Write(r.ADTag) buf.Write(make([]byte, (4-buf.Len()%4)%4)) - return buf + return buf, flags } func NewProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.ConnectionOpts, adTag []byte) (*ProxyRequest, error) { diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index c56e9a2..8415038 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -107,7 +107,7 @@ func (m *MTProtoProxy) Write(p []byte) (int, error) { "quick_ack", m.req.Options.ReadHacks.QuickAck, ) - header := m.req.MakeHeader(p) + header, flags := m.req.MakeHeader(p) if ce := m.logger.Desugar().Check(zap.DebugLevel, "RPC_PROXY_REQ header"); ce != nil { ce.Write( zap.Int("length", len(p)), @@ -115,6 +115,7 @@ func (m *MTProtoProxy) Write(p []byte) (int, error) { zap.Bool("simple_ack", m.req.Options.ReadHacks.QuickAck), zap.Bool("quick_ack", m.req.Options.ReadHacks.SimpleAck), zap.String("header", fmt.Sprintf("%v", header.Bytes())), + zap.Stringer("flags", flags), ) } header.Write(p) From 71614ee61581d9b469bf8aaff0c10a018037f4a9 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 07:49:28 +0300 Subject: [PATCH 60/71] Set correct simple ack --- wrappers/mtproto_abridged.go | 13 ++++++------- wrappers/mtproto_proxy.go | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index e90233c..2ed116f 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -44,7 +44,7 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { if _, err := io.CopyN(buf, m.conn, 1); err != nil { return nil, errors.Annotate(err, "Cannot read message length") } - msgLength := uint8(buf.Bytes()[0]) + msgLength := uint32(buf.Bytes()[0]) buf.Reset() m.logger.Debugw("Packet first byte", @@ -59,27 +59,26 @@ func (m *MTProtoAbridged) Read() ([]byte, error) { msgLength -= mtprotoAbridgedQuickAckLength } - msgLength32 := uint32(msgLength) if msgLength == mtprotoAbridgedSmallPacketLength { if _, err := io.CopyN(buf, m.conn, 3); err != nil { return nil, errors.Annotate(err, "Cannot read the correct message length") } number := utils.Uint24{} copy(number[:], buf.Bytes()) - msgLength32 = utils.FromUint24(number) + msgLength = utils.FromUint24(number) } - msgLength32 *= 4 + msgLength *= 4 m.logger.Debugw("Packet length", - "length", msgLength32, + "length", msgLength, "simple_ack", m.opts.ReadHacks.SimpleAck, "quick_ack", m.opts.ReadHacks.QuickAck, "counter", m.readCounter, ) buf.Reset() - buf.Grow(int(msgLength32)) - if _, err := io.CopyN(buf, m.conn, int64(msgLength32)); err != nil { + buf.Grow(int(msgLength)) + if _, err := io.CopyN(buf, m.conn, int64(msgLength)); err != nil { return nil, errors.Annotate(err, "Cannot read message") } diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index 8415038..fbbb332 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -80,6 +80,7 @@ func (m *MTProtoProxy) readSimpleAck(data []byte) ([]byte, error) { return nil, errors.Errorf("Incorrect data of simple ack: %d", len(data)) } data = data[8:12] + m.req.Options.WriteHacks.SimpleAck = true m.logger.Debugw("Read RPC_SIMPLE_ACK", "counter", m.readCounter, From ff4be53d942aff04b16ec67e1cd5ee7d12292627 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 09:14:11 +0300 Subject: [PATCH 61/71] Add base stats --- Gopkg.lock | 8 ++++- Gopkg.toml | 4 +++ main.go | 5 +++- stats/server.go | 39 +++++++++++++++++++++++++ stats/stats.go | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 stats/server.go create mode 100644 stats/stats.go diff --git a/Gopkg.lock b/Gopkg.lock index 88ad0e6..cb43617 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -22,6 +22,12 @@ revision = "346938d642f2ec3594ed81d874461961cd0faa76" version = "v1.1.0" +[[projects]] + branch = "master" + name = "github.com/dustin/go-humanize" + packages = ["."] + revision = "02af3965c54e8cacf948b97fef38925c4120652c" + [[projects]] branch = "master" name = "github.com/juju/errors" @@ -80,6 +86,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "24afdd6b64331aeba47fed75918d04032e13e404612cac107bad1d68a5038b72" + inputs-digest = "312c9fb15085cbe9660443b15a07981990e1f70ec3ddfcce1b7e6cd5902307da" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index ed9feb4..ec43ce8 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -44,3 +44,7 @@ [[constraint]] name = "github.com/satori/go.uuid" version = "1.2.0" + +[[constraint]] + branch = "master" + name = "github.com/dustin/go-humanize" diff --git a/main.go b/main.go index c0ffba1..2e06549 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/proxy" + "github.com/9seconds/mtg/stats" "github.com/juju/errors" ) @@ -115,13 +116,15 @@ func main() { zap.ReplaceGlobals(logger) defer logger.Sync() + printURLs(conf.GetURLs()) + if conf.UseMiddleProxy() { zap.S().Infow("Use middle proxy connection to Telegram") } else { zap.S().Infow("Use direct connection to Telegram") } - printURLs(conf.GetURLs()) + go stats.Start(conf) server := proxy.NewProxy(conf) if err := server.Serve(); err != nil { diff --git a/stats/server.go b/stats/server.go new file mode 100644 index 0000000..c3935d2 --- /dev/null +++ b/stats/server.go @@ -0,0 +1,39 @@ +package stats + +import ( + "encoding/json" + "net/http" + "sync" + "time" + + "github.com/9seconds/mtg/config" +) + +var instance *stats + +func Start(conf *config.Config) { + instance = &stats{ + URLs: conf.GetURLs(), + Uptime: uptime(time.Now()), + speedCurrent: &speed{}, + mutex: &sync.RWMutex{}, + } + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + instance.mutex.Lock() + first, _ := json.Marshal(instance) + instance.mutex.Unlock() + + interm := map[string]interface{}{} + json.Unmarshal(first, &interm) + + encoder := json.NewEncoder(w) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + encoder.Encode(interm) + }) + + http.ListenAndServe(conf.StatAddr(), nil) +} diff --git a/stats/stats.go b/stats/stats.go new file mode 100644 index 0000000..4754674 --- /dev/null +++ b/stats/stats.go @@ -0,0 +1,77 @@ +package stats + +import ( + "encoding/json" + "fmt" + "strconv" + "sync" + "time" + + humanize "github.com/dustin/go-humanize" + + "github.com/9seconds/mtg/config" +) + +type uptime time.Time + +func (u uptime) MarshalJSON() ([]byte, error) { + duration := time.Since(time.Time(u)) + value := map[string]string{ + "seconds": strconv.Itoa(int(duration.Seconds())), + "human": humanize.Time(time.Time(u)), + } + + return json.Marshal(value) +} + +type trafficValue uint64 + +func (t trafficValue) MarshalJSON() ([]byte, error) { + tv := uint64(t) + value := map[string]interface{}{ + "bytes": tv, + "human": humanize.Bytes(tv), + } + + return json.Marshal(value) +} + +type trafficSpeedValue uint64 + +func (t trafficSpeedValue) MarshalJSON() ([]byte, error) { + speed := uint64(t) + value := map[string]interface{}{ + "bytes/s": speed, + "human": fmt.Sprintf("%s/S", humanize.Bytes(speed)), + } + + return json.Marshal(value) +} + +type connections struct { + All uint32 `json:"all"` + Abridged uint32 `json:"abridged"` + Intermediate uint32 `json:"intermediate"` +} + +type traffic struct { + Ingress trafficValue `json:"ingress"` + Egress trafficValue `json:"egress"` +} + +type speed struct { + Ingress trafficSpeedValue `json:"ingress"` + Egress trafficSpeedValue `json:"egress"` +} + +type stats struct { + URLs config.IPURLs `json:"urls"` + ActiveConnections connections `json:"active_connections"` + AllConnections connections `json:"all_connections"` + Traffic traffic `json:"traffic"` + Speed speed `json:"speed"` + Uptime uptime `json:"uptime"` + + speedCurrent *speed + mutex *sync.RWMutex +} From c66e30042550e06ab43e87088da16abcff9a1f85 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 11:29:47 +0300 Subject: [PATCH 62/71] Stats management utilities --- proxy/proxy.go | 2 +- stats/channels.go | 147 ++++++++++++++++++++++++++++++++++++++++++++++ stats/server.go | 11 ++-- stats/stats.go | 31 ++++++++-- 4 files changed, 182 insertions(+), 9 deletions(-) create mode 100644 stats/channels.go diff --git a/proxy/proxy.go b/proxy/proxy.go index a8399eb..e0f3a4e 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -39,7 +39,7 @@ func (p *Proxy) Serve() error { func (p *Proxy) accept(conn net.Conn) { connID := uuid.NewV4().String() - log := zap.S().With("connection_id", connID) + log := zap.S().With("connection_id", connID).Named("main") defer func() { conn.Close() diff --git a/stats/channels.go b/stats/channels.go new file mode 100644 index 0000000..775bd2e --- /dev/null +++ b/stats/channels.go @@ -0,0 +1,147 @@ +package stats + +import ( + "net" + "sync/atomic" + "time" + + "github.com/9seconds/mtg/mtproto" +) + +const ( + crashesChanLength = 1 + connectionsChanLength = 20 + trafficChanLength = 5000 +) + +var ( + CrashesChan = make(chan struct{}, crashesChanLength) + ConnectionsChan = make(chan *connectionData, connectionsChanLength) + TrafficChan = make(chan *trafficData, trafficChanLength) +) + +type connectionData struct { + connectionType mtproto.ConnectionType + addr *net.TCPAddr + connected bool +} + +type trafficData struct { + traffic int + ingress bool +} + +func crashManager() { + for range CrashesChan { + instance.mutex.RLock() + + instance.Crashes++ + + instance.mutex.RUnlock() + } +} + +func connectionManager() { + for event := range ConnectionsChan { + instance.mutex.RLock() + + isIPv4 := event.addr.IP.To4() == nil + var inc uint32 = 1 + if !event.connected { + inc = ^uint32(0) + } + + switch event.connectionType { + case mtproto.ConnectionTypeAbridged: + if isIPv4 { + atomic.AddUint32(&instance.ActiveConnections.Abridged.IPv4, inc) + if event.connected { + atomic.AddUint32(&instance.AllConnections.Abridged.IPv4, inc) + } + } else { + atomic.AddUint32(&instance.ActiveConnections.Abridged.IPv6, inc) + if event.connected { + atomic.AddUint32(&instance.AllConnections.Abridged.IPv6, inc) + } + } + default: + if isIPv4 { + atomic.AddUint32(&instance.ActiveConnections.Intermediate.IPv4, inc) + if event.connected { + atomic.AddUint32(&instance.AllConnections.Intermediate.IPv4, inc) + } + } else { + atomic.AddUint32(&instance.ActiveConnections.Intermediate.IPv6, inc) + if event.connected { + atomic.AddUint32(&instance.AllConnections.Intermediate.IPv6, inc) + } + } + } + + instance.mutex.RUnlock() + } +} + +func trafficManager() { + speedChan := time.Tick(time.Second) + + for { + select { + case event := <-TrafficChan: + instance.mutex.RLock() + + if event.ingress { + instance.Traffic.Ingress += trafficValue(event.traffic) + instance.speedCurrent.Ingress += trafficSpeedValue(event.traffic) + } else { + instance.Traffic.Egress += trafficValue(event.traffic) + instance.speedCurrent.Egress += trafficSpeedValue(event.traffic) + } + + instance.mutex.RUnlock() + case <-speedChan: + instance.mutex.RLock() + + instance.Speed.Ingress = instance.speedCurrent.Ingress + instance.Speed.Egress = instance.speedCurrent.Egress + instance.speedCurrent.Ingress = trafficSpeedValue(0) + instance.speedCurrent.Egress = trafficSpeedValue(0) + + instance.mutex.RUnlock() + } + } +} + +func NewCrash() { + CrashesChan <- struct{}{} +} + +func ClientConnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { + ConnectionsChan <- &connectionData{ + connectionType: connectionType, + addr: addr, + connected: true, + } +} + +func ClientDisconnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { + ConnectionsChan <- &connectionData{ + connectionType: connectionType, + addr: addr, + connected: false, + } +} + +func IngressTraffic(traffic int) { + TrafficChan <- &trafficData{ + traffic: traffic, + ingress: true, + } +} + +func EgressTraffic(traffic int) { + TrafficChan <- &trafficData{ + traffic: traffic, + ingress: false, + } +} diff --git a/stats/server.go b/stats/server.go index c3935d2..3fb6f99 100644 --- a/stats/server.go +++ b/stats/server.go @@ -13,12 +13,15 @@ var instance *stats func Start(conf *config.Config) { instance = &stats{ - URLs: conf.GetURLs(), - Uptime: uptime(time.Now()), - speedCurrent: &speed{}, - mutex: &sync.RWMutex{}, + URLs: conf.GetURLs(), + Uptime: uptime(time.Now()), + mutex: &sync.RWMutex{}, } + go crashManager() + go connectionManager() + go trafficManager() + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/stats/stats.go b/stats/stats.go index 4754674..5c6099c 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -49,9 +49,31 @@ func (t trafficSpeedValue) MarshalJSON() ([]byte, error) { } type connections struct { - All uint32 `json:"all"` - Abridged uint32 `json:"abridged"` - Intermediate uint32 `json:"intermediate"` + All connectionType `json:"all"` + Abridged connectionType `json:"abridged"` + Intermediate connectionType `json:"intermediate"` +} + +func (c connections) MarshalJSON() ([]byte, error) { + c.All.IPv4 = c.Abridged.IPv4 + c.Intermediate.IPv4 + c.All.IPv6 = c.Abridged.IPv6 + c.Intermediate.IPv6 + + value := struct { + All connectionType `json:"all"` + Abridged connectionType `json:"abridged"` + Intermediate connectionType `json:"intermediate"` + }{ + All: c.All, + Abridged: c.Abridged, + Intermediate: c.Intermediate, + } + + return json.Marshal(value) +} + +type connectionType struct { + IPv6 uint32 `json:"ipv6"` + IPv4 uint32 `json:"ipv4"` } type traffic struct { @@ -71,7 +93,8 @@ type stats struct { Traffic traffic `json:"traffic"` Speed speed `json:"speed"` Uptime uptime `json:"uptime"` + Crashes uint32 `json:"crashes"` - speedCurrent *speed + speedCurrent speed mutex *sync.RWMutex } From 045d417ecece26d972d3b14c4949bd95b39f3adb Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 11:32:42 +0300 Subject: [PATCH 63/71] Add logging to stats --- stats/server.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/stats/server.go b/stats/server.go index 3fb6f99..d0a13b9 100644 --- a/stats/server.go +++ b/stats/server.go @@ -6,12 +6,16 @@ import ( "sync" "time" + "go.uber.org/zap" + "github.com/9seconds/mtg/config" ) var instance *stats func Start(conf *config.Config) { + log := zap.S().Named("stats") + instance = &stats{ URLs: conf.GetURLs(), Uptime: uptime(time.Now()), @@ -26,17 +30,27 @@ func Start(conf *config.Config) { w.Header().Set("Content-Type", "application/json") instance.mutex.Lock() - first, _ := json.Marshal(instance) + first, err := json.Marshal(instance) instance.mutex.Unlock() + if err != nil { + log.Errorw("Cannot encode json", "error", err) + http.Error(w, "Internal server error", 500) + return + } + interm := map[string]interface{}{} json.Unmarshal(first, &interm) encoder := json.NewEncoder(w) encoder.SetEscapeHTML(false) encoder.SetIndent("", " ") - encoder.Encode(interm) + if err = encoder.Encode(interm); err != nil { + log.Errorw("Cannot encode json", "error", err) + } }) - http.ListenAndServe(conf.StatAddr(), nil) + if err := http.ListenAndServe(conf.StatAddr(), nil); err != nil { + log.Fatalw("Stats server has been stopped", "error", err) + } } From 1aa8cfe036e7e437187ee54dd7d773e4f62ab9a3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 11:42:44 +0300 Subject: [PATCH 64/71] Stats callbacks --- proxy/proxy.go | 5 +++++ stats/channels.go | 2 +- wrappers/conn.go | 11 ++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index e0f3a4e..1f2df20 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -12,6 +12,7 @@ import ( "github.com/9seconds/mtg/client" "github.com/9seconds/mtg/config" "github.com/9seconds/mtg/mtproto" + "github.com/9seconds/mtg/stats" "github.com/9seconds/mtg/telegram" "github.com/9seconds/mtg/wrappers" ) @@ -45,6 +46,7 @@ func (p *Proxy) accept(conn net.Conn) { conn.Close() if err := recover(); err != nil { + stats.NewCrash() log.Errorw("Crash of accept handler", "error", err) } }() @@ -58,6 +60,9 @@ func (p *Proxy) accept(conn net.Conn) { } defer client.(io.Closer).Close() + stats.ClientConnected(opts.ConnectionType, client.RemoteAddr()) + defer stats.ClientDisconnected(opts.ConnectionType, client.RemoteAddr()) + server, err := p.getTelegramConn(opts, connID) if err != nil { log.Errorw("Cannot initialize server connection", "error", err) diff --git a/stats/channels.go b/stats/channels.go index 775bd2e..1ece855 100644 --- a/stats/channels.go +++ b/stats/channels.go @@ -45,7 +45,7 @@ func connectionManager() { for event := range ConnectionsChan { instance.mutex.RLock() - isIPv4 := event.addr.IP.To4() == nil + isIPv4 := event.addr.IP.To4() != nil var inc uint32 = 1 if !event.connected { inc = ^uint32(0) diff --git a/wrappers/conn.go b/wrappers/conn.go index ad853af..d770cb4 100644 --- a/wrappers/conn.go +++ b/wrappers/conn.go @@ -5,6 +5,8 @@ import ( "time" "go.uber.org/zap" + + "github.com/9seconds/mtg/stats" ) type ConnPurpose uint8 @@ -31,9 +33,10 @@ const ( ) type Conn struct { - connID string - conn net.Conn - logger *zap.SugaredLogger + connID string + conn net.Conn + logger *zap.SugaredLogger + publicIPv4 net.IP publicIPv6 net.IP } @@ -43,6 +46,7 @@ func (c *Conn) Write(p []byte) (int, error) { n, err := c.conn.Write(p) c.logger.Debugw("Write to stream", "bytes", n, "error", err) + stats.EgressTraffic(n) return n, err } @@ -52,6 +56,7 @@ func (c *Conn) Read(p []byte) (int, error) { n, err := c.conn.Read(p) c.logger.Debugw("Read from stream", "bytes", n, "error", err) + stats.IngressTraffic(n) return n, err } From 7468aec8a38d851d0189970dad8542a66a2dbf21 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 11:55:45 +0300 Subject: [PATCH 65/71] Do not use atomic if not necessary --- stats/channels.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/stats/channels.go b/stats/channels.go index 1ece855..7387b6d 100644 --- a/stats/channels.go +++ b/stats/channels.go @@ -2,7 +2,6 @@ package stats import ( "net" - "sync/atomic" "time" "github.com/9seconds/mtg/mtproto" @@ -54,26 +53,26 @@ func connectionManager() { switch event.connectionType { case mtproto.ConnectionTypeAbridged: if isIPv4 { - atomic.AddUint32(&instance.ActiveConnections.Abridged.IPv4, inc) + instance.ActiveConnections.Abridged.IPv4 += inc if event.connected { - atomic.AddUint32(&instance.AllConnections.Abridged.IPv4, inc) + instance.AllConnections.Abridged.IPv4 += inc } } else { - atomic.AddUint32(&instance.ActiveConnections.Abridged.IPv6, inc) + instance.ActiveConnections.Abridged.IPv6 += inc if event.connected { - atomic.AddUint32(&instance.AllConnections.Abridged.IPv6, inc) + instance.AllConnections.Abridged.IPv6 += inc } } default: if isIPv4 { - atomic.AddUint32(&instance.ActiveConnections.Intermediate.IPv4, inc) + instance.ActiveConnections.Intermediate.IPv4 += inc if event.connected { - atomic.AddUint32(&instance.AllConnections.Intermediate.IPv4, inc) + instance.AllConnections.Intermediate.IPv4 += inc } } else { - atomic.AddUint32(&instance.ActiveConnections.Intermediate.IPv6, inc) + instance.ActiveConnections.Intermediate.IPv6 += inc if event.connected { - atomic.AddUint32(&instance.AllConnections.Intermediate.IPv6, inc) + instance.AllConnections.Intermediate.IPv6 += inc } } } From 2a6cc7e1953a9368774d698a782e4d9d3f61bda7 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 11:57:23 +0300 Subject: [PATCH 66/71] Unpublic stats chans --- stats/channels.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/stats/channels.go b/stats/channels.go index 7387b6d..c9bab87 100644 --- a/stats/channels.go +++ b/stats/channels.go @@ -14,9 +14,9 @@ const ( ) var ( - CrashesChan = make(chan struct{}, crashesChanLength) - ConnectionsChan = make(chan *connectionData, connectionsChanLength) - TrafficChan = make(chan *trafficData, trafficChanLength) + crashesChan = make(chan struct{}, crashesChanLength) + connectionsChan = make(chan *connectionData, connectionsChanLength) + trafficChan = make(chan *trafficData, trafficChanLength) ) type connectionData struct { @@ -31,7 +31,7 @@ type trafficData struct { } func crashManager() { - for range CrashesChan { + for range crashesChan { instance.mutex.RLock() instance.Crashes++ @@ -41,7 +41,7 @@ func crashManager() { } func connectionManager() { - for event := range ConnectionsChan { + for event := range connectionsChan { instance.mutex.RLock() isIPv4 := event.addr.IP.To4() != nil @@ -86,7 +86,7 @@ func trafficManager() { for { select { - case event := <-TrafficChan: + case event := <-trafficChan: instance.mutex.RLock() if event.ingress { @@ -112,11 +112,11 @@ func trafficManager() { } func NewCrash() { - CrashesChan <- struct{}{} + crashesChan <- struct{}{} } func ClientConnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { - ConnectionsChan <- &connectionData{ + connectionsChan <- &connectionData{ connectionType: connectionType, addr: addr, connected: true, @@ -124,7 +124,7 @@ func ClientConnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { } func ClientDisconnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { - ConnectionsChan <- &connectionData{ + connectionsChan <- &connectionData{ connectionType: connectionType, addr: addr, connected: false, @@ -132,14 +132,14 @@ func ClientDisconnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr } func IngressTraffic(traffic int) { - TrafficChan <- &trafficData{ + trafficChan <- &trafficData{ traffic: traffic, ingress: true, } } func EgressTraffic(traffic int) { - TrafficChan <- &trafficData{ + trafficChan <- &trafficData{ traffic: traffic, ingress: false, } From fb00cd11210b0eff90ff672e47ff6257a8ef6309 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 13:01:21 +0300 Subject: [PATCH 67/71] Set buffer sizes to sockets --- client/direct.go | 17 ++++++++++++++--- config/config.go | 17 ----------------- telegram/dialer.go | 18 +++++++++++++++--- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/client/direct.go b/client/direct.go index 9f0df10..266071e 100644 --- a/client/direct.go +++ b/client/direct.go @@ -12,11 +12,22 @@ import ( "github.com/9seconds/mtg/wrappers" ) -const handshakeTimeout = 10 * time.Second +const ( + handshakeTimeout = 10 * time.Second + readBufferSize = 64 * 1024 + writeBufferSize = 64 * 1024 +) func DirectInit(socket net.Conn, connID string, conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { - if err := config.SetSocketOptions(socket); err != nil { - return nil, nil, errors.Annotate(err, "Cannot set socket options") + tcpSocket := socket.(*net.TCPConn) + if err := tcpSocket.SetNoDelay(false); err != nil { + return nil, nil, errors.Annotate(err, "Cannot disable NO_DELAY to client socket") + } + if err := tcpSocket.SetReadBuffer(readBufferSize); err != nil { + return nil, nil, errors.Annotate(err, "Cannot set read buffer size of client socket") + } + if err := tcpSocket.SetWriteBuffer(writeBufferSize); err != nil { + return nil, nil, errors.Annotate(err, "Cannot set write buffer size of client socket") } socket.SetReadDeadline(time.Now().Add(handshakeTimeout)) diff --git a/config/config.go b/config/config.go index 59d06cd..27c11d0 100644 --- a/config/config.go +++ b/config/config.go @@ -149,20 +149,3 @@ func NewConfig(debug, verbose bool, // nolint: gocyclo return conf, nil } - -// SetSocketOptions makes socket keepalive, sets buffer sizes -func SetSocketOptions(conn net.Conn) error { - socket := conn.(*net.TCPConn) - - if err := socket.SetReadBuffer(BufferReadSize); err != nil { - return errors.Annotate(err, "Cannot set read buffer size") - } - if err := socket.SetWriteBuffer(BufferWriteSize); err != nil { - return errors.Annotate(err, "Cannot set write buffer size") - } - if err := socket.SetNoDelay(true); err != nil { - return errors.Annotate(err, "Cannot activate nodelay for the socket") - } - - return nil -} diff --git a/telegram/dialer.go b/telegram/dialer.go index fc70d64..58b4ff5 100644 --- a/telegram/dialer.go +++ b/telegram/dialer.go @@ -10,7 +10,11 @@ import ( "github.com/9seconds/mtg/wrappers" ) -const telegramDialTimeout = 10 * time.Second +const ( + telegramDialTimeout = 10 * time.Second + readBufferSize = 64 * 1024 + writeBufferSize = 64 * 1024 +) type tgDialer struct { net.Dialer @@ -23,8 +27,16 @@ func (t *tgDialer) dial(addr string) (net.Conn, error) { if err != nil { return nil, errors.Annotate(err, "Cannot connect to Telegram") } - if err = config.SetSocketOptions(conn); err != nil { - return nil, errors.Annotate(err, "Cannot set socket options") + + tcpSocket := conn.(*net.TCPConn) + if err = tcpSocket.SetNoDelay(true); err != nil { + return nil, errors.Annotate(err, "Cannot set NO_DELAY to Telegram") + } + if err = tcpSocket.SetReadBuffer(readBufferSize); err != nil { + return nil, errors.Annotate(err, "Cannot set read buffer size on telegram socket") + } + if err = tcpSocket.SetWriteBuffer(writeBufferSize); err != nil { + return nil, errors.Annotate(err, "Cannot set write buffer size on telegram socket") } return conn, nil From 29b5130c958df7466ad42a6857c5f277e095c276 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 14:44:02 +0300 Subject: [PATCH 68/71] Fix lint errors --- Gopkg.lock | 2 +- client/client.go | 1 + client/direct.go | 8 +++++--- client/middle.go | 2 ++ config/config.go | 2 ++ main.go | 2 +- mtproto/connection_options.go | 13 ++++++++++--- mtproto/rpc/handshake_request.go | 4 ++++ mtproto/rpc/handshake_response.go | 6 ++++++ mtproto/rpc/nonce_request.go | 4 ++++ mtproto/rpc/nonce_response.go | 4 ++++ mtproto/rpc/proxy_request.go | 5 +++++ mtproto/rpc/rpc.go | 3 +++ proxy/proxy.go | 21 ++++++++++++--------- stats/channels.go | 9 +++++++-- stats/server.go | 3 ++- telegram/direct.go | 8 ++++---- telegram/middle.go | 17 ++++++++++------- telegram/middle_caller.go | 2 +- telegram/telegram.go | 1 + utils/read_current_data.go | 1 + utils/reverse_bytes.go | 1 + utils/uint24.go | 4 ++++ wrappers/blockcipher.go | 9 +++++++++ wrappers/conn.go | 25 ++++++++++++++++++------- wrappers/mtproto_abridged.go | 7 +++++++ wrappers/mtproto_cipher.go | 24 +++++++++++++----------- wrappers/mtproto_frame.go | 27 ++++++++++++++++++++++----- wrappers/mtproto_intermediate.go | 8 ++++++++ wrappers/mtproto_proxy.go | 6 ++++++ wrappers/streamcipher.go | 7 +++++++ wrappers/wrap.go | 26 ++++++++++++++++++++++++++ 32 files changed, 207 insertions(+), 55 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index cb43617..7a8f120 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -86,6 +86,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "312c9fb15085cbe9660443b15a07981990e1f70ec3ddfcce1b7e6cd5902307da" + inputs-digest = "c4fdd3664f683342ad0c2509f4a8bcfe5b267a6e8cdaf36f70d39536bbf89834" solver-name = "gps-cdcl" solver-version = 1 diff --git a/client/client.go b/client/client.go index 648fec2..fd503ff 100644 --- a/client/client.go +++ b/client/client.go @@ -8,4 +8,5 @@ import ( "github.com/9seconds/mtg/wrappers" ) +// Init defines common method for initializing client connections. type Init func(net.Conn, string, *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) diff --git a/client/direct.go b/client/direct.go index 266071e..de8b66f 100644 --- a/client/direct.go +++ b/client/direct.go @@ -18,6 +18,8 @@ const ( writeBufferSize = 64 * 1024 ) +// DirectInit initializes client connection for proxy which connects to +// Telegram directly. func DirectInit(socket net.Conn, connID string, conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { tcpSocket := socket.(*net.TCPConn) if err := tcpSocket.SetNoDelay(false); err != nil { @@ -30,14 +32,14 @@ func DirectInit(socket net.Conn, connID string, conf *config.Config) (wrappers.W return nil, nil, errors.Annotate(err, "Cannot set write buffer size of client socket") } - socket.SetReadDeadline(time.Now().Add(handshakeTimeout)) + socket.SetReadDeadline(time.Now().Add(handshakeTimeout)) // nolint: errcheck frame, err := obfuscated2.ExtractFrame(socket) if err != nil { return nil, nil, errors.Annotate(err, "Cannot extract frame") } - socket.SetReadDeadline(time.Time{}) - conn := wrappers.NewConn(socket, connID, wrappers.ConnPurposeClient, conf.PublicIPv4, conf.PublicIPv6) + socket.SetReadDeadline(time.Time{}) // nolint: errcheck + conn := wrappers.NewConn(socket, connID, wrappers.ConnPurposeClient, conf.PublicIPv4, conf.PublicIPv6) obfs2, connOpts, err := obfuscated2.ParseObfuscated2ClientFrame(conf.Secret, frame) if err != nil { return nil, nil, errors.Annotate(err, "Cannot parse obfuscated frame") diff --git a/client/middle.go b/client/middle.go index b85deef..53bb64c 100644 --- a/client/middle.go +++ b/client/middle.go @@ -8,6 +8,8 @@ import ( "github.com/9seconds/mtg/wrappers" ) +// MiddleInit initializes client connection for proxy which has to +// support promoted channels, connect to Telegram middle proxies etc. func MiddleInit(socket net.Conn, connID string, conf *config.Config) (wrappers.Wrap, *mtproto.ConnectionOpts, error) { conn, opts, err := DirectInit(socket, connID, conf) if err != nil { diff --git a/config/config.go b/config/config.go index 27c11d0..f5fd97f 100644 --- a/config/config.go +++ b/config/config.go @@ -58,6 +58,8 @@ func (c *Config) StatAddr() string { return getAddr(c.StatsIP, c.StatsPort) } +// UseMiddleProxy defines if this proxy has to connect middle proxies +// which supports promoted channels or directly access Telegram. func (c *Config) UseMiddleProxy() bool { return len(c.AdTag) > 0 } diff --git a/main.go b/main.go index 2e06549..e7d8a16 100644 --- a/main.go +++ b/main.go @@ -114,7 +114,7 @@ func main() { atom, )) zap.ReplaceGlobals(logger) - defer logger.Sync() + defer logger.Sync() // nolint: errcheck printURLs(conf.GetURLs()) diff --git a/mtproto/connection_options.go b/mtproto/connection_options.go index 0285488..76d2bfe 100644 --- a/mtproto/connection_options.go +++ b/mtproto/connection_options.go @@ -11,8 +11,10 @@ import ( // by the user. type ConnectionType uint8 +// ConnectionProtocol is a type of IP protocol to use. type ConnectionProtocol uint8 +// Hacks is a simple structure to store flags for packet transmission. type Hacks struct { SimpleAck bool QuickAck bool @@ -24,9 +26,12 @@ type ConnectionOpts struct { DC int16 ConnectionType ConnectionType ConnectionProto ConnectionProtocol - ReadHacks Hacks - WriteHacks Hacks - ClientAddr *net.TCPAddr + // Read and Write means direction related to the client. + // ReadHacks are meant to be flushed on client read + // WriteHacks are meant to be flushed on client write. + ReadHacks Hacks + WriteHacks Hacks + ClientAddr *net.TCPAddr } // Different connection types which user requests from Telegram. @@ -36,6 +41,8 @@ const ( ConnectionTypeIntermediate ) +// ConnectionProtocol* define which connection protocols to use. +// ConnectionProtocolAny means that any is suitable. const ( ConnectionProtocolIPv4 ConnectionProtocol = 1 ConnectionProtocolIPv6 = ConnectionProtocolIPv4 << 1 diff --git a/mtproto/rpc/handshake_request.go b/mtproto/rpc/handshake_request.go index 924859d..acf3807 100644 --- a/mtproto/rpc/handshake_request.go +++ b/mtproto/rpc/handshake_request.go @@ -2,9 +2,12 @@ package rpc import "bytes" +// HandshakeRequest is the data type which is responsible for +// constructing of correct handshake request. type HandshakeRequest struct { } +// Bytes returns serialized handshake request. func (r *HandshakeRequest) Bytes() []byte { buf := &bytes.Buffer{} buf.Grow(len(TagHandshake) + len(HandshakeFlags) + len(HandshakeSenderPID) + len(HandshakePeerPID)) @@ -17,6 +20,7 @@ func (r *HandshakeRequest) Bytes() []byte { return buf.Bytes() } +// NewHandshakeRequest creates new HandshakeRequest instance. func NewHandshakeRequest() *HandshakeRequest { return &HandshakeRequest{} } diff --git a/mtproto/rpc/handshake_response.go b/mtproto/rpc/handshake_response.go index 40534d4..a8c522c 100644 --- a/mtproto/rpc/handshake_response.go +++ b/mtproto/rpc/handshake_response.go @@ -6,6 +6,8 @@ import ( "github.com/juju/errors" ) +// HandshakeResponse defines data structure which is used for storage of +// handshake response. type HandshakeResponse struct { Type []byte Flags []byte @@ -13,6 +15,7 @@ type HandshakeResponse struct { PeerPID []byte } +// Bytes returns a serialized handshake response. func (r *HandshakeResponse) Bytes() []byte { buf := &bytes.Buffer{} @@ -24,6 +27,7 @@ func (r *HandshakeResponse) Bytes() []byte { return buf.Bytes() } +// Valid checks that handshake response compliments request. func (r *HandshakeResponse) Valid(req *HandshakeRequest) error { if !bytes.Equal(r.Type, TagHandshake) { return errors.New("Unexpected handshake tag") @@ -35,6 +39,8 @@ func (r *HandshakeResponse) Valid(req *HandshakeRequest) error { return nil } +// NewHandshakeResponse constructs new handshake response from the given +// data. func NewHandshakeResponse(data []byte) (*HandshakeResponse, error) { if len(data) != 32 { return nil, errors.New("Incorrect handshake response length") diff --git a/mtproto/rpc/nonce_request.go b/mtproto/rpc/nonce_request.go index e4fb8a0..b714a9c 100644 --- a/mtproto/rpc/nonce_request.go +++ b/mtproto/rpc/nonce_request.go @@ -9,12 +9,15 @@ import ( "github.com/juju/errors" ) +// NonceRequest is the data type which contains all the data for correct +// nonce request. type NonceRequest struct { KeySelector []byte CryptoTS []byte Nonce []byte } +// Bytes returns serialized nonce request. func (r *NonceRequest) Bytes() []byte { buf := &bytes.Buffer{} @@ -27,6 +30,7 @@ func (r *NonceRequest) Bytes() []byte { return buf.Bytes() } +// NewNonceRequest builds new none request based on proxy secret. func NewNonceRequest(proxySecret []byte) (*NonceRequest, error) { nonce := make([]byte, 16) keySelector := make([]byte, 4) diff --git a/mtproto/rpc/nonce_response.go b/mtproto/rpc/nonce_response.go index 85c9383..f76bd44 100644 --- a/mtproto/rpc/nonce_response.go +++ b/mtproto/rpc/nonce_response.go @@ -6,6 +6,7 @@ import ( "github.com/juju/errors" ) +// NonceResponse is the data type which contains data of nonce response. type NonceResponse struct { NonceRequest @@ -13,6 +14,7 @@ type NonceResponse struct { Crypto []byte } +// Bytes returns serialized form of the nonce response. func (r *NonceResponse) Bytes() []byte { buf := &bytes.Buffer{} @@ -25,6 +27,7 @@ func (r *NonceResponse) Bytes() []byte { return buf.Bytes() } +// Valid checks that nonce response compliments nonce request. func (r *NonceResponse) Valid(req *NonceRequest) error { if !bytes.Equal(r.Type, TagNonce) { return errors.New("Unexpected RPC type") @@ -39,6 +42,7 @@ func (r *NonceResponse) Valid(req *NonceRequest) error { return nil } +// NewNonceResponse build new nonce response based on the given data. func NewNonceResponse(data []byte) (*NonceResponse, error) { if len(data) != 32 { return nil, errors.New("Unexpected message length") diff --git a/mtproto/rpc/proxy_request.go b/mtproto/rpc/proxy_request.go index 40ab3cf..9e2dacf 100644 --- a/mtproto/rpc/proxy_request.go +++ b/mtproto/rpc/proxy_request.go @@ -12,6 +12,8 @@ import ( "github.com/9seconds/mtg/mtproto" ) +// ProxyRequest is the data type for storing data required to compose +// RPC_PROXY_REQ request. type ProxyRequest struct { Flags proxyRequestFlags ConnectionID []byte @@ -21,6 +23,8 @@ type ProxyRequest struct { Options *mtproto.ConnectionOpts } +// MakeHeader makes RPC_PROXY_REQ header. We need only to append the +// data for it. func (r *ProxyRequest) MakeHeader(message []byte) (*bytes.Buffer, fmt.Stringer) { bufferLength := len(TagProxyRequest) + 4 + // len(flags) @@ -59,6 +63,7 @@ func (r *ProxyRequest) MakeHeader(message []byte) (*bytes.Buffer, fmt.Stringer) return buf, flags } +// NewProxyRequest build new ProxyRequest data structure. func NewProxyRequest(clientAddr, ownAddr *net.TCPAddr, opts *mtproto.ConnectionOpts, adTag []byte) (*ProxyRequest, error) { flags := proxyRequestFlagsHasAdTag | proxyRequestFlagsMagic | proxyRequestFlagsExtMode2 diff --git a/mtproto/rpc/rpc.go b/mtproto/rpc/rpc.go index bfb0555..5d242f5 100644 --- a/mtproto/rpc/rpc.go +++ b/mtproto/rpc/rpc.go @@ -1,10 +1,13 @@ package rpc +// SeqNo* is the number of the sequence which have special meaning for +// the Telegram. const ( SeqNoNonce = -2 SeqNoHandshake = -1 ) +// Different constants for RPC protocol var ( TagCloseExt = []byte{0xa2, 0x34, 0xb6, 0x5e} TagProxyAns = []byte{0x0d, 0xda, 0x03, 0x44} diff --git a/proxy/proxy.go b/proxy/proxy.go index 1f2df20..42f5d2b 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -17,12 +17,14 @@ import ( "github.com/9seconds/mtg/wrappers" ) +// Proxy is a core of this program. type Proxy struct { clientInit client.Init tg telegram.Telegram conf *config.Config } +// Serve runs TCP proxy server. func (p *Proxy) Serve() error { lsock, err := net.Listen("tcp", p.conf.BindAddr()) if err != nil { @@ -43,7 +45,7 @@ func (p *Proxy) accept(conn net.Conn) { log := zap.S().With("connection_id", connID).Named("main") defer func() { - conn.Close() + conn.Close() // nolint: errcheck if err := recover(); err != nil { stats.NewCrash() @@ -58,7 +60,7 @@ func (p *Proxy) accept(conn net.Conn) { log.Errorw("Cannot initialize client connection", "error", err) return } - defer client.(io.Closer).Close() + defer client.(io.Closer).Close() // nolint: errcheck stats.ClientConnected(opts.ConnectionType, client.RemoteAddr()) defer stats.ClientDisconnected(opts.ConnectionType, client.RemoteAddr()) @@ -68,7 +70,7 @@ func (p *Proxy) accept(conn net.Conn) { log.Errorw("Cannot initialize server connection", "error", err) return } - defer server.(io.Closer).Close() + defer server.(io.Closer).Close() // nolint: errcheck wait := &sync.WaitGroup{} wait.Add(2) @@ -104,10 +106,10 @@ func (p *Proxy) getTelegramConn(opts *mtproto.ConnectionOpts, connID string) (wr return packetConn, nil } -func (p *Proxy) middlePipe(src wrappers.PacketReadCloser, dst wrappers.PacketWriteCloser, wait *sync.WaitGroup, hacks *mtproto.Hacks) { +func (p *Proxy) middlePipe(src wrappers.PacketReadCloser, dst io.WriteCloser, wait *sync.WaitGroup, hacks *mtproto.Hacks) { defer func() { - src.Close() - dst.Close() + src.Close() // nolint: errcheck + dst.Close() // nolint: errcheck wait.Done() }() @@ -127,10 +129,10 @@ func (p *Proxy) middlePipe(src wrappers.PacketReadCloser, dst wrappers.PacketWri } } -func (p *Proxy) directPipe(src wrappers.StreamReadCloser, dst wrappers.StreamWriteCloser, wait *sync.WaitGroup) { +func (p *Proxy) directPipe(src wrappers.StreamReadCloser, dst io.WriteCloser, wait *sync.WaitGroup) { defer func() { - src.Close() - dst.Close() + src.Close() // nolint: errcheck + dst.Close() // nolint: errcheck wait.Done() }() @@ -139,6 +141,7 @@ func (p *Proxy) directPipe(src wrappers.StreamReadCloser, dst wrappers.StreamWri } } +// NewProxy returns new proxy instance. func NewProxy(conf *config.Config) *Proxy { var clientInit client.Init var tg telegram.Telegram diff --git a/stats/channels.go b/stats/channels.go index c9bab87..ee2911b 100644 --- a/stats/channels.go +++ b/stats/channels.go @@ -21,8 +21,8 @@ var ( type connectionData struct { connectionType mtproto.ConnectionType - addr *net.TCPAddr connected bool + addr *net.TCPAddr } type trafficData struct { @@ -40,7 +40,7 @@ func crashManager() { } } -func connectionManager() { +func connectionManager() { // nolint: gocyclo for event := range connectionsChan { instance.mutex.RLock() @@ -111,10 +111,12 @@ func trafficManager() { } } +// NewCrash indicates new crash. func NewCrash() { crashesChan <- struct{}{} } +// ClientConnected indicates that new client was connected. func ClientConnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { connectionsChan <- &connectionData{ connectionType: connectionType, @@ -123,6 +125,7 @@ func ClientConnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { } } +// ClientDisconnected indicates that client was disconnected. func ClientDisconnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr) { connectionsChan <- &connectionData{ connectionType: connectionType, @@ -131,6 +134,7 @@ func ClientDisconnected(connectionType mtproto.ConnectionType, addr *net.TCPAddr } } +// IngressTraffic accounts new ingress traffic. func IngressTraffic(traffic int) { trafficChan <- &trafficData{ traffic: traffic, @@ -138,6 +142,7 @@ func IngressTraffic(traffic int) { } } +// EgressTraffic accounts new ingress traffic. func EgressTraffic(traffic int) { trafficChan <- &trafficData{ traffic: traffic, diff --git a/stats/server.go b/stats/server.go index d0a13b9..a5369e4 100644 --- a/stats/server.go +++ b/stats/server.go @@ -13,6 +13,7 @@ import ( var instance *stats +// Start starts new statisitcs server. func Start(conf *config.Config) { log := zap.S().Named("stats") @@ -40,7 +41,7 @@ func Start(conf *config.Config) { } interm := map[string]interface{}{} - json.Unmarshal(first, &interm) + json.Unmarshal(first, &interm) // nolint: errcheck encoder := json.NewEncoder(w) encoder.SetEscapeHTML(false) diff --git a/telegram/direct.go b/telegram/direct.go index 749aa07..0656a50 100644 --- a/telegram/direct.go +++ b/telegram/direct.go @@ -28,11 +28,11 @@ var ( } ) -type DirectTelegram struct { +type directTelegram struct { baseTelegram } -func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.StreamReadWriteCloser, error) { +func (t *directTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) (wrappers.StreamReadWriteCloser, error) { dc := connOpts.DC if dc < 0 { dc = -dc @@ -43,7 +43,7 @@ func (t *DirectTelegram) Dial(connID string, connOpts *mtproto.ConnectionOpts) ( return t.baseTelegram.dial(dc-1, connID, connOpts.ConnectionProto) } -func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) { +func (t *directTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) { obfs2, frame := obfuscated2.MakeTelegramObfuscated2Frame(connOpts) if _, err := conn.Write(frame); err != nil { @@ -56,7 +56,7 @@ func (t *DirectTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.St // NewDirectTelegram returns Telegram instance which connects directly // to Telegram bypassing middleproxies. func NewDirectTelegram(conf *config.Config) Telegram { - return &DirectTelegram{baseTelegram{ + return &directTelegram{baseTelegram{ dialer: tgDialer{ Dialer: net.Dialer{Timeout: telegramDialTimeout}, conf: conf, diff --git a/telegram/middle.go b/telegram/middle.go index 7404e3d..1724bf0 100644 --- a/telegram/middle.go +++ b/telegram/middle.go @@ -1,6 +1,7 @@ package telegram import ( + "io" "net" "net/http" "sync" @@ -13,13 +14,13 @@ import ( "github.com/9seconds/mtg/wrappers" ) -type MiddleTelegram struct { +type middleTelegram struct { middleTelegramCaller conf *config.Config } -func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) { +func (t *middleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) { rpcNonceConn := wrappers.NewMTProtoFrame(conn, rpc.SeqNoNonce) rpcNonceReq, err := t.sendRPCNonceRequest(rpcNonceConn) @@ -52,7 +53,7 @@ func (t *MiddleTelegram) Init(connOpts *mtproto.ConnectionOpts, conn wrappers.St return proxyConn, nil } -func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.PacketWriter) (*rpc.NonceRequest, error) { +func (t *middleTelegram) sendRPCNonceRequest(conn io.Writer) (*rpc.NonceRequest, error) { rpcNonceReq, err := rpc.NewNonceRequest(t.proxySecret) if err != nil { return nil, errors.Annotate(err, "Cannot create RPC nonce request") @@ -64,7 +65,7 @@ func (t *MiddleTelegram) sendRPCNonceRequest(conn wrappers.PacketWriter) (*rpc.N return rpcNonceReq, nil } -func (t *MiddleTelegram) receiveRPCNonceResponse(conn wrappers.PacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { +func (t *middleTelegram) receiveRPCNonceResponse(conn wrappers.PacketReader, req *rpc.NonceRequest) (*rpc.NonceResponse, error) { packet, err := conn.Read() if err != nil { return nil, errors.Annotate(err, "Cannot read RPC nonce response") @@ -81,7 +82,7 @@ func (t *MiddleTelegram) receiveRPCNonceResponse(conn wrappers.PacketReader, req return rpcNonceResp, nil } -func (t *MiddleTelegram) sendRPCHandshakeRequest(conn wrappers.PacketWriter) (*rpc.HandshakeRequest, error) { +func (t *middleTelegram) sendRPCHandshakeRequest(conn io.Writer) (*rpc.HandshakeRequest, error) { req := rpc.NewHandshakeRequest() if _, err := conn.Write(req.Bytes()); err != nil { return nil, errors.Annotate(err, "Cannot send RPC handshake request") @@ -90,7 +91,7 @@ func (t *MiddleTelegram) sendRPCHandshakeRequest(conn wrappers.PacketWriter) (*r return req, nil } -func (t *MiddleTelegram) receiveRPCHandshakeResponse(conn wrappers.PacketReader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { +func (t *middleTelegram) receiveRPCHandshakeResponse(conn wrappers.PacketReader, req *rpc.HandshakeRequest) (*rpc.HandshakeResponse, error) { packet, err := conn.Read() if err != nil { return nil, errors.Annotate(err, "Cannot read RPC handshake response") @@ -107,8 +108,10 @@ func (t *MiddleTelegram) receiveRPCHandshakeResponse(conn wrappers.PacketReader, return rpcHandshakeResp, nil } +// NewMiddleTelegram creates new instance of Telegram which works with +// middle proxies. func NewMiddleTelegram(conf *config.Config) Telegram { - tg := &MiddleTelegram{ + tg := &middleTelegram{ middleTelegramCaller: middleTelegramCaller{ baseTelegram: baseTelegram{ dialer: tgDialer{ diff --git a/telegram/middle_caller.go b/telegram/middle_caller.go index 832430a..bb5961a 100644 --- a/telegram/middle_caller.go +++ b/telegram/middle_caller.go @@ -104,7 +104,7 @@ func (t *middleTelegramCaller) getTelegramAddresses(url string) (map[int16][]str if err != nil { return nil, errors.Annotate(err, "Cannot access telegram server") } - defer resp.Body.Close() + defer resp.Body.Close() // nolint: errcheck scanner := bufio.NewScanner(resp.Body) data := map[int16][]string{} diff --git a/telegram/telegram.go b/telegram/telegram.go index 43f436d..b6b34ef 100644 --- a/telegram/telegram.go +++ b/telegram/telegram.go @@ -9,6 +9,7 @@ import ( "github.com/9seconds/mtg/wrappers" ) +// Telegram is an interface for different Telegram work modes. type Telegram interface { Dial(string, *mtproto.ConnectionOpts) (wrappers.StreamReadWriteCloser, error) Init(*mtproto.ConnectionOpts, wrappers.StreamReadWriteCloser) (wrappers.Wrap, error) diff --git a/utils/read_current_data.go b/utils/read_current_data.go index d66802b..284369c 100644 --- a/utils/read_current_data.go +++ b/utils/read_current_data.go @@ -4,6 +4,7 @@ import "io" const readCurrentDataBufferSize = 1024 + 1 // + 1 because telegram operates with blocks mod 4 +// ReadCurrentData reads all data from io.Reader which is ready to be read. func ReadCurrentData(src io.Reader) (rv []byte, err error) { buf := make([]byte, readCurrentDataBufferSize) n := readCurrentDataBufferSize diff --git a/utils/reverse_bytes.go b/utils/reverse_bytes.go index 447adbd..ab7cd5b 100644 --- a/utils/reverse_bytes.go +++ b/utils/reverse_bytes.go @@ -1,5 +1,6 @@ package utils +// ReverseBytes is a common slice reverser. func ReverseBytes(data []byte) []byte { dataLen := len(data) rv := make([]byte, dataLen) diff --git a/utils/uint24.go b/utils/uint24.go index 350f3d5..66c5f1b 100644 --- a/utils/uint24.go +++ b/utils/uint24.go @@ -1,11 +1,15 @@ package utils +// Uint24 is a replacement for the absent Go uint24 data type. +// This data type is little endian. type Uint24 [3]byte +// ToUint24 converts number to Uint24. func ToUint24(number uint32) Uint24 { return Uint24{byte(number), byte(number >> 8), byte(number >> 16)} } +// FromUint24 converts Uint24 to number. func FromUint24(number Uint24) uint32 { return uint32(number[0]) + (uint32(number[1]) << 8) + (uint32(number[2]) << 16) } diff --git a/wrappers/blockcipher.go b/wrappers/blockcipher.go index a4a7149..39fe7e8 100644 --- a/wrappers/blockcipher.go +++ b/wrappers/blockcipher.go @@ -12,6 +12,10 @@ import ( "github.com/juju/errors" ) +// BlockCipher is a stream writer which encrypts/decrypts blocks of data +// with AES CBC. This also is buffered reader. It means, that block +// reading is transparent for it, you can assume you are working with +// good old io.Reader. type BlockCipher struct { buf *bytes.Buffer @@ -63,22 +67,27 @@ func (b *BlockCipher) Write(p []byte) (int, error) { return b.conn.Write(encrypted) } +// Logger returns an instance of the logger for this wrapper. func (b *BlockCipher) Logger() *zap.SugaredLogger { return b.logger } +// LocalAddr returns local address of the underlying net.Conn. func (b *BlockCipher) LocalAddr() *net.TCPAddr { return b.conn.LocalAddr() } +// RemoteAddr returns remote address of the underlying net.Conn. func (b *BlockCipher) RemoteAddr() *net.TCPAddr { return b.conn.RemoteAddr() } +// Close closes underlying net.Conn. func (b *BlockCipher) Close() error { return b.conn.Close() } +// NewBlockCipher creates new instance of BlockCipher based on given data. func NewBlockCipher(conn StreamReadWriteCloser, encryptor, decryptor cipher.BlockMode) StreamReadWriteCloser { return &BlockCipher{ buf: &bytes.Buffer{}, diff --git a/wrappers/conn.go b/wrappers/conn.go index d770cb4..b8ec4e3 100644 --- a/wrappers/conn.go +++ b/wrappers/conn.go @@ -9,6 +9,9 @@ import ( "github.com/9seconds/mtg/stats" ) +// ConnPurpose is intented to be identifier of connection purpose. We +// sometimes want to treat client/telegram connection differently (for +// logging for example). type ConnPurpose uint8 func (c ConnPurpose) String() string { @@ -22,6 +25,7 @@ func (c ConnPurpose) String() string { return "" } +// ConnPurpose* define different connection types. const ( ConnPurposeClient = iota ConnPurposeTelegram @@ -32,6 +36,8 @@ const ( connTimeoutWrite = 5 * time.Minute ) +// Conn is a basic wrapper for net.Conn providing the most low-level +// logic and management as possible. type Conn struct { connID string conn net.Conn @@ -42,7 +48,7 @@ type Conn struct { } func (c *Conn) Write(p []byte) (int, error) { - c.conn.SetWriteDeadline(time.Now().Add(connTimeoutWrite)) + c.conn.SetWriteDeadline(time.Now().Add(connTimeoutWrite)) // nolint: errcheck n, err := c.conn.Write(p) c.logger.Debugw("Write to stream", "bytes", n, "error", err) @@ -52,7 +58,7 @@ func (c *Conn) Write(p []byte) (int, error) { } func (c *Conn) Read(p []byte) (int, error) { - c.conn.SetReadDeadline(time.Now().Add(connTimeoutRead)) + c.conn.SetReadDeadline(time.Now().Add(connTimeoutRead)) // nolint: errcheck n, err := c.conn.Read(p) c.logger.Debugw("Read from stream", "bytes", n, "error", err) @@ -61,11 +67,18 @@ func (c *Conn) Read(p []byte) (int, error) { return n, err } +// Close closes underlying net.Conn instance. func (c *Conn) Close() error { - defer c.logger.Debugw("Closed connection") + defer c.logger.Debugw("Close connection") return c.conn.Close() } +// Logger returns an instance of the logger for this wrapper. +func (c *Conn) Logger() *zap.SugaredLogger { + return c.logger +} + +// LocalAddr returns local address of the underlying net.Conn. func (c *Conn) LocalAddr() *net.TCPAddr { addr := c.conn.LocalAddr().(*net.TCPAddr) newAddr := *addr @@ -81,14 +94,12 @@ func (c *Conn) LocalAddr() *net.TCPAddr { return &newAddr } +// RemoteAddr returns remote address of the underlying net.Conn. func (c *Conn) RemoteAddr() *net.TCPAddr { return c.conn.RemoteAddr().(*net.TCPAddr) } -func (c *Conn) Logger() *zap.SugaredLogger { - return c.logger -} - +// NewConn initializes Conn wrapper for net.Conn. func NewConn(conn net.Conn, connID string, purpose ConnPurpose, publicIPv4, publicIPv6 net.IP) StreamReadWriteCloser { logger := zap.S().With( "connection_id", connID, diff --git a/wrappers/mtproto_abridged.go b/wrappers/mtproto_abridged.go index 2ed116f..e4775d0 100644 --- a/wrappers/mtproto_abridged.go +++ b/wrappers/mtproto_abridged.go @@ -18,6 +18,8 @@ const ( mtprotoAbridgedLargePacketLength = 16777216 // 256 ^ 3 ) +// MTProtoAbridged presents abridged connection between client and +// middle proxy. type MTProtoAbridged struct { conn StreamReadWriteCloser opts *mtproto.ConnectionOpts @@ -127,22 +129,27 @@ func (m *MTProtoAbridged) Write(p []byte) (int, error) { return 0, errors.Errorf("Packet is too big %d", len(p)) } +// Logger returns an instance of the logger for this wrapper. func (m *MTProtoAbridged) Logger() *zap.SugaredLogger { return m.logger } +// LocalAddr returns local address of the underlying net.Conn. func (m *MTProtoAbridged) LocalAddr() *net.TCPAddr { return m.conn.LocalAddr() } +// RemoteAddr returns remote address of the underlying net.Conn. func (m *MTProtoAbridged) RemoteAddr() *net.TCPAddr { return m.conn.RemoteAddr() } +// Close closes underlying net.Conn instance. func (m *MTProtoAbridged) Close() error { return m.conn.Close() } +// NewMTProtoAbridged creates new wrapper for abridged client connection. func NewMTProtoAbridged(conn StreamReadWriteCloser, opts *mtproto.ConnectionOpts) PacketReadWriteCloser { return &MTProtoAbridged{ conn: conn, diff --git a/wrappers/mtproto_cipher.go b/wrappers/mtproto_cipher.go index 72dad3f..bd26eb3 100644 --- a/wrappers/mtproto_cipher.go +++ b/wrappers/mtproto_cipher.go @@ -4,7 +4,7 @@ import ( "bytes" "crypto/aes" "crypto/cipher" - "crypto/md5" + "crypto/md5" // nolint: gas "crypto/sha1" "encoding/binary" "net" @@ -13,21 +13,23 @@ import ( "github.com/9seconds/mtg/utils" ) -type CipherPurpose uint8 +type cipherPurpose uint8 const ( - CipherPurposeClient CipherPurpose = iota - CipherPurposeServer + cipherPurposeClient cipherPurpose = iota + cipherPurposeServer ) var emptyIP = [4]byte{0x00, 0x00, 0x00, 0x00} +// NewMiddleProxyCipher creates new block cipher to proxy<->telegram +// connection. func NewMiddleProxyCipher(conn StreamReadWriteCloser, req *rpc.NonceRequest, resp *rpc.NonceResponse, secret []byte) StreamReadWriteCloser { localAddr := conn.LocalAddr() remoteAddr := conn.RemoteAddr() - encKey, encIV := deriveKeys(CipherPurposeClient, req, resp, localAddr, remoteAddr, secret) - decKey, decIV := deriveKeys(CipherPurposeServer, req, resp, localAddr, remoteAddr, secret) + encKey, encIV := deriveKeys(cipherPurposeClient, req, resp, localAddr, remoteAddr, secret) + decKey, decIV := deriveKeys(cipherPurposeServer, req, resp, localAddr, remoteAddr, secret) enc, _ := makeEncrypterDecrypter(encKey, encIV) _, dec := makeEncrypterDecrypter(decKey, decIV) @@ -35,7 +37,7 @@ func NewMiddleProxyCipher(conn StreamReadWriteCloser, req *rpc.NonceRequest, res return NewBlockCipher(conn, enc, dec) } -func deriveKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { +func deriveKeys(purpose cipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceResponse, client *net.TCPAddr, remote *net.TCPAddr, secret []byte) ([]byte, []byte) { message := bytes.Buffer{} message.Write(resp.Nonce[:]) message.Write(req.Nonce[:]) @@ -54,9 +56,9 @@ func deriveKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceRes message.Write(port[:]) switch purpose { - case CipherPurposeClient: + case cipherPurposeClient: message.WriteString("CLIENT") - case CipherPurposeServer: + case cipherPurposeServer: message.WriteString("SERVER") default: panic("Unexpected cipher purpose") @@ -75,11 +77,11 @@ func deriveKeys(purpose CipherPurpose, req *rpc.NonceRequest, resp *rpc.NonceRes message.Write(req.Nonce[:]) data := message.Bytes() - md5sum := md5.Sum(data[1:]) + md5sum := md5.Sum(data[1:]) // nolint: gas sha1sum := sha1.Sum(data) key := append(md5sum[:12], sha1sum[:]...) - iv := md5.Sum(data[2:]) + iv := md5.Sum(data[2:]) // nolint: gas return key, iv[:] } diff --git a/wrappers/mtproto_frame.go b/wrappers/mtproto_frame.go index 71ee084..58a523c 100644 --- a/wrappers/mtproto_frame.go +++ b/wrappers/mtproto_frame.go @@ -20,6 +20,18 @@ const ( var mtprotoFramePadding = []byte{0x04, 0x00, 0x00, 0x00} +// MTProtoFrame is a wrapper which converts written data to the MTProtoFrame. +// The format of the frame: +// +// [ MSGLEN(4) | SEQNO(4) | MSG(...) | CRC32(4) | PADDING(4*x) ] +// +// MSGLEN is the length of the message + len of seqno and msglen. +// SEQNO is the number of frame in the receive/send sequence. If client +// sends a message with SeqNo 18, it has to receive message with SeqNo 18. +// MSG is the data which has to be written +// CRC32 is the CRC32 checksum of MSGLEN + SEQNO + MSG +// PADDING is custom padding schema to complete frame length to such that +// len(frame) % 16 == 0 type MTProtoFrame struct { conn StreamReadWriteCloser logger *zap.SugaredLogger @@ -28,7 +40,7 @@ type MTProtoFrame struct { writeSeqNo int32 } -func (m *MTProtoFrame) Read() ([]byte, error) { +func (m *MTProtoFrame) Read() ([]byte, error) { // nolint: gocyclo buf := &bytes.Buffer{} sum := crc32.NewIEEE() writer := io.MultiWriter(buf, sum) @@ -60,7 +72,7 @@ func (m *MTProtoFrame) Read() ([]byte, error) { } var seqNo int32 - binary.Read(buf, binary.LittleEndian, &seqNo) + binary.Read(buf, binary.LittleEndian, &seqNo) // nolint: errcheck if seqNo != m.readSeqNo { return nil, errors.Errorf("Unexpected sequence number %d (wait for %d)", seqNo, m.readSeqNo) } @@ -96,12 +108,12 @@ func (m *MTProtoFrame) Write(p []byte) (int, error) { buf := &bytes.Buffer{} buf.Grow(messageLength + paddingLength) - binary.Write(buf, binary.LittleEndian, uint32(messageLength)) - binary.Write(buf, binary.LittleEndian, m.writeSeqNo) + binary.Write(buf, binary.LittleEndian, uint32(messageLength)) // nolint: errcheck + binary.Write(buf, binary.LittleEndian, m.writeSeqNo) // nolint: errcheck buf.Write(p) checksum := crc32.ChecksumIEEE(buf.Bytes()) - binary.Write(buf, binary.LittleEndian, checksum) + binary.Write(buf, binary.LittleEndian, checksum) // nolint: errcheck buf.Write(bytes.Repeat(mtprotoFramePadding, paddingLength/4)) m.logger.Debugw("Write MTProto frame", @@ -117,22 +129,27 @@ func (m *MTProtoFrame) Write(p []byte) (int, error) { return len(p), err } +// Logger returns an instance of the logger for this wrapper. func (m *MTProtoFrame) Logger() *zap.SugaredLogger { return m.logger } +// LocalAddr returns local address of the underlying net.Conn. func (m *MTProtoFrame) LocalAddr() *net.TCPAddr { return m.conn.LocalAddr() } +// RemoteAddr returns remote address of the underlying net.Conn. func (m *MTProtoFrame) RemoteAddr() *net.TCPAddr { return m.conn.RemoteAddr() } +// Close closes underlying net.Conn instance. func (m *MTProtoFrame) Close() error { return m.conn.Close() } +// NewMTProtoFrame creates new PacketWrapper for underlying connection. func NewMTProtoFrame(conn StreamReadWriteCloser, seqNo int32) PacketReadWriteCloser { return &MTProtoFrame{ conn: conn, diff --git a/wrappers/mtproto_intermediate.go b/wrappers/mtproto_intermediate.go index 5c29c79..ab0761c 100644 --- a/wrappers/mtproto_intermediate.go +++ b/wrappers/mtproto_intermediate.go @@ -14,6 +14,8 @@ import ( const mtprotoIntermediateQuickAckLength = 0x80000000 +// MTProtoIntermediate presents intermediate connection between client +// and Telegram. type MTProtoIntermediate struct { conn StreamReadWriteCloser opts *mtproto.ConnectionOpts @@ -88,22 +90,28 @@ func (m *MTProtoIntermediate) Write(p []byte) (int, error) { return m.conn.Write(append(length[:], p...)) } +// Logger returns an instance of the logger for this wrapper. func (m *MTProtoIntermediate) Logger() *zap.SugaredLogger { return m.logger } +// LocalAddr returns local address of the underlying net.Conn. func (m *MTProtoIntermediate) LocalAddr() *net.TCPAddr { return m.conn.LocalAddr() } +// RemoteAddr returns remote address of the underlying net.Conn. func (m *MTProtoIntermediate) RemoteAddr() *net.TCPAddr { return m.conn.RemoteAddr() } +// Close closes underlying net.Conn instance. func (m *MTProtoIntermediate) Close() error { return m.conn.Close() } +// NewMTProtoIntermediate creates new PacketWrapper for intermediate +// client connection. func NewMTProtoIntermediate(conn StreamReadWriteCloser, opts *mtproto.ConnectionOpts) PacketReadWriteCloser { return &MTProtoIntermediate{ conn: conn, diff --git a/wrappers/mtproto_proxy.go b/wrappers/mtproto_proxy.go index fbbb332..1babb95 100644 --- a/wrappers/mtproto_proxy.go +++ b/wrappers/mtproto_proxy.go @@ -12,6 +12,7 @@ import ( "github.com/9seconds/mtg/mtproto/rpc" ) +// MTProtoProxy is a wrapper which creates/reads RPC responses from Telegram. type MTProtoProxy struct { conn PacketReadWriteCloser req *rpc.ProxyRequest @@ -128,22 +129,27 @@ func (m *MTProtoProxy) Write(p []byte) (int, error) { return len(p), nil } +// Logger returns an instance of the logger for this wrapper. func (m *MTProtoProxy) Logger() *zap.SugaredLogger { return m.logger } +// LocalAddr returns local address of the underlying net.Conn. func (m *MTProtoProxy) LocalAddr() *net.TCPAddr { return m.conn.LocalAddr() } +// RemoteAddr returns remote address of the underlying net.Conn. func (m *MTProtoProxy) RemoteAddr() *net.TCPAddr { return m.conn.RemoteAddr() } +// Close closes underlying net.Conn instance. func (m *MTProtoProxy) Close() error { return m.conn.Close() } +// NewMTProtoProxy creates new RPC wrapper. func NewMTProtoProxy(conn PacketReadWriteCloser, connOpts *mtproto.ConnectionOpts, adTag []byte) (PacketReadWriteCloser, error) { req, err := rpc.NewProxyRequest(connOpts.ClientAddr, conn.LocalAddr(), connOpts, adTag) if err != nil { diff --git a/wrappers/streamcipher.go b/wrappers/streamcipher.go index da89535..b50a351 100644 --- a/wrappers/streamcipher.go +++ b/wrappers/streamcipher.go @@ -8,6 +8,8 @@ import ( "go.uber.org/zap" ) +// StreamCipher is a wrapper which encrypts/decrypts stream with AES-CTR +// (as a part of obfuscated2 protocol). type StreamCipher struct { encryptor cipher.Stream decryptor cipher.Stream @@ -32,22 +34,27 @@ func (s *StreamCipher) Write(p []byte) (int, error) { return s.conn.Write(encrypted) } +// Logger returns an instance of the logger for this wrapper. func (s *StreamCipher) Logger() *zap.SugaredLogger { return s.logger } +// LocalAddr returns local address of the underlying net.Conn. func (s *StreamCipher) LocalAddr() *net.TCPAddr { return s.conn.LocalAddr() } +// RemoteAddr returns remote address of the underlying net.Conn. func (s *StreamCipher) RemoteAddr() *net.TCPAddr { return s.conn.RemoteAddr() } +// Close closes underlying net.Conn instance. func (s *StreamCipher) Close() error { return s.conn.Close() } +// NewStreamCipher creates new stream cipher wrapper. func NewStreamCipher(conn StreamReadWriteCloser, encryptor, decryptor cipher.Stream) StreamReadWriteCloser { return &StreamCipher{ conn: conn, diff --git a/wrappers/wrap.go b/wrappers/wrap.go index 2023694..5e9581b 100644 --- a/wrappers/wrap.go +++ b/wrappers/wrap.go @@ -7,78 +7,104 @@ import ( "go.uber.org/zap" ) +// Wrap is a base interface for all wrappers in this package. type Wrap interface { Logger() *zap.SugaredLogger LocalAddr() *net.TCPAddr RemoteAddr() *net.TCPAddr } +// Writer is a base interface for writers of this package. type Writer interface { io.Writer Wrap } +// Closer is a base interface for wrappers of this package which can +// close connections. type Closer interface { io.Closer Wrap } +// WriteCloser is a base interface for wrappers of this package which +// can write to and close connections. type WriteCloser interface { io.Closer Writer } +// StreamReader is a base interface for wrappers which can read from the +// stream. type StreamReader interface { io.Reader Wrap } +// StreamReadCloser is a base interface for wrappers which can read from +// and close the connections. type StreamReadCloser interface { io.Closer StreamReader } +// StreamReadWriter is a base interface for wrappers which can read from +// and write to the connections. type StreamReadWriter interface { io.Writer StreamReader } +// StreamWriteCloser is a base interface for wrappers which can write to +// and close the connections. type StreamWriteCloser interface { io.WriteCloser Wrap } +// StreamReadWriteCloser is a base interface for stream processors. type StreamReadWriteCloser interface { io.Closer StreamReadWriter } +// PacketReader is a base interface for wrappers which reads 'packets'. +// packets are atoms so you either get a packet or you get an error You +// cannot resume reading from packet. type PacketReader interface { Read() ([]byte, error) Wrap } +// PacketWriter is a base interface for wrappers which can write packets. type PacketWriter interface { io.Writer Wrap } +// PacketReadWriter is a base interface for wrappers which can read from +// and write packets. type PacketReadWriter interface { io.Writer PacketReader } +// PacketReadCloser is a base interface for wrappers which can read +// packets and close the connection. type PacketReadCloser interface { io.Closer PacketReader } +// PacketWriteCloser is a base interface for wrappers which can write +// packets and close the connection. type PacketWriteCloser interface { io.Writer io.Closer Wrap } +// PacketReadWriteCloser is a base interface for packet processors. type PacketReadWriteCloser interface { io.Closer PacketReadWriter From 0a3dca90d245e6339e20ba3dfe84fb573ba94fc1 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 16:31:49 +0300 Subject: [PATCH 69/71] Update README --- README.md | 12 ++++++++++++ run-mtg.sh | 1 - 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2ecccd9..980855d 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,18 @@ mtg is an implementation in golang which is intended to be: * **No management WebUI** This is an implementation of simple lightweight proxy. I won't do that. +This proxy supports 2 modes of work: direct connection to Telegram and +promoted channel mode. If you do not need promoted channels, I would +recommend you to go with direct mode: this is way more robust. + +To run proxy in direct mode, all you need to do is just provide a +secret. If you do not provide ADTag as a second parameter, promoted +channels mode won't be activated. + +To get promoted channel, please contact +[@MTProxybot|https://t.me/MTProxybot] and provide generated adtag as a +second parameter. + # How to build diff --git a/run-mtg.sh b/run-mtg.sh index 96d72d3..60459c0 100755 --- a/run-mtg.sh +++ b/run-mtg.sh @@ -21,7 +21,6 @@ docker run \ --sysctl net.ipv4.tcp_congestion_control=bbr \ --sysctl net.ipv4.tcp_fastopen=3 \ --sysctl net.ipv4.tcp_fin_timeout=30 \ - --sysctl net.ipv4.tcp_keepalive_time=1200 \ --sysctl net.ipv4.tcp_max_syn_backlog=4096 \ --sysctl net.ipv4.tcp_max_tw_buckets=5000 \ --sysctl net.ipv4.tcp_mtu_probing=1 \ From ce8168bc28b8733d6501c859c776056b42726252 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 16:35:50 +0300 Subject: [PATCH 70/71] Fix small lint problem --- proxy/proxy.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 42f5d2b..3fba774 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -55,34 +55,34 @@ func (p *Proxy) accept(conn net.Conn) { log.Infow("Client connected", "addr", conn.RemoteAddr()) - client, opts, err := p.clientInit(conn, connID, p.conf) + clientConn, opts, err := p.clientInit(conn, connID, p.conf) if err != nil { log.Errorw("Cannot initialize client connection", "error", err) return } - defer client.(io.Closer).Close() // nolint: errcheck + defer clientConn.(io.Closer).Close() // nolint: errcheck - stats.ClientConnected(opts.ConnectionType, client.RemoteAddr()) - defer stats.ClientDisconnected(opts.ConnectionType, client.RemoteAddr()) + stats.ClientConnected(opts.ConnectionType, clientConn.RemoteAddr()) + defer stats.ClientDisconnected(opts.ConnectionType, clientConn.RemoteAddr()) - server, err := p.getTelegramConn(opts, connID) + serverConn, err := p.getTelegramConn(opts, connID) if err != nil { log.Errorw("Cannot initialize server connection", "error", err) return } - defer server.(io.Closer).Close() // nolint: errcheck + defer serverConn.(io.Closer).Close() // nolint: errcheck wait := &sync.WaitGroup{} wait.Add(2) if p.conf.UseMiddleProxy() { - clientPacket := client.(wrappers.PacketReadWriteCloser) - serverPacket := server.(wrappers.PacketReadWriteCloser) + clientPacket := clientConn.(wrappers.PacketReadWriteCloser) + serverPacket := serverConn.(wrappers.PacketReadWriteCloser) go p.middlePipe(clientPacket, serverPacket, wait, &opts.ReadHacks) go p.middlePipe(serverPacket, clientPacket, wait, &opts.WriteHacks) } else { - clientStream := client.(wrappers.StreamReadWriteCloser) - serverStream := server.(wrappers.StreamReadWriteCloser) + clientStream := clientConn.(wrappers.StreamReadWriteCloser) + serverStream := serverConn.(wrappers.StreamReadWriteCloser) go p.directPipe(clientStream, serverStream, wait) go p.directPipe(serverStream, clientStream, wait) } From c74d4d65dbafb32fe0abc6011cf552e59d656cb3 Mon Sep 17 00:00:00 2001 From: 9seconds Date: Mon, 9 Jul 2018 16:41:18 +0300 Subject: [PATCH 71/71] Put timeouts for 2 minutes stale connections --- wrappers/conn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wrappers/conn.go b/wrappers/conn.go index b8ec4e3..408483a 100644 --- a/wrappers/conn.go +++ b/wrappers/conn.go @@ -32,8 +32,8 @@ const ( ) const ( - connTimeoutRead = 5 * time.Minute - connTimeoutWrite = 5 * time.Minute + connTimeoutRead = 2 * time.Minute + connTimeoutWrite = 2 * time.Minute ) // Conn is a basic wrapper for net.Conn providing the most low-level