diff --git a/config/config.go b/config/config.go index 702c5f7..3176867 100644 --- a/config/config.go +++ b/config/config.go @@ -27,6 +27,12 @@ type Config struct { MaxSize TypeBytes `json:"max-size"` ErrorRate TypeErrorRate `json:"error-rate"` } `json:"anti-replay"` + Blocklist struct { + Enabled bool `json:"enabled"` + DownloadConcurrency uint `json:"download-concurrency"` + URLs []TypeBlocklistURI `json:"urls"` + UpdateEach TypeDuration `json:"update-each"` + } `json:"blocklist"` } `json:"defense"` Network struct { PublicIP struct { @@ -60,6 +66,7 @@ func (c *Config) Validate() error { if !c.Secret.Valid() { return fmt.Errorf("invalid secret %s", c.Secret.String()) } + if len(c.BindTo.HostValue(nil)) == 0 || c.BindTo.PortValue(0) == 0 { return fmt.Errorf("incorrect bind-to parameter %s", c.BindTo.String()) } @@ -98,6 +105,12 @@ type configRaw struct { MaxSize string `toml:"max-size" json:"max-size,omitempty"` ErrorRate float64 `toml:"error-rate" json:"error-rate,omitempty"` } `toml:"anti-replay" json:"anti-replay,omitempty"` + Blocklist struct { + Enabled bool `toml:"enabled" json:"enabled,omitempty"` + DownloadConcurrency uint `toml:"download-concurrency" json:"download-concurrency,omitempty"` + URLs []string `toml:"urls" json:"urls,omitempty"` + UpdateEach string `toml:"update-each" json:"update-each,omitempty"` + } `toml:"blocklist" json:"blocklist,omitempty"` } `toml:"defense" json:"defense,omitempty"` Network struct { PublicIP struct { diff --git a/config/config_test.go b/config/config_test.go index 513ad7b..3233c13 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -39,7 +39,7 @@ func (suite *ConfigTestSuite) TestParseMinimalConfig() { conf, err := config.Parse(suite.ReadConfig("minimal.toml")) suite.NoError(err) suite.Equal("7oe1GqLy6TBc38CV3jx7q09nb29nbGUuY29t", conf.Secret.Base64()) - suite.Equal("0.0.0.0:3128", conf.BindTo.String()) + suite.Equal("0.0.0.0:3128", conf.BindTo.String()) } func TestConfig(t *testing.T) { diff --git a/config/type_blocklist_uri.go b/config/type_blocklist_uri.go new file mode 100644 index 0000000..6bcf6b5 --- /dev/null +++ b/config/type_blocklist_uri.go @@ -0,0 +1,68 @@ +package config + +import ( + "fmt" + "net/url" + "os" + "path/filepath" +) + +type TypeBlocklistURI struct { + value string +} + +func (c *TypeBlocklistURI) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + + text := string(data) + if filepath.IsAbs(text) { + if _, err := os.Stat(text); os.IsNotExist(err) { + return fmt.Errorf("filepath %s does not exist", text) + } + + c.value = text + + return nil + } + + parsedURL, err := url.Parse(text) + if err != nil { + return fmt.Errorf("incorrect url: %w", err) + } + + switch parsedURL.Scheme { + case "http", "https": // nolint: goconst + default: + return fmt.Errorf("unknown schema %s", parsedURL.Scheme) + } + + if parsedURL.Host == "" { + return fmt.Errorf("incorrect url %s", text) + } + + c.value = parsedURL.String() + + return nil +} + +func (c TypeBlocklistURI) MarshalText() ([]byte, error) { + return []byte(c.value), nil +} + +func (c TypeBlocklistURI) String() string { + return c.value +} + +func (c TypeBlocklistURI) IsRemote() bool { + return !filepath.IsAbs(c.value) +} + +func (c TypeBlocklistURI) Value(defaultValue string) string { + if c.value == "" { + return defaultValue + } + + return c.value +} diff --git a/config/type_blocklist_uri_test.go b/config/type_blocklist_uri_test.go new file mode 100644 index 0000000..e0b8974 --- /dev/null +++ b/config/type_blocklist_uri_test.go @@ -0,0 +1,156 @@ +package config_test + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeBlocklistURITestStruct struct { + Value config.TypeBlocklistURI `json:"value"` +} + +type TypeBlocklistURITestSuite struct { + suite.Suite +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalFail() { + rnd := make([]byte, 48) + + rand.Read(rnd) // nolint: errcheck + + unknownPath := base64.StdEncoding.EncodeToString(rnd) + + testData := []string{ + "1", + unknownPath, + "/" + unknownPath, + "http:/", + "gopher://lalalal", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeBlocklistURITestStruct{})) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestUnmarshalOk() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := []string{ + "http://lalala", + filepath.Join(dir, "config.go"), + "https://lalala", + } + + for _, v := range testData { + value := v + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.EqualValues(t, value, testStruct.Value.Value("")) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestMarshalOk() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := []string{ + "http://lalalal", + filepath.Join(dir, "config.go"), + } + + for _, v := range testData { + name := v + + data, err := json.Marshal(map[string]string{ + "value": name, + }) + suite.NoError(err) + + suite.T().Run(name, func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, name, testStruct.Value.String()) + + marshalled, err := testStruct.Value.MarshalText() + assert.NoError(t, err) + assert.Equal(t, name, string(marshalled)) + }) + } +} + +func (suite *TypeBlocklistURITestSuite) TestValue() { + testStruct := &typeBlocklistURITestStruct{} + + suite.Equal("http://lalala", testStruct.Value.Value("http://lalala")) + + data, err := json.Marshal(map[string]string{ + "value": "http://blablabla", + }) + suite.NoError(err) + suite.NoError(json.Unmarshal(data, testStruct)) + + suite.Equal("http://blablabla", testStruct.Value.Value("")) +} + +func (suite *TypeBlocklistURITestSuite) TestIsRemote() { + dir, _ := os.Getwd() + dir, _ = filepath.Abs(dir) + + testData := map[bool]string{ + true: "http://lalalal", + false: filepath.Join(dir, "config.go"), + } + + for k, v := range testData { + ok := k + + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(strconv.FormatBool(ok), func(t *testing.T) { + testStruct := &typeBlocklistURITestStruct{} + assert.NoError(t, json.Unmarshal(data, testStruct)) + + if ok { + assert.True(t, testStruct.Value.IsRemote()) + } else { + assert.False(t, testStruct.Value.IsRemote()) + } + }) + } +} + +func TestTypeBlocklistURI(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeBlocklistURITestSuite{}) +} diff --git a/example.config.toml b/example.config.toml index a9ff45b..0522909 100644 --- a/example.config.toml +++ b/example.config.toml @@ -149,6 +149,30 @@ max-size = "16mb" # to maintain a desired error ratio. error-rate = 0.0001 +# You can protect proxies by using different blocklists. If client has +# ip from the given range, we do not try to do a proper handshake. We +# actually route it to fronting domain. So, this client will never ever +# have a chance to use mtg to access Telegram. +# +# Please remember that blocklists are initialized in async way. So, +# when you start a proxy, blocklists are empty, they are populated and +# processed in backgrounds. An error in any URL is ignored. +[defense.blocklist] +# You can enable/disable this feature. +enabled = true +# This is a limiter for concurrency. In order to protect website +# from overloading, we download files in this number of threads. +download-concurrency = 2 +# A list of URLs in FireHOL format (https://iplists.firehol.org/) +# You can provider links here (starts with https:// or http://) or +# path to a local file, but in this case it should be absolute. +urls = [ + # "https://iplists.firehol.org/files/firehol_level1.netset", + # "/local.file" +] +# How often do we need to update a blocklist set. +update-each = "1d" + # statsd statistics integration. [stats.statsd] # enabled/disabled