ScuroNeko/mtg
1
0
Fork 0
mirror of https://github.com/ScuroNeko/mtg.git synced 2026-08-31 22:44:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add EventTraffic

Browse Source
This commit is contained in:
9seconds
2021-03-22 17:54:46 +03:00
parent 925a02dac3
commit 42160a08fe
18 changed files with 569 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
stats/init.go
+15 -7
View File
@@ -6,13 +6,21 @@ const (
DefaultStatsdMetricPrefix = DefaultMetricPrefix + "."
DefaultStatsdTagFormat = "datadog"
MetricActiveConnection = "active_connections"
MetricSessionDuration = "session_duration"
MetricConcurrencyLimited = "concurrency_limited"
MetricIPBlocklisted = "ip_blocklisted"
MetricClientConnections = "client_connections"
MetricTelegramConnections = "telegram_connections"
MetricTraffic = "traffic"
MetricSessionDuration = "session_duration"
MetricSessionTraffic = "session_traffic"
MetricConcurrencyLimited = "concurrency_limited"
MetricIPBlocklisted = "ip_blocklisted"
TagIPType = "ip_type"
TagIPType = "ip_type"
TagTelegramIP = "ip"
TagDC = "dc"
TagDirection = "direction"
TagIPTypeIPv4 = "ipv4"
TagIPTypeIPv6 = "ipv6"
TagIPTypeIPv4 = "ipv4"
TagIPTypeIPv6 = "ipv6"
TagDirectionTelegram = "telegram"
TagDirectionClient = "client"
)
stats/prometheus.go
+105 -10
View File
@@ -4,6 +4,7 @@ import (
"context"
"net"
"net/http"
"strconv"
"time"
"github.com/9seconds/mtg/v2/events"
@@ -24,7 +25,47 @@ func (p prometheusProcessor) EventStart(evt mtglib.EventStart) {
}
p.streams[evt.StreamID()] = sInfo
p.factory.metricActiveConnections.WithLabelValues(sInfo.IPType()).Inc()
p.factory.metricClientConnections.WithLabelValues(sInfo.GetClientIPType()).Inc()
}
func (p prometheusProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) {
sInfo, ok := p.streams[evt.StreamID()]
if !ok {
return
}
sInfo.remoteIP = evt.RemoteIP
sInfo.dc = evt.DC
p.factory.metricTelegramConnections.WithLabelValues(
sInfo.GetRemoteIPType(),
sInfo.remoteIP.String(),
strconv.Itoa(sInfo.dc)).Inc()
}
func (p prometheusProcessor) EventTraffic(evt mtglib.EventTraffic) {
sInfo, ok := p.streams[evt.StreamID()]
if !ok {
return
}
labels := []string{
sInfo.GetRemoteIPType(),
sInfo.remoteIP.String(),
strconv.Itoa(sInfo.dc),
}
if evt.IsRead {
sInfo.bytesRecvFromTelegram += evt.Traffic
labels = append(labels, TagDirectionClient)
} else {
sInfo.bytesSentToTelegram += evt.Traffic
labels = append(labels, TagDirectionTelegram)
}
p.factory.metricTraffic.WithLabelValues(labels...).Add(float64(evt.Traffic))
}
func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) {
@@ -37,8 +78,30 @@ func (p prometheusProcessor) EventFinish(evt mtglib.EventFinish) {
duration := evt.CreatedAt.Sub(sInfo.createdAt)
p.factory.metricActiveConnections.WithLabelValues(sInfo.IPType()).Dec()
p.factory.metricClientConnections.WithLabelValues(sInfo.GetRemoteIPType()).Dec()
p.factory.metricSessionDuration.Observe(float64(duration) / float64(time.Second))
if sInfo.remoteIP == nil {
return
}
labels := []string{
sInfo.GetRemoteIPType(),
sInfo.remoteIP.String(),
strconv.Itoa(sInfo.dc),
}
p.factory.metricTelegramConnections.WithLabelValues(labels...).Dec()
labels = append(labels, TagDirectionClient)
p.factory.metricSessionTraffic.
WithLabelValues(labels...).
Observe(float64(sInfo.bytesRecvFromTelegram))
labels[3] = TagDirectionTelegram
p.factory.metricSessionTraffic.
WithLabelValues(labels...).
Observe(float64(sInfo.bytesSentToTelegram))
}
func (p prometheusProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {
@@ -60,10 +123,13 @@ func (p prometheusProcessor) Shutdown() {
type PrometheusFactory struct {
httpServer *http.Server
metricActiveConnections *prometheus.GaugeVec
metricIPBlocklisted *prometheus.CounterVec
metricConcurrencyLimited prometheus.Counter
metricSessionDuration prometheus.Histogram
metricClientConnections *prometheus.GaugeVec
metricTelegramConnections *prometheus.GaugeVec
metricTraffic *prometheus.CounterVec
metricIPBlocklisted *prometheus.CounterVec
metricSessionTraffic *prometheus.HistogramVec
metricConcurrencyLimited prometheus.Counter
metricSessionDuration prometheus.Histogram
}
func (p *PrometheusFactory) Make() events.Observer {
@@ -81,7 +147,7 @@ func (p *PrometheusFactory) Close() error {
return p.httpServer.Shutdown(context.Background())
}
func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory {
func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory { // nolint: funlen
registry := prometheus.NewPedanticRegistry()
httpHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{
EnableOpenMetrics: true,
@@ -95,11 +161,16 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory {
Handler: mux,
},
metricActiveConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
metricClientConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: metricPrefix,
Name: MetricActiveConnection,
Name: MetricClientConnections,
Help: "A number of connections under active processing.",
}, []string{TagIPType}),
metricTelegramConnections: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: metricPrefix,
Name: MetricTelegramConnections,
Help: "A number of connections to Telegram servers.",
}, []string{TagIPType, TagTelegramIP, TagDC}),
metricSessionDuration: prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: metricPrefix,
Name: MetricSessionDuration,
@@ -117,6 +188,27 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory {
300,
},
}),
metricSessionTraffic: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricPrefix,
Name: MetricSessionTraffic,
Help: "A traffic size which flew via proxy within a single session.",
Buckets: []float64{ // per 1mb
1 * 1024 * 1024,
2 * 1024 * 1024,
3 * 1024 * 1024,
4 * 1024 * 1024,
5 * 1024 * 1024,
6 * 1024 * 1024,
7 * 1024 * 1024,
8 * 1024 * 1024,
9 * 1024 * 1024,
},
}, []string{TagIPType, TagTelegramIP, TagDC, TagDirection}),
metricTraffic: prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: metricPrefix,
Name: MetricTraffic,
Help: "Traffic which is sent through this proxy.",
}, []string{TagIPType, TagTelegramIP, TagDC, TagDirection}),
metricConcurrencyLimited: prometheus.NewCounter(prometheus.CounterOpts{
Namespace: metricPrefix,
Name: MetricConcurrencyLimited,
@@ -129,7 +221,10 @@ func NewPrometheus(metricPrefix, httpPath string) *PrometheusFactory {
}, []string{TagIPType}),
}
registry.MustRegister(factory.metricActiveConnections)
registry.MustRegister(factory.metricClientConnections)
registry.MustRegister(factory.metricTelegramConnections)
registry.MustRegister(factory.metricTraffic)
registry.MustRegister(factory.metricSessionTraffic)
registry.MustRegister(factory.metricSessionDuration)
registry.MustRegister(factory.metricConcurrencyLimited)
registry.MustRegister(factory.metricIPBlocklisted)
stats/prometheus_test.go
+41 -4
View File
@@ -60,23 +60,60 @@ func (suite *PrometheusTestSuite) TestEventStartFinish() {
ConnID: "connID",
RemoteIP: net.ParseIP("10.0.0.10"),
})
time.Sleep(100 * time.Millisecond)
data, err := suite.Get()
suite.NoError(err)
suite.Contains(data, `mtg_active_connections{ip_type="ipv4"} 1`)
suite.Contains(data, `mtg_client_connections{ip_type="ipv4"} 1`)
suite.prometheus.EventConnectedToDC(mtglib.EventConnectedToDC{
CreatedAt: time.Now(),
ConnID: "connID",
RemoteIP: net.ParseIP("10.0.0.1"),
DC: 4,
})
time.Sleep(100 * time.Millisecond)
data, err = suite.Get()
suite.NoError(err)
suite.Contains(data, `mtg_telegram_connections{dc="4",ip="10.0.0.1",ip_type="ipv4"} 1`)
suite.prometheus.EventTraffic(mtglib.EventTraffic{
CreatedAt: time.Now(),
ConnID: "connID",
Traffic: 200,
IsRead: true,
})
time.Sleep(100 * time.Millisecond)
data, err = suite.Get()
suite.NoError(err)
suite.Contains(data, `mtg_traffic{dc="4",direction="client",ip="10.0.0.1",ip_type="ipv4"} 200`)
suite.prometheus.EventTraffic(mtglib.EventTraffic{
CreatedAt: time.Now(),
ConnID: "connID",
Traffic: 100,
IsRead: false,
})
time.Sleep(100 * time.Millisecond)
data, err = suite.Get()
suite.NoError(err)
suite.Contains(data, `mtg_traffic{dc="4",direction="telegram",ip="10.0.0.1",ip_type="ipv4"} 100`)
suite.prometheus.EventFinish(mtglib.EventFinish{
CreatedAt: time.Now(),
ConnID: "connID",
})
time.Sleep(100 * time.Millisecond)
data, err = suite.Get()
suite.NoError(err)
suite.Contains(data, `mtg_active_connections{ip_type="ipv4"} 0`)
suite.Contains(data, `mtg_client_connections{ip_type="ipv4"} 0`)
suite.Contains(data, `mtg_telegram_connections{dc="4",ip="10.0.0.1",ip_type="ipv4"} 0`)
suite.Contains(data, `mtg_traffic{dc="4",direction="client",ip="10.0.0.1",ip_type="ipv4"} 200`)
suite.Contains(data, `mtg_traffic{dc="4",direction="telegram",ip="10.0.0.1",ip_type="ipv4"} 100`)
}
func (suite *PrometheusTestSuite) TestEventConcurrencyLimited() {
stats/statsd.go
+52 -6
View File
@@ -22,9 +22,47 @@ func (s statsdProcessor) EventStart(evt mtglib.EventStart) {
clientIP: evt.RemoteIP,
}
s.streams[evt.StreamID()] = sInfo
ipTypeTag := statsd.StringTag(TagIPType, sInfo.IPType())
s.client.GaugeDelta(MetricActiveConnection, 1, ipTypeTag)
s.client.GaugeDelta(MetricClientConnections,
1,
statsd.StringTag(TagIPType, sInfo.GetClientIPType()))
}
func (s statsdProcessor) EventConnectedToDC(evt mtglib.EventConnectedToDC) {
sInfo, ok := s.streams[evt.StreamID()]
if !ok {
return
}
sInfo.remoteIP = evt.RemoteIP
sInfo.dc = evt.DC
s.client.GaugeDelta(MetricTelegramConnections,
1,
statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()),
statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()),
statsd.IntTag(TagDC, sInfo.dc))
}
func (s statsdProcessor) EventTraffic(evt mtglib.EventTraffic) {
sInfo, ok := s.streams[evt.StreamID()]
if !ok {
return
}
tags := []statsd.Tag{
statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()),
statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()),
statsd.IntTag(TagDC, sInfo.dc),
}
if evt.IsRead {
tags = append(tags, statsd.StringTag(TagDirection, TagDirectionClient))
s.client.Incr(MetricTraffic, int64(evt.Traffic), tags...)
} else {
tags = append(tags, statsd.StringTag(TagDirection, TagDirectionTelegram))
s.client.Incr(MetricTraffic, int64(evt.Traffic), tags...)
}
}
func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) {
@@ -35,11 +73,19 @@ func (s statsdProcessor) EventFinish(evt mtglib.EventFinish) {
defer delete(s.streams, evt.StreamID())
duration := evt.CreatedAt.Sub(sInfo.createdAt)
ipTypeTag := statsd.StringTag(TagIPType, sInfo.IPType())
s.client.GaugeDelta(MetricClientConnections,
-1,
statsd.StringTag(TagIPType, sInfo.GetClientIPType()))
s.client.PrecisionTiming(MetricSessionDuration,
evt.CreatedAt.Sub(sInfo.createdAt))
s.client.GaugeDelta(MetricActiveConnection, -1, ipTypeTag)
s.client.PrecisionTiming(MetricSessionDuration, duration)
if sInfo.remoteIP != nil {
s.client.GaugeDelta(MetricTelegramConnections,
-1,
statsd.StringTag(TagIPType, sInfo.GetRemoteIPType()),
statsd.StringTag(TagTelegramIP, sInfo.remoteIP.String()),
statsd.IntTag(TagDC, sInfo.dc))
}
}
func (s statsdProcessor) EventConcurrencyLimited(_ mtglib.EventConcurrencyLimited) {
stats/statsd_test.go
+35 -3
View File
@@ -103,17 +103,49 @@ func (suite *StatsdTestSuite) TestEventStartFinish() {
ConnID: "connID",
RemoteIP: net.ParseIP("10.0.0.10"),
})
time.Sleep(statsdSleepTime)
suite.Equal("mtg.active_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String())
suite.Equal("mtg.client_connections:+1|g|#ip_type:ipv4", suite.statsdServer.String())
suite.statsd.EventConnectedToDC(mtglib.EventConnectedToDC{
CreatedAt: time.Now(),
ConnID: "connID",
RemoteIP: net.ParseIP("10.1.0.10"),
DC: 2,
})
time.Sleep(statsdSleepTime)
suite.Contains(suite.statsdServer.String(),
"mtg.telegram_connections:+1|g|#ip_type:ipv4,ip:10.1.0.10,dc:2")
suite.statsd.EventTraffic(mtglib.EventTraffic{
CreatedAt: time.Now(),
ConnID: "connID",
Traffic: 30,
IsRead: true,
})
time.Sleep(statsdSleepTime)
suite.Contains(suite.statsdServer.String(),
"mtg.traffic:30|c|#ip_type:ipv4,ip:10.1.0.10,dc:2,direction:client")
suite.statsd.EventTraffic(mtglib.EventTraffic{
CreatedAt: time.Now(),
ConnID: "connID",
Traffic: 90,
IsRead: false,
})
time.Sleep(statsdSleepTime)
suite.Contains(suite.statsdServer.String(),
"mtg.traffic:90|c|#ip_type:ipv4,ip:10.1.0.10,dc:2,direction:telegram")
suite.statsd.EventFinish(mtglib.EventFinish{
CreatedAt: time.Now(),
ConnID: "connID",
})
time.Sleep(statsdSleepTime)
suite.Contains(suite.statsdServer.String(), "mtg.session_duration")
suite.Contains(suite.statsdServer.String(),
"mtg.telegram_connections:-1|g|#ip_type:ipv4,ip:10.1.0.10,dc:2")
suite.Contains(suite.statsdServer.String(),
"mtg.client_connections:-1|g|#ip_type:ipv4")
}
func (suite *StatsdTestSuite) TestEventConcurrencyLimited() {
stats/stream_info.go
+16 -4
View File
@@ -6,12 +6,24 @@ import (
)
type streamInfo struct {
createdAt time.Time
clientIP net.IP
createdAt time.Time
clientIP net.IP
remoteIP net.IP
dc int
bytesSentToTelegram uint
bytesRecvFromTelegram uint
}
func (s *streamInfo) IPType() string {
if s.clientIP.To4() == nil {
func (s *streamInfo) GetClientIPType() string {
return s.getIPType(s.clientIP)
}
func (s *streamInfo) GetRemoteIPType() string {
return s.getIPType(s.remoteIP)
}
func (s *streamInfo) getIPType(ip net.IP) string {
if ip.To4() == nil {
return TagIPTypeIPv6
}