From 21036c3e307d9aab0c85d8f0d4c5daa6593184e9 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Wed, 6 Nov 2024 00:36:54 -0300 Subject: [PATCH] Support for prometheus /metrics endpoint --- go.mod | 9 +++++ go.sum | 18 +++++++++ httppaths.go | 3 ++ main.go | 103 +++++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 126 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index d3a170a..5619475 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,18 @@ toolchain go1.23.0 require github.com/quic-go/quic-go v0.48.1 require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/conduitio/bwlimit v0.1.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/pprof v0.0.0-20241029010322-833c56d90c8e // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.20.2 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect go.uber.org/mock v0.5.0 // indirect golang.org/x/crypto v0.28.0 // indirect @@ -22,4 +30,5 @@ require ( golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.26.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/go.sum b/go.sum index 37f8a01..9102fc9 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= @@ -20,14 +24,26 @@ github.com/google/pprof v0.0.0-20241029010322-833c56d90c8e h1:v7R0PZoC2p1KWQmv1+ github.com/google/pprof v0.0.0-20241029010322-833c56d90c8e/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE= github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kolesa-team/go-webp v1.0.4 h1:wQvU4PLG/X7RS0vAeyhiivhLRoxfLVRlDq4I3frdxIQ= github.com/kolesa-team/go-webp v1.0.4/go.mod h1:oMvdivD6K+Q5qIIkVC2w4k2ZUnI1H+MyP7inwgWq9aA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.47.0 h1:yXs3v7r2bm1wmPTYNLKAAJTHMYkPEsfYJmTazXrCZ7Y= @@ -78,6 +94,8 @@ golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/httppaths.go b/httppaths.go index 1652091..4f1b1f3 100644 --- a/httppaths.go +++ b/httppaths.go @@ -97,6 +97,7 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { if resp.StatusCode == 403 { atomic.AddInt64(&stats_.RequestsForbidden.Videoplayback, 1) + metrics.RequestForbidden.Videoplayback.Inc() io.WriteString(w, "Forbidden 403\n") io.WriteString(w, "Maybe Youtube blocked the IP of this proxy?\n") return @@ -178,6 +179,7 @@ func vi(w http.ResponseWriter, req *http.Request) { w.WriteHeader(resp.StatusCode) if resp.StatusCode == 403 { atomic.AddInt64(&stats_.RequestsForbidden.Vi, 1) + metrics.RequestForbidden.Vi.Inc() io.WriteString(w, "Forbidden 403") return } @@ -217,6 +219,7 @@ func ggpht(w http.ResponseWriter, req *http.Request) { w.WriteHeader(resp.StatusCode) if resp.StatusCode == 403 { atomic.AddInt64(&stats_.RequestsForbidden.Ggpht, 1) + metrics.RequestForbidden.Ggpht.Inc() io.WriteString(w, "Forbidden 403") return } diff --git a/main.go b/main.go index bd01b5b..7e85202 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,8 @@ import ( "time" "github.com/conduitio/bwlimit" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" ) @@ -113,12 +115,15 @@ type ConnectionWatcher struct { func (cw *ConnectionWatcher) OnStateChange(conn net.Conn, state http.ConnState) { switch state { case http.StateNew: - atomic.AddInt64(&cw.established, 1) - atomic.AddInt64(&cw.totalEstablished, 1) + atomic.AddInt64(&stats_.EstablishedConnections, 1) + metrics.EstablishedConnections.Inc() + atomic.AddInt64(&stats_.TotalConnEstablished, 1) + metrics.TotalConnEstablished.Inc() // case http.StateActive: // atomic.AddInt64(&cw.active, 1) case http.StateClosed, http.StateHijacked: - atomic.AddInt64(&cw.established, -1) + atomic.AddInt64(&stats_.EstablishedConnections, -1) + metrics.EstablishedConnections.Dec() } } @@ -141,7 +146,7 @@ type statusJson struct { RequestCount int64 `json:"requestCount"` RequestPerSecond int64 `json:"requestPerSecond"` RequestPerMinute int64 `json:"requestPerMinute"` - TotalEstablished int64 `json:"totalEstablished"` + TotalConnEstablished int64 `json:"totalEstablished"` EstablishedConnections int64 `json:"establishedConnections"` ActiveConnections int64 `json:"activeConnections"` IdleConnections int64 `json:"idleConnections"` @@ -158,7 +163,7 @@ var stats_ = statusJson{ RequestCount: 0, RequestPerSecond: 0, RequestPerMinute: 0, - TotalEstablished: 0, + TotalConnEstablished: 0, EstablishedConnections: 0, ActiveConnections: 0, IdleConnections: 0, @@ -173,6 +178,65 @@ var stats_ = statusJson{ }, } +type Metrics struct { + Uptime prometheus.Gauge + RequestCount prometheus.Counter + RequestPerSecond prometheus.Gauge + RequestPerMinute prometheus.Gauge + TotalConnEstablished prometheus.Counter + EstablishedConnections prometheus.Gauge + ActiveConnections prometheus.Gauge + IdleConnections prometheus.Gauge + RequestForbidden struct { + Videoplayback prometheus.Counter + Vi prometheus.Counter + Ggpht prometheus.Counter + } +} + +var metrics = Metrics{ + Uptime: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "uptime", + }), + RequestCount: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "request_count", + }), + RequestPerSecond: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "request_per_second", + }), + RequestPerMinute: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "request_per_minute", + }), + TotalConnEstablished: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "total_conn_established", + }), + EstablishedConnections: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "established_conns", + }), + ActiveConnections: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "active_conns", + }), + IdleConnections: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "idle_conns", + }), + + RequestForbidden: struct { + Videoplayback prometheus.Counter + Vi prometheus.Counter + Ggpht prometheus.Counter + }{ + Videoplayback: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "request_forbidden_videoplayback", + }), + Vi: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "request_forbidden_vi", + }), + Ggpht: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "request_forbidden_ggpht", + }), + }, +} + func root(w http.ResponseWriter, req *http.Request) { const msg = ` HTTP youtube proxy for https://inv.nadeko.net @@ -184,11 +248,19 @@ func root(w http.ResponseWriter, req *http.Request) { io.WriteString(w, msg) } +// CustomHandler wraps the default promhttp.Handler with custom logic +func metricsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + metrics.Uptime.Set(float64(time.Duration(time.Since(programInit).Seconds()))) + promhttp.Handler().ServeHTTP(w, req) + }) +} + func stats(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") stats_.Uptime = time.Duration(time.Since(programInit).Seconds()) - stats_.TotalEstablished = int64(cw.totalEstablished) - stats_.EstablishedConnections = int64(cw.established) + // stats_.TotalEstablished = int64(cw.totalEstablished) + // stats_.EstablishedConnections = int64(cw.established) // stats_.ActiveConnections = int64(cw.active) // stats_.IdleConnections = int64(cw.idle) @@ -208,6 +280,7 @@ func requestPerSecond() { time.Sleep(1 * time.Second) current := stats_.RequestCount stats_.RequestPerSecond = current - last + metrics.RequestPerSecond.Set(float64(stats_.RequestPerSecond)) last = current } } @@ -218,6 +291,7 @@ func requestPerMinute() { time.Sleep(60 * time.Second) current := stats_.RequestCount stats_.RequestPerMinute = current - last + metrics.RequestPerMinute.Set(float64(stats_.RequestPerMinute)) last = current } } @@ -249,6 +323,7 @@ func beforeAll(next http.HandlerFunc) http.HandlerFunc { w.Header().Set("Access-Control-Max-Age", "1728000") atomic.AddInt64(&stats_.RequestCount, 1) + metrics.RequestCount.Inc() next(w, req) } } @@ -307,6 +382,20 @@ func main() { mux.HandleFunc("/health", health) mux.HandleFunc("/stats", stats) + prometheus.MustRegister(metrics.Uptime) + prometheus.MustRegister(metrics.ActiveConnections) + prometheus.MustRegister(metrics.IdleConnections) + prometheus.MustRegister(metrics.EstablishedConnections) + prometheus.MustRegister(metrics.TotalConnEstablished) + prometheus.MustRegister(metrics.RequestCount) + prometheus.MustRegister(metrics.RequestPerSecond) + prometheus.MustRegister(metrics.RequestPerMinute) + prometheus.MustRegister(metrics.RequestForbidden.Videoplayback) + prometheus.MustRegister(metrics.RequestForbidden.Vi) + prometheus.MustRegister(metrics.RequestForbidden.Ggpht) + + mux.Handle("/metrics", metricsHandler()) + mux.HandleFunc("/videoplayback", beforeAll(videoplayback)) mux.HandleFunc("/vi/", beforeAll(vi)) mux.HandleFunc("/vi_webp/", beforeAll(vi))