From 8821540bd9b4ea86d305bf432ed08ce7bf976945 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 20 Feb 2025 01:13:40 -0300 Subject: [PATCH] style: refactor project to use the standard go project layout --- main.go => cmd/http3-ytproxy/main.go | 302 ++---------------- go.mod | 2 +- internal/httpc/client.go | 59 ++++ internal/httpc/server.go | 1 + internal/metrics/metrics.go | 76 +++++ internal/metrics/stats.go | 52 +++ internal/paths/consts.go | 33 ++ internal/paths/ggpht.go | 50 +++ internal/paths/health.go | 11 + internal/paths/metrics.go | 27 ++ internal/paths/root.go | 17 + internal/paths/stats.go | 20 ++ internal/paths/vi.go | 62 ++++ .../paths/videoplayback.go | 160 +++------- utils.go => internal/utils/utils.go | 31 +- nginx.conf | 19 -- 16 files changed, 506 insertions(+), 416 deletions(-) rename main.go => cmd/http3-ytproxy/main.go (54%) create mode 100644 internal/httpc/client.go create mode 100644 internal/httpc/server.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/metrics/stats.go create mode 100644 internal/paths/consts.go create mode 100644 internal/paths/ggpht.go create mode 100644 internal/paths/health.go create mode 100644 internal/paths/metrics.go create mode 100644 internal/paths/root.go create mode 100644 internal/paths/stats.go create mode 100644 internal/paths/vi.go rename httppaths.go => internal/paths/videoplayback.go (68%) rename utils.go => internal/utils/utils.go (69%) delete mode 100644 nginx.conf diff --git a/main.go b/cmd/http3-ytproxy/main.go similarity index 54% rename from main.go rename to cmd/http3-ytproxy/main.go index 4c28b8f..0080ef7 100644 --- a/main.go +++ b/cmd/http3-ytproxy/main.go @@ -2,16 +2,13 @@ package main import ( "crypto/tls" - "encoding/json" "errors" "flag" "io" "log" "net" "net/http" - "net/url" "os" - "regexp" "runtime" "strconv" "strings" @@ -19,9 +16,11 @@ import ( "syscall" "time" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/httpc" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/metrics" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/paths" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/utils" "github.com/conduitio/bwlimit" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/procfs" "github.com/quic-go/quic-go" "github.com/quic-go/quic-go/http3" @@ -32,94 +31,12 @@ var ( rl = flag.Int("r", 8000, "Read limit in Kbps") ) -// QUIC doesn't seem to support HTTP nor SOCKS5 proxies due to how it's made. -// (Since it's UDP) -var h3client = &http.Client{ - Transport: &http3.Transport{}, - Timeout: 10 * time.Second, -} - -var dialer = &net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, -} - -var proxy string - -// http/2 client -var h2client = &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - Transport: &http.Transport{ - Dial: func(network, addr string) (net.Conn, error) { - var net string - if ipv6_only { - net = "tcp6" - } else { - net = "tcp4" - } - return dialer.Dial(net, addr) - }, - TLSHandshakeTimeout: 10 * time.Second, - ResponseHeaderTimeout: 20 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - IdleConnTimeout: 30 * time.Second, - ReadBufferSize: 16 * 1024, - ForceAttemptHTTP2: true, - MaxConnsPerHost: 0, - MaxIdleConnsPerHost: 10, - MaxIdleConns: 0, - Proxy: func(r *http.Request) (*url.URL, error) { - if proxy != "" { - return url.Parse(proxy) - } - return nil, nil - }, - }, -} - -var client *http.Client - -var default_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" - -var allowed_hosts = []string{ - "youtube.com", - "googlevideo.com", - "ytimg.com", - "ggpht.com", - "googleusercontent.com", -} - -var strip_headers = []string{ - "Accept-Encoding", - "Authorization", - "Origin", - "Referer", - "Cookie", - "Set-Cookie", - "Etag", - "Alt-Svc", - "Server", - "Cache-Control", - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-to - "report-to", -} - -var path_prefix = "" - -var manifest_re = regexp.MustCompile(`(?m)URI="([^"]+)"`) - -var ipv6_only = false - var version string var h3s bool var domain_only_access bool = false -var programInit = time.Now() - type ConnectionWatcher struct { totalEstablished int64 established int64 @@ -136,14 +53,14 @@ func (cw *ConnectionWatcher) OnStateChange(conn net.Conn, state http.ConnState) switch state { case http.StateNew: atomic.AddInt64(&stats_.EstablishedConnections, 1) - metrics.EstablishedConnections.Inc() + metrics.Metrics.EstablishedConnections.Inc() atomic.AddInt64(&stats_.TotalConnEstablished, 1) - metrics.TotalConnEstablished.Inc() + metrics.Metrics.TotalConnEstablished.Inc() // case http.StateActive: // atomic.AddInt64(&cw.active, 1) case http.StateClosed, http.StateHijacked: atomic.AddInt64(&stats_.EstablishedConnections, -1) - metrics.EstablishedConnections.Dec() + metrics.Metrics.EstablishedConnections.Dec() } } @@ -160,161 +77,13 @@ func (cw *ConnectionWatcher) OnStateChange(conn net.Conn, state http.ConnState) var cw ConnectionWatcher -type statusJson struct { - Version string `json:"version"` - Uptime time.Duration `json:"uptime"` - RequestCount int64 `json:"requestCount"` - RequestPerSecond int64 `json:"requestPerSecond"` - RequestPerMinute int64 `json:"requestPerMinute"` - TotalConnEstablished int64 `json:"totalEstablished"` - EstablishedConnections int64 `json:"establishedConnections"` - ActiveConnections int64 `json:"activeConnections"` - IdleConnections int64 `json:"idleConnections"` - RequestsForbiddenPerSec struct { - Videoplayback int64 `json:"videoplayback"` - } - RequestsForbidden struct { - Videoplayback int64 `json:"videoplayback"` - Vi int64 `json:"vi"` - Ggpht int64 `json:"ggpht"` - } `json:"requestsForbidden"` -} - -var stats_ = statusJson{ - Version: version + "-" + runtime.GOARCH, - Uptime: 0, - RequestCount: 0, - RequestPerSecond: 0, - RequestPerMinute: 0, - TotalConnEstablished: 0, - EstablishedConnections: 0, - ActiveConnections: 0, - IdleConnections: 0, - RequestsForbiddenPerSec: struct { - Videoplayback int64 `json:"videoplayback"` - }{ - Videoplayback: 0, - }, - RequestsForbidden: struct { - Videoplayback int64 `json:"videoplayback"` - Vi int64 `json:"vi"` - Ggpht int64 `json:"ggpht"` - }{ - Videoplayback: 0, - Vi: 0, - Ggpht: 0, - }, -} - -type Metrics struct { - Uptime prometheus.Gauge - RequestCount prometheus.Counter - RequestPerSecond prometheus.Gauge - RequestPerMinute prometheus.Gauge - TotalConnEstablished prometheus.Counter - EstablishedConnections prometheus.Gauge - ActiveConnections prometheus.Gauge - IdleConnections prometheus.Gauge - RequestForbidden struct { - Videoplayback prometheus.Counter - Vi prometheus.Counter - Ggpht prometheus.Counter - } -} - -var metrics = Metrics{ - Uptime: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_uptime", - }), - RequestCount: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_request_count", - }), - RequestPerSecond: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_request_per_second", - }), - RequestPerMinute: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_request_per_minute", - }), - TotalConnEstablished: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_total_conn_established", - }), - EstablishedConnections: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_established_conns", - }), - ActiveConnections: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_active_conns", - }), - IdleConnections: prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "http3_ytproxy_idle_conns", - }), - - RequestForbidden: struct { - Videoplayback prometheus.Counter - Vi prometheus.Counter - Ggpht prometheus.Counter - }{ - Videoplayback: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "http3_ytproxy_request_forbidden_videoplayback", - }), - Vi: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "http3_ytproxy_request_forbidden_vi", - }), - Ggpht: prometheus.NewCounter(prometheus.CounterOpts{ - Name: "http3_ytproxy_request_forbidden_ggpht", - }), - }, -} - -func root(w http.ResponseWriter, req *http.Request) { - const msg = ` - HTTP youtube proxy for https://inv.nadeko.net - https://git.nadeko.net/Fijxu/http3-ytproxy - - Routes: - /stats - /health` - io.WriteString(w, msg) -} - -// CustomHandler wraps the default promhttp.Handler with custom logic -func metricsHandler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - // To prevent accessing from the bare IP address - if req.Host == "" || net.ParseIP(strings.Split(req.Host, ":")[0]) != nil { - w.WriteHeader(444) - return - } - - metrics.Uptime.Set(float64(time.Duration(time.Since(programInit).Seconds()))) - promhttp.Handler().ServeHTTP(w, req) - }) -} - -func stats(w http.ResponseWriter, req *http.Request) { - w.Header().Set("Content-Type", "application/json") - stats_.Uptime = time.Duration(time.Since(programInit).Seconds()) - // stats_.TotalEstablished = int64(cw.totalEstablished) - // stats_.EstablishedConnections = int64(cw.established) - // stats_.ActiveConnections = int64(cw.active) - // stats_.IdleConnections = int64(cw.idle) - - if err := json.NewEncoder(w).Encode(stats_); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -func health(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(200) - io.WriteString(w, "OK") -} - func requestPerSecond() { var last int64 for { time.Sleep(1 * time.Second) current := stats_.RequestCount stats_.RequestPerSecond = current - last - metrics.RequestPerSecond.Set(float64(stats_.RequestPerSecond)) + metrics.Metrics.RequestPerSecond.Set(float64(stats_.RequestPerSecond)) last = current } } @@ -325,7 +94,7 @@ func requestPerMinute() { time.Sleep(60 * time.Second) current := stats_.RequestCount stats_.RequestPerMinute = current - last - metrics.RequestPerMinute.Set(float64(stats_.RequestPerMinute)) + metrics.Metrics.RequestPerMinute.Set(float64(stats_.RequestPerMinute)) last = current } } @@ -371,7 +140,7 @@ func blockChecker(gh string, cooldown int) { body := "{\"status\":\"stopped\"}\"" // This should never fail too request, _ := http.NewRequest("PUT", url, strings.NewReader(body)) - _, err = client.Do(request) + _, err = httpc.Client.Do(request) if err != nil { log.Printf("[ERROR] Failed to send request to gluetun.") } else { @@ -383,7 +152,7 @@ func blockChecker(gh string, cooldown int) { func beforeMisc(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - defer panicHandler(w) + defer utils.PanicHandler(w) // To prevent accessing from the bare IP address if domain_only_access && (req.Host == "" || net.ParseIP(strings.Split(req.Host, ":")[0]) != nil) { @@ -397,7 +166,7 @@ func beforeMisc(next http.HandlerFunc) http.HandlerFunc { func beforeProxy(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - defer panicHandler(w) + defer utils.PanicHandler(w) // To prevent accessing from the bare IP address if domain_only_access && (req.Host == "" || net.ParseIP(strings.Split(req.Host, ":")[0]) != nil) { @@ -428,7 +197,7 @@ func beforeProxy(next http.HandlerFunc) http.HandlerFunc { } atomic.AddInt64(&stats_.RequestCount, 1) - metrics.RequestCount.Inc() + metrics.Metrics.RequestCount.Inc() next(w, req) } } @@ -493,23 +262,24 @@ func main() { if bc_cooldown == "" { bc_cooldown = "60" } - proxy = os.Getenv("PROXY") + httpc.Proxy = os.Getenv("PROXY") flag.BoolVar(&https, "https", https, "Use built-in https server (recommended)") flag.BoolVar(&h3c, "h3c", h3c, "Use HTTP/3 for client requests (high CPU usage)") flag.BoolVar(&h3s, "h3s", h3s, "Use HTTP/3 for server requests, (requires HTTPS)") - flag.BoolVar(&ipv6_only, "ipv6_only", ipv6_only, "Only use ipv6 for requests") + flag.BoolVar(&httpc.Ipv6_only, "ipv6_only", httpc.Ipv6_only, "Only use ipv6 for requests") flag.StringVar(&tls_cert, "tls-cert", tls_cert, "TLS Certificate path") flag.StringVar(&tls_key, "tls-key", tls_key, "TLS Certificate Key path") flag.StringVar(&sock, "s", sock, "Specify a socket name") flag.StringVar(&port, "p", port, "Specify a port number") flag.StringVar(&host, "l", host, "Specify a listen address") flag.Parse() + httpc.Ipv6_only = ipv6 if h3c { - client = h3client + httpc.Client = httpc.H3client } else { - client = h2client + httpc.Client = httpc.H2client } if https { @@ -522,37 +292,25 @@ func main() { } } - ipv6_only = ipv6 - mux := http.NewServeMux() // MISC ROUTES - mux.HandleFunc("/", beforeMisc(root)) - mux.HandleFunc("/health", beforeMisc(health)) - mux.HandleFunc("/stats", beforeMisc(stats)) + mux.HandleFunc("/", beforeMisc(paths.Root)) + mux.HandleFunc("/health", beforeMisc(paths.Health)) + mux.HandleFunc("/stats", beforeMisc(paths.Stats)) - prometheus.MustRegister(metrics.Uptime) - prometheus.MustRegister(metrics.ActiveConnections) - prometheus.MustRegister(metrics.IdleConnections) - prometheus.MustRegister(metrics.EstablishedConnections) - prometheus.MustRegister(metrics.TotalConnEstablished) - prometheus.MustRegister(metrics.RequestCount) - prometheus.MustRegister(metrics.RequestPerSecond) - prometheus.MustRegister(metrics.RequestPerMinute) - prometheus.MustRegister(metrics.RequestForbidden.Videoplayback) - prometheus.MustRegister(metrics.RequestForbidden.Vi) - prometheus.MustRegister(metrics.RequestForbidden.Ggpht) + metrics.Register() - mux.Handle("/metrics", metricsHandler()) + mux.Handle("/metrics", paths.MetricsHandler()) // PROXY ROUTES - mux.HandleFunc("/videoplayback", beforeProxy(videoplayback)) - mux.HandleFunc("/vi/", beforeProxy(vi)) - mux.HandleFunc("/vi_webp/", beforeProxy(vi)) - mux.HandleFunc("/sb/", beforeProxy(vi)) - mux.HandleFunc("/ggpht/", beforeProxy(ggpht)) - mux.HandleFunc("/a/", beforeProxy(ggpht)) - mux.HandleFunc("/ytc/", beforeProxy(ggpht)) + mux.HandleFunc("/videoplayback", beforeProxy(paths.Videoplayback)) + mux.HandleFunc("/vi/", beforeProxy(paths.Vi)) + mux.HandleFunc("/vi_webp/", beforeProxy(paths.Vi)) + mux.HandleFunc("/sb/", beforeProxy(paths.Vi)) + mux.HandleFunc("/ggpht/", beforeProxy(paths.Ggpht)) + mux.HandleFunc("/a/", beforeProxy(paths.Ggpht)) + mux.HandleFunc("/ytc/", beforeProxy(paths.Ggpht)) go requestPerSecond() go requestPerMinute() diff --git a/go.mod b/go.mod index 5619475..40fb84f 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module git.nadeko.net/Fijxu/http3-ytproxy/v3 +module git.nadeko.net/Fijxu/http3-ytproxy go 1.22.0 diff --git a/internal/httpc/client.go b/internal/httpc/client.go new file mode 100644 index 0000000..f3b891f --- /dev/null +++ b/internal/httpc/client.go @@ -0,0 +1,59 @@ +package httpc + +import ( + "net" + "net/http" + "net/url" + "time" + + "github.com/quic-go/quic-go/http3" +) + +var Ipv6_only = false +var Proxy string +var Client *http.Client + +var dialer = &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, +} + +// QUIC doesn't seem to support HTTP nor SOCKS5 proxies due to how it's made. +// (Since it's UDP) +var H3client = &http.Client{ + Transport: &http3.Transport{}, + Timeout: 10 * time.Second, +} + +// http/2 client +var H2client = &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: &http.Transport{ + Dial: func(network, addr string) (net.Conn, error) { + var net string + if Ipv6_only { + net = "tcp6" + } else { + net = "tcp4" + } + return dialer.Dial(net, addr) + }, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 20 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + IdleConnTimeout: 30 * time.Second, + ReadBufferSize: 16 * 1024, + ForceAttemptHTTP2: true, + MaxConnsPerHost: 0, + MaxIdleConnsPerHost: 10, + MaxIdleConns: 0, + Proxy: func(r *http.Request) (*url.URL, error) { + if Proxy != "" { + return url.Parse(Proxy) + } + return nil, nil + }, + }, +} diff --git a/internal/httpc/server.go b/internal/httpc/server.go new file mode 100644 index 0000000..574ebac --- /dev/null +++ b/internal/httpc/server.go @@ -0,0 +1 @@ +package httpc diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..de0acbd --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,76 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +type metrics struct { + Uptime prometheus.Gauge + RequestCount prometheus.Counter + RequestPerSecond prometheus.Gauge + RequestPerMinute prometheus.Gauge + TotalConnEstablished prometheus.Counter + EstablishedConnections prometheus.Gauge + ActiveConnections prometheus.Gauge + IdleConnections prometheus.Gauge + RequestForbidden struct { + Videoplayback prometheus.Counter + Vi prometheus.Counter + Ggpht prometheus.Counter + } +} + +var Metrics = metrics{ + Uptime: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_uptime", + }), + RequestCount: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_request_count", + }), + RequestPerSecond: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_request_per_second", + }), + RequestPerMinute: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_request_per_minute", + }), + TotalConnEstablished: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_total_conn_established", + }), + EstablishedConnections: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_established_conns", + }), + ActiveConnections: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_active_conns", + }), + IdleConnections: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http3_ytproxy_idle_conns", + }), + + RequestForbidden: struct { + Videoplayback prometheus.Counter + Vi prometheus.Counter + Ggpht prometheus.Counter + }{ + Videoplayback: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "http3_ytproxy_request_forbidden_videoplayback", + }), + Vi: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "http3_ytproxy_request_forbidden_vi", + }), + Ggpht: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "http3_ytproxy_request_forbidden_ggpht", + }), + }, +} + +func Register() { + prometheus.MustRegister(Metrics.Uptime) + prometheus.MustRegister(Metrics.ActiveConnections) + prometheus.MustRegister(Metrics.IdleConnections) + prometheus.MustRegister(Metrics.EstablishedConnections) + prometheus.MustRegister(Metrics.TotalConnEstablished) + prometheus.MustRegister(Metrics.RequestCount) + prometheus.MustRegister(Metrics.RequestPerSecond) + prometheus.MustRegister(Metrics.RequestPerMinute) + prometheus.MustRegister(Metrics.RequestForbidden.Videoplayback) + prometheus.MustRegister(Metrics.RequestForbidden.Vi) + prometheus.MustRegister(Metrics.RequestForbidden.Ggpht) +} diff --git a/internal/metrics/stats.go b/internal/metrics/stats.go new file mode 100644 index 0000000..74d8b02 --- /dev/null +++ b/internal/metrics/stats.go @@ -0,0 +1,52 @@ +package metrics + +import ( + "runtime" + "time" +) + +type statusJson struct { + Version string `json:"version"` + Uptime time.Duration `json:"uptime"` + RequestCount int64 `json:"requestCount"` + RequestPerSecond int64 `json:"requestPerSecond"` + RequestPerMinute int64 `json:"requestPerMinute"` + TotalConnEstablished int64 `json:"totalEstablished"` + EstablishedConnections int64 `json:"establishedConnections"` + ActiveConnections int64 `json:"activeConnections"` + IdleConnections int64 `json:"idleConnections"` + RequestsForbiddenPerSec struct { + Videoplayback int64 `json:"videoplayback"` + } + RequestsForbidden struct { + Videoplayback int64 `json:"videoplayback"` + Vi int64 `json:"vi"` + Ggpht int64 `json:"ggpht"` + } `json:"requestsForbidden"` +} + +var stats_ = statusJson{ + Version: version + "-" + runtime.GOARCH, + Uptime: 0, + RequestCount: 0, + RequestPerSecond: 0, + RequestPerMinute: 0, + TotalConnEstablished: 0, + EstablishedConnections: 0, + ActiveConnections: 0, + IdleConnections: 0, + RequestsForbiddenPerSec: struct { + Videoplayback int64 `json:"videoplayback"` + }{ + Videoplayback: 0, + }, + RequestsForbidden: struct { + Videoplayback int64 `json:"videoplayback"` + Vi int64 `json:"vi"` + Ggpht int64 `json:"ggpht"` + }{ + Videoplayback: 0, + Vi: 0, + Ggpht: 0, + }, +} diff --git a/internal/paths/consts.go b/internal/paths/consts.go new file mode 100644 index 0000000..ffa60ec --- /dev/null +++ b/internal/paths/consts.go @@ -0,0 +1,33 @@ +package paths + +import "regexp" + +const ( + default_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" + ggpht_host = "yt3.ggpht.com" +) + +var manifest_re = regexp.MustCompile(`(?m)URI="([^"]+)"`) + +var allowed_hosts = []string{ + "youtube.com", + "googlevideo.com", + "ytimg.com", + "ggpht.com", + "googleusercontent.com", +} + +var strip_headers = []string{ + "Accept-Encoding", + "Authorization", + "Origin", + "Referer", + "Cookie", + "Set-Cookie", + "Etag", + "Alt-Svc", + "Server", + "Cache-Control", + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-to + "report-to", +} diff --git a/internal/paths/ggpht.go b/internal/paths/ggpht.go new file mode 100644 index 0000000..a2344cf --- /dev/null +++ b/internal/paths/ggpht.go @@ -0,0 +1,50 @@ +package paths + +import ( + "io" + "log" + "net/http" + "net/url" + "strings" + "sync/atomic" + + "git.nadeko.net/Fijxu/http3-ytproxy/internal/httpc" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/utils" +) + +func Ggpht(w http.ResponseWriter, req *http.Request) { + path := req.URL.EscapedPath() + path = strings.Replace(path, "/ggpht", "", 1) + path = strings.Replace(path, "/i/", "/", 1) + + proxyURL, err := url.Parse("https://" + ggpht_host + path) + if err != nil { + log.Panic(err) + } + + request, err := http.NewRequest(req.Method, proxyURL.String(), nil) + utils.CopyHeaders(req.Header, request.Header, false) + request.Header.Set("User-Agent", default_ua) + if err != nil { + log.Panic(err) + } + + resp, err := httpc.Client.Do(request) + if err != nil { + log.Panic(err) + } + + if err := forbiddenChecker(resp, w); err != nil { + atomic.AddInt64(&stats_.RequestsForbidden.Ggpht, 1) + metrics.RequestForbidden.Ggpht.Inc() + return + } + + defer resp.Body.Close() + + NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video") + utils.CopyHeaders(resp.Header, w.Header(), NoRewrite) + w.WriteHeader(resp.StatusCode) + + io.Copy(w, resp.Body) +} diff --git a/internal/paths/health.go b/internal/paths/health.go new file mode 100644 index 0000000..1d3cb08 --- /dev/null +++ b/internal/paths/health.go @@ -0,0 +1,11 @@ +package paths + +import ( + "io" + "net/http" +) + +func Health(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(200) + io.WriteString(w, "OK") +} diff --git a/internal/paths/metrics.go b/internal/paths/metrics.go new file mode 100644 index 0000000..ec7df66 --- /dev/null +++ b/internal/paths/metrics.go @@ -0,0 +1,27 @@ +package paths + +import ( + "net" + "net/http" + "strings" + "time" + + "git.nadeko.net/Fijxu/http3-ytproxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var programInit = time.Now() + +// CustomHandler wraps the default promhttp.Handler with custom logic +func MetricsHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + // To prevent accessing from the bare IP address + if req.Host == "" || net.ParseIP(strings.Split(req.Host, ":")[0]) != nil { + w.WriteHeader(444) + return + } + + metrics.Metrics.Uptime.Set(float64(time.Duration(time.Since(programInit).Seconds()))) + promhttp.Handler().ServeHTTP(w, req) + }) +} diff --git a/internal/paths/root.go b/internal/paths/root.go new file mode 100644 index 0000000..be6d2cc --- /dev/null +++ b/internal/paths/root.go @@ -0,0 +1,17 @@ +package paths + +import ( + "io" + "net/http" +) + +func Root(w http.ResponseWriter, req *http.Request) { + const msg = ` + HTTP youtube proxy for https://inv.nadeko.net + https://git.nadeko.net/Fijxu/http3-ytproxy + + Routes: + /stats + /health` + io.WriteString(w, msg) +} diff --git a/internal/paths/stats.go b/internal/paths/stats.go new file mode 100644 index 0000000..3c34439 --- /dev/null +++ b/internal/paths/stats.go @@ -0,0 +1,20 @@ +package paths + +import ( + "encoding/json" + "net/http" + "time" +) + +func Stats(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "application/json") + stats_.Uptime = time.Duration(time.Since(programInit).Seconds()) + // stats_.TotalEstablished = int64(cw.totalEstablished) + // stats_.EstablishedConnections = int64(cw.established) + // stats_.ActiveConnections = int64(cw.active) + // stats_.IdleConnections = int64(cw.idle) + + if err := json.NewEncoder(w).Encode(stats_); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} diff --git a/internal/paths/vi.go b/internal/paths/vi.go new file mode 100644 index 0000000..4ec22d2 --- /dev/null +++ b/internal/paths/vi.go @@ -0,0 +1,62 @@ +package paths + +import ( + "io" + "log" + "net/http" + "net/url" + "strings" + "sync/atomic" + + "git.nadeko.net/Fijxu/http3-ytproxy/internal/httpc" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/utils" +) + +func Vi(w http.ResponseWriter, req *http.Request) { + const host string = "i.ytimg.com" + q := req.URL.Query() + + path := req.URL.EscapedPath() + + proxyURL, err := url.Parse("https://" + host + path) + if err != nil { + log.Panic(err) + } + + if strings.HasSuffix(proxyURL.EscapedPath(), "maxres.jpg") { + proxyURL.Path = utils.GetBestThumbnail(proxyURL.EscapedPath()) + } + + /* + Required for /sb/ endpoints + You can't access https://i.ytimg.com/sb//storyboard3_L2/M3.jpg + without it's parameters `sqp` and `sigh` + */ + proxyURL.RawQuery = q.Encode() + + request, err := http.NewRequest(req.Method, proxyURL.String(), nil) + if err != nil { + log.Panic(err) + } + + request.Header.Set("User-Agent", default_ua) + + resp, err := httpc.Client.Do(request) + if err != nil { + log.Panic(err) + } + + if err := forbiddenChecker(resp, w); err != nil { + atomic.AddInt64(&stats_.RequestsForbidden.Vi, 1) + metrics.RequestForbidden.Vi.Inc() + return + } + + defer resp.Body.Close() + + // NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video") + // copyHeaders(resp.Header, w.Header(), NoRewrite) + w.WriteHeader(resp.StatusCode) + + io.Copy(w, resp.Body) +} diff --git a/httppaths.go b/internal/paths/videoplayback.go similarity index 68% rename from httppaths.go rename to internal/paths/videoplayback.go index 7ee113d..f26e8cb 100644 --- a/httppaths.go +++ b/internal/paths/videoplayback.go @@ -1,4 +1,4 @@ -package main +package paths import ( "bytes" @@ -11,6 +11,9 @@ import ( "strings" "sync/atomic" "time" + + "git.nadeko.net/Fijxu/http3-ytproxy/internal/httpc" + "git.nadeko.net/Fijxu/http3-ytproxy/internal/utils" ) func forbiddenChecker(resp *http.Response, w http.ResponseWriter) error { @@ -23,9 +26,38 @@ func forbiddenChecker(resp *http.Response, w http.ResponseWriter) error { return nil } -func videoplayback(w http.ResponseWriter, req *http.Request) { +func checkRequest(w http.ResponseWriter, req *http.Request, params url.Values) { + host := params.Get("host") + + parts := strings.Split(strings.ToLower(host), ".") + if len(parts) < 2 { + w.WriteHeader(400) + io.WriteString(w, "Invalid hostname.") + return + } + + domain := parts[len(parts)-2] + "." + parts[len(parts)-1] + disallowed := true + for _, value := range allowed_hosts { + if domain == value { + disallowed = false + break + } + } + + if disallowed { + w.WriteHeader(401) + io.WriteString(w, "Non YouTube domains are not supported.") + return + } + +} + +func Videoplayback(w http.ResponseWriter, req *http.Request) { q := req.URL.Query() + checkRequest(w, req, q) + expire, err := strconv.ParseInt(q.Get("expire"), 10, 64) if err != nil { w.WriteHeader(500) @@ -71,28 +103,6 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { // host = "rr" + mvi + "---" + mn[0] + ".googlevideo.com" // } - parts := strings.Split(strings.ToLower(host), ".") - if len(parts) < 2 { - w.WriteHeader(400) - io.WriteString(w, "Invalid hostname.") - return - } - - domain := parts[len(parts)-2] + "." + parts[len(parts)-1] - disallowed := true - for _, value := range allowed_hosts { - if domain == value { - disallowed = false - break - } - } - - if disallowed { - w.WriteHeader(401) - io.WriteString(w, "Non YouTube domains are not supported.") - return - } - // if c == "WEB" { // q.Set("alr", "yes") // } @@ -120,8 +130,8 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { if err != nil { log.Panic("Failed to create headRequest:", err) } - copyHeaders(req.Header, postRequest.Header, false) - copyHeaders(req.Header, headRequest.Header, false) + utils.CopyHeaders(req.Header, postRequest.Header, false) + utils.CopyHeaders(req.Header, headRequest.Header, false) switch c { case "ANDROID": @@ -146,7 +156,7 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { resp := &http.Response{} for i := 0; i < 5; i++ { - resp, err = client.Do(headRequest) + resp, err = httpc.Client.Do(headRequest) if err != nil { log.Panic("Failed to do HEAD request:", err) } @@ -162,7 +172,7 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { } } - resp, err = client.Do(postRequest) + resp, err = httpc.Client.Do(postRequest) if err != nil { log.Panic("Failed to do POST request:", err) } @@ -176,7 +186,7 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { defer resp.Body.Close() NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video") - copyHeaders(resp.Header, w.Header(), NoRewrite) + utils.CopyHeaders(resp.Header, w.Header(), NoRewrite) w.WriteHeader(resp.StatusCode) @@ -196,12 +206,12 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { line = "https://" + reqUrl.Hostname() + path + line } if strings.HasPrefix(line, "https://") { - lines[i] = RelativeUrl(line) + lines[i] = utils.RelativeUrl(line) } if manifest_re.MatchString(line) { url := manifest_re.FindStringSubmatch(line)[1] - lines[i] = strings.Replace(line, url, RelativeUrl(url), 1) + lines[i] = strings.Replace(line, url, utils.RelativeUrl(url), 1) } } @@ -210,91 +220,3 @@ func videoplayback(w http.ResponseWriter, req *http.Request) { io.Copy(w, resp.Body) } } - -func vi(w http.ResponseWriter, req *http.Request) { - const host string = "i.ytimg.com" - q := req.URL.Query() - - path := req.URL.EscapedPath() - - proxyURL, err := url.Parse("https://" + host + path) - if err != nil { - log.Panic(err) - } - - if strings.HasSuffix(proxyURL.EscapedPath(), "maxres.jpg") { - proxyURL.Path = getBestThumbnail(proxyURL.EscapedPath()) - } - - /* - Required for /sb/ endpoints - You can't access https://i.ytimg.com/sb//storyboard3_L2/M3.jpg - without it's parameters `sqp` and `sigh` - */ - proxyURL.RawQuery = q.Encode() - - request, err := http.NewRequest(req.Method, proxyURL.String(), nil) - if err != nil { - log.Panic(err) - } - - request.Header.Set("User-Agent", default_ua) - - resp, err := client.Do(request) - if err != nil { - log.Panic(err) - } - - if err := forbiddenChecker(resp, w); err != nil { - atomic.AddInt64(&stats_.RequestsForbidden.Vi, 1) - metrics.RequestForbidden.Vi.Inc() - return - } - - defer resp.Body.Close() - - // NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video") - // copyHeaders(resp.Header, w.Header(), NoRewrite) - w.WriteHeader(resp.StatusCode) - - io.Copy(w, resp.Body) -} - -func ggpht(w http.ResponseWriter, req *http.Request) { - const host string = "yt3.ggpht.com" - - path := req.URL.EscapedPath() - path = strings.Replace(path, "/ggpht", "", 1) - path = strings.Replace(path, "/i/", "/", 1) - - proxyURL, err := url.Parse("https://" + host + path) - if err != nil { - log.Panic(err) - } - - request, err := http.NewRequest(req.Method, proxyURL.String(), nil) - copyHeaders(req.Header, request.Header, false) - request.Header.Set("User-Agent", default_ua) - if err != nil { - log.Panic(err) - } - - resp, err := client.Do(request) - if err != nil { - log.Panic(err) - } - - if err := forbiddenChecker(resp, w); err != nil { - atomic.AddInt64(&stats_.RequestsForbidden.Ggpht, 1) - metrics.RequestForbidden.Ggpht.Inc() - return - } - - defer resp.Body.Close() - - NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video") - copyHeaders(resp.Header, w.Header(), NoRewrite) - w.WriteHeader(resp.StatusCode) - - io.Copy(w, resp.Body) -} diff --git a/utils.go b/internal/utils/utils.go similarity index 69% rename from utils.go rename to internal/utils/utils.go index 6db8c0f..6718101 100644 --- a/utils.go +++ b/internal/utils/utils.go @@ -1,13 +1,34 @@ -package main +package utils import ( "log" "net/http" "net/url" "strings" + + "git.nadeko.net/Fijxu/http3-ytproxy/internal/httpc" ) -func copyHeaders(from http.Header, to http.Header, length bool) { +const ( + path_prefix = "" +) + +var strip_headers = []string{ + "Accept-Encoding", + "Authorization", + "Origin", + "Referer", + "Cookie", + "Set-Cookie", + "Etag", + "Alt-Svc", + "Server", + "Cache-Control", + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy/report-to + "report-to", +} + +func CopyHeaders(from http.Header, to http.Header, length bool) { // Loop over header names outer: for name, values := range from { @@ -28,14 +49,14 @@ outer: } } -func getBestThumbnail(path string) (newpath string) { +func GetBestThumbnail(path string) (newpath string) { formats := [4]string{"maxresdefault.jpg", "sddefault.jpg", "hqdefault.jpg", "mqdefault.jpg"} for _, format := range formats { newpath = strings.Replace(path, "maxres.jpg", format, 1) url := "https://i.ytimg.com" + newpath - resp, _ := h2client.Head(url) + resp, _ := httpc.Client.Head(url) if resp.StatusCode == 200 { return newpath } @@ -56,7 +77,7 @@ func RelativeUrl(in string) (newurl string) { return segment_url.RequestURI() } -func panicHandler(w http.ResponseWriter) { +func PanicHandler(w http.ResponseWriter) { if r := recover(); r != nil { log.Printf("Panic: %v", r) http.Error(w, "Internal Server Error", http.StatusInternalServerError) diff --git a/nginx.conf b/nginx.conf deleted file mode 100644 index fd5ba00..0000000 --- a/nginx.conf +++ /dev/null @@ -1,19 +0,0 @@ -user www-data; -events { - worker_connections 1024; -} -http { - server { - listen 3000; - listen [::]:3000; - access_log off; - location / { - resolver 127.0.0.11; - set $backend "http3-proxy"; - proxy_pass http://$backend:8080; - proxy_http_version 1.1; # to keep alive - proxy_set_header Connection ""; # to keep alive - } - } -} -