meow

httppaths.go

@@ -38,6 +38,8 @@ func connectionChecker(ctx context.Context) bool {
 func videoplayback(w http.ResponseWriter, req *http.Request) {
 	q := req.URL.Query()
 
+	// log.Println(req.URL.RawQuery)
+
 	expire, err := strconv.ParseInt(q.Get("expire"), 10, 64)
 	if err != nil {
 		w.WriteHeader(500)
@@ -53,7 +55,7 @@ func videoplayback(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	c := q.Get("c")
+	// c := q.Get("c")
 	// if c == "" {
 	// 	w.WriteHeader(400)
 	// 	io.WriteString(w, "'c' query string undefined.")
@@ -63,25 +65,25 @@ func videoplayback(w http.ResponseWriter, req *http.Request) {
 	host := q.Get("host")
 	q.Del("host")
 
-	if len(host) <= 0 {
+	// if len(host) <= 0 {
-		// Fallback to use mvi and mn to build a host
+	// 	// Fallback to use mvi and mn to build a host
-		mvi := q.Get("mvi")
+	// 	mvi := q.Get("mvi")
-		mn := strings.Split(q.Get("mn"), ",")
+	// 	mn := strings.Split(q.Get("mn"), ",")
 
-		if len(mvi) <= 0 {
+	// 	if len(mvi) <= 0 {
-			w.WriteHeader(400)
+	// 		w.WriteHeader(400)
-			io.WriteString(w, "'mvi' query string undefined")
+	// 		io.WriteString(w, "'mvi' query string undefined")
-			return
+	// 		return
-		}
+	// 	}
 
-		if len(mn) <= 0 {
+	// 	if len(mn) <= 0 {
-			w.WriteHeader(400)
+	// 		w.WriteHeader(400)
-			io.WriteString(w, "'mn' query string undefined")
+	// 		io.WriteString(w, "'mn' query string undefined")
-			return
+	// 		return
-		}
+	// 	}
 
-		host = "rr" + mvi + "---" + mn[0] + ".googlevideo.com"
+	// 	host = "rr" + mvi + "---" + mn[0] + ".googlevideo.com"
-	}
+	// }
 
 	parts := strings.Split(strings.ToLower(host), ".")
 	if len(parts) < 2 {
@@ -113,100 +115,161 @@ func videoplayback(w http.ResponseWriter, req *http.Request) {
 	// }
 
 	path := req.URL.EscapedPath()
+	// fmt.Println("esc path:", path)
 
 	proxyURL, err := url.Parse("https://" + host + path)
 	if err != nil {
 		log.Panic(err)
 	}
 
+	// fmt.Println("query params:", q)
 	proxyURL.RawQuery = q.Encode()
+	// fmt.Println("esc path:", proxyURL.RawQuery)
 
 	// https://github.com/FreeTubeApp/FreeTube/blob/5a4cd981cdf2c2a20ab68b001746658fd0c6484e/src/renderer/components/ft-shaka-video-player/ft-shaka-video-player.js#L1097
 	body := []byte{0x78, 0} // protobuf body
 
-	request, err := http.NewRequest("POST", proxyURL.String(), bytes.NewReader(body))
+	postRequest, err := http.NewRequest("POST", proxyURL.String(), bytes.NewReader(body))
 	if err != nil {
-		log.Panic(err)
+		log.Panic("Failed to create postRequest:", err)
 	}
-	copyHeaders(req.Header, request.Header, false)
+	// headRequest, err := http.Head(proxyURL.String())
-
+	headRequest, err := http.NewRequest("HEAD", proxyURL.String(), nil)
-	switch c {
-	case "ANDROID":
-		request.Header.Set("User-Agent", "com.google.android.youtube/1537338816 (Linux; U; Android 13; en_US; ; Build/TQ2A.230505.002; Cronet/113.0.5672.24)")
-	case "IOS":
-		request.Header.Set("User-Agent", "com.google.ios.youtube/19.32.8 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)")
-	case "WEB":
-		request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
-	default:
-		request.Header.Set("User-Agent", default_ua)
-	}
-
-	request.Header.Add("Origin", "https://www.youtube.com")
-	request.Header.Add("Referer", "https://www.youtube.com/")
-
-	if connectionChecker(req.Context()) {
-		return
-	}
-
-	resp, err := client.Do(request)
 	if err != nil {
-		log.Panic(err)
+		log.Panic("Failed to create headRequest:", err)
 	}
+	copyHeaders(req.Header, postRequest.Header, false)
+	copyHeaders(req.Header, headRequest.Header, false)
 
-	if resp.Header.Get("location") != "" {
+	// switch c {
-		new_url, err := url.Parse(resp.Header.Get("location"))
+	// case "ANDROID":
+	// 	request.Header.Set("User-Agent", "com.google.android.youtube/1537338816 (Linux; U; Android 13; en_US; ; Build/TQ2A.230505.002; Cronet/113.0.5672.24)")
+	// case "IOS":
+	// 	request.Header.Set("User-Agent", "com.google.ios.youtube/19.32.8 (iPhone14,5; U; CPU iOS 17_6 like Mac OS X;)")
+	// case "WEB":
+	// 	request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+	// default:
+	// 	request.Header.Set("User-Agent", default_ua)
+	// }
+
+	// request.Header.Add("Origin", "https://www.youtube.com")
+	// request.Header.Add("Referer", "https://www.youtube.com/")
+
+	// if connectionChecker(req.Context()) {
+	// 	return
+	// }
+
+	resp := &http.Response{}
+
+	// doPostRequest := func () {
+
+	// }
+
+	for i := 0; i < 5; i++ {
+		// log.Println("headRequest URL:", headRequest.URL)
+		resp, err = client.Do(headRequest)
 		if err != nil {
-			log.Panic(err)
+			log.Panic("Failed to do HEAD request:", err)
 		}
-		request.URL = new_url
+		// log.Println("HEAD request made to:", headRequest.URL, "returned", strconv.Itoa(resp.StatusCode))
-		resp, err = client.Do(request)
+		// log.Println("HEAD Location header:", resp.Header.Get("Location"))
-		if err != nil {
+		if resp.Header.Get("Location") != "" {
-			log.Panic(err)
+			log.Println("")
+			location := resp.Header.Get("Location")
+			// log.Println("location:", location)
+			new_url, _ := url.Parse(location)
+			new_host := new_url.Host
+			log.Println("new_host:", new_host)
+			// log.Println("new_url:", new_url.Host)
+			// log.Println("old_request:", proxyURL.String())
+
+			postRequest.URL = new_url
+			headRequest.URL = new_url
+			// headRequest.Header.Set("Host", new_url.Host)
+			// postRequest.Header.Set("Host", new_url.Host)
+			postRequest.Host = new_url.Host
+			headRequest.Host = new_url.Host
+			log.Println("postRequest.Host after location:", postRequest.Host)
+			log.Println("headRequest.Host after location:", headRequest.Host)
+			// log.Println("postrequest new_url:", postRequest.URL)
+			// resp, err = client.Do(postRequest)
+			// if err != nil {
+			// 	log.Panic("Failed to do POST request")
+			// }
+			continue
+		} else {
+			break
 		}
 	}
 
-	if err := forbiddenChecker(resp, w); err != nil {
+	resp, err = client.Do(postRequest)
-		atomic.AddInt64(&stats_.RequestsForbidden.Videoplayback, 1)
+	if err != nil {
-		metrics.RequestForbidden.Videoplayback.Inc()
+		log.Panic("Failed to do POST request:", err)
-		return
 	}
 
 	defer resp.Body.Close()
-
-	NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video")
-	copyHeaders(resp.Header, w.Header(), NoRewrite)
-
 	w.WriteHeader(resp.StatusCode)
+	io.Copy(w, resp.Body)
 
-	if req.Method == "GET" && (resp.Header.Get("Content-Type") == "application/x-mpegurl" || resp.Header.Get("Content-Type") == "application/vnd.apple.mpegurl") {
+	// resp, err = client.Do(postRequest)
-		bytes, err := io.ReadAll(resp.Body)
+	// if err != nil {
-		if err != nil {
+	// 	log.Panic("Failed to do POST request:", err)
-			log.Panic(err)
+	// }
-		}
 
-		lines := strings.Split(string(bytes), "\n")
+	// if resp.Header.Get("location") != "" {
-		reqUrl := resp.Request.URL
+	// 	new_url, err := url.Parse(resp.Header.Get("location"))
-		for i := 0; i < len(lines); i++ {
+	// 	if err != nil {
-			line := lines[i]
+	// 		log.Panic(err)
-			if !strings.HasPrefix(line, "https://") && (strings.HasSuffix(line, ".m3u8") || strings.HasSuffix(line, ".ts")) {
+	// 	}
-				path := reqUrl.EscapedPath()
+	// 	request.URL = new_url
-				path = path[0 : strings.LastIndex(path, "/")+1]
+	// 	resp, err = client.Do(request)
-				line = "https://" + reqUrl.Hostname() + path + line
+	// 	if err != nil {
-			}
+	// 		log.Panic(err)
-			if strings.HasPrefix(line, "https://") {
+	// 	}
-				lines[i] = RelativeUrl(line)
+	// }
-			}
 
-			if manifest_re.MatchString(line) {
+	// if err := forbiddenChecker(resp, w); err != nil {
-				url := manifest_re.FindStringSubmatch(line)[1]
+	// 	atomic.AddInt64(&stats_.RequestsForbidden.Videoplayback, 1)
-				lines[i] = strings.Replace(line, url, RelativeUrl(url), 1)
+	// 	metrics.RequestForbidden.Videoplayback.Inc()
-			}
+	// 	return
-		}
+	// }
 
-		io.WriteString(w, strings.Join(lines, "\n"))
+	// defer resp.Body.Close()
-	} else {
+
-		io.Copy(w, resp.Body)
+	// NoRewrite := strings.HasPrefix(resp.Header.Get("Content-Type"), "audio") || strings.HasPrefix(resp.Header.Get("Content-Type"), "video")
-	}
+	// copyHeaders(resp.Header, w.Header(), NoRewrite)
+
+	// w.WriteHeader(resp.StatusCode)
+
+	// if req.Method == "GET" && (resp.Header.Get("Content-Type") == "application/x-mpegurl" || resp.Header.Get("Content-Type") == "application/vnd.apple.mpegurl") {
+	// 	bytes, err := io.ReadAll(resp.Body)
+	// 	if err != nil {
+	// 		log.Panic(err)
+	// 	}
+
+	// 	lines := strings.Split(string(bytes), "\n")
+	// 	reqUrl := resp.Request.URL
+	// 	for i := 0; i < len(lines); i++ {
+	// 		line := lines[i]
+	// 		if !strings.HasPrefix(line, "https://") && (strings.HasSuffix(line, ".m3u8") || strings.HasSuffix(line, ".ts")) {
+	// 			path := reqUrl.EscapedPath()
+	// 			path = path[0 : strings.LastIndex(path, "/")+1]
+	// 			line = "https://" + reqUrl.Hostname() + path + line
+	// 		}
+	// 		if strings.HasPrefix(line, "https://") {
+	// 			lines[i] = RelativeUrl(line)
+	// 		}
+
+	// 		if manifest_re.MatchString(line) {
+	// 			url := manifest_re.FindStringSubmatch(line)[1]
+	// 			lines[i] = strings.Replace(line, url, RelativeUrl(url), 1)
+	// 		}
+	// 	}
+
+	// 	io.WriteString(w, strings.Join(lines, "\n"))
+	// } else {
+	io.Copy(w, resp.Body)
+	// }
 }
 
 func vi(w http.ResponseWriter, req *http.Request) {

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -13,6 +14,7 @@ import (
 	"os"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
@@ -21,6 +23,7 @@ import (
 	"github.com/conduitio/bwlimit"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/procfs"
 	"github.com/quic-go/quic-go"
 	"github.com/quic-go/quic-go/http3"
 )
@@ -33,47 +36,41 @@ var (
 // QUIC doesn't seem to support HTTP nor SOCKS5 proxies due to how it's made.
 // (Since it's UDP)
 var h3client = &http.Client{
-	Transport: &http3.Transport{},
+	Transport: &http3.Transport{
-	Timeout:   10 * time.Second,
+		EnableDatagrams: false,
+	},
+	Timeout: 10 * time.Second,
 }
 
 var dialer = &net.Dialer{
-	Timeout:   30 * time.Second,
+	Timeout: 30 * time.Second,
-	KeepAlive: 30 * time.Second,
 }
 
 var proxy string
 
+// var proxyUrl, _ = url.Parse("http://127.0.0.1:8080")
+var proxyUrl, _ = url.Parse("socks5://127.0.0.1:1080")
+
+var protocols *http.Protocols
+
 // http/2 client
 var h2client = &http.Client{
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	},
 	Transport: &http.Transport{
-		Dial: func(network, addr string) (net.Conn, error) {
+		TLSNextProto:    make(map[string]func(string, *tls.Conn) http.RoundTripper),
-			var net string
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true, VerifyPeerCertificate: func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { return nil }},
-			if ipv6_only {
+		// Proxy:           http.ProxyURL(proxyUrl),
-				net = "tcp6"
-			} else {
-				net = "tcp4"
-			}
-			return dialer.Dial(net, addr)
-		},
-		TLSHandshakeTimeout:   10 * time.Second,
-		ResponseHeaderTimeout: 20 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-		IdleConnTimeout:       30 * time.Second,
-		ReadBufferSize:        16 * 1024,
-		ForceAttemptHTTP2:     true,
-		MaxConnsPerHost:       0,
-		MaxIdleConnsPerHost:   10,
-		MaxIdleConns:          0,
-		Proxy: func(r *http.Request) (*url.URL, error) {
-			if proxy != "" {
-				return url.Parse(proxy)
-			}
-			return nil, nil
-		},
 	},
 }
 
+// var h2client_test = &http.Client{
+// 	// Yeah I know I can just use http.Transport and it will use HTTP/2 automatically
+// 	// I prefer to set it up explicitly
+// 	Transport: &http.Transport{},
+// }
+
 var client *http.Client
 
 var default_ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
@@ -325,27 +322,57 @@ func requestPerMinute() {
 	}
 }
 
-func forbiddenRequestsPerSec() {
+var tx uint64
-	var last int64
+
+func blockCheckerCalc(p *procfs.Proc) {
+	var last uint64
 	for {
 		time.Sleep(1 * time.Second)
-		current := stats_.RequestsForbidden.Videoplayback
+		// p.NetDev should never fail.
-		stats_.RequestsForbiddenPerSec.Videoplayback = current - last
+		stat, _ := p.NetDev()
-		// fmt.Println(aux)
+		current := stat.Total().TxBytes
-		// if aux > 15 {
+		tx = current - last
-		// 	body := "{\"status\":\"stopped\"}\""
-		// 	request, err := http.NewRequest("PUT", "http://gluetun:8000", strings.NewReader(body))
-		// 	if err != nil {
-		// 		log.Panic(err)
-		// 	}
-		// 	client.Do(request)
-		// }
-		// metrics.RequestPerSecond.Set(float64(stats_.RequestPerSecond))
-		// time.Sleep(60 * time.Second)
 		last = current
 	}
 }
 
+// Detects if a backend has been blocked based on the amount of bandwidth
+// reported by procfs.
+// This may be the best way to detect if the IP has been blocked from googlevideo
+// servers. I would like to detect blockages using the status code that googlevideo
+// returns, which most of the time is 403 (Forbidden). But this error code is not
+// exclusive to IP blocks, it's also returned for other reasons like a wrong
+// query parameter like `pot` (po_token) or anything like that.
+func blockChecker(gh string, cooldown int) {
+	log.Println("[INFO] Starting blockchecker")
+	// Sleep for 60 seconds before commencing the loop
+	time.Sleep(60 * time.Second)
+	url := "http://" + gh + "/v1/openvpn/status"
+
+	p, err := procfs.Self()
+	if err != nil {
+		log.Printf("[ERROR] [procfs]: Could not get process: %s\n", err)
+		log.Println("[INFO] Blockchecker will not run, so if the VPN IP used on gluetun gets blocked, it will not be rotated!")
+		return
+	}
+	go blockCheckerCalc(&p)
+
+	for {
+		time.Sleep(time.Duration(cooldown) * time.Second)
+		if float64(tx)*0.000008 < 2.0 {
+			body := "{\"status\":\"stopped\"}\""
+			// This should never fail too
+			request, _ := http.NewRequest("PUT", url, strings.NewReader(body))
+			_, err = client.Do(request)
+			if err != nil {
+				log.Printf("[ERROR] Failed to send request to gluetun.")
+			} else {
+				log.Printf("[INFO] Request to change IP sent to gluetun successfully")
+			}
+		}
+	}
+}
+
 func beforeMisc(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		defer panicHandler(w)
@@ -399,6 +426,11 @@ func beforeProxy(next http.HandlerFunc) http.HandlerFunc {
 }
 
 func main() {
+	protocols = &http.Protocols{}
+	protocols.SetHTTP1(true)
+	protocols.SetHTTP2(false)
+	protocols.SetUnencryptedHTTP2(false)
+
 	defaultHost := "0.0.0.0"
 	defaultPort := "8080"
 	defaultSock := "/tmp/http-ytproxy.sock"
@@ -408,6 +440,7 @@ func main() {
 	var https bool = false
 	var h3c bool = false
 	var ipv6 bool = false
+	var bc bool = true
 
 	if strings.ToLower(os.Getenv("HTTPS")) == "true" {
 		https = true
@@ -421,6 +454,9 @@ func main() {
 	if strings.ToLower(os.Getenv("IPV6_ONLY")) == "true" {
 		ipv6 = true
 	}
+	if strings.ToLower(os.Getenv("BLOCK_CHECKER")) == "false" {
+		bc = false
+	}
 	if strings.ToLower(os.Getenv("DOMAIN_ONLY_ACCESS")) == "true" {
 		domain_only_access = true
 	}
@@ -445,6 +481,15 @@ func main() {
 	if host == "" {
 		host = defaultHost
 	}
+	// gh is where the gluetun api is located
+	gh := os.Getenv("GLUETUN_HOSTNAME")
+	if gh == "" {
+		gh = "127.0.0.1:8000"
+	}
+	bc_cooldown := os.Getenv("BLOCK_CHECKER_COOLDOWN")
+	if bc_cooldown == "" {
+		bc_cooldown = "60"
+	}
 	proxy = os.Getenv("PROXY")
 
 	flag.BoolVar(&https, "https", https, "Use built-in https server (recommended)")
@@ -459,8 +504,10 @@ func main() {
 	flag.Parse()
 
 	if h3c {
+		log.Println("[INFO] Using HTTP/3 client")
 		client = h3client
 	} else {
+		log.Println("[INFO] Using HTTP/2 client")
 		client = h2client
 	}
 
@@ -508,7 +555,13 @@ func main() {
 
 	go requestPerSecond()
 	go requestPerMinute()
-	go forbiddenRequestsPerSec()
+	if bc {
+		num, err := strconv.Atoi(bc_cooldown)
+		if err != nil {
+			log.Fatalf("[FATAL] Error while setting BLOCK_CHECKER_COOLDOWN: %s", err)
+		}
+		go blockChecker(gh, num)
+	}
 
 	ln, err := net.Listen("tcp", host+":"+port)
 	if err != nil {