From 0c40e04a771274e1a2e75a307c25b7bf7eea33a9 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 13 Sep 2024 13:19:06 -0300 Subject: [PATCH] Cleanup in haproxy --- invidious/invidious-vpn/docker-compose.yml | 90 +++++++++++++++++++++ invidious/invidious-vpn/haproxy/haproxy.cfg | 75 +++++------------ invidious/invidious/haproxy/haproxy.cfg | 68 +++++----------- 3 files changed, 127 insertions(+), 106 deletions(-) diff --git a/invidious/invidious-vpn/docker-compose.yml b/invidious/invidious-vpn/docker-compose.yml index 5abc648..e0c8ffe 100644 --- a/invidious/invidious-vpn/docker-compose.yml +++ b/invidious/invidious-vpn/docker-compose.yml @@ -446,6 +446,93 @@ services: timeout: 5s retries: 2 + # INVIDIOUS CLEARNET + invidious-6: + env_file: + - .env + image: "git.nadeko.net/fijxu/invidious:${TAG}" + network_mode: "service:gluetun" + restart: unless-stopped + volumes: + - /var/run/postgresql/.s.PGSQL.5432:/var/run/postgresql/.s.PGSQL.5432:rw + environment: + INVIDIOUS_CONFIG: | + feed_threads: 0 + log_level: Warn + # db: + # dbname: "${PGSQL_DBNAME}" + # user: "${PGSQL_USER}" + # password: "${PGSQL_PASS}" + # host: 172.80.0.22 + # port: 5432 + port: 3005 + database_url: postgres://kemal:kemal@/invidious + redis_url: tcp://172.80.0.20:6379 + donation_url: "https://nadeko.net/donate" + contact_url: "https://nadeko.net/contact" + home_domain: "nadeko.net" + materialious_domain: "materialious.nadeko.net" + full_refresh: false + https_only: true + hmac_key: "${HMAC_KEY}" + domain: "${DOMAIN}" + use_pubsub_feeds: true + popular_enabled: true + captcha_enabled: true + login_enabled: true + registration_enabled: false + statistics_enabled: true + external_port: 443 + default_user_preferences: + annotations: true + disable_proxy: ["downloads"] + annotations_subscribed: true + autoplay: true + captions: ["English", "Spanish", "English (auto-generated)"] + comments: ["youtube", ""] + dark_mode: auto + latest_only: false + listen: false + locale: en-US + watch_history: true + max_results: 60 + notifications_only: false + banner: "${BANNER}" + player_style: invidious + quality: "${QUALITY}" + quality_dash: auto + default_home: Popular + feed_menu: ["Popular", "Trending", "Subscriptions", "Playlists"] + automatic_instance_redirect: true + region: CL + sort: published + extend_desc: true + save_player_pos: true + cache_annotations: true + hsts: true + enable_user_notifications: false + modified_source_code_url: "${MODIFIED_SOURCE_CODE}" + force_resolve: ipv4 + pool_size: "${POOL_SIZE}" + use_innertube_for_captions: true + jobs: + refresh_channels: + enable: false + subscribe_to_feeds: + enable: false + po_token: "${PO_TOKEN}" + visitor_data: "${VISITOR_DATA}" + signature_server: "172.80.0.30:12999" + depends_on: + - valkey + - inv_sig_helper + - gluetun + healthcheck: + test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/trending || exit 1 + interval: 30s + timeout: 5s + retries: 2 + # SIGNATURE HELPER inv_sig_helper: image: quay.io/invidious/inv-sig-helper @@ -498,8 +585,11 @@ services: - WIREGUARD_PRIVATE_KEY="${WIREGUARD_KEY}" - WIREGUARD_ADDRESSES="${WIREGUARD_ADDRESSES}" - SERVER_HOSTNAMES="${SERVER_HOSTNAMES}" + - DNS_KEEP_NAMESERVER=on networks: gluetun_network: + depends_on: + - haproxy # gluetun-updater: # image: qmcgaw/gluetun diff --git a/invidious/invidious-vpn/haproxy/haproxy.cfg b/invidious/invidious-vpn/haproxy/haproxy.cfg index ae6ec78..863097e 100644 --- a/invidious/invidious-vpn/haproxy/haproxy.cfg +++ b/invidious/invidious-vpn/haproxy/haproxy.cfg @@ -1,6 +1,6 @@ global # This because haproxy is dumb and likes to use the maxconnection from the kernel and that is super mega huge making haproxy oom - maxconn 32768 + maxconn 16384 # 777 perms for all sockets unix-bind mode 777 stats socket ipv4@0.0.0.0:8405 level admin @@ -17,6 +17,7 @@ defaults timeout server 5s timeout http-request 10s +# Prometheus for graphs frontend prometheus bind *:8404 mode http @@ -45,90 +46,50 @@ cache invidious-ggpht-cache total-max-size 96 # 32kB max-object-size 32768 - # 30min + # 1h max-age 3600 process-vary on -# cache api-cache -# # 32MB -# total-max-size 32 -# # 4kB -# max-object-size 32000 -# # 5min -# max-age 300 -# process-vary on - frontend invidious - http-request set-var(txn.ggpht) path - bind /tmp/invidious-vpn.sock bind *:8001 - #filter cache invidious-api-cache - #http-request cache-use invidious-api-cache if { path_beg /api/v1 } - #http-response cache-store invidious-api-cache default_backend invidious -# frontend invidious-tor -# bind /tmp/invidious-tor.sock -# bind *:8002 -# default_backend invidious-tor -# -# frontend invidious-i2p -# bind /tmp/invidious-i2p.sock -# bind *:8003 -# default_backend invidious-i2p - frontend http3-proxy http-request set-var(txn.vipath) path - # acl vi path_beg /vi/ - bind /tmp/http3-proxy-vpn.sock bind *:8004 - #http-response set-header Cache-Control public - #http-response set-header Expires -1 - #http-response set-header Pragma cache - # acl youtube_thumbnails path_beg /vi/ - # default_backend http3-proxy backend invidious balance leastconn - #http-request cache-use api-cache if { path_beg /api/v1/channels/ } - #http-response cache-store api-cache - # acl inv_api var(txn.api) -m beg /api/ - # http-response set-header X-Cache-Status HIT if !{ srv_id -m found } inv_api - # http-response set-header X-Cache-Status MISS if { srv_id -m found } inv_api - # - # http-request cache-use invidious-ggpht-cache if { path_beg /ggpht/ } - # http-response cache-store invidious-ggpht-cache - - # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 - # acl youtube_ggpht var(txn.ggpht) -m beg /ggpht/ - # http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_ggpht - # http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_ggpht server invidious-1-vpn gluetun:3000 check init-addr libc,none server invidious-2-vpn gluetun:3001 check init-addr libc,none server invidious-3-vpn gluetun:3002 check init-addr libc,none server invidious-4-vpn gluetun:3003 check init-addr libc,none server invidious-5-vpn gluetun:3004 check init-addr libc,none server invidious-6-vpn gluetun:3005 check init-addr libc,none - # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ - option redispatch -# backend invidious-tor -# server-template invidious-tor 2 invidious-tor:3000 check resolvers docker init-addr libc,none -# -# backend invidious-i2p -# server-template invidious-i2p 2 invidious-i2p:3000 check resolvers docker init-addr libc,none + # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ + option redispatch 2 backend http3-proxy - # Unix socket for http3-proxy for later - #server http3-proxy-unix-socket /http3-proxy/http-proxy.sock - # balance source + # Caches video thumbnails, profile pictures and others http-request cache-use invidious-thumbnail-cache if { path_beg /vi/ } http-response cache-store invidious-thumbnail-cache + http-request cache-use invidious-ggpht-cache if { path_beg /ggpht/ } + http-response cache-store invidious-ggpht-cache # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 - acl youtube_thumbnails var(txn.vipath) -m beg /vi/ + acl youtube_thumbnails var(txn.path) -m beg /vi/ + acl youtube_ggpht var(txn.path) -m beg /ggpht/ + http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_thumbnails http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_thumbnails + http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_ggpht + http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_ggpht + server http3-proxy-1-vpn gluetun:8080 check init-addr libc,none + # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ + # Use another server if one takes too long to reply + # option redispatch 2 + # Only 1 server, not neccessary diff --git a/invidious/invidious/haproxy/haproxy.cfg b/invidious/invidious/haproxy/haproxy.cfg index 71c1961..a4e38f5 100644 --- a/invidious/invidious/haproxy/haproxy.cfg +++ b/invidious/invidious/haproxy/haproxy.cfg @@ -1,6 +1,6 @@ global # This because haproxy is dumb and likes to use the maxconnection from the kernel and that is super mega huge making haproxy oom - maxconn 32768 + maxconn 16384 # 777 perms for all sockets unix-bind mode 777 stats socket ipv4@0.0.0.0:8405 level admin @@ -17,6 +17,7 @@ defaults timeout server 5s timeout http-request 10s +# Prometheus for graphs frontend prometheus bind *:8404 mode http @@ -49,67 +50,30 @@ cache invidious-ggpht-cache max-age 3600 process-vary on -# cache api-cache -# # 32MB -# total-max-size 32 -# # 4kB -# max-object-size 32000 -# # 5min -# max-age 300 -# process-vary on - frontend invidious - http-request set-var(txn.ggpht) path - bind /tmp/invidious.sock bind *:8001 - #filter cache invidious-api-cache - #http-request cache-use invidious-api-cache if { path_beg /api/v1 } - #http-response cache-store invidious-api-cache - # use_backend http3-proxy if { path_beg /videoplayback } default_backend invidious frontend invidious-tor - bind /tmp/invidious-tor.sock bind *:8002 default_backend invidious-tor frontend invidious-i2p - bind /tmp/invidious-i2p.sock bind *:8003 default_backend invidious-i2p frontend http3-proxy - http-request set-var(txn.vipath) path - # acl vi path_beg /vi/ - bind /tmp/http3-proxy.sock + http-request set-var(txn.path) path bind *:8004 - #http-response set-header Cache-Control public - #http-response set-header Expires -1 - #http-response set-header Pragma cache - # acl youtube_thumbnails path_beg /vi/ - # default_backend http3-proxy backend invidious balance leastconn - #http-request cache-use api-cache if { path_beg /api/v1/channels/ } - #http-response cache-store api-cache - # acl inv_api var(txn.api) -m beg /api/ - # http-response set-header X-Cache-Status HIT if !{ srv_id -m found } inv_api - # http-response set-header X-Cache-Status MISS if { srv_id -m found } inv_api - # - http-request cache-use invidious-ggpht-cache if { path_beg /ggpht/ } - http-response cache-store invidious-ggpht-cache - - # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 - acl youtube_ggpht var(txn.ggpht) -m beg /ggpht/ - http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_ggpht - http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_ggpht server-template invidious 8 invidious:3000 check resolvers docker init-addr libc,none + # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ - option redispatch - # server invidious-1 gluetun:3000 check resolvers docker init-addr libc,none - # server invidious-2 gluetun:3001 check resolvers docker init-addr libc,none + # Use another server if one takes too long to reply + option redispatch 2 backend invidious-tor server-template invidious-tor 2 invidious-tor:3000 check resolvers docker init-addr libc,none @@ -118,19 +82,25 @@ backend invidious-i2p server-template invidious-i2p 2 invidious-i2p:3000 check resolvers docker init-addr libc,none backend http3-proxy - # Unix socket for http3-proxy for later - #server http3-proxy-unix-socket /http3-proxy/http-proxy.sock - # balance source + # Caches video thumbnails, profile pictures and others http-request cache-use invidious-thumbnail-cache if { path_beg /vi/ } http-response cache-store invidious-thumbnail-cache + http-request cache-use invidious-ggpht-cache if { path_beg /ggpht/ } + http-response cache-store invidious-ggpht-cache # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 - acl youtube_thumbnails var(txn.vipath) -m beg /vi/ + acl youtube_thumbnails var(txn.path) -m beg /vi/ + acl youtube_ggpht var(txn.path) -m beg /ggpht/ + http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_thumbnails http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_thumbnails - #server http3-proxy-1 gluetun:8080 check resolvers docker init-addr libc,none + http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_ggpht + http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_ggpht + server http3-proxy-1 http3-proxy:8080 check resolvers docker init-addr libc,none server http3-proxy-2 http3-proxy:8080 check resolvers docker init-addr libc,none - # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ - option redispatch + + # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ + # Use another server if one takes too long to reply + option redispatch 2