From ca75b9a1797a044dd9696a06dd73ef95a24c07fd Mon Sep 17 00:00:00 2001 From: root Date: Sun, 15 Sep 2024 20:19:54 -0300 Subject: [PATCH] Fix redis databases --- invidious/invidious-vpn/docker-compose.yml | 37 +++++++++++++------ invidious/invidious-vpn/haproxy/haproxy.cfg | 32 +++++++++++++--- invidious/invidious/docker-compose.yml | 2 + invidious/invidious/haproxy/haproxy.cfg | 26 ++++++++++++- invidious/valkey-invidious/docker-compose.yml | 35 +++++++++++++++++- 5 files changed, 110 insertions(+), 22 deletions(-) diff --git a/invidious/invidious-vpn/docker-compose.yml b/invidious/invidious-vpn/docker-compose.yml index 18273b0..cbee5ce 100644 --- a/invidious/invidious-vpn/docker-compose.yml +++ b/invidious/invidious-vpn/docker-compose.yml @@ -25,6 +25,7 @@ services: environment: INVIDIOUS_CONFIG: | feed_threads: 0 + backends: ["1", "2"] log_level: Warn # db: # dbname: "${PGSQL_DBNAME}" @@ -34,7 +35,7 @@ services: # port: 5432 port: 3000 database_url: postgres://kemal:kemal@/invidious - redis_url: tcp://valkey:6379 + redis_url: tcp://valkey-vpn:6379 donation_url: "https://nadeko.net/donate" contact_url: "https://nadeko.net/contact" home_domain: "nadeko.net" @@ -91,8 +92,10 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "172.80.0.30:12999" + max_dash_resolution: ${MAX_RES} + backends: ["1", "2"] depends_on: - # - valkey + # - valkey-vpn - inv_sig_helper - gluetun healthcheck: @@ -113,6 +116,7 @@ services: environment: INVIDIOUS_CONFIG: | feed_threads: 0 + backends: ["1", "2"] log_level: Warn # db: # dbname: "${PGSQL_DBNAME}" @@ -122,7 +126,7 @@ services: # port: 5432 port: 3001 database_url: postgres://kemal:kemal@/invidious - redis_url: tcp://valkey:6379 + redis_url: tcp://valkey-vpn:6379 donation_url: "https://nadeko.net/donate" contact_url: "https://nadeko.net/contact" home_domain: "nadeko.net" @@ -179,8 +183,9 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "172.80.0.30:12999" + max_dash_resolution: ${MAX_RES} depends_on: - # - valkey + # - valkey-vpn - inv_sig_helper - gluetun healthcheck: @@ -201,6 +206,7 @@ services: environment: INVIDIOUS_CONFIG: | feed_threads: 0 + backends: ["1", "2"] log_level: Warn # db: # dbname: "${PGSQL_DBNAME}" @@ -210,7 +216,7 @@ services: # port: 5432 port: 3002 database_url: postgres://kemal:kemal@/invidious - redis_url: tcp://valkey:6379 + redis_url: tcp://valkey-vpn:6379 donation_url: "https://nadeko.net/donate" contact_url: "https://nadeko.net/contact" home_domain: "nadeko.net" @@ -267,8 +273,9 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "172.80.0.30:12999" + max_dash_resolution: ${MAX_RES} depends_on: - # - valkey + # - valkey-vpn - inv_sig_helper - gluetun healthcheck: @@ -289,6 +296,7 @@ services: environment: INVIDIOUS_CONFIG: | feed_threads: 0 + backends: ["1", "2"] log_level: Warn # db: # dbname: "${PGSQL_DBNAME}" @@ -298,7 +306,7 @@ services: # port: 5432 port: 3003 database_url: postgres://kemal:kemal@/invidious - redis_url: tcp://valkey:6379 + redis_url: tcp://valkey-vpn:6379 donation_url: "https://nadeko.net/donate" contact_url: "https://nadeko.net/contact" home_domain: "nadeko.net" @@ -355,8 +363,9 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "172.80.0.30:12999" + max_dash_resolution: ${MAX_RES} depends_on: - # - valkey + # - valkey-vpn - inv_sig_helper - gluetun healthcheck: @@ -377,6 +386,7 @@ services: environment: INVIDIOUS_CONFIG: | feed_threads: 0 + backends: ["1", "2"] log_level: Warn # db: # dbname: "${PGSQL_DBNAME}" @@ -386,7 +396,7 @@ services: # port: 5432 port: 3004 database_url: postgres://kemal:kemal@/invidious - redis_url: tcp://valkey:6379 + redis_url: tcp://valkey-vpn:6379 donation_url: "https://nadeko.net/donate" contact_url: "https://nadeko.net/contact" home_domain: "nadeko.net" @@ -443,8 +453,9 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "172.80.0.30:12999" + max_dash_resolution: ${MAX_RES} depends_on: - # - valkey + # - valkey-vpn - inv_sig_helper - gluetun healthcheck: @@ -465,6 +476,7 @@ services: environment: INVIDIOUS_CONFIG: | feed_threads: 0 + backends: ["1", "2"] log_level: Warn # db: # dbname: "${PGSQL_DBNAME}" @@ -474,7 +486,7 @@ services: # port: 5432 port: 3005 database_url: postgres://kemal:kemal@/invidious - redis_url: tcp://valkey:6379 + redis_url: tcp://valkey-vpn:6379 donation_url: "https://nadeko.net/donate" contact_url: "https://nadeko.net/contact" home_domain: "nadeko.net" @@ -531,8 +543,9 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "172.80.0.30:12999" + max_dash_resolution: ${MAX_RES} depends_on: - # - valkey + # - valkey-vpn - inv_sig_helper - gluetun healthcheck: diff --git a/invidious/invidious-vpn/haproxy/haproxy.cfg b/invidious/invidious-vpn/haproxy/haproxy.cfg index cb57283..7407e78 100644 --- a/invidious/invidious-vpn/haproxy/haproxy.cfg +++ b/invidious/invidious-vpn/haproxy/haproxy.cfg @@ -13,9 +13,8 @@ resolvers docker defaults mode http timeout client 10s - timeout connect 10s + timeout connect 20s timeout server 30s - timeout http-request 10s # Prometheus for graphs frontend prometheus @@ -50,17 +49,38 @@ cache invidious-ggpht-cache max-age 3600 process-vary on +cache invidious-sb-cache + # 128MB + total-max-size 128 + # 128kB + max-object-size 131072 + # 1h + max-age 86400 + process-vary on + frontend invidious + http-request set-var(txn.ipath) path bind *:8001 default_backend invidious frontend http3-proxy - http-request set-var(txn.vipath) path + http-request set-var(txn.hpath) path bind *:8004 default_backend http3-proxy backend invidious balance leastconn + + # Caches video storyboards + http-request cache-use invidious-sb-cache if { path_beg /sb/ } + http-response cache-store invidious-sb-cache + + # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 + acl sb var(txn.ipath) -m beg /sb/ + + http-response set-header X-Cache-Status HIT if !{ srv_id -m found } sb + http-response set-header X-Cache-Status MISS if { srv_id -m found } sb + server invidious-1-vpn gluetun:3000 check init-addr libc,none server invidious-2-vpn gluetun:3001 check init-addr libc,none server invidious-3-vpn gluetun:3002 check init-addr libc,none @@ -72,7 +92,7 @@ backend invidious option redispatch 2 backend http3-proxy - timeout server 15s + timeout queue 30s # Caches video thumbnails, profile pictures and others http-request cache-use invidious-thumbnail-cache if { path_beg /vi/ } http-response cache-store invidious-thumbnail-cache @@ -80,8 +100,8 @@ backend http3-proxy http-response cache-store invidious-ggpht-cache # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 - acl youtube_thumbnails var(txn.path) -m beg /vi/ - acl youtube_ggpht var(txn.path) -m beg /ggpht/ + acl youtube_thumbnails var(txn.hpath) -m beg /vi/ + acl youtube_ggpht var(txn.hpath) -m beg /ggpht/ http-response set-header X-Cache-Status HIT if !{ srv_id -m found } youtube_thumbnails http-response set-header X-Cache-Status MISS if { srv_id -m found } youtube_thumbnails diff --git a/invidious/invidious/docker-compose.yml b/invidious/invidious/docker-compose.yml index 9daacab..6ecdda1 100644 --- a/invidious/invidious/docker-compose.yml +++ b/invidious/invidious/docker-compose.yml @@ -47,6 +47,7 @@ services: hmac_key: "${HMAC_KEY}" domain: "inv.nadeko.net" alternative_domains: ["inv.nadeko.net", "inv.nadekonw7plitnjuawu6ytjsl7jlglk2t6pyq6eftptmiv3dvqndwvyd.onion", "zzlsbhhfvwg3oh36tcvx4r7n6jrw7zibvyvfxqlodcwn3mfrvzuq.b32.i2p", "inv.zzls.i2p"] + backends: ["1", "2"] use_pubsub_feeds: true popular_enabled: true captcha_enabled: true @@ -94,6 +95,7 @@ services: po_token: "${PO_TOKEN}" visitor_data: "${VISITOR_DATA}" signature_server: "inv_sig_helper:12999" + max_dash_resolution: 1080 depends_on: # - valkey - inv_sig_helper diff --git a/invidious/invidious/haproxy/haproxy.cfg b/invidious/invidious/haproxy/haproxy.cfg index a76ad84..82203b1 100644 --- a/invidious/invidious/haproxy/haproxy.cfg +++ b/invidious/invidious/haproxy/haproxy.cfg @@ -13,9 +13,9 @@ resolvers docker defaults mode http timeout client 10s - timeout connect 10s + timeout connect 20s timeout server 30s - timeout http-request 10s + timeout queue 30s # Prometheus for graphs frontend prometheus @@ -50,7 +50,17 @@ cache invidious-ggpht-cache max-age 3600 process-vary on +cache invidious-sb-cache + # 128MB + total-max-size 128 + # 128kB + max-object-size 131072 + # 1h + max-age 86400 + process-vary on + frontend invidious + http-request set-var(txn.ipath) path bind *:8001 default_backend invidious @@ -69,6 +79,17 @@ frontend http3-proxy backend invidious balance leastconn + + # Caches video storyboards + http-request cache-use invidious-sb-cache if { path_beg /sb/ } + http-response cache-store invidious-sb-cache + + # https://discourse.haproxy.org/t/http-response-set-header-with-condition-not-working/3108 + acl sb var(txn.ipath) -m beg /sb/ + + http-response set-header X-Cache-Status HIT if !{ srv_id -m found } sb + http-response set-header X-Cache-Status MISS if { srv_id -m found } sb + server-template invidious 8 invidious:3000 check resolvers docker init-addr libc,none # https://www.haproxy.com/documentation/haproxy-configuration-tutorials/service-reliability/retries/ @@ -82,6 +103,7 @@ backend invidious-i2p server-template invidious-i2p 2 invidious-i2p:3000 check resolvers docker init-addr libc,none backend http3-proxy + timeout queue 30s # Caches video thumbnails, profile pictures and others http-request cache-use invidious-thumbnail-cache if { path_beg /vi/ } http-response cache-store invidious-thumbnail-cache diff --git a/invidious/valkey-invidious/docker-compose.yml b/invidious/valkey-invidious/docker-compose.yml index 1e063e1..2ec5178 100644 --- a/invidious/valkey-invidious/docker-compose.yml +++ b/invidious/valkey-invidious/docker-compose.yml @@ -1,14 +1,29 @@ services: - # One valkey instance for all invidious instances, to share cache and prevent duplicates + # Cache needs to be separated if one backend is using other IP address + # Ex: If the user request a video from the Backend 1, the IP address attached + # to the video information will be the IP of the Backend 1, so if the user changes + # to Backend 2, the returned IP attached to the video information will be from the + # IP address of the Backend 1, which will make videoplayback fail. + valkey: image: valkey/valkey:7.2-alpine restart: unless-stopped volumes: + - invidious-valkey:/data + - ./valkey.conf:/usr/local/etc/valkey/valkey.conf + command: "valkey-server /usr/local/etc/valkey/valkey.conf" + networks: + - invidious + + valkey-vpn: + image: valkey/valkey:7.2-alpine + restart: unless-stopped + volumes: + - invidious-valkey-vpn:/data - ./valkey.conf:/usr/local/etc/valkey/valkey.conf command: "valkey-server /usr/local/etc/valkey/valkey.conf" networks: - invidious-vpn - - invidious - gluetun_network # INVIDIOUS VALKEY METRICS @@ -21,6 +36,17 @@ services: networks: - invidious + # INVIDIOUS VALKEY METRICS + invidious-redis-exporter-vpn: + image: oliver006/redis_exporter + restart: unless-stopped + ports: + - "127.0.0.1:9122:9121" + command: "--redis.addr=redis://valkey-vpn:6379" + networks: + - invidious-vpn + - gluetun_network + networks: invidious-vpn: external: true @@ -28,3 +54,8 @@ networks: external: true gluetun_network: external: true + +volumes: + invidious-valkey: + invidious-valkey-vpn: +