feat: add ganymede for stream archiving

This commit is contained in:
SeanOMik 2024-07-23 18:34:42 -04:00
parent 05faa34efe
commit 277cbe789a
Signed by: SeanOMik
GPG Key ID: FEC9E2FC15235964
8 changed files with 480 additions and 1 deletions

View File

@ -0,0 +1,77 @@
apiVersion: v1
kind: Secret
metadata:
name: ganymede-env
namespace: default
stringData:
dbPassword: ENC[AES256_GCM,data:sze6VLYEriMaKWDRHq/2q1CfX6p5HFqUDK9UQBAwhsk=,iv:RoBLiJWWeDhtNAkw6y86aDAqNpi5MHCTC4dXaosCFPE=,tag:tj1rhVowe4vwtk5bADpVJQ==,type:str]
jwtSecret: ENC[AES256_GCM,data:/duysBwm3rY8y1g4xOwB5q2FuTvsQLfFhq4Ciz5zFO0hX90OI4k5ugG/QOEA1Eo8AM+aQlrAjY7c8G2eaH8+2C5Ieh9L3L1Xqw817OcGCP2N9uAAzz112eJ1DMa+Pk+6TQ4W2ksQNJv4eeM+FeiLcWB4P3kAafR3YZTm5wbYl2JQdyafFLVBA/wOWhBTexWJkIyyTsvjpjjFcAYNmqSwfkI5XMAehiVpELhdDcC8yxqN3WK0uLxd/1m0McB0/rmq8NDuVjPtgNR7B+dfVCVLO0xDeWE27RYblACh4DZuqubSweJPxGk6NBTjr9ZIIiN3gjtdMhFKyFcDk2+/ThJ0STKEnifMEllyzOvvhQx+hZ+R0fPK2BgpLqYZycOOc47yDvBbMft4jdh76AH+05H8WPyX3gI4m+C6Ys7QiCc+pM12a+XbjNNmrqSo3l6Re/A4GrU8m4kwuTI=,iv:f/p2IXZtOlD3N/hyjIdYmNqWHsLc3jpa5sfV31lGiSc=,tag:c+OzXzPnyMOitK2dDRuqtQ==,type:str]
jwtRefreshSecret: ENC[AES256_GCM,data:f6vWk11QSE/FIMtIizAg7uxwiEK9MgiiUAdevkn5yQMw5SXlfwk4tcqikksdgJ/w/beVKzRKksCYjKMGLVvQwXYP7OZvyewwAW552Mnr9RAw4etwMAsLew==,iv:dHR/GoqODvDrESkLUG+CMoNk21GRaTqruUe/U2WPl1Q=,tag:AdW6fHa1ByzIy0sLMf8ULA==,type:str]
twitchClientId: ENC[AES256_GCM,data:PpkalHb9B1m9WOzFXAng8Zus6M/JwxTbm1gsXq6n,iv:XJS+/KdSRI8Z3Pm5dnPBEyHDCPNWET76juAabka6a9Q=,tag:uBogLH6CxL9uVGs+1jtyRQ==,type:str]
twitchClientSecret: ENC[AES256_GCM,data:qys5GCCWpRwRQJsDWu/6x0teenNPiEiz+zpxUyVj,iv:tTkZQ0GoA8ris8w1sxWuGwpHGoXueLdkjd3vTczWwzo=,tag:kL/376ufSxC30gxtJbUf1Q==,type:str]
oauthClientId: ENC[AES256_GCM,data:6IKwl/49W95m0761iusD/dvCYU1ZC5yg/I4c2Qirh936KdB7uiP5HA==,iv:WiGHiRgEohOeJE/iqgam7YX18OBm+geJu3xGnqokXEQ=,tag:22p86e+A8hhcbEI6cwfAaQ==,type:str]
oauthClientSecret: ENC[AES256_GCM,data:N4tbC85GhXsPUy9lkblF7rog1p7ui+MI+IIAguRjkD5t5pMdOmXv0cAWoG6uKZLupdURpSNv8JyGRfRxn878bP3et0ESRYzCbdzxsEGxNynm9a5EtJzBGzvy8gVR2pNGrDQly7uaXWCC0ebYf3S/BxU9RkrE3XNRIJRH4q1RkRo=,iv:F8F9UOtIgIKOeYga5HoLLx7mFScNbgzRAPnRKFTfwTI=,tag:iP9ivH9awttctMCqzjp0/Q==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age: []
lastmodified: "2024-07-23T22:33:48Z"
mac: ENC[AES256_GCM,data:VDeT09BQ8RJRufUViVbCLFrpkMImdaQMcHe3GETGEhh7HXHlkU+uo2uMehUfOUC1MOx824DWAzA2xFNCufU+WUhRRX4p9R42kA59vUDQIvnmF8bDQcgXE4F755ZwQJNFxh4ufP7f5gRddzAVEjxwibgGIcnGByi71JGiHc6Hbo8=,iv:fbWRtnQgkBvp47o5LRSiJXD+SxiecsA5vYX9PQYw79M=,tag:+aeq5DTcRa2su5zILNEu2Q==,type:str]
pgp:
- created_at: "2024-07-23T22:33:48Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMAyqlIeyoxYovARAA1rubuIC/QUpYcXfph1sIe2QAxnQXP4bO35crckNHwvL4
WLrqMSxs9KiZPiwOxefGJGlTTOFJAas0C8WASylWgTnYBA/Qtw4TutcPv/sIUCTk
+b3eDuISJi2EFdEknyH0BzkEaK4BKiheLa7dWOvshMbSOVlXZn9L/7S6+8/mG0UR
HpVRay4Tz34EfshNyhK70dcH3AsWAY7HHXFQiXjxrB3+uebNAEWZg7WBy274AY5E
14PBO/EyeyX/bwEkyazIb/23q8TYEqPTBe+4jk830e8uuvWj6N2PVXPHRAk49Kbw
/ATfhJIUQJ/NXYWG6o6kECX3iWZTrTbvapXZl6nRda29INxX967RA/fuIYG06kwa
8igV0k0zxlvFnJmMFd40Vu0yr0DtcQEDhZZ2q4rSYfi6KpmBcgR3W3nQxr6dTg34
E+U0+p9FEkMFWOrLG233jJVgE+ZMU7NzvAB0mO6B2URpNmAGgsfeg5MsFUpuUzs/
/Ee0Fp7fJf4kCyw7inwQrQMjNq2ddEKzxFWt/BGTZCawWq4fekuT20HZPaPWGkoJ
YzLhN33d/xJhtrEmqe9Jf+lJocnCpjsNXoKS05ifPR4RE2o9bB4PZ6Oxa2F2tmk8
yScn12iw8fyFu/yvoF6fGcHRXvzrty9DSZ7s5ao3W8DatKZlInIB+CK7j1TK4tSF
AgwDXjg0p2IN1X8BEAC1gztbByJjfYIHCucoPGCojsxjKNXXDeXnSdJOPwKc1M5P
MvbUMZAQn6+8Cni+/iYBVW5MoE/sbpBQs/+AOamCyiYRdSPmU9/oFEYTojnEiUk2
fCAdC5lyebagd+msT74jwc0/UZ/s6T8jVmdjyxCCkR5Yj3dHUJpMlTYTU32K1YJC
gQG8X0q7vO7s03+5v9micl4QkQmT8vmf3aHq8ltqKgez/lFt8lSxfnjmoxuGYUz0
+xIRQBuDdKt8Nugta94jU3bzZlGWmDfbD7fuFAjF6BN/6dtZxatdDN3dCMPqCiyB
OEkOmRcubkqDOWJmPKGYJKQZc+71606/VHYt6jUqwxru9iIy7wPWewjm01vekUiO
rbE6LIhQTlNkdkv3SHbv9JiZT3CPaHZslJjDtKZ30htV9IVF3uOpb+c2auZrOO8C
d8ZZt2I4FimcErI0CHnZ1VxyQy+ztKrgO7NElcHLRzLs9fYjhT1oTLpK3oRVObJX
RvdKWz/GPzJ71oSByFSQ0zP19zt6DCYAzu3O39Emn5mW96Du58/9wgRe77JMrvOn
QJXlxyx6jruEu5rNrJrMvfE4tlGy0U7zDDjKx8Ho6BfnGAYxe8GKiF8BqMVWDAnS
SpEafDce84Y+2a3kzE2T/bxfpDb/TB2kVUaBxWKgn8SP2NzSpcsynjJ1DUza49Rm
AQkCEPJgTXfHNpb3Ik44AQWBYQoq2a2y5eyEQl5jknMw2yiJ1Rjfwk9hDGTeZ+VW
iV1+tkAAp80CxpwAVpYzWaAGWCb2BMhvASR4ScMui2uJ+KPcyURTN3V/OkFrmaEQ
BRdSfzyO
=bECn
-----END PGP MESSAGE-----
fp: BD1AAF9D8170F4BEE437365FF6F0933799CFEBCD
- created_at: "2024-07-23T22:33:48Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMAy5t8IMoPu4VAQ//aYCLa/lPOdGtT/In1lx8UVRW0R+AcU7fxXco1BsHlJls
voh4GMH8x/3TtXnZ5UYo9WXeR1iOAy1CHDyqdnJl1xkXe4CxVltHyj1Cot1y6Xo9
CQ8woUgof9PNmG+O6+PDxNHh1raU+xqv43vV3/tLQO3DKTWQfg3mSrlrTR6NwJW/
pg0qKUsNBw1kSAWPzxg+0HDLcQgc5DFCd+/CviZShxSyoDTSHOYGLIUiPUBiKtXR
77V9/ZtvGUlW2d+TNJkptDl+ptDrKJVeaJhzxcZRFpouFWEnOfHW/CudXqkuL36l
yRC3m9HfS8kjmT0tNgDLp6TGIXmj/6sDTS/X4eeqjQVUwR4XGek+qdADrky41P0Q
OCleXnZOXkAbLgS57/rGIDgvzQ3DFhp47/MjfnarTxYnH85LskS47GoywwbH8bCF
G4RAv7oqdvB+ANHp5Et11KA1ADUzqbLWq9PcC+92kio3riM52Fopdc/6y0lU66/G
MwzvwJKc0SfOyUNzNK1sMFAtuXGn6uebom9Wkvwzfdi75LEJPGZqFe8sNViU3Qk6
iEJ3gdgnHjp50fo21iEmPXOtpIym1s46D2xKskYBZahIGcItxM2gJeqz2eTCD8kv
UPA+Hh+qK7gvBlXuu65v9GzqXJhxr6NmxgkZPoRmR4ayJZq7NIsrAVrgWm5ac07U
ZgEJAhAvjWnbUlEFvHvFVo/kUO+EKS/djcRUMmH9ueZkW1i/aUFtyHK6iOWRyrhu
E+6egxpRcFeBzImzgg+PP97TG6gG1hWdgVtnqQmUDG4dWp6zyyL8ZOSaLhwgdQtR
f09z2C7FUg==
=emFU
-----END PGP MESSAGE-----
fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D
encrypted_regex: ^(data|stringData)$
version: 3.8.1

View File

@ -0,0 +1,58 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: ganymede-conf
namespace: default
data:
config.json: |
{
"archive": {
"save_as_hls": false
},
"debug": false,
"live_check_interval_seconds": 300,
"livestream": {
"proxies": [
{
"header": "",
"url": "https://eu.luminous.dev"
},
{
"header": "x-donate-to:https://ttv.lol/donate",
"url": "https://api.ttv.lol"
}
],
"proxy_enabled": false,
"proxy_parameters": "%3Fplayer%3Dtwitchweb%26type%3Dany%26allow_source%3Dtrue%26allow_audio_only%3Dtrue%26allow_spectre%3Dfalse%26fast_bread%3Dtrue",
"proxy_whitelist": [
""
]
},
"notifications": {
"error_enabled": true,
"error_template": "⚠️ Error: Queue ID {{queue_id}} for {{channel_display_name}} failed at task {{failed_task}}.",
"error_webhook_url": "",
"is_live_enabled": true,
"is_live_template": "🔴 {{channel_display_name}} is live!",
"is_live_webhook_url": "",
"live_success_enabled": true,
"live_success_template": "✅ Live Stream Archived: {{vod_title}} by {{channel_display_name}}.",
"live_success_webhook_url": "",
"video_success_enabled": true,
"video_success_template": "✅ Video Archived: {{vod_title}} by {{channel_display_name}}.",
"video_success_webhook_url": ""
},
"oauth_enabled": true,
"parameters": {
"chat_render": "-h 1440 -w 340 --framerate 30 --font Inter --font-size 13",
"streamlink_live": "--twitch-low-latency,--twitch-disable-hosting",
"twitch_token": "",
"video_convert": "-c:v copy -c:a copy"
},
"registration_enabled": true,
"storage_templates": {
"file_template": "{{id}}",
"folder_template": "{{date}}-{{id}}-{{type}}-{{uuid}}"
},
"video_check_interval_minutes": 180
}

View File

@ -0,0 +1,222 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: ganymede
namespace: default
spec:
interval: 5m
chart:
spec:
chart: app-template
version: 3.1.0
sourceRef:
kind: HelmRepository
name: bjws-charts
namespace: flux-system
values:
controllers:
main:
# pod:
# securityContext:
# runAsNonRoot: true
# runAsUser: 10000
# runAsGroup: 10000
# fsGroup: 10000
# fsGroupChangePolicy: OnRootMismatch
initContainers:
copy-config:
image:
repository: alpine
tag: 3.20
command: [ "sh", "-c", "cat /ganymede-config.json && cp -v /ganymede-config.json /data/config.json" ]
containers:
api:
image:
repository: ghcr.io/zibbp/ganymede
tag: v2.3.2
env:
- name: TZ
value: "America/New_York" # Set to your timezone
- name: DB_HOST
value: "postgresql.database"
- name: DB_PORT
value: "5432"
- name: DB_USER
value: "ganymede"
- name: DB_PASS
secretKeyRef:
name: ganymede-env
key: dbPassword
- name: DB_NAME
value: "ganymede"
- name: DB_SSL
value: "disable"
- name: JWT_SECRET
secretKeyRef:
name: ganymede-env
key: jwtSecret
- name: JWT_REFRESH_SECRET
secretKeyRef:
name: ganymede-env
key: jwtRefreshSecret
- name: TWITCH_CLIENT_ID
secretKeyRef:
name: ganymede-env
key: twitchClientId
- name: TWITCH_CLIENT_SECRET
secretKeyRef:
name: ganymede-env
key: twitchClientSecret
- name: FRONTEND_HOST
value: https://twvods.${SECRET_NEW_DOMAIN}
- name: OAUTH_PROVIDER_URL
value: "https://auth.${SECRET_NEW_DOMAIN}/application/o/ganymede/.well-known/openid-configuration"
- name: OAUTH_CLIENT_ID
secretKeyRef:
name: ganymede-env
key: oauthClientId
- name: OAUTH_CLIENT_SECRET
secretKeyRef:
name: ganymede-env
key: oauthClientSecret
- name: OAUTH_REDIRECT_URL
value: "https://twvods.${SECRET_NEW_DOMAIN}/api/v1/auth/oauth/callback"
- name: TEMPORAL_URL
value: "temporal:7233"
# WORKER
- name: MAX_CHAT_DOWNLOAD_EXECUTIONS
value: "5"
- name: MAX_CHAT_RENDER_EXECUTIONS
value: "3"
- name: MAX_VIDEO_DOWNLOAD_EXECUTIONS
value: "5"
- name: MAX_VIDEO_CONVERT_EXECUTIONS
value: "3"
frontend:
image:
repository: ghcr.io/zibbp/ganymede-frontend
tag: v2.3.1
env:
- name: API_URL
# /api will be added to this
value: "https://twvods.${SECRET_NEW_DOMAIN}" # Points to the API service
- name: CDN_URL
# /vods will be added to this
value: "https://twvods.${SECRET_NEW_DOMAIN}" # Points to the CDN service
- name: SHOW_SSO_LOGIN_BUTTON
value: "true" # show/hide SSO login button on login page
- name: FORCE_SSO_AUTH
value: "false" # force SSO auth for all users (bypasses login page and redirects to SSO)
- name: REQUIRE_LOGIN
value: "false" # require login to view videos
nginx:
image:
repository: nginxinc/nginx-unprivileged
tag: 1.27.0-alpine
service:
app:
controller: main
ports:
nginx:
port: 8080
frontend:
port: 3000
api:
port: 4000
ingress:
main:
annotations:
cert-manager.io/cluster-issuer: letsencrypt-production
traefik.ingress.kubernetes.io/router.entrypoints: websecure
hosts:
- host: twvods.${SECRET_NEW_DOMAIN}
paths:
- path: /
service:
identifier: app
port: frontend
- path: /api
service:
identifier: app
port: api
- path: /vods
service:
identifier: app
port: nginx
persistence:
vods:
type: persistentVolumeClaim
size: 50Gi
retain: true
storageClass: mainpool-hostpath
accessMode: ReadWriteOnce
globalMounts:
- path: /vods
ganymede-data:
type: persistentVolumeClaim
size: 5Gi
retain: true
storageClass: mainpool-hostpath
accessMode: ReadWriteOnce
advancedMounts:
main: # controller name
api: # container name
- path: /data
ganymede-logs:
type: persistentVolumeClaim
size: 5Gi
retain: true
storageClass: mainpool-hostpath
accessMode: ReadWriteOnce
advancedMounts:
main: # controller name
api: # container name
- path: /logs
nginx-conf:
name: ganymede-nginx-conf
type: configMap
defaultMode: 0664
advancedMounts:
main: # controller name
nginx: # container name
- subPath: nginx.conf
path: /etc/nginx/nginx.conf
ganymede-temp-conf:
type: emptyDir
advancedMounts:
main: # controller name
api: # container name
- path: /data
copy-config: # container name
- path: /data
ganymede-conf:
name: ganymede-conf
type: configMap
defaultMode: 0777
advancedMounts:
main: # controller name
copy-config: # container name
- subPath: config.json
path: /ganymede-config.json

View File

@ -0,0 +1,8 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./env-secret.sops.yaml
- ./nginx-conf.yaml
- ./ganymede-conf.yaml
- ./temporal-helm-release.yaml
- ./helm-release.yaml

View File

@ -0,0 +1,54 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: ganymede-nginx-conf
namespace: default
data:
nginx.conf: |
worker_processes auto;
worker_rlimit_nofile 65535;
error_log /var/log/nginx/error.log info;
pid /tmp/nginx.pid;
events {
multi_accept on;
worker_connections 65535;
}
http {
sendfile on;
sendfile_max_chunk 1m;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
gzip on;
server {
listen 8080;
root /vods;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always;
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
location ^~ /vods {
autoindex on;
alias /vods;
location ~* \.(ico|css|js|gif|jpeg|jpg|png|svg|webp)$ {
expires 30d;
add_header Pragma "public";
add_header Cache-Control "public";
}
location ~* \.(mp4)$ {
add_header Content-Type "video/mp4";
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS' always;
add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range' always;
add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range' always;
}
}
}
}

View File

@ -0,0 +1,52 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: temporal
namespace: default
spec:
interval: 5m
chart:
spec:
chart: app-template
version: 3.1.0
sourceRef:
kind: HelmRepository
name: bjws-charts
namespace: flux-system
values:
controllers:
main:
containers:
main:
image:
repository: temporalio/auto-setup
tag: 1.23
env:
- name: DB
value: postgres12
- name: DBNAME
value: ganymede
- name: DB_PORT
value: "5432"
- name: POSTGRES_USER
value: ganymede
- name: POSTGRES_PWD
secretKeyRef:
name: ganymede-env
key: dbPassword
- name: POSTGRES_SEEDS # postgres hostname, idk why its called SEEDS
value: postgresql.database
service:
app:
controller: main
ports:
temporal:
port: 7233
protocol: TCP

View File

@ -8,4 +8,5 @@ resources:
- ./huginn
- ./exim
- ./well-known-site
- ./dendrite
- ./dendrite
- ./ganymede

7
docs/apps/ganymede.md Normal file
View File

@ -0,0 +1,7 @@
# Ganymede
## Temporal IO
Temporal IO requires some annoying permissions in the database. It requires permission to create databases.
```sql
ALTER USER temporal CREATEDB;
```