SeanOMik/k3s-cluster
1
0
Fork 0
Code Issues 3 Pull requests 4 Projects Releases Packages Wiki Activity
k3s-cluster/kubernetes/main/apps/dev/airflow/helm-release.yaml

297 lines
No EOL
10 KiB
YAML
Raw Blame History

apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: apache-airflow
namespace: dev
spec:
interval: 5m
chart:
spec:
chart: airflow
version: "8.9.0"
sourceRef:
kind: HelmRepository
name: apache-airflow-charts
namespace: flux-system
values:
airflow:
executor: KubernetesExecutor
## environment variables for airflow configs
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/airflow-configs.md
config:
AIRFLOW__CELERY__FLOWER_URL_PREFIX: &flowerUrlPrefix "/flower"
# AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
AIRFLOW__CORE__LOAD_EXAMPLES: "True"
extraEnv:
- name: AIRFLOW__CORE__FERNET_KEY
valueFrom:
secretKeyRef:
name: airflow
key: fernetKey
- name: AIRFLOW__WEBSERVER__SECRET_KEY
valueFrom:
secretKeyRef:
name: airflow
key: webserverSecretKey
## a list of users to create
## not creating any since openid is setup
users: []
## a list airflow connections to create
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-connections.md
connections: []
## a list airflow variables to create
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-variables.md
variables: []
## a list airflow pools to create
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-pools.md
pools: []
## extra pip packages to install in airflow Pods
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/extra-python-packages.md
## [WARNING] this feature is not recommended for production use, see docs
extraPipPackages:
- fake_useragent
- bs4
- selenium
## extra VolumeMounts for the airflow Pods
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumeMounts: []
## extra Volumes for the airflow Pods
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumes: []
## configs generating the `pod_template.yaml` file for `AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE`
## [NOTE] the `dags.gitSync` values will create a git-sync init-container in the pod
## [NOTE] the `airflow.extraPipPackages` will NOT be installed
kubernetesPodTemplate:
## the full content of the pod-template file (as a string)
## [NOTE] all other `kubernetesPodTemplate.*` are disabled when this is set
stringOverride: ""
## resource requests/limits for the Pod template "base" container
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## extra pip packages to install in the Pod template
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/extra-python-packages.md
## [WARNING] this feature is not recommended for production use, see docs
extraPipPackages:
- fake_useragent
- bs4
- selenium
## extra VolumeMounts for the Pod template
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumeMounts: []
## extra Volumes for the Pod template
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumes: []
###################################
## COMPONENT | Airflow Scheduler
###################################
scheduler:
## the number of scheduler Pods to run
replicas: 1
## resource requests/limits for the scheduler Pods
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## configs for the log-cleanup sidecar of the scheduler
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-cleanup.md
logCleanup:
enabled: true
retentionMinutes: 21600
## configs for the scheduler Pods' liveness probe
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
livenessProbe:
enabled: true
## configs for an additional check that ensures tasks are being created by the scheduler
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
taskCreationCheck:
enabled: false
thresholdSeconds: 300
schedulerAgeBeforeCheck: 180
###################################
## COMPONENT | Airflow Webserver
###################################
web:
## the number of web Pods to run
replicas: 1
## resource requests/limits for the web Pods
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## configs for the Service of the web Pods
service:
type: ClusterIP
externalPort: 8080
webserverConfig:
enabled: true
## the name of a Secret containing a `webserver_config.py` key
existingSecret: "airflow"
# Airflow Workers
workers:
## if the airflow workers StatefulSet should be deployed
enabled: false
# Triggerer
triggerer:
## if the airflow triggerer should be deployed
enabled: true
## the number of triggerer Pods to run
replicas: 1
## resource requests/limits for the triggerer Pods
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## maximum number of triggers each triggerer will run at once (sets `AIRFLOW__TRIGGERER__DEFAULT_CAPACITY`)
capacity: 1000
# Flower
flower:
## if the airflow flower UI should be deployed
enabled: false
# Airflow Logs
logs:
## the airflow logs folder
path: /opt/airflow/logs
## configs for the logs PVC
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-persistence.md
persistence:
enabled: false
# Airflow DAGs
dags:
## the airflow dags folder
path: /opt/airflow/dags
## configs for the dags PVC
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/load-dag-definitions.md
persistence:
enabled: false
## configs for the git-sync sidecar
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/load-dag-definitions.md
gitSync:
enabled: true
repo: "https://git.${SECRET_NEW_DOMAIN}/SeanOMik/airflow-dags.git"
branch: main
revision: HEAD
repoSubPath: "dags/"
syncWait: 60
httpSecret: airflow
httpSecretUsernameKey: gitSyncUsername
httpSecretPasswordKey: gitSyncToken
# Kubernetes Ingress
ingress:
## if we should deploy Ingress resources
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/ingress.md
enabled: true
web:
annotations: {}
host: "airflow.${SECRET_NEW_DOMAIN}"
path: ""
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
#ingressClassName: "nginx"
## flower ingress configs
# flower is currently disabled
flower:
annotations: {}
host: "airflow.${SECRET_NEW_DOMAIN}"
path: *flowerUrlPrefix
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
#ingressClassName: "nginx"
# Kubernetes ServiceAccount
serviceAccount:
## if a Kubernetes ServiceAccount is created
create: true
## the name of the ServiceAccount
name: ""
## annotations for the ServiceAccount
annotations: {}
# disable embedded postgres
postgresql:
enabled: false
pgbouncer:
enabled: true
externalDatabase:
type: postgres
host: postgres16-rw.database.svc
port: 5432
## the schema which will contain the airflow tables
database: airflow
## (username - option 1) a plain-text helm value
user: airflow
## (username - option 2) a Kubernetes secret in your airflow namespace
#userSecret: "airflow-cluster1-database-credentials"
#userSecretKey: "username"
## (password - option 1) a plain-text helm value
#password: my_airflow_password
## (password - option 2) a Kubernetes secret in your airflow namespace
passwordSecret: "airflow"
passwordSecretKey: "databasePassword"
## use this for any extra connection-string settings, e.g. ?sslmode=disable
properties: ""
# disable embedded redis
redis:
enabled: false
externalRedis:
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/external-redis.md
host: redis-master.database
port: 6379
databaseNumber: 2
passwordSecret: airflow
passwordSecretKey: redisPassword
Reference in a new issue View git blame Copy permalink