feat: add apache airflow

This commit is contained in:
SeanOMik 2024-07-21 10:06:55 -04:00
parent d9c34fdf34
commit 0ddc7a8573
Signed by: SeanOMik
GPG Key ID: FEC9E2FC15235964
5 changed files with 369 additions and 0 deletions

View File

@ -0,0 +1,279 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: apache-airflow
namespace: dev
spec:
interval: 5m
chart:
spec:
chart: airflow
version: "8.9.0"
sourceRef:
kind: HelmRepository
name: apache-airflow-charts
namespace: flux-system
values:
airflow:
executor: KubernetesExecutor
## environment variables for airflow configs
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/airflow-configs.md
config:
AIRFLOW__CELERY__FLOWER_URL_PREFIX: &flowerUrlPrefix "/flower"
# AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
AIRFLOW__CORE__LOAD_EXAMPLES: "True"
extraEnv:
- name: AIRFLOW__CORE__FERNET_KEY
valueFrom:
secretKeyRef:
name: airflow
key: fernetKey
- name: AIRFLOW__WEBSERVER__SECRET_KEY
valueFrom:
secretKeyRef:
name: airflow
key: webserverSecretKey
## a list of users to create
## not creating any since openid is setup
users: []
## a list airflow connections to create
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-connections.md
connections: []
## a list airflow variables to create
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-variables.md
variables: []
## a list airflow pools to create
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-pools.md
pools: []
## extra pip packages to install in airflow Pods
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/extra-python-packages.md
## [WARNING] this feature is not recommended for production use, see docs
extraPipPackages: []
## extra VolumeMounts for the airflow Pods
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumeMounts: []
## extra Volumes for the airflow Pods
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumes: []
## configs generating the `pod_template.yaml` file for `AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE`
## [NOTE] the `dags.gitSync` values will create a git-sync init-container in the pod
## [NOTE] the `airflow.extraPipPackages` will NOT be installed
kubernetesPodTemplate:
## the full content of the pod-template file (as a string)
## [NOTE] all other `kubernetesPodTemplate.*` are disabled when this is set
stringOverride: ""
## resource requests/limits for the Pod template "base" container
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## extra pip packages to install in the Pod template
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/extra-python-packages.md
## [WARNING] this feature is not recommended for production use, see docs
extraPipPackages: []
## extra VolumeMounts for the Pod template
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumeMounts: []
## extra Volumes for the Pod template
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
extraVolumes: []
###################################
## COMPONENT | Airflow Scheduler
###################################
scheduler:
## the number of scheduler Pods to run
replicas: 1
## resource requests/limits for the scheduler Pods
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## configs for the log-cleanup sidecar of the scheduler
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-cleanup.md
logCleanup:
enabled: true
retentionMinutes: 21600
## configs for the scheduler Pods' liveness probe
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
livenessProbe:
enabled: true
## configs for an additional check that ensures tasks are being created by the scheduler
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
taskCreationCheck:
enabled: false
thresholdSeconds: 300
schedulerAgeBeforeCheck: 180
###################################
## COMPONENT | Airflow Webserver
###################################
web:
## the number of web Pods to run
replicas: 1
## resource requests/limits for the web Pods
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## configs for the Service of the web Pods
service:
type: ClusterIP
externalPort: 8080
webserverConfig:
enabled: true
## the name of a Secret containing a `webserver_config.py` key
existingSecret: "airflow"
# Airflow Workers
workers:
## if the airflow workers StatefulSet should be deployed
enabled: false
# Triggerer
triggerer:
## if the airflow triggerer should be deployed
enabled: true
## the number of triggerer Pods to run
replicas: 1
## resource requests/limits for the triggerer Pods
## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
resources: {}
## maximum number of triggers each triggerer will run at once (sets `AIRFLOW__TRIGGERER__DEFAULT_CAPACITY`)
capacity: 1000
# Flower
flower:
## if the airflow flower UI should be deployed
enabled: false
# Airflow Logs
logs:
## the airflow logs folder
path: /opt/airflow/logs
## configs for the logs PVC
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-persistence.md
persistence:
enabled: false
# Airflow DAGs
dags:
## the airflow dags folder
path: /opt/airflow/dags
## configs for the dags PVC
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/load-dag-definitions.md
persistence:
enabled: false
## configs for the git-sync sidecar
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/load-dag-definitions.md
gitSync:
enabled: false
# Kubernetes Ingress
ingress:
## if we should deploy Ingress resources
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/ingress.md
enabled: true
web:
annotations: {}
host: "airflow.seanomik.net"
path: ""
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
#ingressClassName: "nginx"
## flower ingress configs
# flower is currently disabled
flower:
annotations: {}
host: "airflow.seanomik.net"
path: *flowerUrlPrefix
## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
#ingressClassName: "nginx"
# Kubernetes ServiceAccount
serviceAccount:
## if a Kubernetes ServiceAccount is created
create: true
## the name of the ServiceAccount
name: ""
## annotations for the ServiceAccount
annotations: {}
# disable embedded postgres
postgresql:
enabled: false
pgbouncer:
enabled: true
externalDatabase:
type: postgres
host: postgresql.database
port: 5432
## the schema which will contain the airflow tables
database: airflow
## (username - option 1) a plain-text helm value
user: airflow
## (username - option 2) a Kubernetes secret in your airflow namespace
#userSecret: "airflow-cluster1-database-credentials"
#userSecretKey: "username"
## (password - option 1) a plain-text helm value
#password: my_airflow_password
## (password - option 2) a Kubernetes secret in your airflow namespace
passwordSecret: "airflow"
passwordSecretKey: "databasePassword"
## use this for any extra connection-string settings, e.g. ?sslmode=disable
properties: ""
# disable embedded redis
redis:
enabled: false
externalRedis:
## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/external-redis.md
host: redis-master.database
port: 6379
databaseNumber: 2
passwordSecret: airflow
passwordSecretKey: redisPassword

View File

@ -0,0 +1,8 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: apache-airflow-charts
namespace: flux-system
spec:
interval: 1m
url: https://airflow-helm.github.io/charts

View File

@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./helm-repository.yaml
- ./secret.sops.yaml
- ./helm-release.yaml

View File

@ -0,0 +1,75 @@
apiVersion: v1
kind: Secret
metadata:
name: airflow
namespace: dev
stringData:
databasePassword: ENC[AES256_GCM,data:MXuGOUob61OjphobhnNF2RcFfsKSqCJLjzzmGORRqeM=,iv:q8FXp8TiGDnie72gBDNMmfczvbMkd1n+G9G38/JVeOM=,tag:vCNeyps8x4G5pqR4m+jT2A==,type:str]
redisPassword: ENC[AES256_GCM,data:G61bmd2Yk5LQkbnXMgrZYLZJZKzTH5rFFZW0CrdV480=,iv:9ipy9Q95aTt6C2uabXHN2MhMdbCZ0x+hMqFo02GLN2c=,tag:lMkq4JU9n53KXE/rmnfICg==,type:str]
fernetKey: ENC[AES256_GCM,data:1UTaFc8GyiEjQhAlxjcamWBOMDrSkP5NZ3VwD/feVKiZ/CT43nrUbs6qKns=,iv:2LT+BWNDk9/lkbVfX6vRpRgW99DOM9d/ZoW/z2Ylq7A=,tag:qnBM6zXJI/mX/LtPJQspPA==,type:str]
webserverSecretKey: ENC[AES256_GCM,data:+HVGC4/BGbn3MR65jXohFmOUiTlUAtiY7wGLv35LVWs=,iv:XipTb+UG1yUaUHdyQumnUFRO9QNFIhOBUxxeuX0A4kA=,tag:l0oq3C3BReNqI5nia2AhKw==,type:str]
webserver_config.py: ENC[AES256_GCM,data:2bJhtQKL/WW38qaTjMTo88TavieIn27ConlzVJp2fqGmMymfttwGq8fkUYPj8+Bij8vfVeYaH3DOAr6+rACSsFJgNdIFhAtJcYsYVEPFJN1MtbNWmqZL/ZV3AU8ySkXbyrFu1b3+u1ZKkDzQRYjOx/ZtUsiZkGOHqtxysH7pm7z9qRyalpYg3px2Hdza5nAME0+JaMq0qSa9gkMsNLyiG4a6I5gxsaY55CJaJCfEuvj6ijv1JRYbCb6CjWkAZJGfLeJFzC/LlXqG6dUm8i4IChddnvd7Hg+JZMaUCm0yYmgNnntmDi3l+y31LbW7B6hD+5JZ6x6DNL533uZiRq7vNfFNnpbimy17etIK2enpeeXQYMg1ctMJy074NsrQSjd3makb/7kL3WXyRVt+OIDnNUcabmaZpuwgQHJdnRnfKfhDxFmbZ9TCbLTOXmXGjjVddza7Eo8TuoWw334JktEU9ceHHVn5F4guYaFe90I5OFpX4JTpW5WKzelyiivFWWubIgTt/p1S4IYb1dOGyfGtttumvPuAYe6u3HLQAE/TE/LUT2naN7Bof5iEhTmR3QihM98yWq9tPI0QDiA7xZHur7RyK9t06pFldW4JB8fMKfQ2a+Y4gMo6MKEU2Ibnd61mZ6AkYLCcdiZauJ8dNPf3EI2VpZfgTIuVY0qEyZ40zmxnxKu6TNLd/rjrfOHExcwZGR3yf3oWm/GClv26yznb2xtf+I/gkOHKNhxZVT53qDNtiqR0Lp2ou/LITp5W9jbgMtJH+Z/lAk9aWwTyb1yZgzA4HcP7hkH15Gzn4Bj1KPy+MvWPAdBycQKf93bPvU30Zh7+bCaHLG5079BMZCnEnS07U8QanS0lvX7IMrj45I5EmOy9EapUGjHaxUGg+4PFiip3y84T1lZLbQhHK3EFBh6HICXNuwzc+y9jtwBFvCFjh7HwhdUyukB2EpwFNveDtquFseNPjgExhfPjkTIXD5rMDda8qnmyl5Z06gBg/SiiGeGxtD4tC/oUSHyx8f740XevUo4eH0ui9ET6hA8yoAm0dt0FQ7yPZHEDYsPmmuM46BfScIapxyRWwy6eQwkcNJelT6OUlG6/LL5QMuJfcFekkzbAa32Mu6hKTwjgSv3NPJHa6zla8+Jk/+lDUyIiVszg+WUWUHnNy6GIVLs/QBFhHIdZq0Dxn6c+Wmm+C6tEGeosz8GU8AyXVOBpNkhJnVDL6olybQeFWN7ljPEXJF8P+jh8kOQKEzZJvkHb5a9chyKku5aOelei913jJUPTXMNpnv222SgJyN5zjRbm5cx0QSq8vV3uYaYCjEl4SB3ZQ5qVJmI6l2HIUp/yyisdgNVVji0J8zx4DHEgsVs1TC4goRwBP32t4uSwcuovD5qR2cvmxeW8sUDwRZohVoQOHUfNkbnlxWBGCXDOJJWB/osflA/WnYdNV5GgEB+/THqKhkrnPi0D2TfonMA9kw1zqfiUk71hIKzNKsiUTYz18XS5CogYcLRIc5/5/wSGJtIpt5haW8p8pWO1XZVuL5dmxLzcXpLIVTYzbeFcnod/dpQ9HdJHQRetuOwGBYgHK08TafSGaGi6++JeYUpUGQFhmfBdPjP1wFqmyNXbyWY4KbIb641gdy+/o4YCJu78blXEX/xiBTi/7JaFW8IRA0cgluE6Y0J/ktJuNDJjTZbx2ygIjlggPKSbJ+1doCNV818UTyh1qAY6AqMti4jhWLmBtrzauLNg1VxFxNglzqcs+Q2HeBur99ReohOOI4wpC+InE9uER8k9vAll6o3wmUkrIPh/E591ltSESPPdCFkiYOaCvGpu2BCyItI3glGxFbO0V/qPXVPulvikL5UVY11CP8J3NzEKwRibG0GU/xUEhr2dSIBZ0o9yjth++Sx5TfRBIGmtg1Ts5cL24XyqYsCDKWQnUYvUDYDfJgH2grDBxBlA6JTL44GBlnZRcQ0VRkANz1xn0KzfSqgzE1dO5AkEmKfhjxkfppcNJhj6KVR+V7c1XxO+omItkk4FtbsTfKbUwrVRz3jdMiNqzg3Qury1tsrxhqEfdWZs5mgrnPM4oskcn4glWCheFHq0ZLbRExVmTwEx,iv:SmWNrexD8SIE3Km7hXYrRt+t6llcktHXb8cNwEGNUko=,tag:Q/czK0O+eYdIX/oQIDC33Q==,type:str]
sops:
kms: []
gcp_kms: []
azure_kv: []
hc_vault: []
age: []
lastmodified: "2024-07-21T14:04:54Z"
mac: ENC[AES256_GCM,data:2DvS62F/ryDpLa7apr7WrIxMcr/FlGkBhHBrE0xKC+TwZdw5zgWtI3K0l+M7WbnWEUXxrE8VVyEHU+Nx1uaPaajleDA8ayDXlCqnzRr9nBecEHpZF0xY7d7NWChXhCkWpt8i7mhWuJzRL2K44Ql0WYoBIAdy2drzYhSCFj+B/pI=,iv:gooMtGgwGfMbwd4mik+M3WPAHmnEl44CnEVR7R+GwpQ=,tag:99Pg7n6mXOjv6s+UXUKpBg==,type:str]
pgp:
- created_at: "2024-07-21T14:04:54Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMAyqlIeyoxYovARAAhH0qXjQVA7IPWYm+/AqkG333nUOi1T1Am5BCjbO2dIGb
h6HBGZfFm9v7Nnrzp92Vitp0OzubNwdIHsqjdCbJLYcNIyxr7fQnrOYi1AizkFOT
NoVfTZ3NZa25Ff9Tg0jOkRrdMXKeH5f6RkI/pI4KGzRpJNjqU/zfmuARLNxBmVMr
iGFWQvyU7NAiGa5bW46gPw/v3N8Y278RsAmBZfZawz7GlCzlNksub++eTAU3AzMo
mx8ajPGe/4m4hj0s2toOq5h7nofCbzGYan8VXSJY/2aPqgVvyuapEbuR32UuL1YZ
zIOLrq361oWmjrVGft5oWisYtvu2IcS+lKqXgM3hiroyLh9JBQR6iN3ZoBqmh8YR
/i7Ktk+gObqFGyBEJLgu7SHjbGzLK4v61efnhdfGb1RugZrL7Nqqmd24oPGuTWki
ZOF58vVCIZVf//3iB1YG8apaUsk8kBa6LN6bOrTCvSv+P6B8FBN0VXZSbOKVwdGD
F0rk9fW+Us/aQXdsYtpCCGzNR0v/oMFLrvcIgIYwm2tEIGgKllq7j9ixUF4QIc/v
Ces9CdcO9dtMDUS23nblklHQdmhtqhMAw4+KKdo7MYC8RQpAO3kSVc0cNDMcnxl3
NmMZLfNeVkTDQAEOxvwiMM0YIy6dE5kSSJTbS0HMo3l4Dp71IZY3y+Q36K62SNGF
AgwDXjg0p2IN1X8BEADH9JXMrswMMDoyJ7qjN3wrI+X9MuvMihdI58HMygPzMhLF
EomLFZxBPNBfsbHnW5V1l+5gDt4q0UA4v201Xu0DhPq/km8ZCi8LaVhpRbtVn59r
Sz1aAxIh5/m9ttG0LN8I48SAN5VlS4/N23XuNIIXpro3kWwrgn4NyfkmWEWeIOPb
tYIEzCTjwoQq7duSgno/LrlPzbR17kGgnuQS3ktZy3p27y/NruJ9yOBltoQbaKbn
3uC8qtD6TX7i/hulnUggNB7sBTCrs8fG1CuaJHEO9nwAuQ2vTxFmkXRrNDnkEZqG
A8RRKjkKlfJEStalJAOsYV2EwPBlTnXPpJSfT+43YGFJx64GaNv/REmdpU8tme30
8fIdAo1oB2wcmn7DjpUZhZ3s49yfGFd++ia+PSnRpRM3rsXLmSOHLiHPKZOGgfhk
7R3kW/9ZXxOIl3aEdY9kAO1pse7W5iL38qm8+pIrkzqYvjTWmEmSdKe8mGqhyexj
Pkn/Qja51j6/pl11/Hm996S6EJFop+MBc16nQeWhWJ8wOkdHEW3YJR5Z7jaat1rK
ZAVPSEKLLUL8To8nd2XR2JV5cGYlBv3WDWiVkbs4GToySpk3dj0GvcCv7owLz6bh
sROTF/jXecgn/d3hVDccxim/LPgkbR8DwvX+C2ZLk3KR6PfuHBWwvy0ppW5It9Ro
AQkCEOU1PjQgkj9F4zpr5JhVMX5DBh9DD29mSal778jaZEJog1fEqhKfuW+AhqSF
yViuMb1EgNY4a3RkHuDf9OQ3jetl3MY1DVIhXo3TMMTZqNqRjs8lp/iar64iAOS8
OMtzC7tGkiM=
=8KNn
-----END PGP MESSAGE-----
fp: BD1AAF9D8170F4BEE437365FF6F0933799CFEBCD
- created_at: "2024-07-21T14:04:54Z"
enc: |-
-----BEGIN PGP MESSAGE-----
hQIMAy5t8IMoPu4VAQ/+I7WynQpLtIhTVtO/gN8+w+SH3GWAeWP+UgSvmkpe6KIl
tDU6PBM0mwjU1tqb8O7MjpVZEA7Hf+Cc0mlHRyXmGvF5U2qA8wOnzVyufL4ahRfj
X9C1ehmLXguaZNoS0mjeWekS62Ec0H5HfUqCag/wKDk8+yg8vNIEi/sBfeW8bQ/v
JZltq1p2r/RarSKlxzSAf/fVfAIKQQ8pp8U5s1mCUWFU2h/77QsyKroNcMqzzuFz
nR925Z5uKqDiZsjIdz0CFSG6S8E+psf1leFvutBDxstSxB7+/a/l5pdtSoqKGP9S
sa3UMPTHly0UTGmHqT1qCcrZdxGvsvJ+/14xaGhclt1o4S8un2CjWuZLD6rc807K
snHZ5qLtysPsLPF1p2P2I9tTvDxJ+7jbbmVAxU6FtzHRDvPfKwXCYX/QWa6XdEes
u/eAwuAvuCkKR6Dx0c/UZxG4OYgwKuqABkVDh6eltod9eXDl+W3xc9t/1IjWNPT3
znucpqg1ZKGsgKHSp35BQ/fnr5HDgMDAkcnLqMi1wY5+nBDVwHeq7Y9HT87mKTLS
Z8Xc3Pw09LKg3EYtBkfReCb4qZOH9eTD6CZkN0/Q02CkNd25AN6c0MC9Agh/wbcK
L8GmO1f4tWByu+zOvPHJ8YUBNFSBtTzqnRVy/cLmGF56/mw9NvCD7cQc43h+a8/U
aAEJAhDb+4ENov4tJS8cWGQ1q+0q3fT5FJLqhACLCpX3GAXDJh6b4Z6HlZvD6Yn7
CxUOAKVeWzFM1i1NduLLCI15W9H7UtXkQ6lbXfea3QG1fG2pHS4GF2ip4G17GSHD
bUvfh8e5wCC4
=grUU
-----END PGP MESSAGE-----
fp: 687802D4DFD8AA82EA55666CF7DADAC782D7663D
encrypted_regex: ^(data|stringData)$
version: 3.8.1

View File

@ -4,3 +4,4 @@ resources:
- ./namespace.yaml - ./namespace.yaml
- ./woodpecker - ./woodpecker
- ./forgejo-runner - ./forgejo-runner
- ./airflow