apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: apache-airflow
  namespace: dev
spec:
  interval: 5m
  chart:
    spec:
      chart: airflow
      version: "8.9.0"
      sourceRef:
        kind: HelmRepository
        name: apache-airflow-charts
        namespace: flux-system

  values:
    airflow:
      executor: KubernetesExecutor

      ## environment variables for airflow configs
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/airflow-configs.md
      config:
        AIRFLOW__CELERY__FLOWER_URL_PREFIX: &flowerUrlPrefix "/flower"
#        AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
        AIRFLOW__CORE__LOAD_EXAMPLES: "True"

      extraEnv:
      - name: AIRFLOW__CORE__FERNET_KEY
        valueFrom:
          secretKeyRef:
            name: airflow
            key: fernetKey

      - name: AIRFLOW__WEBSERVER__SECRET_KEY
        valueFrom:
          secretKeyRef:
            name: airflow
            key: webserverSecretKey

      ## a list of users to create
      ## not creating any since openid is setup
      users: []

      ## a list airflow connections to create
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-connections.md
      connections: []

      ## a list airflow variables to create
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-variables.md
      variables: []

      ## a list airflow pools to create
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/airflow-pools.md
      pools: []

      ## extra pip packages to install in airflow Pods
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/extra-python-packages.md
      ## [WARNING] this feature is not recommended for production use, see docs
      extraPipPackages:
      - fake_useragent
      - bs4
      - selenium

      ## extra VolumeMounts for the airflow Pods
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
      extraVolumeMounts: []

      ## extra Volumes for the airflow Pods
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
      extraVolumes: []

      ## configs generating the `pod_template.yaml` file for `AIRFLOW__KUBERNETES__POD_TEMPLATE_FILE`
      ## [NOTE] the `dags.gitSync` values will create a git-sync init-container in the pod
      ## [NOTE] the `airflow.extraPipPackages` will NOT be installed
      kubernetesPodTemplate:

        ## the full content of the pod-template file (as a string)
        ## [NOTE] all other `kubernetesPodTemplate.*` are disabled when this is set
        stringOverride: ""

        ## resource requests/limits for the Pod template "base" container
        ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
        resources: {}

        ## extra pip packages to install in the Pod template
        ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/configuration/extra-python-packages.md
        ## [WARNING] this feature is not recommended for production use, see docs
        extraPipPackages:
        - fake_useragent
        - bs4
        - selenium

        ## extra VolumeMounts for the Pod template
        ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
        ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
        extraVolumeMounts: []

        ## extra Volumes for the Pod template
        ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-persistent-volumes.md
        ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/mount-files.md
        extraVolumes: []

    ###################################
    ## COMPONENT | Airflow Scheduler
    ###################################
    scheduler:
      ## the number of scheduler Pods to run
      replicas: 1

      ## resource requests/limits for the scheduler Pods
      ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
      resources: {}

      ## configs for the log-cleanup sidecar of the scheduler
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-cleanup.md
      logCleanup:
        enabled: true
        retentionMinutes: 21600

      ## configs for the scheduler Pods' liveness probe
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
      livenessProbe:
        enabled: true

        ## configs for an additional check that ensures tasks are being created by the scheduler
        ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/scheduler-liveness-probe.md
        taskCreationCheck:
          enabled: false
          thresholdSeconds: 300
          schedulerAgeBeforeCheck: 180

    ###################################
    ## COMPONENT | Airflow Webserver
    ###################################
    web:
      ## the number of web Pods to run
      replicas: 1

      ## resource requests/limits for the web Pods
      ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
      resources: {}

      ## configs for the Service of the web Pods
      service:
        type: ClusterIP
        externalPort: 8080

      webserverConfig:
        enabled: true

        ## the name of a Secret containing a `webserver_config.py` key
        existingSecret: "airflow"

    # Airflow Workers
    workers:
      ## if the airflow workers StatefulSet should be deployed
      enabled: false

    # Triggerer
    triggerer:
      ## if the airflow triggerer should be deployed
      enabled: true

      ## the number of triggerer Pods to run
      replicas: 1

      ## resource requests/limits for the triggerer Pods
      ## [SPEC] https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#resourcerequirements-v1-core
      resources: {}

      ## maximum number of triggers each triggerer will run at once (sets `AIRFLOW__TRIGGERER__DEFAULT_CAPACITY`)
      capacity: 1000

    # Flower
    flower:
      ## if the airflow flower UI should be deployed
      enabled: false

    # Airflow Logs
    logs:
      ## the airflow logs folder
      path: /opt/airflow/logs

      ## configs for the logs PVC
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/monitoring/log-persistence.md
      persistence:
        enabled: false

    # Airflow DAGs
    dags:
      ## the airflow dags folder
      path: /opt/airflow/dags

      ## configs for the dags PVC
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/load-dag-definitions.md
      persistence:
        enabled: false

      ## configs for the git-sync sidecar
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/dags/load-dag-definitions.md
      gitSync:
        enabled: true

        repo: "https://git.${SECRET_NEW_DOMAIN}/SeanOMik/airflow-dags.git"
        branch: main
        revision: HEAD

        repoSubPath: "dags/"

        syncWait: 60

        httpSecret: airflow
        httpSecretUsernameKey: gitSyncUsername
        httpSecretPasswordKey: gitSyncToken

    # Kubernetes Ingress
    ingress:
      ## if we should deploy Ingress resources
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/kubernetes/ingress.md
      enabled: true

      web:
        annotations: {}
        host: "airflow.${SECRET_NEW_DOMAIN}"
        path: ""
        ## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
        #ingressClassName: "nginx"
        
      ## flower ingress configs
      # flower is currently disabled
      flower:
        annotations: {}
        host: "airflow.${SECRET_NEW_DOMAIN}"
        path: *flowerUrlPrefix
        ## WARNING: requires Kubernetes 1.18 or later, use "kubernetes.io/ingress.class" annotation for older versions
        #ingressClassName: "nginx"

    # Kubernetes ServiceAccount
    serviceAccount:
      ## if a Kubernetes ServiceAccount is created
      create: true

      ## the name of the ServiceAccount
      name: ""

      ## annotations for the ServiceAccount
      annotations: {}

    # disable embedded postgres
    postgresql:
      enabled: false

    pgbouncer:
      enabled: true

    externalDatabase:
      type: postgres
      
      host: postgres16-rw.database.svc
      port: 5432
      
      ## the schema which will contain the airflow tables
      database: airflow

      ## (username - option 1) a plain-text helm value
      user: airflow
      
      ## (username - option 2) a Kubernetes secret in your airflow namespace
      #userSecret: "airflow-cluster1-database-credentials"
      #userSecretKey: "username"

      ## (password - option 1) a plain-text helm value
      #password: my_airflow_password

      ## (password - option 2) a Kubernetes secret in your airflow namespace
      passwordSecret: "airflow"
      passwordSecretKey: "databasePassword"

      ## use this for any extra connection-string settings, e.g. ?sslmode=disable
      properties: ""

    # disable embedded redis
    redis:
      enabled: false

    externalRedis:
      ## [FAQ] https://github.com/airflow-helm/charts/blob/main/charts/airflow/docs/faq/database/external-redis.md
      host: redis-master.database
      port: 6379

      databaseNumber: 2

      passwordSecret: airflow
      passwordSecretKey: redisPassword