From 1de87716dc64cb6a304afe8570dd6f57c0edac39 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:25:51 +0200 Subject: [PATCH 01/27] feat(health): add livez and readyz endpoints with tests --- src/ahc/apps/homepage/tests/test_health.py | 31 ++++++++++++++++++++++ src/ahc/urls.py | 4 +++ src/ahc/views.py | 20 ++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 src/ahc/apps/homepage/tests/test_health.py create mode 100644 src/ahc/views.py diff --git a/src/ahc/apps/homepage/tests/test_health.py b/src/ahc/apps/homepage/tests/test_health.py new file mode 100644 index 0000000..276e4c5 --- /dev/null +++ b/src/ahc/apps/homepage/tests/test_health.py @@ -0,0 +1,31 @@ +from unittest.mock import patch + +import pytest +from django.db import DatabaseError + + +@pytest.mark.unit +def test_livez_returns_ok_without_touching_the_database(client): + response = client.get("/livez") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +@pytest.mark.unit +@pytest.mark.django_db +def test_readyz_returns_ok_when_database_is_reachable(client): + response = client.get("/readyz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +@pytest.mark.unit +def test_readyz_returns_503_when_database_is_down(client): + with patch("ahc.views.connection") as mock_connection: + mock_connection.cursor.side_effect = DatabaseError("connection refused") + response = client.get("/readyz") + + assert response.status_code == 503 + assert response.json() == {"status": "unavailable"} diff --git a/src/ahc/urls.py b/src/ahc/urls.py index b95d4e1..cb782b2 100644 --- a/src/ahc/urls.py +++ b/src/ahc/urls.py @@ -21,8 +21,12 @@ from django.views.generic.base import RedirectView from django.views.static import serve +from ahc.views import livez, readyz + urlpatterns = [ path("admin/", admin.site.urls), + path("livez", livez, name="livez"), + path("readyz", readyz, name="readyz"), path("", include("ahc.apps.homepage.urls")), path("user/", include("ahc.apps.users.urls")), path("pet/", include("ahc.apps.animals.urls")), diff --git a/src/ahc/views.py b/src/ahc/views.py new file mode 100644 index 0000000..1878041 --- /dev/null +++ b/src/ahc/views.py @@ -0,0 +1,20 @@ +"""Project-level operational views (health probes).""" + +from django.db import DatabaseError, connection +from django.http import HttpRequest, JsonResponse + + +def livez(request: HttpRequest) -> JsonResponse: + """Liveness probe: the process is up. No external dependencies.""" + return JsonResponse({"status": "ok"}) + + +def readyz(request: HttpRequest) -> JsonResponse: + """Readiness probe: the app can serve traffic (database reachable).""" + try: + with connection.cursor() as cursor: + cursor.execute("SELECT 1") + cursor.fetchone() + except DatabaseError: + return JsonResponse({"status": "unavailable"}, status=503) + return JsonResponse({"status": "ok"}) From 9f3955083b5e3b874693406b2dadf7742603b891 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:26:14 +0200 Subject: [PATCH 02/27] feat(docker): add gunicorn CMD and EXPOSE to app image --- docker/Dockerfile-app | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile-app b/docker/Dockerfile-app index feb5414..5e2b854 100644 --- a/docker/Dockerfile-app +++ b/docker/Dockerfile-app @@ -21,3 +21,6 @@ ENV PATH="/opt/venv/bin:$PATH" \ COPY --chown=appuser:appuser . . USER appuser + +EXPOSE 8000 +CMD ["gunicorn", "ahc.wsgi:application", "--bind", "0.0.0.0:8000", "--workers", "4"] From 4f1290912d42c34c9fd6c0749f83c2760bb15822 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:26:51 +0200 Subject: [PATCH 03/27] refactor(k8s): restructure into base and overlays with sync-waves --- .gitignore | 4 + kubernetes/backend/web/deployment.yaml | 37 ----- kubernetes/backend/web/kustomization.yaml | 4 - kubernetes/backend/web/secret.yaml.template | 30 ---- kubernetes/base/celery-beat/deployment.yaml | 52 +++++++ kubernetes/base/celery/deployment.yaml | 69 +++++++++ kubernetes/base/config/app-configmap.yaml | 25 ++++ kubernetes/base/couchdb/init-job.yaml | 54 +++++++ kubernetes/base/couchdb/service.yaml | 16 ++ kubernetes/base/couchdb/statefulset.yaml | 58 ++++++++ kubernetes/base/flower/deployment.yaml | 43 ++++++ .../{queue => base}/flower/service.yaml | 8 +- kubernetes/{ => base}/ingress.yaml | 3 +- kubernetes/base/jobs/migrate-job.yaml | 65 ++++++++ kubernetes/base/kustomization.yaml | 29 ++++ kubernetes/base/namespace.yaml | 4 + kubernetes/base/postgres/service.yaml | 14 ++ kubernetes/base/postgres/statefulset.yaml | 59 ++++++++ kubernetes/base/redis/deployment.yaml | 40 +++++ kubernetes/{queue => base}/redis/service.yaml | 10 +- .../storage/app-media-pvc.yaml} | 9 +- kubernetes/base/storage/app-private-pvc.yaml | 12 ++ kubernetes/base/web/deployment.yaml | 139 ++++++++++++++++++ kubernetes/{backend => base}/web/service.yaml | 8 +- kubernetes/db/couchdb/configmap.yaml | 6 - kubernetes/db/couchdb/deployment.yaml | 53 ------- kubernetes/db/couchdb/kustomization.yaml | 7 - kubernetes/db/couchdb/pv.yaml | 17 --- kubernetes/db/couchdb/pvc.yaml | 14 -- kubernetes/db/couchdb/secret.yaml.template | 9 -- kubernetes/db/couchdb/service.yaml | 12 -- kubernetes/db/couchdb/storage-class.yaml | 6 - kubernetes/db/postgres/deployment.yaml | 39 ----- kubernetes/db/postgres/kustomization.yaml | 7 - kubernetes/db/postgres/pv.yaml | 17 --- kubernetes/db/postgres/service.yaml | 12 -- kubernetes/db/postgres/storage-class.yaml | 6 - kubernetes/kustomization.yaml | 9 -- .../overlays/home/backup/backup-pvc.yaml | 12 ++ .../home/backup/couchdb-export-cronjob.yaml | 49 ++++++ .../home/backup/files-backup-cronjob.yaml | 55 +++++++ .../home/backup/pg-backup-cronjob.yaml | 45 ++++++ .../overlays/home/celery-beat-patch.yaml | 9 ++ kubernetes/overlays/home/configmap-patch.yaml | 9 ++ kubernetes/overlays/home/ingress-patch.yaml | 19 +++ kubernetes/overlays/home/kustomization.yaml | 31 ++++ kubernetes/overlays/home/smoke-test-job.yaml | 30 ++++ .../minikube-argocd/ingress-patch.yaml | 6 + .../minikube-argocd/kustomization.yaml | 24 +++ .../minikube-local/ingress-patch.yaml | 6 + .../minikube-local/kustomization.yaml | 21 +++ .../secrets/app-secret.yaml.template | 12 ++ .../secrets/couchdb-secret.yaml.template | 10 ++ .../secrets/flower-secret.yaml.template | 9 ++ .../secrets/postgres-secret.yaml.template} | 5 +- kubernetes/queue/celery-beat/deployment.yaml | 42 ------ .../queue/celery-beat/kustomization.yaml | 2 - kubernetes/queue/celery/configmap.yaml | 6 - kubernetes/queue/celery/deployment.yaml | 45 ------ kubernetes/queue/celery/kustomization.yaml | 3 - kubernetes/queue/celery/secret.yaml.template | 30 ---- kubernetes/queue/celery/service.yaml | 12 -- kubernetes/queue/flower/deployment.yaml | 32 ---- kubernetes/queue/flower/kustomization.yaml | 3 - kubernetes/queue/kustomization.yaml | 6 - kubernetes/queue/redis/deployment.yaml | 32 ---- kubernetes/queue/redis/kustomization.yaml | 3 - kubernetes/queue/secret.yaml.template | 8 - 68 files changed, 1054 insertions(+), 528 deletions(-) delete mode 100644 kubernetes/backend/web/deployment.yaml delete mode 100644 kubernetes/backend/web/kustomization.yaml delete mode 100644 kubernetes/backend/web/secret.yaml.template create mode 100644 kubernetes/base/celery-beat/deployment.yaml create mode 100644 kubernetes/base/celery/deployment.yaml create mode 100644 kubernetes/base/config/app-configmap.yaml create mode 100644 kubernetes/base/couchdb/init-job.yaml create mode 100644 kubernetes/base/couchdb/service.yaml create mode 100644 kubernetes/base/couchdb/statefulset.yaml create mode 100644 kubernetes/base/flower/deployment.yaml rename kubernetes/{queue => base}/flower/service.yaml (69%) rename kubernetes/{ => base}/ingress.yaml (74%) create mode 100644 kubernetes/base/jobs/migrate-job.yaml create mode 100644 kubernetes/base/kustomization.yaml create mode 100644 kubernetes/base/namespace.yaml create mode 100644 kubernetes/base/postgres/service.yaml create mode 100644 kubernetes/base/postgres/statefulset.yaml create mode 100644 kubernetes/base/redis/deployment.yaml rename kubernetes/{queue => base}/redis/service.yaml (53%) rename kubernetes/{db/postgres/pvc.yaml => base/storage/app-media-pvc.yaml} (51%) create mode 100644 kubernetes/base/storage/app-private-pvc.yaml create mode 100644 kubernetes/base/web/deployment.yaml rename kubernetes/{backend => base}/web/service.yaml (69%) delete mode 100644 kubernetes/db/couchdb/configmap.yaml delete mode 100644 kubernetes/db/couchdb/deployment.yaml delete mode 100644 kubernetes/db/couchdb/kustomization.yaml delete mode 100644 kubernetes/db/couchdb/pv.yaml delete mode 100644 kubernetes/db/couchdb/pvc.yaml delete mode 100644 kubernetes/db/couchdb/secret.yaml.template delete mode 100644 kubernetes/db/couchdb/service.yaml delete mode 100644 kubernetes/db/couchdb/storage-class.yaml delete mode 100644 kubernetes/db/postgres/deployment.yaml delete mode 100644 kubernetes/db/postgres/kustomization.yaml delete mode 100644 kubernetes/db/postgres/pv.yaml delete mode 100644 kubernetes/db/postgres/service.yaml delete mode 100644 kubernetes/db/postgres/storage-class.yaml delete mode 100644 kubernetes/kustomization.yaml create mode 100644 kubernetes/overlays/home/backup/backup-pvc.yaml create mode 100644 kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml create mode 100644 kubernetes/overlays/home/backup/files-backup-cronjob.yaml create mode 100644 kubernetes/overlays/home/backup/pg-backup-cronjob.yaml create mode 100644 kubernetes/overlays/home/celery-beat-patch.yaml create mode 100644 kubernetes/overlays/home/configmap-patch.yaml create mode 100644 kubernetes/overlays/home/ingress-patch.yaml create mode 100644 kubernetes/overlays/home/kustomization.yaml create mode 100644 kubernetes/overlays/home/smoke-test-job.yaml create mode 100644 kubernetes/overlays/minikube-argocd/ingress-patch.yaml create mode 100644 kubernetes/overlays/minikube-argocd/kustomization.yaml create mode 100644 kubernetes/overlays/minikube-local/ingress-patch.yaml create mode 100644 kubernetes/overlays/minikube-local/kustomization.yaml create mode 100644 kubernetes/overlays/minikube-local/secrets/app-secret.yaml.template create mode 100644 kubernetes/overlays/minikube-local/secrets/couchdb-secret.yaml.template create mode 100644 kubernetes/overlays/minikube-local/secrets/flower-secret.yaml.template rename kubernetes/{db/postgres/secret.yaml.template => overlays/minikube-local/secrets/postgres-secret.yaml.template} (54%) delete mode 100644 kubernetes/queue/celery-beat/deployment.yaml delete mode 100644 kubernetes/queue/celery-beat/kustomization.yaml delete mode 100644 kubernetes/queue/celery/configmap.yaml delete mode 100644 kubernetes/queue/celery/deployment.yaml delete mode 100644 kubernetes/queue/celery/kustomization.yaml delete mode 100644 kubernetes/queue/celery/secret.yaml.template delete mode 100644 kubernetes/queue/celery/service.yaml delete mode 100644 kubernetes/queue/flower/deployment.yaml delete mode 100644 kubernetes/queue/flower/kustomization.yaml delete mode 100644 kubernetes/queue/kustomization.yaml delete mode 100644 kubernetes/queue/redis/deployment.yaml delete mode 100644 kubernetes/queue/redis/kustomization.yaml delete mode 100644 kubernetes/queue/secret.yaml.template diff --git a/.gitignore b/.gitignore index 044c3dd..9e3a5d2 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,10 @@ CLAUDE.md /tars/ /kubernetes/**/*.tar secret.yaml +# Filled plain Secrets (templates stay tracked; SealedSecrets in sealed/ stay tracked) +kubernetes/overlays/*/secrets/*.yaml +# Fetched sealed-secrets controller certificates +*.pem # Volumes /db/ diff --git a/kubernetes/backend/web/deployment.yaml b/kubernetes/backend/web/deployment.yaml deleted file mode 100644 index 7eef713..0000000 --- a/kubernetes/backend/web/deployment.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ahc-app-backend - labels: - app: ahc_app_backend -spec: - replicas: 1 - selector: - matchLabels: - app: ahc_app_backend - template: - metadata: - name: ahc-app-backend - labels: - app: ahc_app_backend - spec: - containers: - - name: ahc-app-backend - image: ahc-app:latest - imagePullPolicy: Never - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: PYTHONPATH - value: "/app" - envFrom: - - secretRef: - name: web-secrets - ports: - - containerPort: 8000 - protocol: TCP - resources: - limits: - memory: "256Mi" - cpu: "500m" - restartPolicy: Always diff --git a/kubernetes/backend/web/kustomization.yaml b/kubernetes/backend/web/kustomization.yaml deleted file mode 100644 index 46986f6..0000000 --- a/kubernetes/backend/web/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: - - deployment.yaml - - secret.yaml - - service.yaml diff --git a/kubernetes/backend/web/secret.yaml.template b/kubernetes/backend/web/secret.yaml.template deleted file mode 100644 index 9b64c58..0000000 --- a/kubernetes/backend/web/secret.yaml.template +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: web-secrets - -stringData: - DB_USER: "..." - DB_PASSWORD: "..." - - EMAIL_HOST_USER: "..." - EMAIL_HOST_PASSWORD: "..." - - DISCORD_TOKEN: "..." - - DB_NAME: "..." - DB_HOST: "..." - DB_PORT: "5432" - - COUCHDB_USER: "..." - COUCHDB_PASSWORD: "..." - COUCHDB_PORT: "5982" - - COUCH_CONNECTOR: "Server(\"http://127.0.0.1:5982\")" - CELERY_BACKEND: "redis://redis:6379/0" - DJANGO_SETTINGS_MODULE: "ahc.settings" - - EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" - EMAIL_HOST: "sandbox.smtp.mailtrap.io" - EMAIL_PORT: "2525" - EMAIL_USE_TLS: "True" diff --git a/kubernetes/base/celery-beat/deployment.yaml b/kubernetes/base/celery-beat/deployment.yaml new file mode 100644 index 0000000..69917e8 --- /dev/null +++ b/kubernetes/base/celery-beat/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-beat + labels: + app: celery-beat +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: celery-beat + template: + metadata: + labels: + app: celery-beat + spec: + containers: + - name: celery-beat + image: ghcr.io/cybernetic-ransomware/ahc-app:prod + command: ["celery", "-A", "celery_notifications.config:celery_obj", "beat", "-l", "info"] + envFrom: + - configMapRef: + name: ahc-app-config + - secretRef: + name: ahc-app-secrets + - secretRef: + name: couchdb-credentials + env: + - name: DB_NAME + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + - name: DB_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + resources: + requests: + memory: "64Mi" + cpu: "25m" + limits: + memory: "128Mi" + cpu: "100m" diff --git a/kubernetes/base/celery/deployment.yaml b/kubernetes/base/celery/deployment.yaml new file mode 100644 index 0000000..a06992f --- /dev/null +++ b/kubernetes/base/celery/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-worker + labels: + app: celery-worker +spec: + replicas: 1 + selector: + matchLabels: + app: celery-worker + template: + metadata: + labels: + app: celery-worker + spec: + containers: + - name: celery-worker + image: ghcr.io/cybernetic-ransomware/ahc-app:prod + # Explicit concurrency: the default equals the node CPU count, + # which forks more Django processes than the memory limit can hold. + command: ["celery", "-A", "celery_notifications.config:celery_obj", "worker", "-l", "info", "--concurrency=2"] + envFrom: + - configMapRef: + name: ahc-app-config + - secretRef: + name: ahc-app-secrets + - secretRef: + name: couchdb-credentials + env: + - name: DB_NAME + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + - name: DB_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + volumeMounts: + - name: app-private + mountPath: /app/private + livenessProbe: + exec: + command: + - sh + - -c + - celery -A celery_notifications.config:celery_obj inspect ping -d "celery@$HOSTNAME" --timeout 10 + initialDelaySeconds: 30 + periodSeconds: 120 + timeoutSeconds: 20 + failureThreshold: 3 + resources: + requests: + memory: "128Mi" + cpu: "50m" + limits: + memory: "384Mi" + cpu: "300m" + volumes: + - name: app-private + persistentVolumeClaim: + claimName: app-private-pvc diff --git a/kubernetes/base/config/app-configmap.yaml b/kubernetes/base/config/app-configmap.yaml new file mode 100644 index 0000000..0baf4bf --- /dev/null +++ b/kubernetes/base/config/app-configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ahc-app-config + annotations: + argocd.argoproj.io/sync-wave: "-3" +data: + DEBUG: "False" + ALLOWED_HOSTS: "localhost,127.0.0.1" + CSRF_TRUSTED_ORIGINS: "" + DB_HOST: "postgres-service" + DB_PORT: "5432" + COUCHDB_HOST: "appendixes-db" + COUCHDB_PORT: "5982" + COUCHDB_DB_NAME: "appendixes" + CELERY_BROKER_URL: "redis://redis:6379/0" + CELERY_BACKEND: "redis://redis:6379/0" + EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" + EMAIL_HOST: "sandbox.smtp.mailtrap.io" + EMAIL_PORT: "2525" + EMAIL_USE_TLS: "True" + DJANGO_SETTINGS_MODULE: "ahc.settings" + PRIVATE_STORAGE_ROOT: "/app/private" + FLOWER_PORT: "5555" + PYTHONUNBUFFERED: "1" diff --git a/kubernetes/base/couchdb/init-job.yaml b/kubernetes/base/couchdb/init-job.yaml new file mode 100644 index 0000000..83ad7ca --- /dev/null +++ b/kubernetes/base/couchdb/init-job.yaml @@ -0,0 +1,54 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: couchdb-init + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded +spec: + backoffLimit: 4 + activeDeadlineSeconds: 300 + template: + spec: + restartPolicy: Never + initContainers: + - name: wait-for-couchdb + image: busybox:1.37 + command: + - sh + - -c + - until nc -z appendixes-db 5982; do echo "waiting for couchdb"; sleep 2; done + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "32Mi" + cpu: "50m" + containers: + - name: init + image: curlimages/curl:8.10.1 + envFrom: + - secretRef: + name: couchdb-credentials + command: + - sh + - -c + - | + set -u + for db in _users _replicator appendixes; do + code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT \ + "http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@appendixes-db:5982/${db}") + case "$code" in + 201|202|412) echo "$db: $code (ok)" ;; + *) echo "$db: $code (fail)"; exit 1 ;; + esac + done + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "64Mi" + cpu: "100m" diff --git a/kubernetes/base/couchdb/service.yaml b/kubernetes/base/couchdb/service.yaml new file mode 100644 index 0000000..5c6a771 --- /dev/null +++ b/kubernetes/base/couchdb/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: appendixes-db + annotations: + argocd.argoproj.io/sync-wave: "-2" +spec: + type: ClusterIP + selector: + app: couchdb + ports: + # The app expects COUCHDB_PORT=5982 (legacy custom port); the official + # couchdb image listens on 5984, so the Service translates 5982 -> 5984. + - port: 5982 + targetPort: 5984 + protocol: TCP diff --git a/kubernetes/base/couchdb/statefulset.yaml b/kubernetes/base/couchdb/statefulset.yaml new file mode 100644 index 0000000..f761bea --- /dev/null +++ b/kubernetes/base/couchdb/statefulset.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: couchdb + labels: + app: couchdb + annotations: + argocd.argoproj.io/sync-wave: "-2" +spec: + serviceName: appendixes-db + replicas: 1 + selector: + matchLabels: + app: couchdb + template: + metadata: + labels: + app: couchdb + spec: + containers: + - name: couchdb + image: couchdb:3.5.2 + ports: + - containerPort: 5984 + protocol: TCP + envFrom: + - secretRef: + name: couchdb-credentials + volumeMounts: + - name: couchdb-data + mountPath: /opt/couchdb/data + readinessProbe: + httpGet: + path: /_up + port: 5984 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /_up + port: 5984 + initialDelaySeconds: 30 + periodSeconds: 15 + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumeClaimTemplates: + - metadata: + name: couchdb-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/kubernetes/base/flower/deployment.yaml b/kubernetes/base/flower/deployment.yaml new file mode 100644 index 0000000..ad0c393 --- /dev/null +++ b/kubernetes/base/flower/deployment.yaml @@ -0,0 +1,43 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-flower + labels: + app: flower +spec: + replicas: 1 + selector: + matchLabels: + app: flower + template: + metadata: + labels: + app: flower + spec: + containers: + - name: celery-flower + image: mher/flower:2.0.1 + envFrom: + - configMapRef: + name: ahc-app-config + - secretRef: + name: flower-secrets + ports: + - containerPort: 5555 + protocol: TCP + readinessProbe: + tcpSocket: + port: 5555 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 5555 + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: + memory: "64Mi" + cpu: "25m" + limits: + memory: "128Mi" + cpu: "200m" diff --git a/kubernetes/queue/flower/service.yaml b/kubernetes/base/flower/service.yaml similarity index 69% rename from kubernetes/queue/flower/service.yaml rename to kubernetes/base/flower/service.yaml index 94489bf..62e5e2a 100644 --- a/kubernetes/queue/flower/service.yaml +++ b/kubernetes/base/flower/service.yaml @@ -3,10 +3,10 @@ kind: Service metadata: name: celery-flower-service spec: + type: ClusterIP selector: - app: celery_flower + app: flower ports: - - protocol: TCP - port: 5555 + - port: 5555 targetPort: 5555 - type: ClusterIP + protocol: TCP diff --git a/kubernetes/ingress.yaml b/kubernetes/base/ingress.yaml similarity index 74% rename from kubernetes/ingress.yaml rename to kubernetes/base/ingress.yaml index acb594b..ed99162 100644 --- a/kubernetes/ingress.yaml +++ b/kubernetes/base/ingress.yaml @@ -3,8 +3,9 @@ kind: Ingress metadata: name: ingress-service annotations: - ingressClassName: nginx + argocd.argoproj.io/sync-wave: "1" spec: + # ingressClassName is set per overlay (nginx on minikube, traefik on k3s). rules: - http: paths: diff --git a/kubernetes/base/jobs/migrate-job.yaml b/kubernetes/base/jobs/migrate-job.yaml new file mode 100644 index 0000000..428414a --- /dev/null +++ b/kubernetes/base/jobs/migrate-job.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ahc-migrate + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded +spec: + backoffLimit: 2 + activeDeadlineSeconds: 300 + template: + spec: + restartPolicy: Never + # Outside ArgoCD (plain kubectl apply -k) there is no wave ordering, + # so the job has to wait for the database itself. The headless + # postgres-service has no DNS records until the pod is ready. + initContainers: + - name: wait-for-postgres + image: busybox:1.37 + command: + - sh + - -c + - until nc -z postgres-service 5432; do echo "waiting for postgres"; sleep 2; done + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "32Mi" + cpu: "50m" + containers: + - name: migrate + image: ghcr.io/cybernetic-ransomware/ahc-app:prod + command: ["python", "manage.py", "migrate", "--noinput"] + envFrom: + - configMapRef: + name: ahc-app-config + - secretRef: + name: ahc-app-secrets + - secretRef: + name: couchdb-credentials + env: + - name: DB_NAME + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + - name: DB_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" diff --git a/kubernetes/base/kustomization.yaml b/kubernetes/base/kustomization.yaml new file mode 100644 index 0000000..a6071b1 --- /dev/null +++ b/kubernetes/base/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: ahc + +resources: + - namespace.yaml + - config/app-configmap.yaml + - storage/app-media-pvc.yaml + - storage/app-private-pvc.yaml + - web/deployment.yaml + - web/service.yaml + - celery/deployment.yaml + - celery-beat/deployment.yaml + - flower/deployment.yaml + - flower/service.yaml + - redis/deployment.yaml + - redis/service.yaml + - postgres/statefulset.yaml + - postgres/service.yaml + - couchdb/statefulset.yaml + - couchdb/service.yaml + - couchdb/init-job.yaml + - jobs/migrate-job.yaml + - ingress.yaml + +images: + - name: ghcr.io/cybernetic-ransomware/ahc-app + newTag: prod diff --git a/kubernetes/base/namespace.yaml b/kubernetes/base/namespace.yaml new file mode 100644 index 0000000..408b1b5 --- /dev/null +++ b/kubernetes/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ahc diff --git a/kubernetes/base/postgres/service.yaml b/kubernetes/base/postgres/service.yaml new file mode 100644 index 0000000..f9a27b9 --- /dev/null +++ b/kubernetes/base/postgres/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-service + annotations: + argocd.argoproj.io/sync-wave: "-2" +spec: + clusterIP: None + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/kubernetes/base/postgres/statefulset.yaml b/kubernetes/base/postgres/statefulset.yaml new file mode 100644 index 0000000..4d43a76 --- /dev/null +++ b/kubernetes/base/postgres/statefulset.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + labels: + app: postgres + annotations: + argocd.argoproj.io/sync-wave: "-2" +spec: + serviceName: postgres-service + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:18-alpine + ports: + - containerPort: 5432 + protocol: TCP + envFrom: + - secretRef: + name: postgres-credentials + volumeMounts: + # postgres:18 images keep PGDATA in a versioned subdirectory + # (e.g. /var/lib/postgresql/18/docker), so the volume must cover + # /var/lib/postgresql, not the pre-18 /var/lib/postgresql/data. + - name: pgdata + mountPath: /var/lib/postgresql + readinessProbe: + exec: + command: ["sh", "-c", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""] + periodSeconds: 10 + livenessProbe: + exec: + command: ["sh", "-c", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""] + initialDelaySeconds: 30 + periodSeconds: 15 + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumeClaimTemplates: + - metadata: + name: pgdata + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/kubernetes/base/redis/deployment.yaml b/kubernetes/base/redis/deployment.yaml new file mode 100644 index 0000000..38dca1e --- /dev/null +++ b/kubernetes/base/redis/deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + labels: + app: redis + annotations: + argocd.argoproj.io/sync-wave: "-2" +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7-alpine + ports: + - containerPort: 6379 + protocol: TCP + readinessProbe: + exec: + command: ["redis-cli", "ping"] + periodSeconds: 10 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + requests: + memory: "32Mi" + cpu: "25m" + limits: + memory: "128Mi" + cpu: "200m" diff --git a/kubernetes/queue/redis/service.yaml b/kubernetes/base/redis/service.yaml similarity index 53% rename from kubernetes/queue/redis/service.yaml rename to kubernetes/base/redis/service.yaml index 9a2b430..19f6552 100644 --- a/kubernetes/queue/redis/service.yaml +++ b/kubernetes/base/redis/service.yaml @@ -1,12 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: redis-service + name: redis + annotations: + argocd.argoproj.io/sync-wave: "-2" spec: + type: ClusterIP selector: app: redis ports: - - protocol: TCP - port: 6379 + - port: 6379 targetPort: 6379 - type: ClusterIP + protocol: TCP diff --git a/kubernetes/db/postgres/pvc.yaml b/kubernetes/base/storage/app-media-pvc.yaml similarity index 51% rename from kubernetes/db/postgres/pvc.yaml rename to kubernetes/base/storage/app-media-pvc.yaml index 638704b..57dec1d 100644 --- a/kubernetes/db/postgres/pvc.yaml +++ b/kubernetes/base/storage/app-media-pvc.yaml @@ -1,13 +1,12 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: postgres-volume-claim - labels: - app: postgres + name: app-media-pvc + annotations: + argocd.argoproj.io/sync-wave: "-3" spec: - storageClassName: local-postgres accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 2Gi diff --git a/kubernetes/base/storage/app-private-pvc.yaml b/kubernetes/base/storage/app-private-pvc.yaml new file mode 100644 index 0000000..68b0709 --- /dev/null +++ b/kubernetes/base/storage/app-private-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: app-private-pvc + annotations: + argocd.argoproj.io/sync-wave: "-3" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi diff --git a/kubernetes/base/web/deployment.yaml b/kubernetes/base/web/deployment.yaml new file mode 100644 index 0000000..1e46635 --- /dev/null +++ b/kubernetes/base/web/deployment.yaml @@ -0,0 +1,139 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ahc-app-backend + labels: + app: ahc-web +spec: + replicas: 1 + selector: + matchLabels: + app: ahc-web + template: + metadata: + labels: + app: ahc-web + spec: + initContainers: + - name: collectstatic + image: ghcr.io/cybernetic-ransomware/ahc-app:prod + command: ["python", "manage.py", "collectstatic", "--noinput"] + envFrom: + - configMapRef: + name: ahc-app-config + - secretRef: + name: ahc-app-secrets + - secretRef: + name: couchdb-credentials + env: + - name: DB_NAME + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + - name: DB_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + volumeMounts: + - name: static-collected + mountPath: /app/static_collected + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + containers: + - name: ahc-app-backend + image: ghcr.io/cybernetic-ransomware/ahc-app:prod + envFrom: + - configMapRef: + name: ahc-app-config + - secretRef: + name: ahc-app-secrets + - secretRef: + name: couchdb-credentials + env: + - name: DB_NAME + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_DB + - name: DB_USER + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-credentials + key: POSTGRES_PASSWORD + ports: + - containerPort: 8000 + protocol: TCP + volumeMounts: + - name: static-collected + mountPath: /app/static_collected + - name: app-media + mountPath: /app/static/media/animal_pic + subPath: animal_pic + - name: app-media + mountPath: /app/static/media/attachments + subPath: attachments + - name: app-media + mountPath: /app/static/media/profile_pics + subPath: profile_pics + - name: app-private + mountPath: /app/private + startupProbe: + httpGet: + path: /livez + port: 8000 + httpHeaders: + - name: Host + value: localhost + periodSeconds: 5 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /livez + port: 8000 + httpHeaders: + - name: Host + value: localhost + periodSeconds: 20 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /readyz + port: 8000 + httpHeaders: + - name: Host + value: localhost + periodSeconds: 10 + timeoutSeconds: 5 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "500m" + volumes: + - name: static-collected + emptyDir: {} + - name: app-media + persistentVolumeClaim: + claimName: app-media-pvc + - name: app-private + persistentVolumeClaim: + claimName: app-private-pvc diff --git a/kubernetes/backend/web/service.yaml b/kubernetes/base/web/service.yaml similarity index 69% rename from kubernetes/backend/web/service.yaml rename to kubernetes/base/web/service.yaml index 5f1551c..32d5094 100644 --- a/kubernetes/backend/web/service.yaml +++ b/kubernetes/base/web/service.yaml @@ -3,10 +3,10 @@ kind: Service metadata: name: ahc-app-backend-service spec: + type: ClusterIP selector: - app: ahc_app_backend + app: ahc-web ports: - - protocol: TCP - port: 8000 + - port: 8000 targetPort: 8000 - type: ClusterIP + protocol: TCP diff --git a/kubernetes/db/couchdb/configmap.yaml b/kubernetes/db/couchdb/configmap.yaml deleted file mode 100644 index f617347..0000000 --- a/kubernetes/db/couchdb/configmap.yaml +++ /dev/null @@ -1,6 +0,0 @@ -#apiVersion: v1 -#kind: ConfigMap -#metadata: -# name: couchdb-config -#data: -# COUCHDB_PORT: "5982" diff --git a/kubernetes/db/couchdb/deployment.yaml b/kubernetes/db/couchdb/deployment.yaml deleted file mode 100644 index 7b094a9..0000000 --- a/kubernetes/db/couchdb/deployment.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: couch-db - labels: - app: couch-db -spec: - replicas: 1 - selector: - matchLabels: - app: couch-db-container - template: - metadata: - name: couch-db-container - labels: - app: couch-db-container - tier: backend - spec: - containers: - - name: couch-db-container - image: ahc_app-couch_db:latest - imagePullPolicy: Never - env: - - name: COUCHDB_USER - valueFrom: - secretKeyRef: - name: couchdb-secrets - key: COUCHDB_USER - - name: COUCHDB_PASSWORD - valueFrom: - secretKeyRef: - name: couchdb-secrets - key: COUCHDB_PASSWORD - - name: COUCHDB_PORT - valueFrom: - secretKeyRef: - name: couchdb-secrets - key: COUCHDB_PORT - volumeMounts: - - name: couchdbdata - mountPath: /opt/couchdb/data - resources: - limits: - memory: "256Mi" - cpu: "500m" - ports: - - containerPort: 5982 - protocol: TCP - volumes: - - name: couchdbdata - persistentVolumeClaim: - claimName: couch-db-volume-claim - restartPolicy: Always diff --git a/kubernetes/db/couchdb/kustomization.yaml b/kubernetes/db/couchdb/kustomization.yaml deleted file mode 100644 index bb6417d..0000000 --- a/kubernetes/db/couchdb/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -resources: - - storage-class.yaml - - pv.yaml - - pvc.yaml - - secret.yaml - - deployment.yaml - - service.yaml diff --git a/kubernetes/db/couchdb/pv.yaml b/kubernetes/db/couchdb/pv.yaml deleted file mode 100644 index 2e20965..0000000 --- a/kubernetes/db/couchdb/pv.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: couch-db-volume - labels: - type: local - app: couch-db -spec: - storageClassName: local-couch-db - capacity: - storage: 2Gi -# volumeMode: Filesystem - accessModes: - - ReadWriteMany -# persistentVolumeReclaimPolicy: Retain - hostPath: - path: "/db" diff --git a/kubernetes/db/couchdb/pvc.yaml b/kubernetes/db/couchdb/pvc.yaml deleted file mode 100644 index bcd2932..0000000 --- a/kubernetes/db/couchdb/pvc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: couch-db-volume-claim - labels: - app: couch-db -spec: - storageClassName: local-couch-db - accessModes: - - ReadWriteMany - resources: - requests: - storage: 2Gi - volumeName: couch-db-volume diff --git a/kubernetes/db/couchdb/secret.yaml.template b/kubernetes/db/couchdb/secret.yaml.template deleted file mode 100644 index d907d4b..0000000 --- a/kubernetes/db/couchdb/secret.yaml.template +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: couchdb-secrets -type: Opaque -stringData: - COUCHDB_USER: "..." - COUCHDB_PASSWORD: "..." - COUCHDB_PORT: "5982" diff --git a/kubernetes/db/couchdb/service.yaml b/kubernetes/db/couchdb/service.yaml deleted file mode 100644 index 6fc6990..0000000 --- a/kubernetes/db/couchdb/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: appendixes-db -spec: - selector: - app: couch-db - ports: - - protocol: TCP - port: 5982 - targetPort: 5982 - type: ClusterIP diff --git a/kubernetes/db/couchdb/storage-class.yaml b/kubernetes/db/couchdb/storage-class.yaml deleted file mode 100644 index 15641f3..0000000 --- a/kubernetes/db/couchdb/storage-class.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local-couch-db -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer diff --git a/kubernetes/db/postgres/deployment.yaml b/kubernetes/db/postgres/deployment.yaml deleted file mode 100644 index 19dc611..0000000 --- a/kubernetes/db/postgres/deployment.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres-db - labels: - app: postgres_db -spec: - replicas: 1 - selector: - matchLabels: - app: postgres_db - template: - metadata: - name: postgres-db - labels: - app: postgres_db - spec: - containers: - - name: postgres-db - image: postgres:18-alpine - imagePullPolicy: IfNotPresent - ports: - - containerPort: 5433 - protocol: TCP - envFrom: - - secretRef: - name: postgres-secrets -# volumeMounts: -# - mountPath: /var/lib/postgresql/data -# name: postgresdata - resources: - limits: - memory: "256Mi" - cpu: "500m" - volumes: - - name: postgresdata - persistentVolumeClaim: - claimName: postgres-volume-claim - restartPolicy: Always diff --git a/kubernetes/db/postgres/kustomization.yaml b/kubernetes/db/postgres/kustomization.yaml deleted file mode 100644 index bb6417d..0000000 --- a/kubernetes/db/postgres/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -resources: - - storage-class.yaml - - pv.yaml - - pvc.yaml - - secret.yaml - - deployment.yaml - - service.yaml diff --git a/kubernetes/db/postgres/pv.yaml b/kubernetes/db/postgres/pv.yaml deleted file mode 100644 index f9c3fba..0000000 --- a/kubernetes/db/postgres/pv.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: postgres-volume - labels: - type: local - app: postgres -spec: - storageClassName: local-postgres - capacity: - storage: 2Gi - volumeMode: Filesystem - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - hostPath: - path: db/postgres diff --git a/kubernetes/db/postgres/service.yaml b/kubernetes/db/postgres/service.yaml deleted file mode 100644 index 9a52711..0000000 --- a/kubernetes/db/postgres/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: postgres-service -spec: - selector: - app: ahc_app_backend - ports: - - protocol: TCP - port: 5433 - targetPort: 5432 - type: ClusterIP diff --git a/kubernetes/db/postgres/storage-class.yaml b/kubernetes/db/postgres/storage-class.yaml deleted file mode 100644 index 68dbf8d..0000000 --- a/kubernetes/db/postgres/storage-class.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local-postgres -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer diff --git a/kubernetes/kustomization.yaml b/kubernetes/kustomization.yaml deleted file mode 100644 index 93aa203..0000000 --- a/kubernetes/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ingress.yaml - - backend/web/ - - db/couchdb/ - - db/postgres/ - - queue/ diff --git a/kubernetes/overlays/home/backup/backup-pvc.yaml b/kubernetes/overlays/home/backup/backup-pvc.yaml new file mode 100644 index 0000000..b978310 --- /dev/null +++ b/kubernetes/overlays/home/backup/backup-pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: backup-data + annotations: + argocd.argoproj.io/sync-wave: "-3" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml b/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml new file mode 100644 index 0000000..7ee93d3 --- /dev/null +++ b/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml @@ -0,0 +1,49 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: couchdb-export +spec: + # Diagnostic document export, NOT a full backup: attachments and internal + # state are not fully covered by _all_docs. The real backup strategy is + # one-way replication to a second CouchDB instance or a PVC snapshot — + # see kubernetes/README.md. + schedule: "30 3 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 600 + template: + spec: + restartPolicy: Never + containers: + - name: couchdb-export + image: curlimages/curl:8.10.1 + envFrom: + - secretRef: + name: couchdb-credentials + command: + - sh + - -c + - | + set -eu + f="/backup/couchdb-appendixes-$(date +%Y%m%d-%H%M%S).json.gz" + curl -fsS "http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@appendixes-db:5982/appendixes/_all_docs?include_docs=true&attachments=true" \ + -H "Accept: application/json" | gzip > "$f" + echo "written $f" + find /backup -name 'couchdb-*.json.gz' -mtime +14 -delete + volumeMounts: + - name: backup + mountPath: /backup + resources: + requests: + memory: "32Mi" + cpu: "25m" + limits: + memory: "128Mi" + cpu: "200m" + volumes: + - name: backup + persistentVolumeClaim: + claimName: backup-data diff --git a/kubernetes/overlays/home/backup/files-backup-cronjob.yaml b/kubernetes/overlays/home/backup/files-backup-cronjob.yaml new file mode 100644 index 0000000..9babbc9 --- /dev/null +++ b/kubernetes/overlays/home/backup/files-backup-cronjob.yaml @@ -0,0 +1,55 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: files-backup +spec: + schedule: "0 4 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 600 + template: + spec: + restartPolicy: Never + containers: + - name: files-backup + image: alpine:3.21 + command: + - sh + - -c + - | + set -eu + stamp=$(date +%Y%m%d-%H%M%S) + tar czf "/backup/media-$stamp.tar.gz" -C /data media + tar czf "/backup/private-$stamp.tar.gz" -C /data private + echo "written media-$stamp.tar.gz private-$stamp.tar.gz" + find /backup -name 'media-*.tar.gz' -mtime +14 -delete + find /backup -name 'private-*.tar.gz' -mtime +14 -delete + volumeMounts: + - name: app-media + mountPath: /data/media + readOnly: true + - name: app-private + mountPath: /data/private + readOnly: true + - name: backup + mountPath: /backup + resources: + requests: + memory: "32Mi" + cpu: "25m" + limits: + memory: "128Mi" + cpu: "200m" + volumes: + - name: app-media + persistentVolumeClaim: + claimName: app-media-pvc + - name: app-private + persistentVolumeClaim: + claimName: app-private-pvc + - name: backup + persistentVolumeClaim: + claimName: backup-data diff --git a/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml b/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml new file mode 100644 index 0000000..1c2bfd2 --- /dev/null +++ b/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml @@ -0,0 +1,45 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pg-backup +spec: + schedule: "0 3 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + activeDeadlineSeconds: 600 + template: + spec: + restartPolicy: Never + containers: + - name: pg-dump + image: postgres:18-alpine + envFrom: + - secretRef: + name: postgres-credentials + command: + - sh + - -c + - | + set -eu + export PGPASSWORD="$POSTGRES_PASSWORD" + f="/backup/pg-$(date +%Y%m%d-%H%M%S).dump" + pg_dump -h postgres-service -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc -f "$f" + echo "written $f" + find /backup -name 'pg-*.dump' -mtime +14 -delete + volumeMounts: + - name: backup + mountPath: /backup + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "500m" + volumes: + - name: backup + persistentVolumeClaim: + claimName: backup-data diff --git a/kubernetes/overlays/home/celery-beat-patch.yaml b/kubernetes/overlays/home/celery-beat-patch.yaml new file mode 100644 index 0000000..f9c4f06 --- /dev/null +++ b/kubernetes/overlays/home/celery-beat-patch.yaml @@ -0,0 +1,9 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-beat +spec: + # Kept at 0 until the cutover from the docker-compose/Watchtower stack: + # two live beat schedulers would send duplicate Discord notifications. + # Set to 1 when this cluster becomes the only production instance. + replicas: 0 diff --git a/kubernetes/overlays/home/configmap-patch.yaml b/kubernetes/overlays/home/configmap-patch.yaml new file mode 100644 index 0000000..d8bbcb3 --- /dev/null +++ b/kubernetes/overlays/home/configmap-patch.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ahc-app-config +data: + # TODO: replace ahc.example.home with the real AHC_DOMAIN before the + # first home sync. localhost must stay in ALLOWED_HOSTS for kubelet probes. + ALLOWED_HOSTS: "localhost,127.0.0.1,ahc.example.home" + CSRF_TRUSTED_ORIGINS: "https://ahc.example.home" diff --git a/kubernetes/overlays/home/ingress-patch.yaml b/kubernetes/overlays/home/ingress-patch.yaml new file mode 100644 index 0000000..8b20f47 --- /dev/null +++ b/kubernetes/overlays/home/ingress-patch.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-service +spec: + # k3s ships Traefik as the default ingress controller. + ingressClassName: traefik + rules: + # TODO: replace with the real AHC_DOMAIN before the first home sync. + - host: ahc.example.home + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ahc-app-backend-service + port: + number: 8000 diff --git a/kubernetes/overlays/home/kustomization.yaml b/kubernetes/overlays/home/kustomization.yaml new file mode 100644 index 0000000..a5b715a --- /dev/null +++ b/kubernetes/overlays/home/kustomization.yaml @@ -0,0 +1,31 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Production overlay for the home k3s cluster, synced by the ahc-home +# ArgoCD Application. Secrets are SealedSecrets encrypted with the home +# cluster controller key. The image tag below is managed by CI +# (kustomize edit set image after each image publish). +# +# The files under sealed/ must be generated with kubeseal against the +# home cluster before the first sync — see kubernetes/README.md. + +resources: + - ../../base + - sealed/app-sealedsecret.yaml + - sealed/postgres-sealedsecret.yaml + - sealed/couchdb-sealedsecret.yaml + - sealed/flower-sealedsecret.yaml + - backup/backup-pvc.yaml + - backup/pg-backup-cronjob.yaml + - backup/couchdb-export-cronjob.yaml + - backup/files-backup-cronjob.yaml + - smoke-test-job.yaml + +patches: + - path: ingress-patch.yaml + - path: configmap-patch.yaml + - path: celery-beat-patch.yaml + +images: + - name: ghcr.io/cybernetic-ransomware/ahc-app + newTag: prod diff --git a/kubernetes/overlays/home/smoke-test-job.yaml b/kubernetes/overlays/home/smoke-test-job.yaml new file mode 100644 index 0000000..9dfc32a --- /dev/null +++ b/kubernetes/overlays/home/smoke-test-job.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ahc-smoke-test + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded +spec: + backoffLimit: 3 + activeDeadlineSeconds: 180 + template: + spec: + restartPolicy: Never + containers: + - name: smoke + image: curlimages/curl:8.10.1 + command: + - sh + - -c + - | + set -eu + curl -fsS -H "Host: localhost" http://ahc-app-backend-service:8000/livez + curl -fsS -H "Host: localhost" http://ahc-app-backend-service:8000/readyz + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "64Mi" + cpu: "100m" diff --git a/kubernetes/overlays/minikube-argocd/ingress-patch.yaml b/kubernetes/overlays/minikube-argocd/ingress-patch.yaml new file mode 100644 index 0000000..d1fa3f6 --- /dev/null +++ b/kubernetes/overlays/minikube-argocd/ingress-patch.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-service +spec: + ingressClassName: nginx diff --git a/kubernetes/overlays/minikube-argocd/kustomization.yaml b/kubernetes/overlays/minikube-argocd/kustomization.yaml new file mode 100644 index 0000000..2597cb8 --- /dev/null +++ b/kubernetes/overlays/minikube-argocd/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# GitOps test overlay: SealedSecrets encrypted with the sealed-secrets +# controller key of the local minikube cluster. Used by the +# ahc-minikube-test ArgoCD Application to rehearse the full sync flow +# (waves, hooks, sealed secrets) before touching the home cluster. +# +# The files under sealed/ must be generated with kubeseal against the +# minikube cluster before this overlay can build — see kubernetes/README.md. + +resources: + - ../../base + - sealed/app-sealedsecret.yaml + - sealed/postgres-sealedsecret.yaml + - sealed/couchdb-sealedsecret.yaml + - sealed/flower-sealedsecret.yaml + +patches: + - path: ingress-patch.yaml + +images: + - name: ghcr.io/cybernetic-ransomware/ahc-app + newTag: prod diff --git a/kubernetes/overlays/minikube-local/ingress-patch.yaml b/kubernetes/overlays/minikube-local/ingress-patch.yaml new file mode 100644 index 0000000..d1fa3f6 --- /dev/null +++ b/kubernetes/overlays/minikube-local/ingress-patch.yaml @@ -0,0 +1,6 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-service +spec: + ingressClassName: nginx diff --git a/kubernetes/overlays/minikube-local/kustomization.yaml b/kubernetes/overlays/minikube-local/kustomization.yaml new file mode 100644 index 0000000..9872a3b --- /dev/null +++ b/kubernetes/overlays/minikube-local/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Local development overlay: plain Secrets filled from the committed +# templates and applied with `kubectl apply -k`. The filled secret files are +# git-ignored, so this overlay MUST NOT be used as an ArgoCD Application +# source (ArgoCD checks out the repo and would not find them). + +resources: + - ../../base + - secrets/app-secret.yaml + - secrets/postgres-secret.yaml + - secrets/couchdb-secret.yaml + - secrets/flower-secret.yaml + +patches: + - path: ingress-patch.yaml + +images: + - name: ghcr.io/cybernetic-ransomware/ahc-app + newTag: prod diff --git a/kubernetes/overlays/minikube-local/secrets/app-secret.yaml.template b/kubernetes/overlays/minikube-local/secrets/app-secret.yaml.template new file mode 100644 index 0000000..500745e --- /dev/null +++ b/kubernetes/overlays/minikube-local/secrets/app-secret.yaml.template @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: ahc-app-secrets + namespace: ahc + annotations: + argocd.argoproj.io/sync-wave: "-3" +stringData: + SECRET_KEY: "..." + EMAIL_HOST_USER: "..." + EMAIL_HOST_PASSWORD: "..." + DISCORD_TOKEN: "..." diff --git a/kubernetes/overlays/minikube-local/secrets/couchdb-secret.yaml.template b/kubernetes/overlays/minikube-local/secrets/couchdb-secret.yaml.template new file mode 100644 index 0000000..c976fe7 --- /dev/null +++ b/kubernetes/overlays/minikube-local/secrets/couchdb-secret.yaml.template @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: couchdb-credentials + namespace: ahc + annotations: + argocd.argoproj.io/sync-wave: "-3" +stringData: + COUCHDB_USER: "..." + COUCHDB_PASSWORD: "..." diff --git a/kubernetes/overlays/minikube-local/secrets/flower-secret.yaml.template b/kubernetes/overlays/minikube-local/secrets/flower-secret.yaml.template new file mode 100644 index 0000000..db72966 --- /dev/null +++ b/kubernetes/overlays/minikube-local/secrets/flower-secret.yaml.template @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flower-secrets + namespace: ahc + annotations: + argocd.argoproj.io/sync-wave: "-3" +stringData: + FLOWER_BASIC_AUTH: "user:password" diff --git a/kubernetes/db/postgres/secret.yaml.template b/kubernetes/overlays/minikube-local/secrets/postgres-secret.yaml.template similarity index 54% rename from kubernetes/db/postgres/secret.yaml.template rename to kubernetes/overlays/minikube-local/secrets/postgres-secret.yaml.template index 8cc46b8..b7e4753 100644 --- a/kubernetes/db/postgres/secret.yaml.template +++ b/kubernetes/overlays/minikube-local/secrets/postgres-secret.yaml.template @@ -1,7 +1,10 @@ apiVersion: v1 kind: Secret metadata: - name: postgres-secrets + name: postgres-credentials + namespace: ahc + annotations: + argocd.argoproj.io/sync-wave: "-3" stringData: POSTGRES_DB: "..." POSTGRES_USER: "..." diff --git a/kubernetes/queue/celery-beat/deployment.yaml b/kubernetes/queue/celery-beat/deployment.yaml deleted file mode 100644 index 5952f2e..0000000 --- a/kubernetes/queue/celery-beat/deployment.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-beat - labels: - app: celery-beat -spec: - replicas: 1 - selector: - matchLabels: - app: celery-beat - template: - metadata: - name: celery-beat - labels: - app: celery-beat - spec: - initContainers: - - name: init-myservice - image: busybox - command: [ 'sh', '-c', 'env' ] - envFrom: - - secretRef: - name: web-secrets - containers: - - name: celery-beat - image: ahc_app-queue:latest - imagePullPolicy: Never - command: [ "celery", "-A", "celery_notifications.config:celery_obj", "beat", "-l", "info" ] - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: PYTHONPATH - value: "/app/src" - envFrom: - - secretRef: - name: web-secrets - resources: - limits: - memory: "128Mi" - cpu: "250m" - restartPolicy: Always diff --git a/kubernetes/queue/celery-beat/kustomization.yaml b/kubernetes/queue/celery-beat/kustomization.yaml deleted file mode 100644 index 9519a26..0000000 --- a/kubernetes/queue/celery-beat/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - deployment.yaml diff --git a/kubernetes/queue/celery/configmap.yaml b/kubernetes/queue/celery/configmap.yaml deleted file mode 100644 index 4b28051..0000000 --- a/kubernetes/queue/celery/configmap.yaml +++ /dev/null @@ -1,6 +0,0 @@ -#apiVersion: v1 -#kind: ConfigMap -#metadata: -# name: celery-config -#data: -# DJANGO_SETTINGS_MODULE: ahc.settings diff --git a/kubernetes/queue/celery/deployment.yaml b/kubernetes/queue/celery/deployment.yaml deleted file mode 100644 index e89a7cc..0000000 --- a/kubernetes/queue/celery/deployment.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: queue - labels: - app: queue -spec: - replicas: 1 - selector: - matchLabels: - app: queue - template: - metadata: - name: queue - labels: - app: queue - spec: - initContainers: - - name: init-myservice - image: busybox - command: [ 'sh', '-c', 'env' ] - envFrom: - - secretRef: - name: web-secrets - containers: - - name: queue - image: ahc_app-queue:latest - imagePullPolicy: Never - command: [ "celery", "-A", "celery_notifications.config:celery_obj", "worker", "-l", "info"] - env: - - name: PYTHONUNBUFFERED - value: "1" - - name: PYTHONPATH - value: "/app/src" - envFrom: - - secretRef: - name: web-secrets - ports: - - containerPort: 5000 - protocol: TCP - resources: - limits: - memory: "256Mi" - cpu: "500m" - restartPolicy: Always diff --git a/kubernetes/queue/celery/kustomization.yaml b/kubernetes/queue/celery/kustomization.yaml deleted file mode 100644 index 6d1374a..0000000 --- a/kubernetes/queue/celery/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - deployment.yaml - - service.yaml diff --git a/kubernetes/queue/celery/secret.yaml.template b/kubernetes/queue/celery/secret.yaml.template deleted file mode 100644 index 6e95f60..0000000 --- a/kubernetes/queue/celery/secret.yaml.template +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: queue-secrets - -stringData: - DB_USER: "..." - DB_PASSWORD: "..." - - EMAIL_HOST_USER: "..." - EMAIL_HOST_PASSWORD: "..." - - DISCORD_TOKEN: "..." - - DB_NAME: "..." - DB_HOST: "..." - DB_PORT: "5432" - - COUCHDB_USER: "..." - COUCHDB_PASSWORD: "..." - COUCHDB_PORT: "5982" - - COUCH_CONNECTOR: "Server(\"http://127.0.0.1:5982\")" - CELERY_BACKEND: "redis://redis:6379/0" - DJANGO_SETTINGS_MODULE: "ahc.settings" - - EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" - EMAIL_HOST: "sandbox.smtp.mailtrap.io" - EMAIL_PORT: "2525" - EMAIL_USE_TLS: "True" diff --git a/kubernetes/queue/celery/service.yaml b/kubernetes/queue/celery/service.yaml deleted file mode 100644 index b95fcf3..0000000 --- a/kubernetes/queue/celery/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: queue-service -spec: - selector: - app: queue - ports: - - protocol: TCP - port: 5000 - targetPort: 5000 - type: ClusterIP diff --git a/kubernetes/queue/flower/deployment.yaml b/kubernetes/queue/flower/deployment.yaml deleted file mode 100644 index b02d57f..0000000 --- a/kubernetes/queue/flower/deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: celery-flower - labels: - app: celery_flower -spec: - replicas: 1 - selector: - matchLabels: - app: celery_flower - template: - metadata: - name: celery-flower - labels: - app: celery_flower - spec: - containers: - - name: celery-flower - image: mher/flower:2.0.1 - imagePullPolicy: Always - ports: - - containerPort: 5555 - protocol: TCP - envFrom: - - secretRef: - name: queue-secrets - resources: - limits: - memory: "256Mi" - cpu: "500m" - restartPolicy: Always diff --git a/kubernetes/queue/flower/kustomization.yaml b/kubernetes/queue/flower/kustomization.yaml deleted file mode 100644 index 6d1374a..0000000 --- a/kubernetes/queue/flower/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - deployment.yaml - - service.yaml diff --git a/kubernetes/queue/kustomization.yaml b/kubernetes/queue/kustomization.yaml deleted file mode 100644 index d578b4a..0000000 --- a/kubernetes/queue/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: - - secret.yaml - - celery/ - - celery-beat/ - - flower/ - - redis/ diff --git a/kubernetes/queue/redis/deployment.yaml b/kubernetes/queue/redis/deployment.yaml deleted file mode 100644 index 1bf969f..0000000 --- a/kubernetes/queue/redis/deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: redis - labels: - app: redis -spec: - replicas: 1 - selector: - matchLabels: - app: redis - template: - metadata: - name: redis - labels: - app: redis - spec: - containers: - - name: redis - image: redis:7-alpine - imagePullPolicy: Always - ports: - - containerPort: 6379 - protocol: TCP - envFrom: - - secretRef: - name: queue-secrets - resources: - limits: - memory: "256Mi" - cpu: "500m" - restartPolicy: Always diff --git a/kubernetes/queue/redis/kustomization.yaml b/kubernetes/queue/redis/kustomization.yaml deleted file mode 100644 index 6d1374a..0000000 --- a/kubernetes/queue/redis/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - deployment.yaml - - service.yaml diff --git a/kubernetes/queue/secret.yaml.template b/kubernetes/queue/secret.yaml.template deleted file mode 100644 index 3afb2e2..0000000 --- a/kubernetes/queue/secret.yaml.template +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: queue-secrets -stringData: - FLOWER_BASIC_AUTH: "...:..." - CELERY_BROKER_URL: "redis://redis:6379/0" - FLOWER_PORT: "5555" From f7b458e12cb8a2e35f32820eb1a840ef71e25919 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:27:11 +0200 Subject: [PATCH 04/27] feat(argocd): add applications and deploy-smoke workflowtemplate --- argocd/ahc-home.yaml | 26 ++++++++++++++ argocd/ahc-minikube-test.yaml | 20 +++++++++++ argocd/argo-workflows.yaml | 26 ++++++++++++++ argocd/sealed-secrets.yaml | 22 ++++++++++++ workflows/deploy-smoke-workflowtemplate.yaml | 36 ++++++++++++++++++++ workflows/rbac.yaml | 32 +++++++++++++++++ 6 files changed, 162 insertions(+) create mode 100644 argocd/ahc-home.yaml create mode 100644 argocd/ahc-minikube-test.yaml create mode 100644 argocd/argo-workflows.yaml create mode 100644 argocd/sealed-secrets.yaml create mode 100644 workflows/deploy-smoke-workflowtemplate.yaml create mode 100644 workflows/rbac.yaml diff --git a/argocd/ahc-home.yaml b/argocd/ahc-home.yaml new file mode 100644 index 0000000..020d2d5 --- /dev/null +++ b/argocd/ahc-home.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ahc-home + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git + targetRevision: main + path: kubernetes/overlays/home + destination: + server: https://kubernetes.default.svc + namespace: ahc + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + retry: + limit: 3 + backoff: + duration: 30s + factor: 2 + maxDuration: 5m diff --git a/argocd/ahc-minikube-test.yaml b/argocd/ahc-minikube-test.yaml new file mode 100644 index 0000000..e1f2602 --- /dev/null +++ b/argocd/ahc-minikube-test.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ahc-minikube-test + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git + targetRevision: feat/argocd + path: kubernetes/overlays/minikube-argocd + destination: + server: https://kubernetes.default.svc + namespace: ahc + # Deliberately no automated sync policy: syncs are manual, without prune + # and without selfHeal, until drift/prune/sealed-secret-recovery tests + # pass on minikube. Automation belongs to ahc-home after the merge. + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/argo-workflows.yaml b/argocd/argo-workflows.yaml new file mode 100644 index 0000000..9644f87 --- /dev/null +++ b/argocd/argo-workflows.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: argo-workflows + namespace: argocd +spec: + project: default + source: + repoURL: https://argoproj.github.io/argo-helm + chart: argo-workflows + targetRevision: 0.45.11 + helm: + values: | + server: + # Homelab-grade auth: the server acts with its own service account. + authModes: + - server + destination: + server: https://kubernetes.default.svc + namespace: argo + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/argocd/sealed-secrets.yaml b/argocd/sealed-secrets.yaml new file mode 100644 index 0000000..ebaa99a --- /dev/null +++ b/argocd/sealed-secrets.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: sealed-secrets + namespace: argocd +spec: + project: default + source: + repoURL: https://bitnami-labs.github.io/sealed-secrets + chart: sealed-secrets + targetRevision: 2.17.3 + helm: + values: | + # Default name expected by the kubeseal CLI discovery. + fullnameOverride: sealed-secrets-controller + destination: + server: https://kubernetes.default.svc + namespace: kube-system + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/workflows/deploy-smoke-workflowtemplate.yaml b/workflows/deploy-smoke-workflowtemplate.yaml new file mode 100644 index 0000000..c6b61ee --- /dev/null +++ b/workflows/deploy-smoke-workflowtemplate.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: ahc-deploy-smoke + namespace: argo +spec: + entrypoint: main + serviceAccountName: ahc-deployer + arguments: + parameters: + - name: image-tag + value: "prod" + templates: + - name: main + steps: + - - name: wait-rollout + template: rollout-status + - - name: smoke + template: smoke-test + + - name: rollout-status + container: + image: bitnami/kubectl:1.31 + command: [sh, -c] + args: + - kubectl -n ahc rollout status deployment/ahc-app-backend --timeout=300s + + - name: smoke-test + container: + image: curlimages/curl:8.10.1 + command: [sh, -c] + args: + - | + set -eu + curl -fsS -H "Host: localhost" http://ahc-app-backend-service.ahc.svc:8000/livez + curl -fsS -H "Host: localhost" http://ahc-app-backend-service.ahc.svc:8000/readyz diff --git a/workflows/rbac.yaml b/workflows/rbac.yaml new file mode 100644 index 0000000..b0a2ff9 --- /dev/null +++ b/workflows/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ahc-deployer + namespace: argo +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ahc-deployer + namespace: ahc +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ahc-deployer + namespace: ahc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ahc-deployer +subjects: + - kind: ServiceAccount + name: ahc-deployer + namespace: argo From a42cc5e56976e817ae92d059816b75103e5c4ade Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:27:37 +0200 Subject: [PATCH 05/27] ci: commit kustomize image bump and gated workflow submit --- .github/workflows/django.yml | 66 ++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index c9a825c..7434368 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -114,3 +114,69 @@ jobs: tags: | ghcr.io/${{ env.OWNER }}/ahc-app:prod ghcr.io/${{ env.OWNER }}/ahc-app:sha-${{ github.sha }} + + bump: + name: Bump image tag in home overlay + runs-on: ubuntu-latest + needs: publish + if: github.event_name == 'push' + permissions: + contents: write + concurrency: + group: ahc-home-image-bump + cancel-in-progress: false + env: + KUSTOMIZE_VERSION: "5.8.1" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: main + + - name: Install pinned kustomize + run: | + curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \ + | tar xz -C /usr/local/bin kustomize + kustomize version + + - name: Set lowercase owner + run: echo "OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV + + - name: Set image tag in home overlay + run: | + cd kubernetes/overlays/home + kustomize edit set image "ghcr.io/${OWNER}/ahc-app=ghcr.io/${OWNER}/ahc-app:sha-${{ github.sha }}" + + - name: Commit and push + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add kubernetes/overlays/home/kustomization.yaml + if git diff --cached --quiet; then + echo "No image tag change to commit." + exit 0 + fi + git commit -m "chore(deploy): bump ahc-app to sha-${{ github.sha }} [skip ci]" + git push + + deploy-workflow: + name: Submit deploy-smoke Argo Workflow + runs-on: ubuntu-latest + needs: bump + # Self-disabled until an Argo Server is reachable from CI (self-hosted + # runner or tunnel). ArgoCD auto-sync remains the primary deploy path. + if: github.event_name == 'push' && vars.ARGO_SERVER_URL != '' + steps: + - name: Submit workflow via Argo Server API + run: | + curl -fsS -X POST \ + -H "Authorization: Bearer ${{ secrets.ARGO_TOKEN }}" \ + -H "Content-Type: application/json" \ + "${{ vars.ARGO_SERVER_URL }}/api/v1/workflows/argo/submit" \ + -d '{ + "resourceKind": "WorkflowTemplate", + "resourceName": "ahc-deploy-smoke", + "submitOptions": { + "parameters": ["image-tag=sha-${{ github.sha }}"] + } + }' From ff0834de1d237964f936f77c74acbebb81e4fd88 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:28:01 +0200 Subject: [PATCH 06/27] docs: add ADR-13 gitops deployment and kubernetes ops readme --- doc/13_adr_gitops_argocd.md | 86 ++++++++++++++++++++++ kubernetes/README.md | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 doc/13_adr_gitops_argocd.md create mode 100644 kubernetes/README.md diff --git a/doc/13_adr_gitops_argocd.md b/doc/13_adr_gitops_argocd.md new file mode 100644 index 0000000..166ce19 --- /dev/null +++ b/doc/13_adr_gitops_argocd.md @@ -0,0 +1,86 @@ +## GitOps deployment with ArgoCD + +### Date: +`2026-07-16` + +### Status +In-building + +### Context +Production deployment is pull-based Watchtower polling the `prod` image tag on a docker-compose stack. +The `kubernetes/` manifests were a leftover from a local minikube workflow: `imagePullPolicy: Never`, +image tarballs committed under `kubernetes/tars/`, no probes, no namespace, broken Service selectors, +ephemeral Postgres storage, plaintext secret files (git-ignored) and no way to run Django migrations +inside the cluster. Deploying that state through any GitOps tool would faithfully reproduce every bug. + +The goal is a declarative deployment where git is the single source of truth, the target being a +single-node k3s cluster on the home server, with local minikube used for rehearsal. + +### Decision +1. **ArgoCD** syncs `kubernetes/overlays/home` from `main`. CI never talks to the cluster; it pushes + the image to GHCR and commits a kustomize image-tag bump (`sha-`) to the home overlay, + serialized by a GitHub Actions concurrency group. ArgoCD picks up the git diff. +2. **Kustomize layout**: `kubernetes/base` plus three overlays: + - `minikube-local` — plain git-ignored Secrets, `kubectl apply -k` only (never an ArgoCD source, + because ArgoCD checks out the repo and the ignored files are absent there); + - `minikube-argocd` — SealedSecrets encrypted for the local minikube controller, used by the + manual-sync `ahc-minikube-test` Application to rehearse the full GitOps loop; + - `home` — SealedSecrets encrypted for the home cluster, CI-managed image tag, backups, smoke test. +3. **Sync phasing**: ArgoCD orders resources by phase first and only then by wave, so a PreSync + migration hook would run before the databases exist on a fresh cluster. Migrations and CouchDB + database creation are therefore **Sync-phase hooks at wave -1**, after ConfigMaps/Secrets/PVCs + (wave -3) and the data stores (wave -2), before the app workloads (wave 0) and Ingress (wave 1). + The smoke test is PostSync. Outside ArgoCD the jobs guard themselves with wait initContainers. +4. **Secrets**: Bitnami **Sealed Secrets**. Only true secrets are encrypted (`ahc-app-secrets`, + `postgres-credentials`, `couchdb-credentials`, `flower-secrets`); everything else lives in the + `ahc-app-config` ConfigMap for readable diffs. Credentials are never duplicated across secrets — + the app maps `DB_*` variables from `postgres-credentials` via `secretKeyRef`. A SealedSecret is + bound to one controller's private key: minikube and home have separate `sealed/` directories. +5. **Probes**: split endpoints. `/livez` has no dependencies (liveness + startup), `/readyz` runs + `SELECT 1` against Postgres (readiness). CouchDB is deliberately outside `/readyz`: its outage + degrades attachments, not the whole application. +6. **Data stores**: StatefulSets with volumeClaimTemplates. Postgres 18 mounts the volume at + `/var/lib/postgresql` (the image keeps PGDATA in a versioned subdirectory since 18). Application + file storage gets its own PVCs: `app-media-pvc` (upload dirs mounted via subPath so committed + icons stay visible) and `app-private-pvc` shared by web and the Celery worker (offline snapshots). +7. **Migrations as a Job**, `makemigrations` removed from every deploy path; `collectstatic` runs in + a web-pod initContainer writing to an emptyDir shared with gunicorn (ManifestStaticFilesStorage). +8. **CI → cluster reachability**: GitHub-hosted runners cannot reach the home server, so the deploy + signal is ArgoCD polling git (option A). The `ahc-deploy-smoke` Argo WorkflowTemplate is committed + for manual runs, and the CI submit step is gated on `vars.ARGO_SERVER_URL` (self-disabled until a + self-hosted runner or tunnel exists). + +Rejected alternatives: Helm (kustomize already in place, no templating need), ArgoCD Image Updater +(extra controller; CI commit is simpler and auditable), External Secrets Operator (requires an +external secret store), SOPS/KSOPS (requires an ArgoCD plugin; kubeseal is plain kubectl-side). + +### Consequences +- Git history becomes the deployment history; rollback is `git revert` of a bump commit. +- The Watchtower compose stack keeps running during the transition. Two live Celery beat schedulers + would duplicate Discord notifications, so the home overlay pins `celery-beat` to 0 replicas until + cutover. +- The first sync starts with **empty databases** — migrating data from the compose stack is a + separate manual dump/restore operation. +- Backups: nightly `pg_dump`, a CouchDB **diagnostic export** (`_all_docs`; a real backup is one-way + replication or a PVC snapshot — follow-up), and tar archives of the file PVCs. All land on a PVC on + the same disk, which protects against user error, not disk failure; shipping copies off-host + (restic/rclone to a NAS) is a post-testing follow-up. +- Application PVCs are ReadWriteOnce, which is fine on a single node; a multi-node cluster would need + RWX or object storage. +- Losing the sealed-secrets controller key (cluster rebuild) invalidates all SealedSecrets; the + git-ignored plain files (kept in a password manager) are the re-sealing source. + +### Keywords +- ArgoCD, +- GitOps, +- kustomize, +- sealed-secrets, +- sync-wave, +- k3s. + +### Links +- [ADR-08](08_adr_databases.md) — database responsibilities (PostgreSQL / CouchDB / Redis). +- [ADR-12](12_adr_turso_offline_snapshots.md) — offline snapshots stored under PRIVATE_STORAGE_ROOT. +- [kubernetes/README.md](../kubernetes/README.md) — operational runbook. +- [ArgoCD sync phases and waves](https://argo-cd.readthedocs.io/en/stable/user-guide/sync-waves/). +- [Sealed Secrets](https://github.com/bitnami-labs/sealed-secrets). diff --git a/kubernetes/README.md b/kubernetes/README.md new file mode 100644 index 0000000..37044c5 --- /dev/null +++ b/kubernetes/README.md @@ -0,0 +1,138 @@ +# Kubernetes deployment — ops runbook + +GitOps layout for ArgoCD (see [ADR-13](../doc/13_adr_gitops_argocd.md)). All app images come from +`ghcr.io/cybernetic-ransomware/ahc-app`; the tag is rewritten per overlay. + +## Layout + +| Path | Purpose | Secrets | Applied by | +|---|---|---|---| +| `base/` | all workloads, config, PVCs, jobs | none (referenced only) | — | +| `overlays/minikube-local/` | local dev loop | plain Secrets, git-ignored | `kubectl apply -k` only — **never ArgoCD** | +| `overlays/minikube-argocd/` | GitOps rehearsal | SealedSecrets (minikube key) | `argocd/ahc-minikube-test.yaml`, manual sync | +| `overlays/home/` | production (k3s) | SealedSecrets (home key) | `argocd/ahc-home.yaml`, auto sync from `main` | + +Sync waves: `-3` ConfigMap/Secrets/PVCs → `-2` Postgres/CouchDB/Redis → `-1` Sync hooks +(`ahc-migrate`, `couchdb-init`) → `0` web/celery/beat/flower → `1` Ingress → PostSync smoke test. +Migrations are a **Sync-phase** hook (not PreSync): ArgoCD orders by phase before wave, and PreSync +would run before the databases exist on a fresh cluster. + +## Local run (minikube-local) + +```powershell +# 1. Fill secrets from templates (once); the filled .yaml files are git-ignored +Copy-Item kubernetes/overlays/minikube-local/secrets/app-secret.yaml.template ` + kubernetes/overlays/minikube-local/secrets/app-secret.yaml +# ...same for postgres/couchdb/flower; generate SECRET_KEY with: +uv run python -c "import secrets; print(secrets.token_urlsafe(50))" + +# 2. Render check, then apply +kubectl kustomize kubernetes/overlays/minikube-local +kubectl apply -k kubernetes/overlays/minikube-local + +# 3. Watch +kubectl -n ahc get pods -w +kubectl -n ahc wait --for=condition=complete job/ahc-migrate --timeout=300s +kubectl -n ahc rollout status deploy/ahc-app-backend --timeout=300s + +# 4. Smoke +kubectl -n ahc port-forward svc/ahc-app-backend-service 18000:8000 +curl.exe -f -H "Host: localhost" http://127.0.0.1:18000/livez +curl.exe -f -H "Host: localhost" http://127.0.0.1:18000/readyz +``` + +**Job immutability caveat:** a plain re-apply after an image-tag change fails on the immutable Job +spec. Delete the hook jobs first (ArgoCD's `BeforeHookCreation` policy does this automatically): + +```powershell +kubectl -n ahc delete job ahc-migrate couchdb-init --ignore-not-found +kubectl apply -k kubernetes/overlays/minikube-local +``` + +**Local image iteration:** the GHCR `prod` tag must contain the gunicorn CMD and health endpoints. +To test unmerged code, build straight into minikube and temporarily switch the overlay tag +(never commit `newTag: dev`): + +```powershell +minikube image build -t ghcr.io/cybernetic-ransomware/ahc-app:dev -f docker/Dockerfile-app . +# set newTag: dev in overlays/minikube-local/kustomization.yaml, apply, revert before committing +``` + +## Sealing secrets (kubeseal) + +Install once: `winget install kubeseal`. A SealedSecret decrypts only on the cluster whose +controller key encrypted it — minikube and home have separate `sealed/` directories, sealed from +the same plain files (keep an off-repo copy of those in a password manager). + +```powershell +# against the target cluster context: +kubeseal --controller-namespace kube-system --fetch-cert > cluster.pem # *.pem is git-ignored +Get-Content kubernetes/overlays/minikube-local/secrets/app-secret.yaml | + kubeseal --cert cluster.pem --format yaml | + Set-Content kubernetes/overlays//sealed/app-sealedsecret.yaml +# repeat for postgres-secret, couchdb-secret, flower-secret +``` + +If the cluster is rebuilt, the controller key is gone: re-seal everything from the plain files. + +## GitOps rehearsal on minikube (minikube-argocd) + +```powershell +kubectl apply -f https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.27.3/controller.yaml +# seal the four secrets into overlays/minikube-argocd/sealed/ (see above), commit them +kubectl create ns argocd +kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml +kubectl apply -f argocd/ahc-minikube-test.yaml +``` + +Sync manually from the ArgoCD UI/CLI and verify waves and hooks. Before enabling automation on +home, pass the drift tests: `kubectl edit` a resource → diff shows drift; delete a Deployment → +re-sync restores it; delete a Secret → controller re-creates it from the SealedSecret. + +## Home cluster bootstrap (first time) + +1. Install k3s (keeps default Traefik ingress and local-path default StorageClass). +2. Install ArgoCD (`kubectl apply -n argocd -f .../install.yaml`). +3. `kubectl apply -f argocd/sealed-secrets.yaml` and wait for the controller. +4. Seal the four secrets against the home cluster into `overlays/home/sealed/`, commit to `main`. +5. Replace `ahc.example.home` in `overlays/home/{ingress-patch,configmap-patch}.yaml` with the real + domain, commit. +6. `kubectl apply -f argocd/ahc-home.yaml` — first sync starts with **empty databases**; migrating + data from the compose stack is a manual dump/restore. +7. Cutover note: `celery-beat` is pinned to 0 replicas in the home overlay until the + compose/Watchtower stack is retired (two beats = duplicate Discord notifications). + +## CI / image flow + +`django.yml` publishes `ghcr.io//ahc-app:prod` + `:sha-`, then the `bump` job commits +`kustomize edit set image ...:sha-` to `overlays/home/` (`[skip ci]`, serialized by a +concurrency group). ArgoCD polls `main` and syncs. The `deploy-workflow` job (Argo Workflows submit) +stays disabled until `vars.ARGO_SERVER_URL` is set — requires a self-hosted runner or tunnel. +Manual workflow run: `kubectl apply -f workflows/` then +`argo submit --from workflowtemplate/ahc-deploy-smoke -n argo`. + +If GHCR packages are private, add a `dockerconfigjson` pull secret (sealed for home) and +`imagePullSecrets:` on every pod spec including the hook jobs. + +## Backups (home overlay) + +| CronJob | Schedule | What | Retention | +|---|---|---|---| +| `pg-backup` | 03:00 | `pg_dump -Fc` to `backup-data` PVC | 14 days | +| `couchdb-export` | 03:30 | `_all_docs` JSON dump — **diagnostic export, not a full backup** | 14 days | +| `files-backup` | 04:00 | tar of `app-media-pvc` + `app-private-pvc` | 14 days | + +Restore: `pg_restore -h postgres-service -U $POSTGRES_USER -d $POSTGRES_DB /backup/pg-.dump`; +CouchDB — re-PUT documents from the export or replicate back from a second instance; files — +untar into the PVCs. + +Limitations (deliberate, documented): backups land on the same disk as the data (protects against +user error, not disk failure) and the CouchDB export is not attachment-complete. Follow-ups after +the testing phase: one-way CouchDB replication or PVC snapshots, and shipping copies off-host with +restic/rclone to a NAS. + +## Credential hygiene + +The legacy plaintext secret files contained a live Discord token and Mailtrap credentials — +**rotate both** before the first exposed deployment, and generate a fresh `SECRET_KEY` for home +(do not reuse the minikube one). From a65bc7c30400495d44707c3432ff691caffb399c Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:41:22 +0200 Subject: [PATCH 07/27] feat(k8s): add sealed secrets for minikube-argocd overlay --- .../sealed/app-sealedsecret.yaml | 20 +++++++++++++++++++ .../sealed/couchdb-sealedsecret.yaml | 18 +++++++++++++++++ .../sealed/flower-sealedsecret.yaml | 17 ++++++++++++++++ .../sealed/postgres-sealedsecret.yaml | 19 ++++++++++++++++++ 4 files changed, 74 insertions(+) create mode 100644 kubernetes/overlays/minikube-argocd/sealed/app-sealedsecret.yaml create mode 100644 kubernetes/overlays/minikube-argocd/sealed/couchdb-sealedsecret.yaml create mode 100644 kubernetes/overlays/minikube-argocd/sealed/flower-sealedsecret.yaml create mode 100644 kubernetes/overlays/minikube-argocd/sealed/postgres-sealedsecret.yaml diff --git a/kubernetes/overlays/minikube-argocd/sealed/app-sealedsecret.yaml b/kubernetes/overlays/minikube-argocd/sealed/app-sealedsecret.yaml new file mode 100644 index 0000000..33305a3 --- /dev/null +++ b/kubernetes/overlays/minikube-argocd/sealed/app-sealedsecret.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: ahc-app-secrets + namespace: ahc +spec: + encryptedData: + DISCORD_TOKEN: AgCiLgqLvMlbfSoEUdSdlVOLqR8i95sEoBwhBFWXcfCGF+a3jE+SaWxwobxlA6KzRAX9aG6s7FReXMQr0SNdP1OnUEzTzIKgilJVjpqrDiKii56p+V0McLRLywH8NoANsga8picI38b9eNqTmd5j5ZpaUvt48c5ewt2dwT3gCbJr2sxAEh0FkNMrPFR//ER08UKu+FXMAU7CkuYGOKnGSS7Ai+m5NvUh4174HDP9KawAytcpohV1U5MdkePobzByTK3vSQ00ZTOtQmztlXsI0yxnLr6tjJDA4KyuBRcpFfqTwod0wt3kqoqFyU3rmU6CYw42BrNRDv+mnwkK8hnTEKgKAhiKXUrtKQcBfPbUWKk/++1dLYc9u+jsdugvTKZxR92ko9W90xC+CQDple/h0jPCZesixatZ1fFFRAcxpBzzKXSvBpVF0aEShz30VPg5hC0Odlqeb5Mr1NUiNY1zmdtP5lXE2/lcahx9pqd2tJMG5Ilcu3oAyMHcD1YY1TO73apk5V/D1LhCcxdVLgQB/EeWKA/moBT39grzRlf1eQrbIORyHy/xyKDrArkefPoBrhGNzQN4KmjpFRgGdc43kSE0yt4loqZtrkEAz7I0Y6+5UhQ9JL3WROUwJ9uhmTcmS0PzAndXn8XHHtsb3ZW4WOEo7pD15tCxBJNvuCNdOiQmE2yr0DBhhkbiwNZIbTjFP81vbmXxrYKZhdFR5P+8yQicqi0vOBBjPFmlg1bEeUjJTEoD9Hqp8vzDj49oHNUFcu9T6RCgc3YP8gcOndiDVPg5M0gRbDSr + EMAIL_HOST_PASSWORD: AgA7Kly0hCzVMbh+JGPTBICcfCUgYJYX+/qheWOvY5zVv/y44EKpoqauClr0WwfX99T+Z8WiiUVChBIa1SON4P00ehTPYhQ0YkuvNk60XgloMsOUgKpWIx73eE0AkMGrm0ftdgQ7J++acKr8vQRXhzg/YICrfIFJJ50FTHmhwPrsSKC54G0XNsuXzb+VRCuIuSn7SLW59u1RCff6I9+pXMqcM39H/zJhJP9qdxiEriUPgAksG3GZIPs2agozsgi8JJ1syDoAeszRn4MLdqhffdt+opDTrTEwyRpk1cNxDewZk0b63Nm4D/1gMCSzHmjJAaioVwXX1NKiFUTumyIXNspqS8qUFSthwHdVn6+5msM+fDriCmW+GEjnRoRfvs771h7DX3Gf04nnAQuBvdWDx0NrqxwjwqXP35C7RgTJHbvY5I5Dhr3BPhEZrsIMv2g+EDoLjyu+orXfAITGVIuzF6D15uxFmZqzS0UtnD3k0iEbHmBtqHKStBsukLITA4zCJ3Zrv1nGLk3lY4lwE0Y0gAdwYBa5Ck8cc1iLWDkEt5zXSWG1oKEP619FisniUBE3j5PySe3bpth+pay9qAqUcivcn+mUMXjcpPGUsiM7WtBd1d7vbwkO60+ZfvuqE6xWAi25069aBpBNX/hNv7e0lTc8j1iCUwxjnoyoMLbZnSx9oOqILQSEW+qbyfvevOFAYwQ0EOWZyty6m/Tdy6POOg== + EMAIL_HOST_USER: AgAJUcALDlKSHiEMlEXebHcyVg9jp6mgWZgZgM1iV0QolDCZyrbmtjePaiJrNjeHdmwdlJziyPYzOpqGEVMe3Vkyaho8QUxxTbZOYK68YbQ9IXYpo0KqrsMjoBjYi4cA4crkfM882XU0Qpve5GM3/cLJxpOas5uUx+iLdsxetQdrMT2UYFwDaLamitmMcJb/I0kz7EgUv/m8gGJ+SSLQzdGMZhLBw1Ip+8oN3+evxeFBjDF1cnHH89TjqcNZX+4fsGw0M4k3l6nINZ3h4ktjWm9GRbduU1nAFG17FGJMiahU+vi68A6+C5EKgpbxIMu/GYQimxIjW7McV6xctstb9YSgfOgtx2k1R4W16KVXECPKQ0uuV+rImV4XTDvsTw5aR22Pv5Z51CPmcWFIHcAD4jbbtzMQAcdzitHl4o7mJS69Np7+wyg1ur8eaN7dMxr55Ie+N0EqhecZgdHfyJWj1Sc3VqocK4Yspotn6DZ/ar/QzT3rXRlcVkcZl3TutWzBe/z2jSnx00brhkNpms1oyXThSIHaeKdEmsBcz7oEs/67mHlbvSUcHUUWQ6RIM4At7FMYiLQfCb95mgZNlp9A65jec2mFb3koEZpI17p2sumI73wTJkjc2valtXuJ9pdmMD7yY659o8ojBlOIsaOIrOOuQUqVJ+wCOf2/5ckj7+l+k4Bv3L4Pisq2dL99N1HxJjnnXQ8pkod2ErpdDbHswQ== + SECRET_KEY: AgCArhaFrkfy/ToWXfTDBF1Nqpob0438/SV6gFHwwLNQEiyv0EeL25NISceHQa/sTK9B1UNnalDybbk1ak+Bu12Fh9n26SMHplAsENkFlCgPXeBLnSfJdvIjgkp/1eUcvT2N9z/eYFgFJdBAwHMpdmC5vI3Vt1VjZ5CHoB8WWRrDl7zTBGWY6BugAczDn+PjnWwk2ygllaPO/b+5Z0XYl8+DSs/RoQxVaCzZ/tjwDdkcZ6Yi4jc1cNCml9/pRI9osAfyIwMGcmz97EwnNO8/rZkBmS/dXGyX4q/lro6fHw+Z5i3BP0nf/l7NI/XpfKUvAJyMj4DBeUd/YjnspXfxRUra+6WLdT0tQCXWSCQ8Wa2YUA9D0P8bkF1NcYKTLSEOf/ZVPyAnf4IM4FbERceXZDQZPExPZzAf+RUeHvUWxnx/nunkfQLIPS2vooCdw0brii79YlbaKSjJgWKUgtdXOa+MPFFSKbm362fQAZRT0i79Muq30W/ok9P08najhaSg0RTEXuJrT91e8TfAgqNiiqgpFQX+ml7+fv7Qcv4TXUsREpcWEkKlSiZr9sPJwU5SAwb62J7FAQXNGcQXuvdkQoauSULfGKtnRrGm/ONwi812ybTMvwqVGtxabsgI9aK027nt6kOAZKl1MYDBag7V3mfXOxa1jKGpx/ArXx+8cv723M/o47iK5qPQ4T7/AEaakAuI/eSvMYl3fEemv2J1Lg67yeJuoiRwyw5egEG52LuJk3b+qUAoFAOOjTjeWa/52Bs2tvE7Dki8qmvT971hFnO76V6K + template: + metadata: + annotations: + argocd.argoproj.io/sync-wave: "-3" + creationTimestamp: null + name: ahc-app-secrets + namespace: ahc diff --git a/kubernetes/overlays/minikube-argocd/sealed/couchdb-sealedsecret.yaml b/kubernetes/overlays/minikube-argocd/sealed/couchdb-sealedsecret.yaml new file mode 100644 index 0000000..a866201 --- /dev/null +++ b/kubernetes/overlays/minikube-argocd/sealed/couchdb-sealedsecret.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: couchdb-credentials + namespace: ahc +spec: + encryptedData: + COUCHDB_PASSWORD: AgBLcq7HVmCbUPN7qqLRtgsBtsM+foW5oj6TGra+BWGKQG+AAcZCp8wMFgkqMov4LvYDJHXnG0JII7AAnZBy8PpHk9hmMSxYyI30TfjKsNdvPDRvQepakbWeJs1nW6oPFgoXDuu8y18fKLc+ETnwxzcjwcr26gqt7Vm5JxgiwBbCcAl+0HyUXNtx1hJkm1sLi/OM/ZczTIYC5fO3/9iPmfg9GgtKI2l5LZOLKj00ItGny2qpaufUwHYC8EW7w1VTtBe93CntZJteTWaXpMr535yihZgV3DufB+wlQUct74SoJvwhzgeQn89WoUQKjmlF5n1LgSFHpMr9IQxZl9ort7mnBVErcbnHAgvzC0/nwQLOqeTL6a8pULoggnlZt/hwCu1ZRFg3Rz0EiffffwEDDD56ERt9AnnIATTSz1Pe4VEw6I7d/ov4JhI+NiLvbuIW4zSGa7RCqS4+rcVw+KyfIMNlXAcjNWopHO72QAi9nK7jx68sRq6RzunYgSiKqJNsnjMkH3Qlqy42vMhd448FjDp6wZWZETTIWYiA1754IQtyhJuEwtZfm8VmRkfeSBEBKEAl5q/jQskceX9PJKRW9bdSamGD/RXPmG6+IuvHdwLZOe/ptAQel59j6n8y/joG6osVYN+nn50EJrfnqjTUAF+z36e2uCqp6msfj/Luc4KjGHhlbJjx86gzxkGYidaS130tJxLZTjs= + COUCHDB_USER: AgCU+DwkpeQBDdpjxWc4t2niyVicMQWIa1x8Dyf2OVs5X3Yi26rgfScasgEuUqvnQ4UusZOKa6/f7jSoTaxesBDPQgPLOwg/X2v6lmJq9Mnoi9wD3iy+cs3IjkGherhNn009gNo85Wh1nnmg/00e3MN6rKDEEGiHB+9rTSG4OIX5ycvfakhqNi5sz/FYgOJazU+g4+qPkgBsQAtKjI4njUB/dVxYgQF9IQV5nNi3Jruo9s7Rn2PfBgz9I78syKkrzSb7N7KbuG6NnPQAeQsohqkDx3ULA+QaoTYrnNqHB+N66IlCLYtRH/5w1QHAJbk80Ysp87ucP2OxbmP/07ZdrgDljl8Z2qBE5szLLPJzhEPkJqtonf8vJxi+6RFmC8mrCxktxDz4VBggYzcrQQtZal4f9zps4NnSOYWqlUa657TcSNmnKxFGOnyI7THVJRo9mNT0Ik/T3ETcs9bA2TqjjFRS/rMWGcmqQSM4QE1yfMJZSzuuw2KRv33WXYYOQEH2EHhr6kc5xM9p1oRTtcf1BhvhXm/90yeXJosL2wfARGxELEitO57o08JCjOr1YX/3pCepJCb0JeyGLK84BGT/WoebTQCnxlEur/CuaEVjC3HFYawlvjMXhdVrZtqyKUyJZE664qlDl/ezq9WKAMgw3Sk6mUowWeZdQR8vYAqa0MCxrlnzQZF1YL6I1PVVGcxRGzmTBGHWB7w= + template: + metadata: + annotations: + argocd.argoproj.io/sync-wave: "-3" + creationTimestamp: null + name: couchdb-credentials + namespace: ahc diff --git a/kubernetes/overlays/minikube-argocd/sealed/flower-sealedsecret.yaml b/kubernetes/overlays/minikube-argocd/sealed/flower-sealedsecret.yaml new file mode 100644 index 0000000..d55c731 --- /dev/null +++ b/kubernetes/overlays/minikube-argocd/sealed/flower-sealedsecret.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: flower-secrets + namespace: ahc +spec: + encryptedData: + FLOWER_BASIC_AUTH: AgB51xcbMBCKjaOBUwY0f3K+M9yvXfc99Bdg73riTowf+GYCvI08CFIVq5DEwBDBXlvFVYhhMeLm74tYJlnURk0OAjP8O/KGXC2asNKWI3hKjEz+CKqEyG5p9/zQKcPuT/JehDwNQQGRa2zubphuKySEcoxlvac2GOVn9epkS8ECxxR94Yd8pkOxJ2xitnyRhY1fDZl9ncMIFC2oBE/9iYfVKsX71tCWByFiS2WwDK+W3VmwngFvdBgQVRNuQ7TZBKPKG5XSUiz+VVXrp15HKfD66S8CiwEsbVl/4BNeIVZB7fN0ILlvNcVspiOJpJt+BrCzCuQQdJQ88sotFMKJ+hmEAZme7FGIyerfqKK8hJd/455Md6UF/5aaVq9f1uGYhYDoSu1aFi1uQTtT62eCqEeBxLmKpLEUfrUfVCVYT4fPA2FnXuu5yaajiZ62ynd/y43NmQ4ecjW78qbym+2QuP3gAzCdjdp7t2ZoAmKkxUlwxNxxbur+c/9vpRM6JuVhVlemXR6Ndcw9ydD1qNorsjyjeeIjPzsPpF0SS4WeVqJlo3dIUZZRzJBI3/cgx05nCPXUAu7namwCIQAps2hNm7tIuuuLMAMTXJ9k0BQTVYSYxLJrtRFFYGK8zXMFPHpQw+lxcv+8PK753bkRaKsFu0AnNEPMU5broNX2uZIyVQcKiYkFjiO7TsG1h/0dTl84rmKCcMzawGKI1zOgKQ== + template: + metadata: + annotations: + argocd.argoproj.io/sync-wave: "-3" + creationTimestamp: null + name: flower-secrets + namespace: ahc diff --git a/kubernetes/overlays/minikube-argocd/sealed/postgres-sealedsecret.yaml b/kubernetes/overlays/minikube-argocd/sealed/postgres-sealedsecret.yaml new file mode 100644 index 0000000..4e39716 --- /dev/null +++ b/kubernetes/overlays/minikube-argocd/sealed/postgres-sealedsecret.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: postgres-credentials + namespace: ahc +spec: + encryptedData: + POSTGRES_DB: AgBGkgny6Z5TLw4be9Q7ZuacM9qT1fsHq37r4f1eQk/bKwe4afpxRat4hYop55GkvzHaileIbXIiLQCWd9jNpCIJkcIh1SLN06rW3RWLvGEohJyYjom3o1drfn6cfYGHF06PP1cisekOQeSdHcHOl2wpqPxK3H8fkqajMAEx27OAH8ASKtgQhi/68Tvrpa/D/xQtzvHZWbinxOcNg0JcR4tAefhXAY0OJ29XXzsdkOgT/dB2equUJuHw2xsRlLrBQ2mM6O7xVAIqgUU/3EUOoSwmGei5+hiDEdgBorbyYW7LhZWchy2ajPilxQ3veM5/2dCXI+BuQHEEloX81jTg/TLvStsHYW7ZzVN30bkqMEMZoIzmiqxdtxHMOco5Y5Gk6oUxwTr6Z9i23fgn6bhHHi66s/cA9Tc+Al6oEsfGJNVUv10cfKkd0oRDfHGvPgh6kvBTPDg4RPcIM7M6gc5fTkgqFhE78Gej0ej2U7ToLgL9iw8GP51g/bojzXVZF2VoS9q+jijuW+2D0+j7Rx5be5//OPdkrVJLSVdsiIAy+yN83v9jNAiCHXkPQo/TMadG1kBjTa5/Z6yZvbVr1qDXeqlW2wXbQyDKlAkryq/+ZDPz7VK34zA5HtDnjyXPipDK7wPNFDgcm0+nCq7GoE98DEBnfqMC7LpQGbNmbrQkkt13b3h6tPHjQkJ0uwUcK1IU6hQlJxflfOM= + POSTGRES_PASSWORD: AgCKszMjK5Fe3GHkQt+elcFsaU339N8+iQG5YmmaEjumoWo3rcmaa5H2cKxNJ/SVrzgqQcTS4WzE3lIwOWEQXWFP8N7DY6KnkjGMqWyAwNx0P1p/nT6xH7omJKSHp4U3CT9JV3n4BEga6wZFJnKOM/e0l8OT6p6Xg4YeLg08vZCV3NIgC4La2i8dug/rxqP9SfZuyFK5Hg/CDQtH1p0A10P1/4zSmqIHEs6ACDhXFMzMmGjJgTxOcpFt53NG6cCbEj4QnvIbmr0uiMtNVrD+/5PLxDCt9zrXzDwGCuMpThEZcEcpj8dhTvtPQkMJ7Uv/g6Jtd9Soh/Q1bznqjtN3X8eoEPW66ODM+uw0eyMum2NB3Oy93Ggw+sqI5/kqqI61tzs0W3U0uJKHTwWZBowQmjXatpPFRppwQe6jCrvBjdmMao0gyAGAYjbYdKGVSd9oL9fUqt+ME2s38PhEhZeyY6VCkdLrL1xvWPUrKKx363xaeQ097GzOwK78bfSe3vwkmKKbi9Q9AymVUTD2Z87BoLHdM3+D/UUD5WeP4AZgg0EEHlVJ1GUThWBIgMKFQz0NEM9sxeGlHivNpCgDdahDtg3LKFAmLwJP8gUutCh/08uiP82UN/ahLgqgkyGkNnHyaijlCRRU/R74dPuqtDZdavwwJjetMC5QBxBnbjz5MaV6MGGtmsJIKL3Ud6/1UaIFus+shszGmMjzJw== + POSTGRES_USER: AgCiRvh3pOVXEHl7Q+qgVS5eBLCicTxd70hpCuQ1YpTBKMApcR6i6IWsR0qvcNXmmZCBYNIufTfld/6vUjL69fC/Tfctx4Byh7Fo0/eZBGeZyTJyMDsrPQS2/jH526THB6fweSEGBY5/nSjfgzZ2swBwceE8ZygUz0ED8DaeXU73x0RQyZuBv0Tu3n8wDduQK7YQfDX4De8xKsC17r30zXOslwe4s5NCJ3h+KhD98fWiLi4In2yOnUwpbSh0nviLCtDx1m/DmdKbDJRsKnuZzMorHRzg5ediOESPzW9FDIEhfm/fdsz84cyB22KmTRm8XerPr//jauAHxAkzVEMNNEFwcvIsGoEqxIp0UnkravPh6qRvHdwBZGpxcQsNgc5BARl9RvrAXiN5iEb7uE5AsNeyVqvBbQJFsxvJw4uGwpqTm3bgxtrlbvmxiAn84QprVkLZ1S73zhzGw1SkypF1EfGk2Dj+FWwcOGJZjyrGj6DWO8K0vMhJdxb43MizqZrlbWRJXlUnLePi9X8D1uGzDh9gRdcEq+HmPrY5t89MohZVDyA9+9W4QugSRLfIRCDBSRhWVh9L2c7JpZoAWI7GkppJ0x9e29NTwVY1Ls091K1Sje7/KxUFbAj/jiTSQW28XSMT21ei24gtLIbsCdeRb7MgXFv+2uvhNBwTN3c3uX4QGFwy/ZNXrBywfhobruPuPdqDVpAupmQWaA== + template: + metadata: + annotations: + argocd.argoproj.io/sync-wave: "-3" + creationTimestamp: null + name: postgres-credentials + namespace: ahc From e8cc2fe33d07480bc71e1dbe89cbd4980d54b476 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:49:44 +0200 Subject: [PATCH 08/27] fix(k8s): stamp sync-wave on sealed secrets via kustomize patch --- kubernetes/README.md | 6 ++++++ kubernetes/overlays/home/kustomization.yaml | 12 ++++++++++++ .../overlays/minikube-argocd/kustomization.yaml | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/kubernetes/README.md b/kubernetes/README.md index 37044c5..f0f80a5 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -75,6 +75,12 @@ Get-Content kubernetes/overlays/minikube-local/secrets/app-secret.yaml | If the cluster is rebuilt, the controller key is gone: re-seal everything from the plain files. +Note: kubeseal copies the input Secret's annotations into `spec.template.metadata` (the future +unsealed Secret), not onto the SealedSecret object itself. The overlays therefore carry a kustomize +patch stamping `argocd.argoproj.io/sync-wave: "-3"` on every SealedSecret — without it they would +sync at wave 0, after the databases that need them (found the hard way during the minikube +rehearsal). + ## GitOps rehearsal on minikube (minikube-argocd) ```powershell diff --git a/kubernetes/overlays/home/kustomization.yaml b/kubernetes/overlays/home/kustomization.yaml index a5b715a..2a94630 100644 --- a/kubernetes/overlays/home/kustomization.yaml +++ b/kubernetes/overlays/home/kustomization.yaml @@ -25,6 +25,18 @@ patches: - path: ingress-patch.yaml - path: configmap-patch.yaml - path: celery-beat-patch.yaml + # kubeseal copies input Secret annotations into spec.template.metadata + # (the unsealed Secret), not onto the SealedSecret itself — without this + # patch SealedSecrets land in wave 0, after the databases that need them. + - target: + kind: SealedSecret + patch: |- + apiVersion: bitnami.com/v1alpha1 + kind: SealedSecret + metadata: + name: placeholder-overridden-by-target + annotations: + argocd.argoproj.io/sync-wave: "-3" images: - name: ghcr.io/cybernetic-ransomware/ahc-app diff --git a/kubernetes/overlays/minikube-argocd/kustomization.yaml b/kubernetes/overlays/minikube-argocd/kustomization.yaml index 2597cb8..5c34aa2 100644 --- a/kubernetes/overlays/minikube-argocd/kustomization.yaml +++ b/kubernetes/overlays/minikube-argocd/kustomization.yaml @@ -18,6 +18,18 @@ resources: patches: - path: ingress-patch.yaml + # kubeseal copies input Secret annotations into spec.template.metadata + # (the unsealed Secret), not onto the SealedSecret itself — without this + # patch SealedSecrets land in wave 0, after the databases that need them. + - target: + kind: SealedSecret + patch: |- + apiVersion: bitnami.com/v1alpha1 + kind: SealedSecret + metadata: + name: placeholder-overridden-by-target + annotations: + argocd.argoproj.io/sync-wave: "-3" images: - name: ghcr.io/cybernetic-ransomware/ahc-app From 60e1b6ea52e2bd04c489ffdba07c04ff4d3316db Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:27:11 +0200 Subject: [PATCH 09/27] fix(k8s): prepare app volumes for uid 1000 with fsGroup and init chown --- kubernetes/base/celery/deployment.yaml | 27 ++++++++++++++++++++++ kubernetes/base/web/deployment.yaml | 31 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/kubernetes/base/celery/deployment.yaml b/kubernetes/base/celery/deployment.yaml index a06992f..e7226e0 100644 --- a/kubernetes/base/celery/deployment.yaml +++ b/kubernetes/base/celery/deployment.yaml @@ -14,6 +14,33 @@ spec: labels: app: celery-worker spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + initContainers: + # Same rationale as in the web Deployment: the worker writes offline + # snapshots to the shared PVC as uid 1000, and the worker can start + # before the web pod has prepared the volume. + - name: prepare-storage + image: busybox:1.37 + securityContext: + runAsUser: 0 + command: + - sh + - -c + - | + set -eu + [ "$(stat -c %u /mnt/private)" = "1000" ] || chown -R 1000:1000 /mnt/private + volumeMounts: + - name: app-private + mountPath: /mnt/private + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "32Mi" + cpu: "50m" containers: - name: celery-worker image: ghcr.io/cybernetic-ransomware/ahc-app:prod diff --git a/kubernetes/base/web/deployment.yaml b/kubernetes/base/web/deployment.yaml index 1e46635..2cbedd4 100644 --- a/kubernetes/base/web/deployment.yaml +++ b/kubernetes/base/web/deployment.yaml @@ -14,7 +14,38 @@ spec: labels: app: ahc-web spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch initContainers: + # The app image runs as appuser (uid/gid 1000). hostPath-backed + # provisioners (minikube standard, k3s local-path) do not honor + # fsGroup, and kubelet creates subPath directories as root, so the + # volumes must be prepared explicitly before the app writes to them. + - name: prepare-storage + image: busybox:1.37 + securityContext: + runAsUser: 0 + command: + - sh + - -c + - | + set -eu + mkdir -p /mnt/media/animal_pic /mnt/media/attachments /mnt/media/profile_pics + [ "$(stat -c %u /mnt/media)" = "1000" ] || chown -R 1000:1000 /mnt/media + [ "$(stat -c %u /mnt/private)" = "1000" ] || chown -R 1000:1000 /mnt/private + volumeMounts: + - name: app-media + mountPath: /mnt/media + - name: app-private + mountPath: /mnt/private + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "32Mi" + cpu: "50m" - name: collectstatic image: ghcr.io/cybernetic-ransomware/ahc-app:prod command: ["python", "manage.py", "collectstatic", "--noinput"] From c2dac0b408cc6ec18db6022ca2da39f117a59c35 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:27:42 +0200 Subject: [PATCH 10/27] fix(k8s): guard namespace and data pvcs from prune, set backup timezone and sizing --- kubernetes/base/namespace.yaml | 4 ++++ kubernetes/base/storage/app-media-pvc.yaml | 1 + kubernetes/base/storage/app-private-pvc.yaml | 1 + kubernetes/overlays/home/backup/backup-pvc.yaml | 5 ++++- .../overlays/home/backup/couchdb-export-cronjob.yaml | 1 + kubernetes/overlays/home/backup/files-backup-cronjob.yaml | 1 + kubernetes/overlays/home/backup/pg-backup-cronjob.yaml | 1 + kubernetes/overlays/home/ingress-patch.yaml | 8 ++++++++ 8 files changed, 21 insertions(+), 1 deletion(-) diff --git a/kubernetes/base/namespace.yaml b/kubernetes/base/namespace.yaml index 408b1b5..2275e37 100644 --- a/kubernetes/base/namespace.yaml +++ b/kubernetes/base/namespace.yaml @@ -2,3 +2,7 @@ apiVersion: v1 kind: Namespace metadata: name: ahc + annotations: + # Deleting the namespace takes every PVC (including StatefulSet data) + # with it — require explicit confirmation in ArgoCD. + argocd.argoproj.io/sync-options: Prune=confirm,Delete=confirm diff --git a/kubernetes/base/storage/app-media-pvc.yaml b/kubernetes/base/storage/app-media-pvc.yaml index 57dec1d..e9c5eed 100644 --- a/kubernetes/base/storage/app-media-pvc.yaml +++ b/kubernetes/base/storage/app-media-pvc.yaml @@ -4,6 +4,7 @@ metadata: name: app-media-pvc annotations: argocd.argoproj.io/sync-wave: "-3" + argocd.argoproj.io/sync-options: Prune=false,Delete=false spec: accessModes: - ReadWriteOnce diff --git a/kubernetes/base/storage/app-private-pvc.yaml b/kubernetes/base/storage/app-private-pvc.yaml index 68b0709..33f099c 100644 --- a/kubernetes/base/storage/app-private-pvc.yaml +++ b/kubernetes/base/storage/app-private-pvc.yaml @@ -4,6 +4,7 @@ metadata: name: app-private-pvc annotations: argocd.argoproj.io/sync-wave: "-3" + argocd.argoproj.io/sync-options: Prune=false,Delete=false spec: accessModes: - ReadWriteOnce diff --git a/kubernetes/overlays/home/backup/backup-pvc.yaml b/kubernetes/overlays/home/backup/backup-pvc.yaml index b978310..3b2dcb3 100644 --- a/kubernetes/overlays/home/backup/backup-pvc.yaml +++ b/kubernetes/overlays/home/backup/backup-pvc.yaml @@ -4,9 +4,12 @@ metadata: name: backup-data annotations: argocd.argoproj.io/sync-wave: "-3" + argocd.argoproj.io/sync-options: Prune=false,Delete=false spec: accessModes: - ReadWriteOnce resources: requests: - storage: 5Gi + # 14-day retention of nightly pg dumps + couchdb exports + full tars + # of two PVCs adds up fast; size accordingly rather than optimistically. + storage: 20Gi diff --git a/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml b/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml index 7ee93d3..cae042a 100644 --- a/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml +++ b/kubernetes/overlays/home/backup/couchdb-export-cronjob.yaml @@ -8,6 +8,7 @@ spec: # one-way replication to a second CouchDB instance or a PVC snapshot — # see kubernetes/README.md. schedule: "30 3 * * *" + timeZone: "Europe/Warsaw" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 diff --git a/kubernetes/overlays/home/backup/files-backup-cronjob.yaml b/kubernetes/overlays/home/backup/files-backup-cronjob.yaml index 9babbc9..5445f5e 100644 --- a/kubernetes/overlays/home/backup/files-backup-cronjob.yaml +++ b/kubernetes/overlays/home/backup/files-backup-cronjob.yaml @@ -4,6 +4,7 @@ metadata: name: files-backup spec: schedule: "0 4 * * *" + timeZone: "Europe/Warsaw" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 diff --git a/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml b/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml index 1c2bfd2..3aaf4d0 100644 --- a/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml +++ b/kubernetes/overlays/home/backup/pg-backup-cronjob.yaml @@ -4,6 +4,7 @@ metadata: name: pg-backup spec: schedule: "0 3 * * *" + timeZone: "Europe/Warsaw" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 diff --git a/kubernetes/overlays/home/ingress-patch.yaml b/kubernetes/overlays/home/ingress-patch.yaml index 8b20f47..8db5d33 100644 --- a/kubernetes/overlays/home/ingress-patch.yaml +++ b/kubernetes/overlays/home/ingress-patch.yaml @@ -5,6 +5,14 @@ metadata: spec: # k3s ships Traefik as the default ingress controller. ingressClassName: traefik + # TODO(bootstrap): CSRF_TRUSTED_ORIGINS uses https://, so TLS must be + # terminated here. Uncomment once a certificate exists (cert-manager or a + # manually created ahc-tls Secret); until then Traefik serves its default + # self-signed certificate. + # tls: + # - hosts: + # - ahc.example.home + # secretName: ahc-tls rules: # TODO: replace with the real AHC_DOMAIN before the first home sync. - host: ahc.example.home From 913e5d9858a26f03b5f454bafeac7f0fe28acfc3 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:28:02 +0200 Subject: [PATCH 11/27] ci: validate manifests and allow branch image publish via workflow_dispatch --- .github/workflows/django.yml | 60 ++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 7434368..0404e84 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -5,6 +5,7 @@ on: branches: [ "main" ] pull_request: branches: [ "main" ] + workflow_dispatch: jobs: lint: @@ -83,11 +84,50 @@ jobs: - name: Run tests (pytest) run: uv run pytest -m "not slow" + manifests: + name: Kubernetes manifests + runs-on: ubuntu-latest + env: + KUSTOMIZE_VERSION: "5.8.1" + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install pinned kustomize + run: | + curl -fsSL "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" \ + | tar xz -C /usr/local/bin kustomize + kustomize version + + - name: Render base + run: kustomize build kubernetes/base > /tmp/base.yaml + + - name: Render ArgoCD test overlay + run: kustomize build kubernetes/overlays/minikube-argocd > /tmp/minikube-argocd.yaml + + # overlays/home joins once its sealed/ files exist (requires the home + # cluster's sealed-secrets controller key). + + - name: Assert sync-wave present on SealedSecrets + run: | + python3 - <<'EOF' + import sys, yaml + docs = list(yaml.safe_load_all(open("/tmp/minikube-argocd.yaml"))) + sealed = [d for d in docs if d and d.get("kind") == "SealedSecret"] + assert sealed, "no SealedSecrets rendered" + for d in sealed: + wave = d["metadata"].get("annotations", {}).get("argocd.argoproj.io/sync-wave") + assert wave == "-3", f"{d['metadata']['name']}: sync-wave={wave!r}, expected '-3'" + print(f"OK: {len(sealed)} SealedSecrets at wave -3") + EOF + publish: name: Publish app image to GHCR runs-on: ubuntu-latest - needs: [test, lint] - if: github.event_name == 'push' + needs: [test, lint, manifests] + # workflow_dispatch allows publishing a sha- tag from a feature branch + # (e.g. to test the branch on minikube before merge). + if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' permissions: contents: read packages: write @@ -105,21 +145,29 @@ jobs: - name: Set lowercase owner run: echo "OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV + - name: Set image tags + # The prod tag moves ONLY on main: Watchtower deploys whatever prod + # points at, so a branch dispatch must never touch it. + run: | + TAGS="ghcr.io/${OWNER}/ahc-app:sha-${{ github.sha }}" + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + TAGS="${TAGS},ghcr.io/${OWNER}/ahc-app:prod" + fi + echo "TAGS=${TAGS}" >> $GITHUB_ENV + - name: Build and push ahc-app uses: docker/build-push-action@v6 with: context: . file: docker/Dockerfile-app push: true - tags: | - ghcr.io/${{ env.OWNER }}/ahc-app:prod - ghcr.io/${{ env.OWNER }}/ahc-app:sha-${{ github.sha }} + tags: ${{ env.TAGS }} bump: name: Bump image tag in home overlay runs-on: ubuntu-latest needs: publish - if: github.event_name == 'push' + if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: write concurrency: From 426df5a6a47b01e6ec23344760dd520e994dcec0 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:28:29 +0200 Subject: [PATCH 12/27] feat(argocd): sync workflows via application and verify image tag before rollout --- argocd/ahc-workflows.yaml | 20 ++++++++++++++++++++ workflows/deploy-smoke-workflowtemplate.yaml | 16 ++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 argocd/ahc-workflows.yaml diff --git a/argocd/ahc-workflows.yaml b/argocd/ahc-workflows.yaml new file mode 100644 index 0000000..0173133 --- /dev/null +++ b/argocd/ahc-workflows.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ahc-workflows + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git + targetRevision: main + path: workflows + destination: + server: https://kubernetes.default.svc + namespace: argo + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/workflows/deploy-smoke-workflowtemplate.yaml b/workflows/deploy-smoke-workflowtemplate.yaml index c6b61ee..6780dec 100644 --- a/workflows/deploy-smoke-workflowtemplate.yaml +++ b/workflows/deploy-smoke-workflowtemplate.yaml @@ -13,11 +13,27 @@ spec: templates: - name: main steps: + - - name: verify-image + template: verify-image - - name: wait-rollout template: rollout-status - - name: smoke template: smoke-test + # Guard against blessing a stale deployment: after a CI-triggered submit + # ArgoCD may not have synced the bump commit yet, and rollout status of + # the previous revision would succeed. Wait until the Deployment actually + # references the expected image tag first. + - name: verify-image + container: + image: bitnami/kubectl:1.31 + command: [sh, -c] + args: + - >- + kubectl -n ahc wait deployment/ahc-app-backend + --for=jsonpath='{.spec.template.spec.containers[0].image}'=ghcr.io/cybernetic-ransomware/ahc-app:{{workflow.parameters.image-tag}} + --timeout=600s + - name: rollout-status container: image: bitnami/kubectl:1.31 From d515554252e936a41ee9a6d559732a284d788686 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:28:53 +0200 Subject: [PATCH 13/27] docs: record volume prep, prune guards and workflows wiring in adr and readme --- doc/13_adr_gitops_argocd.md | 9 ++++++++- kubernetes/README.md | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/doc/13_adr_gitops_argocd.md b/doc/13_adr_gitops_argocd.md index 166ce19..553be8e 100644 --- a/doc/13_adr_gitops_argocd.md +++ b/doc/13_adr_gitops_argocd.md @@ -48,7 +48,14 @@ single-node k3s cluster on the home server, with local minikube used for rehears 8. **CI → cluster reachability**: GitHub-hosted runners cannot reach the home server, so the deploy signal is ArgoCD polling git (option A). The `ahc-deploy-smoke` Argo WorkflowTemplate is committed for manual runs, and the CI submit step is gated on `vars.ARGO_SERVER_URL` (self-disabled until a - self-hosted runner or tunnel exists). + self-hosted runner or tunnel exists). The workflow first waits until the Deployment references + the expected `image-tag` — otherwise a CI-triggered run could bless a stale rollout that ArgoCD + has not synced yet. +9. **Data-loss guards**: the `ahc` Namespace carries `Prune=confirm,Delete=confirm` and the data + PVCs (`app-media-pvc`, `app-private-pvc`, `backup-data`) carry `Prune=false,Delete=false`, so + automated prune cannot silently remove them. The app image runs as uid 1000, and hostPath-backed + provisioners ignore `fsGroup`, so web/celery pods run a root `prepare-storage` initContainer + (mkdir + guarded chown) before the app touches the volumes. Rejected alternatives: Helm (kustomize already in place, no templating need), ArgoCD Image Updater (extra controller; CI commit is simpler and auditable), External Secrets Operator (requires an diff --git a/kubernetes/README.md b/kubernetes/README.md index f0f80a5..f88b24e 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -114,7 +114,8 @@ re-sync restores it; delete a Secret → controller re-creates it from the Seale `kustomize edit set image ...:sha-` to `overlays/home/` (`[skip ci]`, serialized by a concurrency group). ArgoCD polls `main` and syncs. The `deploy-workflow` job (Argo Workflows submit) stays disabled until `vars.ARGO_SERVER_URL` is set — requires a self-hosted runner or tunnel. -Manual workflow run: `kubectl apply -f workflows/` then +The `workflows/` directory is synced by the `ahc-workflows` Application (`argocd/ahc-workflows.yaml`); +without ArgoCD apply it manually with `kubectl apply -f workflows/`. Manual workflow run: `argo submit --from workflowtemplate/ahc-deploy-smoke -n argo`. If GHCR packages are private, add a `dockerconfigjson` pull secret (sealed for home) and From 17fd3320b5be9d5e696ea34a709adf2f808197e5 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:36:52 +0200 Subject: [PATCH 14/27] fix(workflows): grant executor rbac for workflowtaskresults --- workflows/rbac.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/workflows/rbac.yaml b/workflows/rbac.yaml index b0a2ff9..0e5ffe2 100644 --- a/workflows/rbac.yaml +++ b/workflows/rbac.yaml @@ -4,6 +4,33 @@ metadata: name: ahc-deployer namespace: argo --- +# Minimum executor permissions required by Argo Workflows >= 3.4: every +# workflow pod reports step results as workflowtaskresults in the workflow +# namespace, regardless of what the steps themselves do. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ahc-workflow-executor + namespace: argo +rules: + - apiGroups: ["argoproj.io"] + resources: ["workflowtaskresults"] + verbs: ["create", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ahc-workflow-executor + namespace: argo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ahc-workflow-executor +subjects: + - kind: ServiceAccount + name: ahc-deployer + namespace: argo +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: From f7f02587f60446132c38a4c821ed315db6828512 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:37:32 +0200 Subject: [PATCH 15/27] fix(k8s): chown storage unconditionally in prepare-storage init --- kubernetes/base/celery/deployment.yaml | 2 +- kubernetes/base/web/deployment.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/kubernetes/base/celery/deployment.yaml b/kubernetes/base/celery/deployment.yaml index e7226e0..1dbd59c 100644 --- a/kubernetes/base/celery/deployment.yaml +++ b/kubernetes/base/celery/deployment.yaml @@ -30,7 +30,7 @@ spec: - -c - | set -eu - [ "$(stat -c %u /mnt/private)" = "1000" ] || chown -R 1000:1000 /mnt/private + chown -R 1000:1000 /mnt/private volumeMounts: - name: app-private mountPath: /mnt/private diff --git a/kubernetes/base/web/deployment.yaml b/kubernetes/base/web/deployment.yaml index 2cbedd4..49941c2 100644 --- a/kubernetes/base/web/deployment.yaml +++ b/kubernetes/base/web/deployment.yaml @@ -32,8 +32,7 @@ spec: - | set -eu mkdir -p /mnt/media/animal_pic /mnt/media/attachments /mnt/media/profile_pics - [ "$(stat -c %u /mnt/media)" = "1000" ] || chown -R 1000:1000 /mnt/media - [ "$(stat -c %u /mnt/private)" = "1000" ] || chown -R 1000:1000 /mnt/private + chown -R 1000:1000 /mnt/media /mnt/private volumeMounts: - name: app-media mountPath: /mnt/media From c196ae250d63ff2d74293f8153dd6733da4e04be Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:38:05 +0200 Subject: [PATCH 16/27] docs(k8s): pin minikube-argocd image to branch sha and document the path --- kubernetes/README.md | 13 +++++++++++++ .../overlays/minikube-argocd/kustomization.yaml | 4 +++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/kubernetes/README.md b/kubernetes/README.md index f88b24e..f7fa3c3 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -95,6 +95,19 @@ Sync manually from the ArgoCD UI/CLI and verify waves and hooks. Before enabling home, pass the drift tests: `kubectl edit` a resource → diff shows drift; delete a Deployment → re-sync restores it; delete a Secret → controller re-creates it from the SealedSecret. +**Testing branch code through ArgoCD (the honest path):** manifests come from the branch, but +`newTag: prod` points at an image built from an older `main` — a hybrid test proves nothing. +Publish and pin the branch image first: + +1. `gh workflow run "AHC CI|CD" --ref ` (workflow_dispatch publishes `sha-` + without moving `prod`). +2. Wait for the publish job, note the `sha-` tag. +3. Set it in `overlays/minikube-argocd/kustomization.yaml` (`newTag: sha-`). +4. Commit and push to the branch — ArgoCD only sees what is in git. + +Revert the pinned tag before merging. The `minikube image build` + `newTag: dev` shortcut from the +local-run section only applies to `minikube-local` (kubectl path), not to ArgoCD. + ## Home cluster bootstrap (first time) 1. Install k3s (keeps default Traefik ingress and local-path default StorageClass). diff --git a/kubernetes/overlays/minikube-argocd/kustomization.yaml b/kubernetes/overlays/minikube-argocd/kustomization.yaml index 5c34aa2..b8e3f0d 100644 --- a/kubernetes/overlays/minikube-argocd/kustomization.yaml +++ b/kubernetes/overlays/minikube-argocd/kustomization.yaml @@ -33,4 +33,6 @@ patches: images: - name: ghcr.io/cybernetic-ransomware/ahc-app - newTag: prod + # Pinned to the branch image for the ArgoCD rehearsal (see README, + # "Testing branch code through ArgoCD"). Revert to prod before merging. + newTag: sha-d515554252e936a41ee9a6d559732a284d788686 From 65c5e72003f020206d1ae8a7b4706fd40d96bd4a Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:54:14 +0200 Subject: [PATCH 17/27] docs: refresh k8s deploy section and add rehearsal learnings to runbook --- README.md | 13 +++++++++---- kubernetes/README.md | 21 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2283c45..b2276de 100644 --- a/README.md +++ b/README.md @@ -81,11 +81,16 @@ The stack exposes: Django app on `:8000`, Flower (Celery monitor) on `:5555`. uv run python manage.py runserver ``` -### Kubernetes Deploy (alternative) +### Kubernetes Deploy (GitOps via ArgoCD) -An alternative to Docker Compose for production-like environments. -See [`kubernetes/`](kubernetes/) for kustomization files and secret templates. -Build and load images, then apply with `kubectl apply -k kubernetes/`. +Kubernetes manifests live in `kubernetes/` as a kustomize base with three overlays +(`minikube-local` for a plain `kubectl apply -k` dev loop, `minikube-argocd` for the GitOps +rehearsal, `home` for the production k3s cluster synced by ArgoCD). Images are pulled from +GHCR; the deployed tag is committed to git by CI. + +- Operational runbook (local run, sealing secrets, rehearsal, home bootstrap, backups): + [`kubernetes/README.md`](kubernetes/README.md) +- Architecture decision record: [`doc/13_adr_gitops_argocd.md`](doc/13_adr_gitops_argocd.md) ## Testing diff --git a/kubernetes/README.md b/kubernetes/README.md index f7fa3c3..fa722f4 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -87,10 +87,26 @@ rehearsal). kubectl apply -f https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.27.3/controller.yaml # seal the four secrets into overlays/minikube-argocd/sealed/ (see above), commit them kubectl create ns argocd -kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml +# --server-side is required: the applicationsets CRD exceeds the 256 KiB +# last-applied-configuration annotation limit of client-side apply. +kubectl apply -n argocd --server-side -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml kubectl apply -f argocd/ahc-minikube-test.yaml ``` +Triggering a sync without the argocd CLI (equivalent of the Sync button): + +```powershell +kubectl -n argocd patch application ahc-minikube-test --type merge -p '{"operation":{"initiatedBy":{"username":"manual"},"sync":{"revision":""}}}' +``` + +A sync stuck waiting on resources that can never become healthy will not stop when the +`operation` field is removed — terminate it explicitly (the Application CRD has no status +subresource, so this patches the main object): + +```powershell +kubectl -n argocd patch application ahc-minikube-test --type merge -p '{"status":{"operationState":{"phase":"Terminating"}}}' +``` + Sync manually from the ArgoCD UI/CLI and verify waves and hooks. Before enabling automation on home, pass the drift tests: `kubectl edit` a resource → diff shows drift; delete a Deployment → re-sync restores it; delete a Secret → controller re-creates it from the SealedSecret. @@ -111,7 +127,8 @@ local-run section only applies to `minikube-local` (kubectl path), not to ArgoCD ## Home cluster bootstrap (first time) 1. Install k3s (keeps default Traefik ingress and local-path default StorageClass). -2. Install ArgoCD (`kubectl apply -n argocd -f .../install.yaml`). +2. Install ArgoCD (`kubectl apply -n argocd --server-side -f .../install.yaml` — see the + rehearsal section for why `--server-side`). 3. `kubectl apply -f argocd/sealed-secrets.yaml` and wait for the controller. 4. Seal the four secrets against the home cluster into `overlays/home/sealed/`, commit to `main`. 5. Replace `ahc.example.home` in `overlays/home/{ingress-patch,configmap-patch}.yaml` with the real From 4f62f3d3e8b7281f4732eb210fd1db09526607f7 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:59:52 +0200 Subject: [PATCH 18/27] docs: pin argocd version and mark status patch as emergency workaround --- kubernetes/README.md | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/kubernetes/README.md b/kubernetes/README.md index fa722f4..2e82f71 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -86,22 +86,28 @@ rehearsal). ```powershell kubectl apply -f https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.27.3/controller.yaml # seal the four secrets into overlays/minikube-argocd/sealed/ (see above), commit them -kubectl create ns argocd +kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - +# Pinned to the exact version this runbook was rehearsed with — a floating +# "stable" URL would change behavior without any change in this repo. # --server-side is required: the applicationsets CRD exceeds the 256 KiB # last-applied-configuration annotation limit of client-side apply. -kubectl apply -n argocd --server-side -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml +$ArgoCdVersion = "v3.4.5" +kubectl apply -n argocd --server-side --force-conflicts -f "https://raw.githubusercontent.com/argoproj/argo-cd/$ArgoCdVersion/manifests/install.yaml" kubectl apply -f argocd/ahc-minikube-test.yaml ``` -Triggering a sync without the argocd CLI (equivalent of the Sync button): +Triggering a sync without the argocd CLI (equivalent of the Sync button; setting the +`operation` field is the documented kubectl-level interface): ```powershell -kubectl -n argocd patch application ahc-minikube-test --type merge -p '{"operation":{"initiatedBy":{"username":"manual"},"sync":{"revision":""}}}' +kubectl -n argocd patch application ahc-minikube-test --type merge -p '{"operation":{"initiatedBy":{"username":"manual"},"sync":{"revision":"","syncStrategy":{"hook":{}}}}}' ``` A sync stuck waiting on resources that can never become healthy will not stop when the -`operation` field is removed — terminate it explicitly (the Application CRD has no status -subresource, so this patches the main object): +`operation` field is removed. The preferred method is `argocd app terminate-op +ahc-minikube-test`. Without the argocd CLI there is an **emergency workaround**, tested on the +pinned version above — note it reaches into the controller's internal state, not a public +interface (the Application CRD has no status subresource, so it patches the main object): ```powershell kubectl -n argocd patch application ahc-minikube-test --type merge -p '{"status":{"operationState":{"phase":"Terminating"}}}' @@ -127,8 +133,8 @@ local-run section only applies to `minikube-local` (kubectl path), not to ArgoCD ## Home cluster bootstrap (first time) 1. Install k3s (keeps default Traefik ingress and local-path default StorageClass). -2. Install ArgoCD (`kubectl apply -n argocd --server-side -f .../install.yaml` — see the - rehearsal section for why `--server-side`). +2. Install ArgoCD pinned to the rehearsed version (see the rehearsal section for the exact + command, the version variable and why `--server-side`). 3. `kubectl apply -f argocd/sealed-secrets.yaml` and wait for the controller. 4. Seal the four secrets against the home cluster into `overlays/home/sealed/`, commit to `main`. 5. Replace `ahc.example.home` in `overlays/home/{ingress-patch,configmap-patch}.yaml` with the real From 3b3bb6ea3d784758fd10fc3a21f73d5f44c48b27 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:06:41 +0200 Subject: [PATCH 19/27] build(k8s): AWS deployment --- README.md | 12 +- argocd/ahc-aws.yaml | 22 +++ doc/14_adr_aws_eks_deployment.md | 133 +++++++++++++ kubernetes/README-aws.md | 180 ++++++++++++++++++ kubernetes/README.md | 1 + kubernetes/overlays/aws/configmap-patch.yaml | 10 + kubernetes/overlays/aws/ingress-patch.yaml | 33 ++++ kubernetes/overlays/aws/kustomization.yaml | 48 +++++ kubernetes/overlays/aws/smoke-test-job.yaml | 30 +++ kubernetes/overlays/aws/storageclass-gp3.yaml | 13 ++ terraform/aws/.gitignore | 16 ++ terraform/aws/dns.tf | 38 ++++ terraform/aws/eks.tf | 45 +++++ terraform/aws/irsa.tf | 35 ++++ terraform/aws/outputs.tf | 28 +++ terraform/aws/providers.tf | 11 ++ terraform/aws/terraform.tfvars.example | 17 ++ terraform/aws/variables.tf | 59 ++++++ terraform/aws/versions.tf | 16 ++ terraform/aws/vpc.tf | 33 ++++ 20 files changed, 776 insertions(+), 4 deletions(-) create mode 100644 argocd/ahc-aws.yaml create mode 100644 doc/14_adr_aws_eks_deployment.md create mode 100644 kubernetes/README-aws.md create mode 100644 kubernetes/overlays/aws/configmap-patch.yaml create mode 100644 kubernetes/overlays/aws/ingress-patch.yaml create mode 100644 kubernetes/overlays/aws/kustomization.yaml create mode 100644 kubernetes/overlays/aws/smoke-test-job.yaml create mode 100644 kubernetes/overlays/aws/storageclass-gp3.yaml create mode 100644 terraform/aws/.gitignore create mode 100644 terraform/aws/dns.tf create mode 100644 terraform/aws/eks.tf create mode 100644 terraform/aws/irsa.tf create mode 100644 terraform/aws/outputs.tf create mode 100644 terraform/aws/providers.tf create mode 100644 terraform/aws/terraform.tfvars.example create mode 100644 terraform/aws/variables.tf create mode 100644 terraform/aws/versions.tf create mode 100644 terraform/aws/vpc.tf diff --git a/README.md b/README.md index b2276de..97b185c 100644 --- a/README.md +++ b/README.md @@ -83,14 +83,18 @@ The stack exposes: Django app on `:8000`, Flower (Celery monitor) on `:5555`. ### Kubernetes Deploy (GitOps via ArgoCD) -Kubernetes manifests live in `kubernetes/` as a kustomize base with three overlays +Kubernetes manifests live in `kubernetes/` as a kustomize base with four overlays (`minikube-local` for a plain `kubectl apply -k` dev loop, `minikube-argocd` for the GitOps -rehearsal, `home` for the production k3s cluster synced by ArgoCD). Images are pulled from -GHCR; the deployed tag is committed to git by CI. +rehearsal, `home` for the production k3s cluster synced by ArgoCD, `aws` for an EKS demo target +provisioned by Terraform). Images are pulled from GHCR; the deployed tag for `home` is committed +to git by CI, other overlays are bumped manually. - Operational runbook (local run, sealing secrets, rehearsal, home bootstrap, backups): [`kubernetes/README.md`](kubernetes/README.md) -- Architecture decision record: [`doc/13_adr_gitops_argocd.md`](doc/13_adr_gitops_argocd.md) +- AWS EKS runbook (Terraform bootstrap, cost, **mandatory teardown**): + [`kubernetes/README-aws.md`](kubernetes/README-aws.md) +- Architecture decision records: [`doc/13_adr_gitops_argocd.md`](doc/13_adr_gitops_argocd.md), + [`doc/14_adr_aws_eks_deployment.md`](doc/14_adr_aws_eks_deployment.md) ## Testing diff --git a/argocd/ahc-aws.yaml b/argocd/ahc-aws.yaml new file mode 100644 index 0000000..7a9108d --- /dev/null +++ b/argocd/ahc-aws.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ahc-aws + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git + targetRevision: main + path: kubernetes/overlays/aws + destination: + server: https://kubernetes.default.svc + namespace: ahc + # Deliberately no automated sync policy: this is a new cluster type (EKS + + # ALB Controller + EBS CSI) with its own untested failure modes. Syncs are + # manual until a full drift/prune/sealed-secret-recovery pass has been + # done here, same bar ahc-home was held to before it got automation + # (see argocd/ahc-minikube-test.yaml for the precedent). + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/doc/14_adr_aws_eks_deployment.md b/doc/14_adr_aws_eks_deployment.md new file mode 100644 index 0000000..11a3dbb --- /dev/null +++ b/doc/14_adr_aws_eks_deployment.md @@ -0,0 +1,133 @@ +## AWS EKS deployment + +### Date: +`2026-07-25` + +### Status +Proposed + +### Context +ADR-13 covers two GitOps targets — local minikube and the home k3s cluster — both assuming a +cluster already exists (minikube runs standalone; k3s is bootstrapped once on the home server +outside this repo). This adds a fourth, cloud-hosted target for demo/portfolio purposes. Unlike +the first two, the cluster itself does not pre-exist and must be provisioned as part of the +deploy story — the first Infrastructure-as-Code layer in this repository. It also needs a cloud +load balancer with real TLS and dynamic block storage, neither of which the home k3s/minikube +overlays had to solve (k3s ships Traefik and local-path storage out of the box). + +### Decision +1. **EKS** (AWS-managed control plane) over self-managed k3s-on-EC2 — offloads control-plane + operations for a target that is not meant to run continuously; the managed control plane is the + part of a from-scratch cluster least worth hand-rolling for a demo. +2. **Terraform** (`terraform/aws/`) using the community modules `terraform-aws-modules/vpc/aws` + and `terraform-aws-modules/eks/aws`, not hand-rolled resources. Correct subnet tagging for the + ALB Controller's auto-discovery and OIDC provider wiring are exactly the boilerplate these + modules exist to get right; re-deriving that from scratch would not teach anything the module + source doesn't already show. **Local Terraform state**, git-ignored, is deliberate: a single + operator and an ephemeral cluster (spun up per demo, torn down after) don't benefit from an S3 + backend, which would add its own bootstrap problem (something has to create the state bucket + before Terraform can use it) for no gain at this scale. +3. **Terraform's scope is AWS-API-only**: VPC, EKS control plane, a managed node group (EC2, not + Fargate — Fargate cannot back a StatefulSet with an EBS volume, and both Postgres and CouchDB + are StatefulSets), IRSA IAM roles/policies, and the EKS-native addons (`vpc-cni`, `coredns`, + `kube-proxy`, `aws-ebs-csi-driver`, the last via its own IRSA role). No `kubernetes`/`helm` + Terraform providers anywhere: configuring one needs the cluster's endpoint, which does not + exist until `module.eks` has already applied — the classic "provider depends on a resource from + the same apply" problem. This also keeps a clean boundary already implicit in ADR-13: Terraform + owns AWS account resources, kustomize/ArgoCD own cluster-internal state. +4. **AWS Load Balancer Controller installed manually via Helm**, not Terraform-managed — the one + deliberate asymmetry against decision 3 (EBS CSI is an EKS addon; ALB Controller has no addon + equivalent). Consistent with ADR-13 already treating ArgoCD's own install as a manual, + version-pinned `kubectl apply --server-side` step rather than something IaC manages. +5. **ALB + ACM** for ingress/TLS (`ingressClassName: alb`, `alb.ingress.kubernetes.io/*` + annotations) over ingress-nginx + cert-manager. Native AWS integration, ACM handles certificate + renewal automatically, and it avoids running an extra ingress controller plus cert-manager pair + on a small demo node group. IAM policies are attached via the module's + `attach_load_balancer_controller_policy` / `attach_ebs_csi_policy` flags rather than vendoring + AWS's published JSON policy documents — the module tracks upstream policy changes across its + own releases. +6. **`gp3` StorageClass** shipped as a plain kustomize resource inside `overlays/aws` (wave `-3`, + `is-default-class: "true"`), not a per-PVC `storageClassName` patch and not Terraform-managed. + No PVC or `volumeClaimTemplate` in `kubernetes/base` sets `storageClassName` — this keeps base + fully cluster-agnostic, exactly like the existing overlays that rely on their own cluster's + default class. +7. **Sealed Secrets reused** for this fourth cluster (not External Secrets Operator) — the same + 4-secret set (`ahc-app-secrets`, `postgres-credentials`, `couchdb-credentials`, + `flower-secrets`) and the same `kubeseal` workflow already documented for minikube/home. + Consistency with a pattern the project is already invested in outweighs ESO's AWS-native + advantages for a single additional cluster. +8. **`prepare-storage` initContainer left unpatched** in the `web`/`celery` Deployments. It exists + (per ADR-13) because hostPath-backed provisioners (minikube, k3s local-path) ignore `fsGroup`. + The EBS CSI driver *does* honor `fsGroup` with `fsGroupChangePolicy: OnRootMismatch` (already + set), so on EKS `chown -R 1000:1000` runs against an already-correctly-owned volume — a fast, + harmless no-op. Removing it would need a JSON-patch deleting an initContainer array element from + a shared base Deployment, which is more overlay complexity than the no-op it would avoid, and it + would make the pod spec structurally diverge between overlays. +9. **`argocd/ahc-aws.yaml` starts manual-sync-only** (no `automated:` block), mirroring the + precedent `ahc-minikube-test` set for `ahc-home`: a new, unrehearsed cluster type gets observed + step by step before any automation, promoted only after a drift/prune/secret-recovery pass. +10. **Not wired into CI's `bump`/`publish` flow.** `django.yml`'s `bump` job stays scoped to + `overlays/home` only. Automating deploys to a cluster that is intermittently created and + destroyed is actively wasteful and risky — a bump commit landing while the cluster doesn't + exist just means a future `terraform apply` and first sync silently pick up whatever tag + happens to be pinned. The image tag for `overlays/aws` is bumped manually + (`kustomize edit set image`) before each demo, the same way `overlays/minikube-argocd` is + pinned manually via `gh workflow run ... --ref ` today. For the same reason, the CI + `manifests` job does not render `overlays/aws` yet — it already skips `overlays/home` today + for the identical reason (`sealed/` doesn't exist until a real cluster has sealed it); adding + the aws render is a follow-up once its `sealed/` exists. +11. **No backup CronJobs on this overlay.** `overlays/home`'s `pg-backup`/`couchdb-export`/ + `files-backup` protect a long-running cluster; this cluster is explicitly spin-up/tear-down — + `terraform destroy` removes any backup PVC along with everything else, so a same-cluster backup + provides no real protection here. +12. **Cost-conscious defaults**: single managed node group pinned to one AZ (control-plane ENIs + still span two AZs — an EKS requirement), one NAT gateway (not one per AZ), 2× `t3.medium` + on-demand (not Spot — StatefulSets plus a short-lived demo aren't worth interruption/draining + complexity). Not production sizing; a starting point for a bump if pods fail to schedule. + +Rejected alternatives: Fargate (cannot back a StatefulSet with an EBS-backed PVC — no CSI node +DaemonSet support); self-managed k3s-on-EC2 (EKS chosen for its managed control plane); External +Secrets Operator + Secrets Manager (Sealed Secrets kept for consistency, decision 7); +ingress-nginx + cert-manager (ALB + ACM chosen, decision 5); Terraform-managed Helm release for the +ALB Controller (kept manual, decision 4); S3 remote state (local state, deliberate, decision 2). + +### Consequences +- **Real, ongoing AWS cost while the cluster exists** — the only target in this project with a + recurring bill. Rough all-in estimate if left running continuously: ~$73/mo EKS control plane + + ~$60/mo (2× `t3.medium` on-demand) + ~$33/mo single NAT gateway + ~$16/mo ALB base + LCU charges + + small EBS/data-transfer costs — roughly $180–200/month. `kubernetes/README-aws.md`'s teardown + section is required reading, not optional, and setting an AWS Budget/billing alarm manually + before the first `apply` is strongly recommended since Terraform cannot guarantee anyone + remembers to destroy the cluster on schedule. +- **Teardown discipline is mandatory**: the ALB and EBS volumes are AWS resources created by + controllers running *inside* the cluster, invisible to Terraform state. Destroying the VPC/EKS + cluster out from under them orphans both — they keep billing with nothing left able to delete + them through the normal path. The runbook requires deleting the Ingress/PVCs (or letting ArgoCD + prune) *before* `terraform destroy`, plus a post-destroy check via `aws elbv2`/`aws ec2` for + orphans. +- ACM + Route53 need a real, owned domain for DNS validation. Ships with `manage_dns = false` and + a placeholder host (`ahc.example-aws-placeholder.com`) by default — `dns.tf`'s resources stay + inert until both a domain and `manage_dns = true` are supplied. +- No backups means data loss on teardown is expected and accepted for this target — it is a + demo/rehearsal environment, not a second production system. +- CI never deploys here automatically; every rollout is a deliberate manual act — slower than + `overlays/home`'s auto-sync, which is the point. + +### Keywords +- EKS, +- Terraform, +- IRSA, +- AWS Load Balancer Controller, +- ACM, +- EBS CSI, +- gp3, +- Sealed Secrets, +- cost-conscious IaC. + +### Links +- [ADR-13](13_adr_gitops_argocd.md) — GitOps/ArgoCD baseline this extends. +- [kubernetes/README-aws.md](../kubernetes/README-aws.md) — bootstrap and teardown runbook. +- [terraform-aws-modules/eks/aws](https://github.com/terraform-aws-modules/terraform-aws-eks). +- [terraform-aws-modules/vpc/aws](https://github.com/terraform-aws-modules/terraform-aws-vpc). +- [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/). +- [Amazon EBS CSI driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver). diff --git a/kubernetes/README-aws.md b/kubernetes/README-aws.md new file mode 100644 index 0000000..9b891b3 --- /dev/null +++ b/kubernetes/README-aws.md @@ -0,0 +1,180 @@ +# AWS EKS deployment — ops runbook + +See [ADR-14](../doc/14_adr_aws_eks_deployment.md) for the decisions behind this target and +[`kubernetes/README.md`](README.md) for the other (free) deployment targets. + +⚠️ **Unlike every other target in this repo, this one costs real money while running.** Read +"Cost" and "Teardown" before running `terraform apply`. + +## Layout + +| Path | Purpose | +|---|---| +| `terraform/aws/` | Provisions the EKS cluster itself: VPC, control plane, managed node group, IRSA roles, EKS addons | +| `kubernetes/overlays/aws/` | Kustomize overlay: ALB ingress, gp3 StorageClass, SealedSecrets for this cluster | +| `argocd/ahc-aws.yaml` | ArgoCD Application, manual sync only (new cluster type, unrehearsed) | + +Terraform's scope is AWS-API-only (VPC, EKS, node group, IRSA, EKS-native addons). It does **not** +install the AWS Load Balancer Controller or ArgoCD — both are manual, version-pinned steps below, +for the same reason ArgoCD's own install is manual in the other runbook (see ADR-14, decision 4). + +## Cost + +Rough estimate if left running continuously: ~$73/mo EKS control plane + ~$60/mo (2× +`t3.medium` on-demand) + ~$33/mo single NAT gateway + ~$16/mo ALB base + LCU charges + small +EBS/data-transfer costs — **roughly $180–200/month**. Set an AWS Budget/billing alarm manually in +the console before your first `apply`; Terraform cannot guarantee anyone remembers to destroy the +cluster on schedule. + +## Prerequisites + +- AWS account and credentials configured locally (`aws configure` or an SSO profile). +- Terraform >= 1.7, `kubectl`, `helm`, `kubeseal` (see `kubernetes/README.md`'s "Sealing secrets" + section for how to get `kubeseal` on Windows). +- Optional: a real, owned domain if you want a working ALB certificate via ACM/Route53. Without + one, leave `manage_dns = false` (the default) — the overlay ships with a placeholder host that + will never get a valid ALB, which is a safe, documented resting state. + +## Bootstrap (first time) + +```powershell +# 1. Provision AWS infrastructure +cd terraform/aws +Copy-Item terraform.tfvars.example terraform.tfvars # edit region/cluster_name/domain as needed +terraform init +terraform plan -out=tfplan +terraform apply tfplan + +# 2. Point kubectl at the new cluster +terraform output -raw configure_kubectl | Invoke-Expression +kubectl get nodes + +# 3. Install the AWS Load Balancer Controller (manual Helm — see ADR-14 decision 4 for why) +$AlbRoleArn = terraform output -raw alb_controller_role_arn +$ClusterName = terraform output -raw cluster_name +$Region = terraform output -raw region +$VpcId = terraform output -raw vpc_id +helm repo add eks https://aws.github.io/eks-charts +helm repo update +helm install aws-load-balancer-controller eks/aws-load-balancer-controller ` + -n kube-system ` + --set clusterName=$ClusterName ` + --set serviceAccount.create=true ` + --set serviceAccount.name=aws-load-balancer-controller ` + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=$AlbRoleArn ` + --set region=$Region ` + --set vpcId=$VpcId +kubectl -n kube-system rollout status deploy/aws-load-balancer-controller + +# 4. Verify the EBS CSI driver is healthy (installed as a Terraform-managed EKS addon) +kubectl -n kube-system get pods -l app=ebs-csi-controller +# gp3 becomes default once the aws overlay's first sync applies storageclass-gp3.yaml (step 8); +# until then this shows only whatever EKS shipped by default (commonly gp2). +kubectl get storageclass + +# 5. Install ArgoCD — same pinned version/flags as the minikube/home rehearsal +# (deliberately NO Ingress for ArgoCD itself: reach the UI via port-forward, +# sidestepping the ALB Controller/ArgoCD chicken-and-egg entirely) +kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - +$ArgoCdVersion = "v3.4.5" # keep in sync with kubernetes/README.md's pinned version +kubectl apply -n argocd --server-side --force-conflicts -f "https://raw.githubusercontent.com/argoproj/argo-cd/$ArgoCdVersion/manifests/install.yaml" +kubectl -n argocd port-forward svc/argocd-server 8080:443 # separate terminal + +# 6. Bootstrap sealed-secrets, then seal the 4 secrets for THIS cluster +kubectl apply -f argocd/sealed-secrets.yaml +kubectl -n kube-system rollout status deploy/sealed-secrets-controller +kubeseal --controller-namespace kube-system --fetch-cert > aws.pem +Get-Content kubernetes/overlays/minikube-local/secrets/app-secret.yaml | + kubeseal --cert aws.pem --format yaml | + Set-Content kubernetes/overlays/aws/sealed/app-sealedsecret.yaml +# repeat for postgres-secret, couchdb-secret, flower-secret +``` + +## Sealing secrets for aws + +Same 4-secret set and workflow as `kubernetes/README.md`'s "Sealing secrets" section +(`ahc-app-secrets`, `postgres-credentials`, `couchdb-credentials`, `flower-secrets`), sealed +against this cluster's controller key into `kubernetes/overlays/aws/sealed/`. `SealedSecret`s are +bound to the private key of the controller that encrypted them — a file sealed for minikube or +home will not decrypt on this cluster, even with an identical name and namespace. + +```powershell +# 7. Fill in the real domain + ACM certificate ARN (or leave the placeholder +# and accept the ALB will fail to provision a listener until a real +# certificate exists — see kubernetes/overlays/aws/ingress-patch.yaml) +# edit kubernetes/overlays/aws/{ingress-patch.yaml,configmap-patch.yaml} + +git add kubernetes/overlays/aws/sealed kubernetes/overlays/aws/ingress-patch.yaml kubernetes/overlays/aws/configmap-patch.yaml +git commit -m "feat(k8s): seal aws overlay secrets and set real domain" +git push +``` + +## First sync + +```powershell +# 8. Apply the Application and trigger a manual sync (no argocd CLI needed, +# same technique as the minikube rehearsal) +kubectl apply -f argocd/ahc-aws.yaml +kubectl -n argocd patch application ahc-aws --type merge -p '{"operation":{"initiatedBy":{"username":"manual"},"sync":{"revision":"main","syncStrategy":{"hook":{}}}}}' + +# A sync stuck on resources that can never become healthy will not stop when +# the operation field is removed. Preferred: `argocd app terminate-op ahc-aws`. +# Emergency workaround if the argocd CLI isn't available (reaches into +# controller-internal state — see kubernetes/README.md's troubleshooting note): +# kubectl -n argocd patch application ahc-aws --type merge -p '{"status":{"operationState":{"phase":"Terminating"}}}' + +# 9. Smoke test +kubectl -n ahc get ingress ingress-service # note the ALB hostname once provisioned +curl.exe -f https:///livez +curl.exe -f https:///readyz +``` + +## Manual image bump + +Not wired into CI (see ADR-14, decision 10) — bump the tag by hand before each demo: + +```powershell +cd kubernetes/overlays/aws +kustomize edit set image "ghcr.io//ahc-app=ghcr.io//ahc-app:sha-" +git add kustomization.yaml +git commit -m "chore(deploy): bump aws overlay to sha-" +git push +# then repeat the manual sync command from "First sync" step 8 +``` + +## Teardown + +**Read this before running `terraform destroy`.** Terraform has no idea an `Ingress` provisioned +an ALB, or that a `PersistentVolumeClaim` provisioned an EBS volume — those are AWS resources +created by controllers running *inside* the cluster, outside Terraform's state. Destroying the +VPC/EKS cluster out from under them orphans the ALB and EBS volumes: they keep billing +indefinitely with nothing left able to delete them through the normal path. + +```powershell +# 1. Let ArgoCD/kubectl clean up cluster-created AWS resources FIRST, while +# the ALB Controller and EBS CSI driver are still alive to process the +# deletions (this is the step that actually removes the ALB and EBS +# volumes, via their controllers' finalizers) +kubectl delete -f argocd/ahc-aws.yaml +kubectl -n ahc delete ingress ingress-service --ignore-not-found +kubectl -n ahc delete pvc --all +kubectl -n ahc get ingress,pvc # confirm empty before proceeding + +# 2. THEN tear down the infrastructure +cd terraform/aws +terraform destroy + +# 3. Verify no orphans (belt-and-suspenders — check even after step 1) +aws elbv2 describe-load-balancers --region | findstr +aws ec2 describe-volumes --region --filters "Name=status,Values=available" +``` + +If step 1 is skipped, or the cluster is already gone before it runs, the ALB and EBS volumes must +be deleted manually via the AWS console/CLI — they will not disappear on their own and will +continue billing. + +## Troubleshooting + +See `kubernetes/README.md`'s "GitOps rehearsal" section for ArgoCD-level troubleshooting (stuck +syncs, `--server-side` apply, terminating an operation) — it applies identically here, this +overlay just points at a different cluster and Application name (`ahc-aws`). diff --git a/kubernetes/README.md b/kubernetes/README.md index 2e82f71..9103aa7 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -11,6 +11,7 @@ GitOps layout for ArgoCD (see [ADR-13](../doc/13_adr_gitops_argocd.md)). All app | `overlays/minikube-local/` | local dev loop | plain Secrets, git-ignored | `kubectl apply -k` only — **never ArgoCD** | | `overlays/minikube-argocd/` | GitOps rehearsal | SealedSecrets (minikube key) | `argocd/ahc-minikube-test.yaml`, manual sync | | `overlays/home/` | production (k3s) | SealedSecrets (home key) | `argocd/ahc-home.yaml`, auto sync from `main` | +| `overlays/aws/` | AWS EKS demo (real cost — see [README-aws.md](README-aws.md)) | SealedSecrets (aws key) | `argocd/ahc-aws.yaml`, manual sync | Sync waves: `-3` ConfigMap/Secrets/PVCs → `-2` Postgres/CouchDB/Redis → `-1` Sync hooks (`ahc-migrate`, `couchdb-init`) → `0` web/celery/beat/flower → `1` Ingress → PostSync smoke test. diff --git a/kubernetes/overlays/aws/configmap-patch.yaml b/kubernetes/overlays/aws/configmap-patch.yaml new file mode 100644 index 0000000..a778873 --- /dev/null +++ b/kubernetes/overlays/aws/configmap-patch.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ahc-app-config +data: + # TODO: replace ahc.example-aws-placeholder.com with the real AHC_DOMAIN + # before the first aws sync. localhost must stay in ALLOWED_HOSTS for + # kubelet probes. + ALLOWED_HOSTS: "localhost,127.0.0.1,ahc.example-aws-placeholder.com" + CSRF_TRUSTED_ORIGINS: "https://ahc.example-aws-placeholder.com" diff --git a/kubernetes/overlays/aws/ingress-patch.yaml b/kubernetes/overlays/aws/ingress-patch.yaml new file mode 100644 index 0000000..6283343 --- /dev/null +++ b/kubernetes/overlays/aws/ingress-patch.yaml @@ -0,0 +1,33 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-service + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' + alb.ingress.kubernetes.io/ssl-redirect: '443' + alb.ingress.kubernetes.io/healthcheck-path: /livez + # TODO: replace with the real ACM certificate ARN (terraform output + # acm_certificate_arn, or a manually requested cert) before the first + # aws sync — the ALB Controller will fail to provision the listener + # without a valid certificate ARN here. + alb.ingress.kubernetes.io/certificate-arn: "arn:aws:acm:REGION:ACCOUNT_ID:certificate/REPLACE_ME" +spec: + # Unlike nginx/Traefik, the ALB terminates TLS at the load balancer via + # the certificate-arn annotation above, not a k8s `tls:` block — there is + # no equivalent of the home overlay's commented-out `tls:` section to + # uncomment later; the annotation IS the TLS wiring. + ingressClassName: alb + rules: + # TODO: replace with the real AHC_DOMAIN before the first aws sync. + - host: ahc.example-aws-placeholder.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: ahc-app-backend-service + port: + number: 8000 diff --git a/kubernetes/overlays/aws/kustomization.yaml b/kubernetes/overlays/aws/kustomization.yaml new file mode 100644 index 0000000..2b89fcf --- /dev/null +++ b/kubernetes/overlays/aws/kustomization.yaml @@ -0,0 +1,48 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# AWS EKS overlay, synced by the ahc-aws ArgoCD Application (manual sync +# only — see argocd/ahc-aws.yaml). Secrets are SealedSecrets encrypted with +# this cluster's controller key. Unlike overlays/home, the image tag is NOT +# managed by CI: this cluster is spun up/torn down per demo (see ADR-14 and +# kubernetes/README-aws.md), so bumps are a manual +# `kustomize edit set image` run before a demo, not an automated main-branch +# side effect. +# +# The files under sealed/ must be generated with kubeseal against a real +# EKS cluster before the first sync — see kubernetes/README-aws.md. Until +# then `kustomize build` on this overlay fails, exactly like +# overlays/home does today before its own sealed/ exists. +# +# No backup CronJobs here (unlike overlays/home): this cluster is +# ephemeral by design — `terraform destroy` removes any backup PVC along +# with everything else, so a same-cluster backup provides no protection. + +resources: + - ../../base + - sealed/app-sealedsecret.yaml + - sealed/postgres-sealedsecret.yaml + - sealed/couchdb-sealedsecret.yaml + - sealed/flower-sealedsecret.yaml + - storageclass-gp3.yaml + - smoke-test-job.yaml + +patches: + - path: ingress-patch.yaml + - path: configmap-patch.yaml + # kubeseal copies input Secret annotations into spec.template.metadata + # (the unsealed Secret), not onto the SealedSecret itself — without this + # patch SealedSecrets land in wave 0, after the databases that need them. + - target: + kind: SealedSecret + patch: |- + apiVersion: bitnami.com/v1alpha1 + kind: SealedSecret + metadata: + name: placeholder-overridden-by-target + annotations: + argocd.argoproj.io/sync-wave: "-3" + +images: + - name: ghcr.io/cybernetic-ransomware/ahc-app + newTag: prod diff --git a/kubernetes/overlays/aws/smoke-test-job.yaml b/kubernetes/overlays/aws/smoke-test-job.yaml new file mode 100644 index 0000000..9dfc32a --- /dev/null +++ b/kubernetes/overlays/aws/smoke-test-job.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: ahc-smoke-test + annotations: + argocd.argoproj.io/hook: PostSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded +spec: + backoffLimit: 3 + activeDeadlineSeconds: 180 + template: + spec: + restartPolicy: Never + containers: + - name: smoke + image: curlimages/curl:8.10.1 + command: + - sh + - -c + - | + set -eu + curl -fsS -H "Host: localhost" http://ahc-app-backend-service:8000/livez + curl -fsS -H "Host: localhost" http://ahc-app-backend-service:8000/readyz + resources: + requests: + memory: "16Mi" + cpu: "10m" + limits: + memory: "64Mi" + cpu: "100m" diff --git a/kubernetes/overlays/aws/storageclass-gp3.yaml b/kubernetes/overlays/aws/storageclass-gp3.yaml new file mode 100644 index 0000000..659bc01 --- /dev/null +++ b/kubernetes/overlays/aws/storageclass-gp3.yaml @@ -0,0 +1,13 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gp3 + annotations: + storageclass.kubernetes.io/is-default-class: "true" + argocd.argoproj.io/sync-wave: "-3" +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +reclaimPolicy: Delete +parameters: + type: gp3 + fsType: ext4 diff --git a/terraform/aws/.gitignore b/terraform/aws/.gitignore new file mode 100644 index 0000000..9fd002e --- /dev/null +++ b/terraform/aws/.gitignore @@ -0,0 +1,16 @@ +# Local state (deliberate — see versions.tf and ADR-14) +*.tfstate +*.tfstate.* +.terraform/ +.terraform.lock.hcl +crash.log +crash.*.log + +# Real variable values — only the .example file is committed +*.tfvars +!terraform.tfvars.example +*.tfvars.json + +# Plan output files (may contain resolved secrets) +*.tfplan +tfplan diff --git a/terraform/aws/dns.tf b/terraform/aws/dns.tf new file mode 100644 index 0000000..70dfa99 --- /dev/null +++ b/terraform/aws/dns.tf @@ -0,0 +1,38 @@ +# Fully gated behind var.manage_dns (default false). Inert until both a +# real, owned domain (var.domain_name) and manage_dns = true are supplied — +# see kubernetes/README-aws.md. Without this, the aws overlay ships with a +# placeholder host and no valid ALB certificate, which is a safe resting +# state, not a broken one. + +data "aws_route53_zone" "this" { + count = var.manage_dns ? 1 : 0 + name = var.domain_name +} + +resource "aws_acm_certificate" "this" { + count = var.manage_dns ? 1 : 0 + domain_name = "ahc.${var.domain_name}" + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "cert_validation" { + for_each = var.manage_dns ? { + for dvo in aws_acm_certificate.this[0].domain_validation_options : dvo.domain_name => dvo + } : {} + + zone_id = data.aws_route53_zone.this[0].zone_id + name = each.value.resource_record_name + type = each.value.resource_record_type + records = [each.value.resource_record_value] + ttl = 60 +} + +resource "aws_acm_certificate_validation" "this" { + count = var.manage_dns ? 1 : 0 + certificate_arn = aws_acm_certificate.this[0].arn + validation_record_fqdns = [for r in aws_route53_record.cert_validation : r.fqdn] +} diff --git a/terraform/aws/eks.tf b/terraform/aws/eks.tf new file mode 100644 index 0000000..21ade14 --- /dev/null +++ b/terraform/aws/eks.tf @@ -0,0 +1,45 @@ +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 20.0" + + cluster_name = var.cluster_name + cluster_version = var.cluster_version + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets # control plane ENIs span both AZs + + enable_irsa = true # required for the ALB controller and EBS CSI IAM roles in irsa.tf + + cluster_endpoint_public_access = true # kubectl from a laptop; no bastion for a demo cluster + + eks_managed_node_groups = { + default = { + # Pinned to one AZ/subnet: StatefulSets (postgres, couchdb) and the + # web/celery pods talking to them stay same-AZ, avoiding cross-AZ + # data transfer charges on top of the NAT/ALB cost already present. + # Fargate is not an option here — it cannot back a StatefulSet with + # an EBS-backed PVC (no CSI node DaemonSet support), so both database + # StatefulSets require an EC2-backed managed node group regardless. + subnet_ids = [module.vpc.private_subnets[0]] + instance_types = var.node_instance_types + min_size = var.node_min_size + max_size = var.node_max_size + desired_size = var.node_desired_size + capacity_type = "ON_DEMAND" # not Spot: StatefulSets + a short-lived demo aren't worth interruption/draining complexity + } + } + + # vpc-cni/coredns/kube-proxy are the baseline EKS addons. aws-ebs-csi-driver + # is required for dynamic PVC provisioning (not preinstalled on EKS) — its + # IRSA role is defined in irsa.tf. The AWS Load Balancer Controller has no + # EKS-addon equivalent and is installed manually via Helm (see + # kubernetes/README-aws.md) — see ADR-14 for why that asymmetry is deliberate. + cluster_addons = { + vpc-cni = {} + coredns = {} + kube-proxy = {} + aws-ebs-csi-driver = { + service_account_role_arn = module.ebs_csi_irsa.iam_role_arn + } + } +} diff --git a/terraform/aws/irsa.tf b/terraform/aws/irsa.tf new file mode 100644 index 0000000..78bafb7 --- /dev/null +++ b/terraform/aws/irsa.tf @@ -0,0 +1,35 @@ +# IAM roles assumable by in-cluster ServiceAccounts (IRSA), scoped to the +# OIDC provider Terraform creates via module.eks (enable_irsa = true). +# Both roles attach an AWS-managed/published policy through the module +# rather than a vendored JSON copy — the module tracks upstream policy +# changes across its own releases. + +module "alb_controller_irsa" { + source = "terraform-aws-modules/eks/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 20.0" + + role_name = "${var.cluster_name}-alb-controller" + attach_load_balancer_controller_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +module "ebs_csi_irsa" { + source = "terraform-aws-modules/eks/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 20.0" + + role_name = "${var.cluster_name}-ebs-csi" + attach_ebs_csi_policy = true # AWS managed policy: AmazonEBSCSIDriverPolicy + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } +} diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf new file mode 100644 index 0000000..3627d92 --- /dev/null +++ b/terraform/aws/outputs.tf @@ -0,0 +1,28 @@ +# EKS addons (vpc-cni, coredns, kube-proxy, aws-ebs-csi-driver) are declared +# via cluster_addons on module "eks" in eks.tf — a single module argument is +# enough, no standalone aws_eks_addon resources needed. This file exposes +# what the manual post-apply steps in kubernetes/README-aws.md need. + +output "cluster_name" { + value = module.eks.cluster_name +} + +output "region" { + value = var.aws_region +} + +output "vpc_id" { + value = module.vpc.vpc_id +} + +output "alb_controller_role_arn" { + value = module.alb_controller_irsa.iam_role_arn +} + +output "acm_certificate_arn" { + value = var.manage_dns ? aws_acm_certificate.this[0].arn : null +} + +output "configure_kubectl" { + value = "aws eks update-kubeconfig --name ${module.eks.cluster_name} --region ${var.aws_region}" +} diff --git a/terraform/aws/providers.tf b/terraform/aws/providers.tf new file mode 100644 index 0000000..4d87b8f --- /dev/null +++ b/terraform/aws/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Project = "animals-healthcare-application" + ManagedBy = "terraform" + Cluster = var.cluster_name + } + } +} diff --git a/terraform/aws/terraform.tfvars.example b/terraform/aws/terraform.tfvars.example new file mode 100644 index 0000000..ce92abb --- /dev/null +++ b/terraform/aws/terraform.tfvars.example @@ -0,0 +1,17 @@ +# Copy to terraform.tfvars and fill in real values before `terraform apply`. +# terraform.tfvars is git-ignored (see .gitignore) — never commit it. + +aws_region = "eu-central-1" +cluster_name = "ahc-eks-demo" +cluster_version = "1.30" +vpc_cidr = "10.60.0.0/16" + +node_instance_types = ["t3.medium"] +node_desired_size = 2 +node_min_size = 1 +node_max_size = 3 + +# Leave manage_dns = false until you have a real, owned domain for +# ACM/Route53 validation — see kubernetes/README-aws.md. +manage_dns = false +domain_name = "" diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf new file mode 100644 index 0000000..92ff8f1 --- /dev/null +++ b/terraform/aws/variables.tf @@ -0,0 +1,59 @@ +variable "aws_region" { + type = string + description = "AWS region for the cluster and its VPC." + default = "eu-central-1" +} + +variable "cluster_name" { + type = string + description = "EKS cluster name, also used as a prefix for related resource names." + default = "ahc-eks-demo" +} + +variable "cluster_version" { + type = string + description = "Kubernetes version for the EKS control plane." + default = "1.30" +} + +variable "vpc_cidr" { + type = string + description = "CIDR block for the cluster VPC." + default = "10.60.0.0/16" +} + +variable "node_instance_types" { + type = list(string) + description = "EC2 instance types for the managed node group." + default = ["t3.medium"] +} + +variable "node_desired_size" { + type = number + description = "Desired node count in the managed node group." + default = 2 +} + +variable "node_min_size" { + type = number + description = "Minimum node count in the managed node group." + default = 1 +} + +variable "node_max_size" { + type = number + description = "Maximum node count in the managed node group." + default = 3 +} + +variable "manage_dns" { + type = bool + description = "Whether to create Route53/ACM resources in dns.tf. Requires a real, owned domain — leave false until one exists." + default = false +} + +variable "domain_name" { + type = string + description = "Route53 hosted zone domain (e.g. example.com). Only used when manage_dns = true." + default = "" +} diff --git a/terraform/aws/versions.tf b/terraform/aws/versions.tf new file mode 100644 index 0000000..228c200 --- /dev/null +++ b/terraform/aws/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.7" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + + # Local state, deliberately: single operator, ephemeral demo cluster + # (spun up for a demo, torn down afterwards). An S3 backend would add its + # own bootstrap problem (something has to create the bucket before + # Terraform can use it) for no benefit at this scale. Revisit only if + # this cluster becomes long-lived or multi-operator — see ADR-14. +} diff --git a/terraform/aws/vpc.tf b/terraform/aws/vpc.tf new file mode 100644 index 0000000..4c7841b --- /dev/null +++ b/terraform/aws/vpc.tf @@ -0,0 +1,33 @@ +data "aws_availability_zones" "available" { + state = "available" +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = "${var.cluster_name}-vpc" + cidr = var.vpc_cidr + + # EKS requires control-plane ENIs across >= 2 AZs, even though the node + # group itself is pinned to a single AZ (see eks.tf) to avoid cross-AZ + # data transfer charges between the app and the postgres/couchdb + # StatefulSets. + azs = slice(data.aws_availability_zones.available.names, 0, 2) + public_subnets = [cidrsubnet(var.vpc_cidr, 4, 0), cidrsubnet(var.vpc_cidr, 4, 1)] + private_subnets = [cidrsubnet(var.vpc_cidr, 4, 8), cidrsubnet(var.vpc_cidr, 4, 9)] + + enable_nat_gateway = true + single_nat_gateway = true # one NAT gateway total, not one per AZ — cost guard + enable_dns_hostnames = true + + # Required for the AWS Load Balancer Controller's subnet auto-discovery. + public_subnet_tags = { + "kubernetes.io/role/elb" = "1" + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + } + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = "1" + "kubernetes.io/cluster/${var.cluster_name}" = "shared" + } +} From 8bd77c074c7e480a9084946306e637916cec8a22 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:33:46 +0200 Subject: [PATCH 20/27] fix(terraform): pin eks cluster version to 1.34 and grant creator admin access --- terraform/aws/eks.tf | 7 +++++++ terraform/aws/terraform.tfvars.example | 2 +- terraform/aws/variables.tf | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/terraform/aws/eks.tf b/terraform/aws/eks.tf index 21ade14..97baa72 100644 --- a/terraform/aws/eks.tf +++ b/terraform/aws/eks.tf @@ -12,6 +12,13 @@ module "eks" { cluster_endpoint_public_access = true # kubectl from a laptop; no bastion for a demo cluster + # The module defaults this to false and defines no access_entries, which + # leaves the cluster creator with zero API access after `apply` — + # `kubectl get nodes` would fail with Unauthorized. Simplicity is + # appropriate for a single-operator demo cluster; a longer-lived, + # multi-operator cluster should use explicit access_entries instead. + enable_cluster_creator_admin_permissions = true + eks_managed_node_groups = { default = { # Pinned to one AZ/subnet: StatefulSets (postgres, couchdb) and the diff --git a/terraform/aws/terraform.tfvars.example b/terraform/aws/terraform.tfvars.example index ce92abb..f32e479 100644 --- a/terraform/aws/terraform.tfvars.example +++ b/terraform/aws/terraform.tfvars.example @@ -3,7 +3,7 @@ aws_region = "eu-central-1" cluster_name = "ahc-eks-demo" -cluster_version = "1.30" +cluster_version = "1.34" vpc_cidr = "10.60.0.0/16" node_instance_types = ["t3.medium"] diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 92ff8f1..1c043ea 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -13,7 +13,7 @@ variable "cluster_name" { variable "cluster_version" { type = string description = "Kubernetes version for the EKS control plane." - default = "1.30" + default = "1.34" } variable "vpc_cidr" { From 9b313f42b1419a2e663c98e37a90c1686344edf0 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:37:22 +0200 Subject: [PATCH 21/27] ci: serialize the full pipeline per ref instead of only the bump job --- .github/workflows/django.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 0404e84..53da251 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -7,6 +7,17 @@ on: branches: [ "main" ] workflow_dispatch: +# Serializes the whole test -> lint -> manifests -> publish -> bump chain per +# ref, not just the bump job. Without this, two pushes to main can race: +# if commit A's pipeline starts first but commit B's finishes publish first, +# job-level concurrency on bump alone would still process B's bump before +# A's, letting A's bump overwrite B's newer image tag — an unintended +# GitOps rollback. Keying by ref keeps PRs on different branches from +# blocking each other while pushes to the same branch queue in order. +concurrency: + group: ahc-ci-${{ github.ref }} + cancel-in-progress: false + jobs: lint: runs-on: ubuntu-latest @@ -170,9 +181,6 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: write - concurrency: - group: ahc-home-image-bump - cancel-in-progress: false env: KUSTOMIZE_VERSION: "5.8.1" steps: From c406bff05205e37762121b11a7d5b40aed3d6abc Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:37:29 +0200 Subject: [PATCH 22/27] chore(k8s): revert minikube-argocd rehearsal pins before merge --- argocd/ahc-minikube-test.yaml | 2 +- kubernetes/overlays/minikube-argocd/kustomization.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/argocd/ahc-minikube-test.yaml b/argocd/ahc-minikube-test.yaml index e1f2602..03da196 100644 --- a/argocd/ahc-minikube-test.yaml +++ b/argocd/ahc-minikube-test.yaml @@ -7,7 +7,7 @@ spec: project: default source: repoURL: https://github.com/Cybernetic-Ransomware/Animals_Healthcare_Application.git - targetRevision: feat/argocd + targetRevision: main path: kubernetes/overlays/minikube-argocd destination: server: https://kubernetes.default.svc diff --git a/kubernetes/overlays/minikube-argocd/kustomization.yaml b/kubernetes/overlays/minikube-argocd/kustomization.yaml index b8e3f0d..5c34aa2 100644 --- a/kubernetes/overlays/minikube-argocd/kustomization.yaml +++ b/kubernetes/overlays/minikube-argocd/kustomization.yaml @@ -33,6 +33,4 @@ patches: images: - name: ghcr.io/cybernetic-ransomware/ahc-app - # Pinned to the branch image for the ArgoCD rehearsal (see README, - # "Testing branch code through ArgoCD"). Revert to prod before merging. - newTag: sha-d515554252e936a41ee9a6d559732a284d788686 + newTag: prod From d182447fe6cdd215489cb5236b9b1d40624dbdc3 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:38:08 +0200 Subject: [PATCH 23/27] ci: validate terraform aws module in the manifests job --- .github/workflows/django.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 53da251..0e3a5ec 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -132,6 +132,23 @@ jobs: print(f"OK: {len(sealed)} SealedSecrets at wave -3") EOF + # overlays/aws joins the render step above once its sealed/ files exist + # (same reasoning as overlays/home). Terraform is validated separately + # below — it needs neither AWS credentials nor sealed secrets. + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9.8" + + - name: Terraform fmt check + run: terraform -chdir=terraform/aws fmt -check -recursive + + - name: Terraform init (no backend, no cloud credentials needed) + run: terraform -chdir=terraform/aws init -backend=false + + - name: Terraform validate + run: terraform -chdir=terraform/aws validate + publish: name: Publish app image to GHCR runs-on: ubuntu-latest From b444e957c1ca7071470c5b34785947d506ad37d4 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:49:26 +0200 Subject: [PATCH 24/27] fix(terraform): align vpc.tf attribute equals signs for terraform fmt --- terraform/aws/vpc.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/aws/vpc.tf b/terraform/aws/vpc.tf index 4c7841b..f5e5ebc 100644 --- a/terraform/aws/vpc.tf +++ b/terraform/aws/vpc.tf @@ -17,8 +17,8 @@ module "vpc" { public_subnets = [cidrsubnet(var.vpc_cidr, 4, 0), cidrsubnet(var.vpc_cidr, 4, 1)] private_subnets = [cidrsubnet(var.vpc_cidr, 4, 8), cidrsubnet(var.vpc_cidr, 4, 9)] - enable_nat_gateway = true - single_nat_gateway = true # one NAT gateway total, not one per AZ — cost guard + enable_nat_gateway = true + single_nat_gateway = true # one NAT gateway total, not one per AZ — cost guard enable_dns_hostnames = true # Required for the AWS Load Balancer Controller's subnet auto-discovery. From 0d5c60a13cd63c8781479b03c8fd5da35f2285b0 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:49:49 +0200 Subject: [PATCH 25/27] docs(ci): correct concurrency comment about queue ordering guarantees --- .github/workflows/django.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/django.yml b/.github/workflows/django.yml index 0e3a5ec..99f9408 100644 --- a/.github/workflows/django.yml +++ b/.github/workflows/django.yml @@ -13,7 +13,15 @@ on: # job-level concurrency on bump alone would still process B's bump before # A's, letting A's bump overwrite B's newer image tag — an unintended # GitOps rollback. Keying by ref keeps PRs on different branches from -# blocking each other while pushes to the same branch queue in order. +# blocking each other. +# +# GitHub does not guarantee strict FIFO ordering across every trigger in a +# concurrency group: with cancel-in-progress: false, the currently running +# workflow is protected, but only the single most recently queued run is +# kept — an older *queued* run is cancelled outright when a newer one +# arrives, rather than waiting its turn. For this race that is actually the +# property we want: a stale intermediate push never runs its bump after a +# newer one, it just never runs at all. concurrency: group: ahc-ci-${{ github.ref }} cancel-in-progress: false From 5cc268b1ea49fc5c7a5f0578cbab96f6e8e0c739 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:54:46 +0200 Subject: [PATCH 26/27] fix(terraform): use terraform-aws-modules/iam/aws for IRSA roles The iam-role-for-service-accounts-eks submodule was removed from terraform-aws-modules/eks/aws in v20.0.0 and moved to a separate iam module as iam-role-for-service-accounts (no -eks suffix, role_name renamed to name). CI's terraform init -backend=false failed trying to resolve the old, now-nonexistent submodule path. --- terraform/aws/irsa.tf | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/terraform/aws/irsa.tf b/terraform/aws/irsa.tf index 78bafb7..ce1a872 100644 --- a/terraform/aws/irsa.tf +++ b/terraform/aws/irsa.tf @@ -3,12 +3,20 @@ # Both roles attach an AWS-managed/published policy through the module # rather than a vendored JSON copy — the module tracks upstream policy # changes across its own releases. +# +# The submodule lived at terraform-aws-modules/eks/aws//modules/ +# iam-role-for-service-accounts-eks through v19; it was removed in the +# eks module's v20.0.0 and now lives in the separate iam module as +# iam-role-for-service-accounts (no "-eks" suffix). Its role_name input +# was also renamed to name. module "alb_controller_irsa" { - source = "terraform-aws-modules/eks/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 20.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "~> 6.0" + + name = "${var.cluster_name}-alb-controller" + use_name_prefix = false # a stable, predictable role name is worth more here than the module's default random suffix - role_name = "${var.cluster_name}-alb-controller" attach_load_balancer_controller_policy = true oidc_providers = { @@ -20,10 +28,12 @@ module "alb_controller_irsa" { } module "ebs_csi_irsa" { - source = "terraform-aws-modules/eks/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 20.0" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "~> 6.0" + + name = "${var.cluster_name}-ebs-csi" + use_name_prefix = false - role_name = "${var.cluster_name}-ebs-csi" attach_ebs_csi_policy = true # AWS managed policy: AmazonEBSCSIDriverPolicy oidc_providers = { From 6d78010eba1c751a5a75429657e8db9dee8d0771 Mon Sep 17 00:00:00 2001 From: Cybernetic-Ransomware <71835339+Cybernetic-Ransomware@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:00:13 +0200 Subject: [PATCH 27/27] fix(terraform): pin iam module to 5.x, not 6.x iam/aws v6 requires aws provider >= 6.28.0, conflicting with the eks/vpc modules' own < 6.0.0 constraint. Reverting to the -eks suffixed submodule and role_name input, which is what 5.x expects. --- terraform/aws/irsa.tf | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/terraform/aws/irsa.tf b/terraform/aws/irsa.tf index ce1a872..a0730de 100644 --- a/terraform/aws/irsa.tf +++ b/terraform/aws/irsa.tf @@ -4,19 +4,18 @@ # rather than a vendored JSON copy — the module tracks upstream policy # changes across its own releases. # -# The submodule lived at terraform-aws-modules/eks/aws//modules/ -# iam-role-for-service-accounts-eks through v19; it was removed in the -# eks module's v20.0.0 and now lives in the separate iam module as -# iam-role-for-service-accounts (no "-eks" suffix). Its role_name input -# was also renamed to name. +# Pinned to the iam module's 5.x line, not 6.x: iam/aws v6 requires AWS +# provider >= 6.28.0, which conflicts with the eks/vpc modules' own +# provider constraints (< 6.0.0) — mixing major lines across these +# community modules isn't possible in a single configuration. v5.x also +# keeps the -eks suffixed submodule name and the role_name input (renamed +# to name, with the module defaulting to prefix behavior, only in v6). module "alb_controller_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" - version = "~> 6.0" - - name = "${var.cluster_name}-alb-controller" - use_name_prefix = false # a stable, predictable role name is worth more here than the module's default random suffix + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + role_name = "${var.cluster_name}-alb-controller" attach_load_balancer_controller_policy = true oidc_providers = { @@ -28,12 +27,10 @@ module "alb_controller_irsa" { } module "ebs_csi_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" - version = "~> 6.0" - - name = "${var.cluster_name}-ebs-csi" - use_name_prefix = false + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.0" + role_name = "${var.cluster_name}-ebs-csi" attach_ebs_csi_policy = true # AWS managed policy: AmazonEBSCSIDriverPolicy oidc_providers = {