diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 36ab8724..cbd17a36 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -30,6 +30,44 @@ jobs: printf '%s\n' "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key chmod 600 ~/.ssh/deploy_key + - name: Write compose .env file + env: + POSTGRES_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }} + ORGANIZATION_DB_PASSWORD: ${{ secrets.DB_ORGANIZATION_PASSWORD }} + MEMBER_DB_PASSWORD: ${{ secrets.DB_MEMBER_PASSWORD }} + EVENT_DB_PASSWORD: ${{ secrets.DB_EVENT_PASSWORD }} + FEEDBACK_DB_PASSWORD: ${{ secrets.DB_FEEDBACK_PASSWORD }} + FINANCE_DB_PASSWORD: ${{ secrets.DB_FINANCE_PASSWORD }} + LETTER_DB_PASSWORD: ${{ secrets.DB_LETTER_PASSWORD }} + REPORTS_DB_PASSWORD: ${{ secrets.DB_REPORTS_PASSWORD }} + KEYCLOAK_ADMIN_PASSWORD: ${{ secrets.KEYCLOAK_ADMIN_PASSWORD }} + KEYCLOAK_DB_PASSWORD: ${{ secrets.KEYCLOAK_DB_PASSWORD }} + KEYCLOAK_REALM_ADMIN_PASSWORD: ${{ secrets.KEYCLOAK_REALM_ADMIN_PASSWORD }} + KEYCLOAK_REALM_USER_PASSWORD: ${{ secrets.KEYCLOAK_REALM_USER_PASSWORD }} + FORWARD_AUTH_COOKIE_SECRET: ${{ secrets.FORWARD_AUTH_COOKIE_SECRET }} + KEYCLOAK_ADMIN_CLIENT_SECRET: ${{ secrets.KEYCLOAK_ADMIN_CLIENT_SECRET }} + FORWARD_AUTH_CLIENT_SECRET: ${{ secrets.FORWARD_AUTH_CLIENT_SECRET }} + GRAFANA_OAUTH_CLIENT_SECRET: ${{ secrets.GRAFANA_OAUTH_CLIENT_SECRET }} + run: | + cat > /tmp/compose.env << ENVFILE + POSTGRES_PASSWORD=$POSTGRES_PASSWORD + ORGANIZATION_DB_PASSWORD=$ORGANIZATION_DB_PASSWORD + MEMBER_DB_PASSWORD=$MEMBER_DB_PASSWORD + EVENT_DB_PASSWORD=$EVENT_DB_PASSWORD + FEEDBACK_DB_PASSWORD=$FEEDBACK_DB_PASSWORD + FINANCE_DB_PASSWORD=$FINANCE_DB_PASSWORD + LETTER_DB_PASSWORD=$LETTER_DB_PASSWORD + REPORTS_DB_PASSWORD=$REPORTS_DB_PASSWORD + KEYCLOAK_ADMIN_PASSWORD=$KEYCLOAK_ADMIN_PASSWORD + KEYCLOAK_DB_PASSWORD=$KEYCLOAK_DB_PASSWORD + KEYCLOAK_REALM_ADMIN_PASSWORD=$KEYCLOAK_REALM_ADMIN_PASSWORD + KEYCLOAK_REALM_USER_PASSWORD=$KEYCLOAK_REALM_USER_PASSWORD + FORWARD_AUTH_COOKIE_SECRET=$FORWARD_AUTH_COOKIE_SECRET + KEYCLOAK_ADMIN_CLIENT_SECRET=$KEYCLOAK_ADMIN_CLIENT_SECRET + FORWARD_AUTH_CLIENT_SECRET=$FORWARD_AUTH_CLIENT_SECRET + GRAFANA_OAUTH_CLIENT_SECRET=$GRAFANA_OAUTH_CLIENT_SECRET + ENVFILE + - name: Write genai .env file env: GENAI_ENV_CONTENT: ${{ secrets.GENAI_ENV_CONTENT }} @@ -59,6 +97,7 @@ jobs: infra/ansible/playbook.yml \ -e "repo_url=https://github.com/${{ github.repository }}.git" \ -e "branch=${{ github.ref_name }}" \ + -e "compose_env_file=/tmp/compose.env" \ -e "genai_env_file=/tmp/genai.env" \ -e "letter_env_file=/tmp/letter.env" @@ -253,5 +292,21 @@ jobs: --namespace "$NAMESPACE" \ --set global.image.tag=${{ github.sha }} \ --set keycloak.hostname=https://ge83mom-devops26.stud.k8s.aet.cit.tum.de/auth \ + --set database.password="${{ secrets.DB_ADMIN_PASSWORD }}" \ + --set database.users.organization.password="${{ secrets.DB_ORGANIZATION_PASSWORD }}" \ + --set database.users.member.password="${{ secrets.DB_MEMBER_PASSWORD }}" \ + --set database.users.event.password="${{ secrets.DB_EVENT_PASSWORD }}" \ + --set database.users.feedback.password="${{ secrets.DB_FEEDBACK_PASSWORD }}" \ + --set database.users.finance.password="${{ secrets.DB_FINANCE_PASSWORD }}" \ + --set database.users.letter.password="${{ secrets.DB_LETTER_PASSWORD }}" \ + --set database.users.reports.password="${{ secrets.DB_REPORTS_PASSWORD }}" \ + --set keycloak.adminPassword="${{ secrets.KEYCLOAK_ADMIN_PASSWORD }}" \ + --set keycloak.db.password="${{ secrets.KEYCLOAK_DB_PASSWORD }}" \ + --set keycloak.users.admin.password="${{ secrets.KEYCLOAK_REALM_ADMIN_PASSWORD }}" \ + --set keycloak.users.user.password="${{ secrets.KEYCLOAK_REALM_USER_PASSWORD }}" \ --set forwardAuth.cookieSecret="${{ secrets.FORWARD_AUTH_COOKIE_SECRET }}" \ + --set forwardAuth.clientSecret="${{ secrets.FORWARD_AUTH_CLIENT_SECRET }}" \ + --set monitoring.grafana.oauthClientSecret="${{ secrets.GRAFANA_OAUTH_CLIENT_SECRET }}" \ + --set services.organization-service.env.KEYCLOAK_ADMIN_CLIENT_SECRET="${{ secrets.KEYCLOAK_ADMIN_CLIENT_SECRET }}" \ + --set services.member-service.env.KEYCLOAK_SERVICE_ACCOUNT_CLIENT_SECRET="${{ secrets.KEYCLOAK_ADMIN_CLIENT_SECRET }}" \ --rollback-on-failure --timeout 15m diff --git a/README.md b/README.md index bc880af9..17f53d2e 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,24 @@ automated; no manual VM access is required for normal deploys. | Secret | `GENAI_ENV_CONTENT` | Contents of `services/py-genai-helper/.env` | | Secret | `LETTER_ENV_CONTENT` | Contents of `services/spring-letter/.env` (mail credentials) | | Secret | `KUBECONFIG` | Kubeconfig for the RKE2 cluster (used by the `deploy-k8s` job) | +| Secret | `DB_ADMIN_PASSWORD` | Shared Postgres admin password. VM: `POSTGRES_PASSWORD` in `infra/.env`. K8s: Helm `database.password` | +| Secret | `DB_ORGANIZATION_PASSWORD` | `organization_user` DB password. VM: `ORGANIZATION_DB_PASSWORD`. K8s: Helm `database.users.organization.password` | +| Secret | `DB_MEMBER_PASSWORD` | `member_user` DB password. VM: `MEMBER_DB_PASSWORD`. K8s: Helm `database.users.member.password` | +| Secret | `DB_EVENT_PASSWORD` | `event_user` DB password. VM: `EVENT_DB_PASSWORD`. K8s: Helm `database.users.event.password` | +| Secret | `DB_FEEDBACK_PASSWORD` | `feedback_user` DB password. VM: `FEEDBACK_DB_PASSWORD`. K8s: Helm `database.users.feedback.password` | +| Secret | `DB_FINANCE_PASSWORD` | `finance_user` DB password. VM: `FINANCE_DB_PASSWORD`. K8s: Helm `database.users.finance.password` | +| Secret | `DB_LETTER_PASSWORD` | `letter_user` DB password. VM: `LETTER_DB_PASSWORD`. K8s: Helm `database.users.letter.password` | +| Secret | `DB_REPORTS_PASSWORD` | `reports_user` DB password. VM: `REPORTS_DB_PASSWORD`. K8s: Helm `database.users.reports.password` | +| Secret | `KEYCLOAK_ADMIN_PASSWORD` | Keycloak bootstrap admin console password. VM: `KEYCLOAK_ADMIN_PASSWORD`. K8s: Helm `keycloak.adminPassword` | +| Secret | `KEYCLOAK_DB_PASSWORD` | Keycloak's own Postgres password. VM: `KEYCLOAK_DB_PASSWORD`. K8s: Helm `keycloak.db.password` | +| Secret | `KEYCLOAK_REALM_ADMIN_PASSWORD` | Password for the seeded `admin` realm user. VM: `KEYCLOAK_REALM_ADMIN_PASSWORD`. K8s: Helm `keycloak.users.admin.password` | +| Secret | `KEYCLOAK_REALM_USER_PASSWORD` | Password for the seeded `user` realm user. VM: `KEYCLOAK_REALM_USER_PASSWORD`. K8s: Helm `keycloak.users.user.password` | +| Secret | `FORWARD_AUTH_COOKIE_SECRET` | 32+ char random string signing forward-auth session cookies. VM: `FORWARD_AUTH_COOKIE_SECRET`. K8s: Helm `forwardAuth.cookieSecret` | +| Secret | `KEYCLOAK_ADMIN_CLIENT_SECRET` | Secret for the `org-role-sync` Keycloak service-account client (`manage-users`/`view-clients` — used by organization-service and member-service to call the Keycloak Admin REST API). VM: `KEYCLOAK_ADMIN_CLIENT_SECRET`. K8s: Helm `services.organization-service.env.KEYCLOAK_ADMIN_CLIENT_SECRET` and `services.member-service.env.KEYCLOAK_SERVICE_ACCOUNT_CLIENT_SECRET` (same value, two consumers) | +| Secret | `FORWARD_AUTH_CLIENT_SECRET` | Secret for the `traefik-forward-auth` Keycloak client (Traefik/oauth2-proxy's OIDC client, gates every proxied route). VM: `FORWARD_AUTH_CLIENT_SECRET`. K8s: Helm `forwardAuth.clientSecret` | +| Secret | `GRAFANA_OAUTH_CLIENT_SECRET` | Secret for the `grafana` Keycloak client (Grafana's own generic-OAuth login). VM: `GRAFANA_OAUTH_CLIENT_SECRET`. K8s: Helm `monitoring.grafana.oauthClientSecret` | + +Each of these 16 secrets is the single source of truth for that value across **both** deploy targets — the `deploy` job's "Write compose .env file" step assembles `infra/.env` from them for the VM, and the `deploy-k8s` job passes them individually via `--set` for Helm. No GitHub secret's value is duplicated by a *different* GitHub secret (`KEYCLOAK_ADMIN_CLIENT_SECRET` is intentionally passed to two `--set` targets, since organization-service and member-service both authenticate to Keycloak as the same `org-role-sync` client). The OIDC service principal needs `Contributor` on the subscription (to manage resources in `rg-team-devoops`) and `Storage Blob Data Contributor` on the @@ -269,8 +287,10 @@ All services are protected by [Keycloak 26](https://www.keycloak.org) via OIDC/J | Realm | `devops` | |---|---| -| Admin user | `admin` / `admin123` (roles: `admin`, `member`) | -| Regular user | `user` / `user123` (role: `member`) | +| Admin user | `admin` / `admin123` locally (roles: `admin`, `member`) | +| Regular user | `user` / `user123` locally (role: `member`) | + +Passwords shown are the local-dev defaults from [`infra/.env.example`](infra/.env.example). On the VM and Kubernetes they come from the `KEYCLOAK_REALM_ADMIN_PASSWORD`/`KEYCLOAK_REALM_USER_PASSWORD` GitHub secrets instead — see [Required GitHub secrets / variables](#required-github-secrets--variables). ("Realm" distinguishes these two application-level accounts from the Keycloak server's own bootstrap console admin, `KEYCLOAK_ADMIN_PASSWORD`.) `infra/keycloak/realm-config.json` stores these as `__KEYCLOAK_REALM_ADMIN_PASSWORD__`/`__KEYCLOAK_REALM_USER_PASSWORD__` placeholders, substituted at container start (Compose) or chart render (Helm). ### Clients @@ -279,6 +299,9 @@ All services are protected by [Keycloak 26](https://www.keycloak.org) via OIDC/J | `devops-client` | public, PKCE S256 | React frontend | | `traefik-forward-auth` | confidential | Traefik forward-auth middleware | | `grafana` | confidential | Grafana's own generic-OAuth login (admin-only, see [Monitoring](#monitoring)) | +| `org-role-sync` | confidential, service account (`manage-users`/`view-clients`) | organization-service and member-service, to call the Keycloak Admin REST API | + +The three confidential clients' secrets are local-dev defaults from `infra/.env.example` (matching what's committed in `infra/keycloak/realm-config.json`'s `__KEYCLOAK_ADMIN_CLIENT_SECRET__`/`__FORWARD_AUTH_CLIENT_SECRET__`/`__GRAFANA_OAUTH_CLIENT_SECRET__` placeholders); on the VM and Kubernetes they come from the `KEYCLOAK_ADMIN_CLIENT_SECRET`/`FORWARD_AUTH_CLIENT_SECRET`/`GRAFANA_OAUTH_CLIENT_SECRET` GitHub secrets — see [Required GitHub secrets / variables](#required-github-secrets--variables). ### Local login diff --git a/infra/.env.example b/infra/.env.example new file mode 100644 index 00000000..c1da31f2 --- /dev/null +++ b/infra/.env.example @@ -0,0 +1,30 @@ +# Local secrets for `docker compose up` (run from infra/, see README "Running Locally"). +# Copy to `.env` (gitignored) and fill in real values. Never commit `.env`. +# +# Each value below has a matching GitHub secret that's the single source of truth across +# both deploy targets: CI assembles infra/.env on the Azure VM from these secrets, and +# passes the same secrets individually via --set to `helm upgrade` for Kubernetes. See +# README "Required GitHub secrets / variables" for the full mapping. + +POSTGRES_PASSWORD=app_admin_password +ORGANIZATION_DB_PASSWORD=organization_password +MEMBER_DB_PASSWORD=member_password +EVENT_DB_PASSWORD=event_password +FEEDBACK_DB_PASSWORD=feedback_password +FINANCE_DB_PASSWORD=finance_password +LETTER_DB_PASSWORD=letter_password +REPORTS_DB_PASSWORD=reports_password +KEYCLOAK_ADMIN_PASSWORD=admin +KEYCLOAK_DB_PASSWORD=keycloak_password +KEYCLOAK_REALM_ADMIN_PASSWORD=admin123 +KEYCLOAK_REALM_USER_PASSWORD=user123 +FORWARD_AUTH_COOKIE_SECRET=VdpPltrnS+cc+1hswmya727K69iG7MpMZH/LOLlmox0= + +# OIDC/OAuth client secrets. Rendered into the matching client's "secret" field in +# infra/keycloak/realm-config.json at container start; must equal whatever the consuming +# service (grafana, traefik-forward-auth, organization-service/member-service) is +# configured with, or the OIDC handshake fails. Values below just mirror what used to be +# hardcoded directly in realm-config.json -- fine for local dev, not real secrets. +KEYCLOAK_ADMIN_CLIENT_SECRET=org-role-sync-secret +FORWARD_AUTH_CLIENT_SECRET=traefik-forward-auth-secret +GRAFANA_OAUTH_CLIENT_SECRET=grafana-oauth-secret diff --git a/infra/ansible/playbook.yml b/infra/ansible/playbook.yml index a45de19a..723dcb3c 100644 --- a/infra/ansible/playbook.yml +++ b/infra/ansible/playbook.yml @@ -83,6 +83,14 @@ group: "{{ ansible_user }}" recurse: true + - name: Write compose .env file + copy: + src: "{{ compose_env_file }}" + dest: "{{ app_dir }}/infra/.env" + mode: "0600" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + - name: Write py-genai-helper .env file copy: src: "{{ genai_env_file }}" diff --git a/infra/docker-compose.override.yml b/infra/docker-compose.override.yml index f9b0ef20..6cffcd0f 100644 --- a/infra/docker-compose.override.yml +++ b/infra/docker-compose.override.yml @@ -129,7 +129,7 @@ services: - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana - - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=grafana-oauth-secret + - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET} - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile email - GF_AUTH_GENERIC_OAUTH_AUTH_URL=http://localhost:8081/auth/realms/devops/protocol/openid-connect/auth - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=http://keycloak:8080/auth/realms/devops/protocol/openid-connect/token diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index a4638803..b6c66d63 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -37,7 +37,7 @@ services: - KEYCLOAK_JWKS_URL=http://keycloak:8080/auth/realms/devops/protocol/openid-connect/certs - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=reports_user - - SPRING_DATASOURCE_PASSWORD=reports_password + - SPRING_DATASOURCE_PASSWORD=${REPORTS_DB_PASSWORD} - OLLAMA_BASE_URL=http://ollama:11434 - FEEDBACK_SERVICE_URL=http://feedback-service:8080 volumes: @@ -70,8 +70,9 @@ services: environment: - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=organization_user - - SPRING_DATASOURCE_PASSWORD=organization_password + - SPRING_DATASOURCE_PASSWORD=${ORGANIZATION_DB_PASSWORD} - SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops + - KEYCLOAK_ADMIN_CLIENT_SECRET=${KEYCLOAK_ADMIN_CLIENT_SECRET} labels: - "traefik.enable=true" - "traefik.http.routers.organization-service.entrypoints=websecure" @@ -97,8 +98,9 @@ services: environment: - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=member_user - - SPRING_DATASOURCE_PASSWORD=member_password + - SPRING_DATASOURCE_PASSWORD=${MEMBER_DB_PASSWORD} - SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops + - KEYCLOAK_SERVICE_ACCOUNT_CLIENT_SECRET=${KEYCLOAK_ADMIN_CLIENT_SECRET} labels: - "traefik.enable=true" - "traefik.http.routers.member-service.entrypoints=websecure" @@ -124,7 +126,7 @@ services: environment: - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=event_user - - SPRING_DATASOURCE_PASSWORD=event_password + - SPRING_DATASOURCE_PASSWORD=${EVENT_DB_PASSWORD} - SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops labels: - "traefik.enable=true" @@ -151,7 +153,7 @@ services: environment: - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=feedback_user - - SPRING_DATASOURCE_PASSWORD=feedback_password + - SPRING_DATASOURCE_PASSWORD=${FEEDBACK_DB_PASSWORD} - SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops labels: - "traefik.enable=true" @@ -178,7 +180,7 @@ services: environment: - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=finance_user - - SPRING_DATASOURCE_PASSWORD=finance_password + - SPRING_DATASOURCE_PASSWORD=${FINANCE_DB_PASSWORD} - SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops labels: - "traefik.enable=true" @@ -207,7 +209,7 @@ services: environment: - SPRING_DATASOURCE_URL=jdbc:postgresql://app-database:5432/app_db - SPRING_DATASOURCE_USERNAME=letter_user - - SPRING_DATASOURCE_PASSWORD=letter_password + - SPRING_DATASOURCE_PASSWORD=${LETTER_DB_PASSWORD} - SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops labels: - "traefik.enable=true" @@ -299,7 +301,7 @@ services: - GF_AUTH_GENERIC_OAUTH_ENABLED=true - GF_AUTH_GENERIC_OAUTH_NAME=Keycloak - GF_AUTH_GENERIC_OAUTH_CLIENT_ID=grafana - - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=grafana-oauth-secret + - GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET=${GRAFANA_OAUTH_CLIENT_SECRET} - GF_AUTH_GENERIC_OAUTH_SCOPES=openid profile email - GF_AUTH_GENERIC_OAUTH_AUTH_URL=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops/protocol/openid-connect/auth - GF_AUTH_GENERIC_OAUTH_TOKEN_URL=http://keycloak:8080/auth/realms/devops/protocol/openid-connect/token @@ -401,8 +403,8 @@ services: - DEFAULT_PROVIDER=oidc - PROVIDERS_OIDC_ISSUER_URL=https://team-devoops.polandcentral.cloudapp.azure.com/auth/realms/devops - PROVIDERS_OIDC_CLIENT_ID=traefik-forward-auth - - PROVIDERS_OIDC_CLIENT_SECRET=traefik-forward-auth-secret - - SECRET=a-random-32-char-secret-changeme! + - PROVIDERS_OIDC_CLIENT_SECRET=${FORWARD_AUTH_CLIENT_SECRET} + - SECRET=${FORWARD_AUTH_COOKIE_SECRET} - INSECURE_COOKIE=false - LOG_LEVEL=debug extra_hosts: @@ -419,7 +421,28 @@ services: keycloak: image: quay.io/keycloak/keycloak:26.0.0 container_name: keycloak - command: ["start-dev", "--import-realm"] + # realm-config.json is mounted as a template (see volumes below) with + # __KEYCLOAK_REALM_ADMIN_PASSWORD__/__KEYCLOAK_REALM_USER_PASSWORD__/ + # __KEYCLOAK_ADMIN_CLIENT_SECRET__/__FORWARD_AUTH_CLIENT_SECRET__/__GRAFANA_OAUTH_CLIENT_SECRET__ + # placeholders. Render it into the real import path from the container's own env (note the `$$` -- + # a single `$` would be interpolated by Compose itself using the host/.env value, + # which happens to be correct today but would silently freeze the substitution at + # `docker compose config` time instead of at container start) before starting Keycloak. + entrypoint: ["/bin/bash", "-c"] + command: + - | + set -e + mkdir -p /opt/keycloak/data/import + # `|` delimiter, not `/` -- these are openssl-rand-base64 secrets, which routinely + # contain `/` and would otherwise break a `/`-delimited s/// (sed has no way to + # escape the delimiter mid-pattern here). + sed -e "s|__KEYCLOAK_REALM_ADMIN_PASSWORD__|$$KEYCLOAK_REALM_ADMIN_PASSWORD|" \ + -e "s|__KEYCLOAK_REALM_USER_PASSWORD__|$$KEYCLOAK_REALM_USER_PASSWORD|" \ + -e "s|__KEYCLOAK_ADMIN_CLIENT_SECRET__|$$KEYCLOAK_ADMIN_CLIENT_SECRET|" \ + -e "s|__FORWARD_AUTH_CLIENT_SECRET__|$$FORWARD_AUTH_CLIENT_SECRET|" \ + -e "s|__GRAFANA_OAUTH_CLIENT_SECRET__|$$GRAFANA_OAUTH_CLIENT_SECRET|" \ + /opt/keycloak/realm-config.template.json > /opt/keycloak/data/import/realm-config.json + exec /opt/keycloak/bin/kc.sh start-dev --import-realm depends_on: keycloak-database: condition: service_healthy @@ -427,9 +450,14 @@ services: KC_DB: postgres KC_DB_URL_HOST: keycloak-database KC_DB_USERNAME: keycloak_user - KC_DB_PASSWORD: keycloak_password + KC_DB_PASSWORD: ${KEYCLOAK_DB_PASSWORD} KC_BOOTSTRAP_ADMIN_USERNAME: admin - KC_BOOTSTRAP_ADMIN_PASSWORD: admin + KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} + KEYCLOAK_REALM_ADMIN_PASSWORD: ${KEYCLOAK_REALM_ADMIN_PASSWORD} + KEYCLOAK_REALM_USER_PASSWORD: ${KEYCLOAK_REALM_USER_PASSWORD} + KEYCLOAK_ADMIN_CLIENT_SECRET: ${KEYCLOAK_ADMIN_CLIENT_SECRET} + FORWARD_AUTH_CLIENT_SECRET: ${FORWARD_AUTH_CLIENT_SECRET} + GRAFANA_OAUTH_CLIENT_SECRET: ${GRAFANA_OAUTH_CLIENT_SECRET} KC_HEALTH_ENABLED: "true" KC_METRICS_ENABLED: "true" KC_HTTP_RELATIVE_PATH: /auth @@ -452,7 +480,7 @@ services: retries: 20 start_period: 30s volumes: - - ./keycloak/realm-config.json:/opt/keycloak/data/import/realm-config.json + - ./keycloak/realm-config.json:/opt/keycloak/realm-config.template.json:ro - ./keycloak/themes:/opt/keycloak/themes ports: - "8081:8080" @@ -468,7 +496,7 @@ services: environment: POSTGRES_DB: keycloak POSTGRES_USER: keycloak_user - POSTGRES_PASSWORD: keycloak_password + POSTGRES_PASSWORD: ${KEYCLOAK_DB_PASSWORD} healthcheck: test: ["CMD-SHELL", "pg_isready -U keycloak_user -d keycloak"] interval: 10s @@ -485,15 +513,15 @@ services: - 5432 environment: POSTGRES_USER: app_admin - POSTGRES_PASSWORD: app_admin_password + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_DB: app_db - ORGANIZATION_USER_PASSWORD: organization_password - MEMBER_USER_PASSWORD: member_password - EVENT_USER_PASSWORD: event_password - FEEDBACK_USER_PASSWORD: feedback_password - FINANCE_USER_PASSWORD: finance_password - LETTER_USER_PASSWORD: letter_password - REPORTS_USER_PASSWORD: reports_password + ORGANIZATION_USER_PASSWORD: ${ORGANIZATION_DB_PASSWORD} + MEMBER_USER_PASSWORD: ${MEMBER_DB_PASSWORD} + EVENT_USER_PASSWORD: ${EVENT_DB_PASSWORD} + FEEDBACK_USER_PASSWORD: ${FEEDBACK_DB_PASSWORD} + FINANCE_USER_PASSWORD: ${FINANCE_DB_PASSWORD} + LETTER_USER_PASSWORD: ${LETTER_DB_PASSWORD} + REPORTS_USER_PASSWORD: ${REPORTS_DB_PASSWORD} volumes: - app_db_data:/var/lib/postgresql/data - ./postgres/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro diff --git a/infra/helm/team-devoops/files/realm-config.json b/infra/helm/team-devoops/files/realm-config.json index 8683d952..7bc6ead3 100644 --- a/infra/helm/team-devoops/files/realm-config.json +++ b/infra/helm/team-devoops/files/realm-config.json @@ -14,7 +14,7 @@ "credentials": [ { "type": "password", - "value": "admin123", + "value": "__KEYCLOAK_REALM_ADMIN_PASSWORD__", "temporary": false } ], @@ -31,7 +31,7 @@ "credentials": [ { "type": "password", - "value": "user123", + "value": "__KEYCLOAK_REALM_USER_PASSWORD__", "temporary": false } ], @@ -114,7 +114,7 @@ "clientId": "org-role-sync", "enabled": true, "publicClient": false, - "secret": "org-role-sync-secret", + "secret": "__KEYCLOAK_ADMIN_CLIENT_SECRET__", "standardFlowEnabled": false, "directAccessGrantsEnabled": false, "serviceAccountsEnabled": true @@ -123,7 +123,7 @@ "clientId": "traefik-forward-auth", "enabled": true, "publicClient": false, - "secret": "traefik-forward-auth-secret", + "secret": "__FORWARD_AUTH_CLIENT_SECRET__", "standardFlowEnabled": true, "directAccessGrantsEnabled": false, "redirectUris": [ @@ -141,7 +141,7 @@ "clientId": "grafana", "enabled": true, "publicClient": false, - "secret": "grafana-oauth-secret", + "secret": "__GRAFANA_OAUTH_CLIENT_SECRET__", "standardFlowEnabled": true, "directAccessGrantsEnabled": false, "redirectUris": [ diff --git a/infra/helm/team-devoops/templates/_helpers.tpl b/infra/helm/team-devoops/templates/_helpers.tpl index 0d7a7deb..a10d4a0b 100644 --- a/infra/helm/team-devoops/templates/_helpers.tpl +++ b/infra/helm/team-devoops/templates/_helpers.tpl @@ -16,6 +16,27 @@ app.kubernetes.io/name: {{ .name }} app.kubernetes.io/part-of: team-devoops {{- end -}} +{{/* +Baseline hardened container securityContext (KICS: privilege escalation, unrestricted +capabilities, NET_RAW not dropped). Pass "runAsNonRoot": true only for images confirmed +(via Dockerfile USER or a `docker run --entrypoint id` check against the real image) to +default to a non-root user -- forcing it on images that start as root breaks them (e.g. +the official postgres entrypoint's chown/gosu dance needs CAP_CHOWN/CAP_SETUID at startup). +Pass "runAsUser" only when the image needs an explicit non-root UID pinned (e.g. an +initContainer with no non-root user of its own). +*/}} +{{- define "team-devoops.containerSecurityContext" -}} +allowPrivilegeEscalation: false +capabilities: + drop: ["NET_RAW"] +{{- if .runAsNonRoot }} +runAsNonRoot: true +{{- end }} +{{- if .runAsUser }} +runAsUser: {{ .runAsUser }} +{{- end }} +{{- end -}} + {{/* Resolve the container image for a service. Uses an explicit per-service image (external images such as swagger-ui) when set, diff --git a/infra/helm/team-devoops/templates/alloy-deployment.yaml b/infra/helm/team-devoops/templates/alloy-deployment.yaml index cd992d73..a499f276 100644 --- a/infra/helm/team-devoops/templates/alloy-deployment.yaml +++ b/infra/helm/team-devoops/templates/alloy-deployment.yaml @@ -18,6 +18,9 @@ spec: {{- include "team-devoops.selectorLabels" (dict "name" "alloy") | nindent 8 }} spec: serviceAccountName: alloy + securityContext: + seccompProfile: + type: RuntimeDefault {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -26,6 +29,8 @@ spec: - name: alloy image: {{ .Values.monitoring.alloy.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" false) | nindent 12 }} args: - "run" - "--server.http.listen-addr=0.0.0.0:12345" diff --git a/infra/helm/team-devoops/templates/deployment.yaml b/infra/helm/team-devoops/templates/deployment.yaml index 60e7e9d5..ee0daafc 100644 --- a/infra/helm/team-devoops/templates/deployment.yaml +++ b/infra/helm/team-devoops/templates/deployment.yaml @@ -23,10 +23,13 @@ spec: labels: {{- include "team-devoops.selectorLabels" (dict "name" $name) | nindent 8 }} spec: - {{- if $svc.persistence }} + automountServiceAccountToken: false securityContext: + seccompProfile: + type: RuntimeDefault + {{- if $svc.persistence }} fsGroup: {{ $svc.persistence.fsGroup }} - {{- end }} + {{- end }} {{- with $root.Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -36,6 +39,8 @@ spec: - name: {{ $name }} image: {{ include "team-devoops.image" (dict "name" $name "svc" $svc "root" $root) }} imagePullPolicy: {{ $root.Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" (ne $svc.runAsNonRoot false)) | nindent 12 }} ports: - containerPort: {{ $svc.port }} {{- /* Gives the endpoints controller time to remove this pod from Service diff --git a/infra/helm/team-devoops/templates/forward-auth.yaml b/infra/helm/team-devoops/templates/forward-auth.yaml index e8d904df..202d797f 100644 --- a/infra/helm/team-devoops/templates/forward-auth.yaml +++ b/infra/helm/team-devoops/templates/forward-auth.yaml @@ -17,6 +17,10 @@ spec: labels: {{- include "team-devoops.selectorLabels" (dict "name" "oauth2-proxy") | nindent 8 }} spec: + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -25,6 +29,8 @@ spec: - name: oauth2-proxy image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 imagePullPolicy: IfNotPresent + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" true) | nindent 12 }} ports: - containerPort: 4180 env: diff --git a/infra/helm/team-devoops/templates/grafana-deployment.yaml b/infra/helm/team-devoops/templates/grafana-deployment.yaml index 1338555f..efad77fc 100644 --- a/infra/helm/team-devoops/templates/grafana-deployment.yaml +++ b/infra/helm/team-devoops/templates/grafana-deployment.yaml @@ -23,6 +23,9 @@ spec: # /var/lib/grafana (verified live against the cluster). securityContext: fsGroup: 472 + seccompProfile: + type: RuntimeDefault + automountServiceAccountToken: false {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -31,6 +34,8 @@ spec: - name: grafana image: {{ .Values.monitoring.grafana.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" true) | nindent 12 }} ports: - containerPort: 3000 env: diff --git a/infra/helm/team-devoops/templates/keycloak-deployment.yaml b/infra/helm/team-devoops/templates/keycloak-deployment.yaml index 6b9ae5ff..7bb6fa17 100644 --- a/infra/helm/team-devoops/templates/keycloak-deployment.yaml +++ b/infra/helm/team-devoops/templates/keycloak-deployment.yaml @@ -15,10 +15,16 @@ spec: labels: {{- include "team-devoops.selectorLabels" (dict "name" "keycloak") | nindent 8 }} spec: + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault {{- if .Values.keycloak.db.enabled }} initContainers: - name: wait-for-db image: busybox:1.36 + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" true "runAsUser" 65534) | nindent 12 }} command: - sh - -c @@ -39,6 +45,10 @@ spec: - name: keycloak image: {{ .Values.keycloak.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + # quay.io/keycloak/keycloak defaults to uid 1000 (confirmed by inspecting the + # image's Config.User directly). + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" true) | nindent 12 }} args: ["start-dev", "--import-realm"] ports: - containerPort: 8080 diff --git a/infra/helm/team-devoops/templates/keycloak-postgres-statefulset.yaml b/infra/helm/team-devoops/templates/keycloak-postgres-statefulset.yaml index 4e228ae6..d30af106 100644 --- a/infra/helm/team-devoops/templates/keycloak-postgres-statefulset.yaml +++ b/infra/helm/team-devoops/templates/keycloak-postgres-statefulset.yaml @@ -16,10 +16,16 @@ spec: labels: {{- include "team-devoops.selectorLabels" (dict "name" .Values.keycloak.db.host) | nindent 8 }} spec: + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: postgres image: {{ .Values.database.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" false) | nindent 12 }} ports: - containerPort: 5432 env: diff --git a/infra/helm/team-devoops/templates/keycloak-realm-configmap.yaml b/infra/helm/team-devoops/templates/keycloak-realm-configmap.yaml index dae94ded..2e53c4de 100644 --- a/infra/helm/team-devoops/templates/keycloak-realm-configmap.yaml +++ b/infra/helm/team-devoops/templates/keycloak-realm-configmap.yaml @@ -6,6 +6,13 @@ metadata: labels: {{- include "team-devoops.labels" (dict "name" "keycloak-realm-config" "root" $) | nindent 4 }} data: + # The three client secrets below have no other canonical home in values.yaml except + # forwardAuth.clientSecret and monitoring.grafana.oauthClientSecret, which already drive + # oauth2-proxy/Grafana's own OIDC config elsewhere -- reused here rather than duplicated. + # org-role-sync's secret has no such existing single source, so it's read from + # services.organization-service.env (the same value organization-service itself consumes). realm-config.json: | - {{ .Files.Get "files/realm-config.json" | nindent 4 | trim }} + {{- $orgRoleSyncSecret := index .Values.services "organization-service" "env" "KEYCLOAK_ADMIN_CLIENT_SECRET" }} + {{- $rendered := .Files.Get "files/realm-config.json" | replace "__KEYCLOAK_REALM_ADMIN_PASSWORD__" .Values.keycloak.users.admin.password | replace "__KEYCLOAK_REALM_USER_PASSWORD__" .Values.keycloak.users.user.password | replace "__KEYCLOAK_ADMIN_CLIENT_SECRET__" $orgRoleSyncSecret | replace "__FORWARD_AUTH_CLIENT_SECRET__" .Values.forwardAuth.clientSecret | replace "__GRAFANA_OAUTH_CLIENT_SECRET__" .Values.monitoring.grafana.oauthClientSecret }} + {{ $rendered | nindent 4 | trim }} {{- end }} diff --git a/infra/helm/team-devoops/templates/loki-deployment.yaml b/infra/helm/team-devoops/templates/loki-deployment.yaml index 4a1eacea..ae7aa5f9 100644 --- a/infra/helm/team-devoops/templates/loki-deployment.yaml +++ b/infra/helm/team-devoops/templates/loki-deployment.yaml @@ -23,6 +23,9 @@ spec: # (verified live against the cluster). securityContext: fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + automountServiceAccountToken: false {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -31,6 +34,8 @@ spec: - name: loki image: {{ .Values.monitoring.loki.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" true) | nindent 12 }} # ConfigMap key matches the source file's basename (infra/loki/loki-config.yaml), # not Loki's own default "local-config.yaml" name -- must be spelled out explicitly. args: ["-config.file=/etc/loki/loki-config.yaml"] diff --git a/infra/helm/team-devoops/templates/ollama-deployment.yaml b/infra/helm/team-devoops/templates/ollama-deployment.yaml index b8dcd17f..6995a60a 100644 --- a/infra/helm/team-devoops/templates/ollama-deployment.yaml +++ b/infra/helm/team-devoops/templates/ollama-deployment.yaml @@ -20,6 +20,10 @@ spec: # Unlike prometheus/grafana/loki, the official ollama/ollama image has no USER directive # (runs as root), so it doesn't hit the CSI PVC's root-owned-mount permission issue those # need fsGroup for -- confirmed via the image's config blob (Config.User is empty). + securityContext: + seccompProfile: + type: RuntimeDefault + automountServiceAccountToken: false {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -28,6 +32,8 @@ spec: - name: ollama image: {{ .Values.ollama.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" false) | nindent 12 }} command: ["/usr/bin/bash", "/entrypoint_ollama.sh"] env: # Keeps the chat and embedding models from being resident in memory at the same diff --git a/infra/helm/team-devoops/templates/postgres-statefulset.yaml b/infra/helm/team-devoops/templates/postgres-statefulset.yaml index e8b8203a..cad6ff3d 100644 --- a/infra/helm/team-devoops/templates/postgres-statefulset.yaml +++ b/infra/helm/team-devoops/templates/postgres-statefulset.yaml @@ -16,10 +16,16 @@ spec: labels: {{- include "team-devoops.selectorLabels" (dict "name" .Values.database.host) | nindent 8 }} spec: + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault containers: - name: postgres image: {{ .Values.database.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" false) | nindent 12 }} ports: - containerPort: {{ .Values.database.port }} env: diff --git a/infra/helm/team-devoops/templates/prometheus-deployment.yaml b/infra/helm/team-devoops/templates/prometheus-deployment.yaml index 46bdaad2..7f2bb747 100644 --- a/infra/helm/team-devoops/templates/prometheus-deployment.yaml +++ b/infra/helm/team-devoops/templates/prometheus-deployment.yaml @@ -23,6 +23,9 @@ spec: # writing to /prometheus (verified live against the cluster). securityContext: fsGroup: 65534 + seccompProfile: + type: RuntimeDefault + automountServiceAccountToken: false {{- with .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} @@ -31,6 +34,13 @@ spec: - name: prometheus image: {{ .Values.monitoring.prometheus.image }} imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + # prom/prometheus:v2.55.1's image USER is the name "nobody", not a numeric UID -- + # the kubelet can't verify runAsNonRoot against a non-numeric image user and + # refuses to start the container ("cannot verify user is non-root") rather than + # resolving the name itself. Same tradeoff as ollama/postgres/alloy above: the + # image already runs as non-root by default, we just can't assert it here. + {{- include "team-devoops.containerSecurityContext" (dict "runAsNonRoot" false) | nindent 12 }} ports: - containerPort: 9090 volumeMounts: diff --git a/infra/helm/team-devoops/values.yaml b/infra/helm/team-devoops/values.yaml index 6e415c36..68349176 100644 --- a/infra/helm/team-devoops/values.yaml +++ b/infra/helm/team-devoops/values.yaml @@ -21,42 +21,51 @@ database: enabled: true name: app_db user: app_admin - # Rendered into a Secret (db-credentials). Override in CI/prod via --set. - password: app_admin_password + # Placeholder only -- must be overridden at deploy time from the DB_ADMIN_PASSWORD + # GitHub secret (see cd.yml), same as forwardAuth.cookieSecret. + password: "CHANGE_ME_DB_ADMIN_PASSWORD" host: app-database port: 5432 - # Per-service users. Override passwords via --set in CI/prod. + # Per-service users. Placeholders only -- real passwords come from the + # DB_ORGANIZATION_PASSWORD / DB_MEMBER_PASSWORD / DB_EVENT_PASSWORD / + # DB_FEEDBACK_PASSWORD / DB_FINANCE_PASSWORD / DB_LETTER_PASSWORD / + # DB_REPORTS_PASSWORD GitHub secrets, set via --set in cd.yml. users: organization: username: organization_user - password: organization_password + password: "CHANGE_ME_DB_ORGANIZATION_PASSWORD" member: username: member_user - password: member_password + password: "CHANGE_ME_DB_MEMBER_PASSWORD" event: username: event_user - password: event_password + password: "CHANGE_ME_DB_EVENT_PASSWORD" feedback: username: feedback_user - password: feedback_password + password: "CHANGE_ME_DB_FEEDBACK_PASSWORD" finance: username: finance_user - password: finance_password + password: "CHANGE_ME_DB_FINANCE_PASSWORD" letter: username: letter_user - password: letter_password + password: "CHANGE_ME_DB_LETTER_PASSWORD" reports: username: reports_user - password: reports_password + password: "CHANGE_ME_DB_REPORTS_PASSWORD" image: postgres:15.6-alpine storageSize: 5Gi resources: requests: cpu: 50m memory: 128Mi + # Live `kubectl top` shows ~110Mi/4m against this limit -- 146Mi/146m of margin even + # though this instance serves 7 services' connection pools concurrently. Trimmed to + # free a bit more of the namespace's tight 6Gi/4-CPU quota (see the top-level + # `resources:` comment above for the incident this is responding to); still leaves + # 114Mi/116m over the observed figure. limits: - cpu: 150m - memory: 256Mi + cpu: 120m + memory: 224Mi # py-genai-helper reads its configuration from a Secret created out-of-band by the # pipeline (kubectl create secret --from-env-file). Helm only references it by name. @@ -93,9 +102,11 @@ monitoring: dashboardsConfigMapName: grafana-dashboards alertingConfigMapName: grafana-alerting oauthClientId: grafana - # Matches the existing forwardAuth.clientSecret pattern: a hardcoded default - # for an internal-only confidential client, not treated as a rotated secret. - oauthClientSecret: grafana-oauth-secret + # Placeholder only -- must be overridden at deploy time from the GRAFANA_OAUTH_CLIENT_SECRET + # GitHub secret (see cd.yml), same as forwardAuth.cookieSecret. Also substituted into the + # Keycloak realm import for the "grafana" client (see keycloak-realm-configmap.yaml) -- + # this is the single source of truth for that secret, not duplicated elsewhere. + oauthClientSecret: "CHANGE_ME_GRAFANA_OAUTH_CLIENT_SECRET" resources: requests: cpu: 50m @@ -182,7 +193,20 @@ keycloak: path: /auth hostname: https://ge83mom-devops26.stud.k8s.aet.cit.tum.de/auth adminUsername: admin - adminPassword: admin + # Placeholder only -- must be overridden at deploy time from the KEYCLOAK_ADMIN_PASSWORD + # GitHub secret (see cd.yml), same as forwardAuth.cookieSecret. + adminPassword: "CHANGE_ME_KEYCLOAK_ADMIN_PASSWORD" + # Passwords for the two seeded realm users (files/realm-config.json substitutes these + # for the __KEYCLOAK_REALM_ADMIN_PASSWORD__/__KEYCLOAK_REALM_USER_PASSWORD__ tokens via + # keycloak-realm-configmap.yaml). "users" here are realm-level accounts, distinct from + # keycloak.adminPassword above (the Keycloak server's own bootstrap console admin). + # Placeholders only -- must be overridden at deploy time from the KEYCLOAK_REALM_ADMIN_PASSWORD/ + # KEYCLOAK_REALM_USER_PASSWORD GitHub secrets (see cd.yml), same as forwardAuth.cookieSecret. + users: + admin: + password: "CHANGE_ME_KEYCLOAK_REALM_ADMIN_PASSWORD" + user: + password: "CHANGE_ME_KEYCLOAK_REALM_USER_PASSWORD" # Constrain JVM heap so Keycloak stays within its memory limit. javaOptsAppend: "-Xms64m -Xmx256m" resources: @@ -199,7 +223,9 @@ keycloak: enabled: true name: keycloak user: keycloak_user - password: keycloak_password + # Placeholder only -- must be overridden at deploy time from the KEYCLOAK_DB_PASSWORD + # GitHub secret (see cd.yml), same as forwardAuth.cookieSecret. + password: "CHANGE_ME_KEYCLOAK_DB_PASSWORD" host: keycloak-database storageSize: 2Gi resources: @@ -233,11 +259,16 @@ forwardAuth: enabled: true oidcIssuerUrl: "https://ge83mom-devops26.stud.k8s.aet.cit.tum.de/auth/realms/devops" clientId: "traefik-forward-auth" - clientSecret: "traefik-forward-auth-secret" + # Placeholder only -- must be overridden at deploy time from the FORWARD_AUTH_CLIENT_SECRET + # GitHub secret (see cd.yml). Also substituted into the Keycloak realm import for the + # "traefik-forward-auth" client (see keycloak-realm-configmap.yaml) -- this is the single + # source of truth for that secret, not duplicated elsewhere. + clientSecret: "CHANGE_ME_FORWARD_AUTH_CLIENT_SECRET" # 32+ character random string used to sign session cookies. - # Override at deploy time: --set forwardAuth.cookieSecret="$FORWARD_AUTH_COOKIE_SECRET" + # Placeholder only -- must be overridden at deploy time from the FORWARD_AUTH_COOKIE_SECRET + # GitHub secret (see cd.yml). # Generate with: openssl rand -base64 32 - cookieSecret: "" + cookieSecret: "CHANGE_ME_FORWARD_AUTH_COOKIE_SECRET" # Rolling update strategy — maxSurge: 0 ensures the old pod is terminated before # scheduling the new one, which is required to stay within the namespace CPU quota. @@ -250,13 +281,26 @@ strategy: maxUnavailable: 1 # Default compute resources applied to every app container (overridable per service). +# Used as-is by the six Spring services below with no per-service override (organization/ +# member/event/feedback/finance/letter-service) -- every other service in this chart has +# already been trimmed to just above its observed peak (see prometheus/loki/grafana/alloy/ +# ollama/py-genai-helper comments); this was the one category still on an untuned default. +# Live `kubectl top` shows ~271-286Mi steady-state per service (not an idle snapshot -- these +# had been running normally for 19+ minutes). JAVA_TOOL_OPTIONS caps JVM heap at -Xmx300m +# already, so 350Mi limit leaves ~50Mi for metaspace/threads/native overhead above that hard +# cap -- deliberately not cut closer to the observed 286Mi the way idle-only numbers would +# tempt (see grafana's rejected 200Mi/34Mi-headroom attempt for why that's the wrong move). +# CPU is cut harder (200m -> 150m) since it's compressible (throttles, doesn't OOM-kill) and +# live usage is ~2-3m -- still >50x headroom. Frees ~204Mi/300m off the namespace's tight 6Gi/ +# 4-CPU quota (was at 6040Mi/3560m used at rest, no room for a multi-service rolling deploy's +# old+new pod overlap -- see the 2026-07-09 ollama FailedCreate/exceeded-quota incident). resources: requests: cpu: 100m memory: 128Mi limits: - cpu: 200m - memory: 384Mi + cpu: 150m + memory: 350Mi # --------------------------------------------------------------------------- # Service catalogue. Keys are the Kubernetes object names. @@ -284,6 +328,13 @@ services: dbUser: organization health: /actuator/health stripPrefix: true + # This service's Dockerfile creates a named "user" account (adduser -D user), not a + # numeric UID -- the kubelet can't verify runAsNonRoot against a non-numeric image user + # and hard-fails container creation ("cannot verify user is non-root") rather than + # resolving the name itself. Same reasoning applies to every other -service below and + # to py-genai-helper. The image already runs as non-root by default; we just can't + # assert it to the kubelet without either this or pinning an explicit numeric runAsUser. + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 @@ -295,11 +346,13 @@ services: JAVA_TOOL_OPTIONS: "-Xmx300m -Xms64m" # Keycloak Admin REST API for syncing membership client-roles (KeycloakRoleService). # The org-role-sync service-account client is defined in files/realm-config.json. - # Override KEYCLOAK_ADMIN_CLIENT_SECRET via --set in CI/prod. + # Placeholder only -- must be overridden at deploy time from the KEYCLOAK_ADMIN_CLIENT_SECRET + # GitHub secret (see cd.yml). Also the single source of truth substituted into the Keycloak + # realm import (keycloak-realm-configmap.yaml) and into member-service's matching env below. KEYCLOAK_BASE_URL: "http://keycloak:8080/auth" KEYCLOAK_REALM: "devops" KEYCLOAK_ADMIN_CLIENT_ID: "org-role-sync" - KEYCLOAK_ADMIN_CLIENT_SECRET: "org-role-sync-secret" + KEYCLOAK_ADMIN_CLIENT_SECRET: "CHANGE_ME_KEYCLOAK_ADMIN_CLIENT_SECRET" KEYCLOAK_ROLES_CLIENT: "devops-client" member-service: path: /api/v1/members @@ -308,6 +361,7 @@ services: dbUser: member health: /actuator/health stripPrefix: true + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 @@ -317,9 +371,14 @@ services: SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_ISSUER_URI: "https://ge83mom-devops26.stud.k8s.aet.cit.tum.de/auth/realms/devops" SPRING_SECURITY_OAUTH2_RESOURCESERVER_JWT_JWK_SET_URI: "http://keycloak:8080/auth/realms/devops/protocol/openid-connect/certs" JAVA_TOOL_OPTIONS: "-Xmx300m -Xms64m" - # Keycloak Admin REST API for user provisioning (KeycloakService). + # Keycloak Admin REST API for user provisioning (KeycloakService). Same org-role-sync + # service-account client/secret as organization-service above (see that entry's comment) -- + # was previously not wired here at all, so this silently ran on application.properties' + # hardcoded fallback in every deploy target. KEYCLOAK_BASE_URL: "http://keycloak:8080/auth" KEYCLOAK_REALM: "devops" + KEYCLOAK_SERVICE_ACCOUNT_CLIENT_ID: "org-role-sync" + KEYCLOAK_SERVICE_ACCOUNT_CLIENT_SECRET: "CHANGE_ME_KEYCLOAK_ADMIN_CLIENT_SECRET" event-service: path: /api/v1/events port: 8080 @@ -327,6 +386,7 @@ services: dbUser: event health: /actuator/health stripPrefix: true + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 @@ -343,6 +403,7 @@ services: dbUser: feedback health: /actuator/health stripPrefix: true + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 @@ -359,6 +420,7 @@ services: dbUser: finance health: /actuator/health stripPrefix: true + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 @@ -375,6 +437,7 @@ services: dbUser: letter health: /actuator/health stripPrefix: true + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 @@ -392,6 +455,7 @@ services: dbUser: reports health: /health stripPrefix: true + runAsNonRoot: false # Flask routes here are bare (e.g. /reports/...), unlike the Spring services' # generated controllers which keep their own name in the path -- needs the # whole "/api/v1/helper" prefix removed, not just "/api/v1". @@ -467,6 +531,7 @@ services: health: /docs/ stripPrefix: false open: true + runAsNonRoot: false autoscaling: enabled: true minReplicas: 1 diff --git a/infra/keycloak/realm-config.json b/infra/keycloak/realm-config.json index 8683d952..7bc6ead3 100644 --- a/infra/keycloak/realm-config.json +++ b/infra/keycloak/realm-config.json @@ -14,7 +14,7 @@ "credentials": [ { "type": "password", - "value": "admin123", + "value": "__KEYCLOAK_REALM_ADMIN_PASSWORD__", "temporary": false } ], @@ -31,7 +31,7 @@ "credentials": [ { "type": "password", - "value": "user123", + "value": "__KEYCLOAK_REALM_USER_PASSWORD__", "temporary": false } ], @@ -114,7 +114,7 @@ "clientId": "org-role-sync", "enabled": true, "publicClient": false, - "secret": "org-role-sync-secret", + "secret": "__KEYCLOAK_ADMIN_CLIENT_SECRET__", "standardFlowEnabled": false, "directAccessGrantsEnabled": false, "serviceAccountsEnabled": true @@ -123,7 +123,7 @@ "clientId": "traefik-forward-auth", "enabled": true, "publicClient": false, - "secret": "traefik-forward-auth-secret", + "secret": "__FORWARD_AUTH_CLIENT_SECRET__", "standardFlowEnabled": true, "directAccessGrantsEnabled": false, "redirectUris": [ @@ -141,7 +141,7 @@ "clientId": "grafana", "enabled": true, "publicClient": false, - "secret": "grafana-oauth-secret", + "secret": "__GRAFANA_OAUTH_CLIENT_SECRET__", "standardFlowEnabled": true, "directAccessGrantsEnabled": false, "redirectUris": [ diff --git a/services/py-genai-helper/db.py b/services/py-genai-helper/db.py index 8e397d4e..9a0199c6 100644 --- a/services/py-genai-helper/db.py +++ b/services/py-genai-helper/db.py @@ -8,11 +8,14 @@ """ import os +import time import uuid from datetime import UTC, datetime +import psycopg2.errors from sqlalchemy import create_engine, text from sqlalchemy.engine import make_url +from sqlalchemy.exc import ProgrammingError _engine = None @@ -37,15 +40,7 @@ def get_engine(): return _engine -def init_db() -> None: - """Create the report tables if they don't exist yet (idempotent). - - The foreign keys into member.members / organization.teams use ON DELETE CASCADE, matching the rest - of the system: deleting a member/team removes their reports. The referenced tables must already - exist, so a first boot that wins the race against the member / organization services raises and the - container retries (restart: on-failure), the same way the Spring services' cross-schema FK - migrations do. - """ +def _create_report_tables() -> None: with get_engine().begin() as conn: conn.execute( text( @@ -79,6 +74,30 @@ def init_db() -> None: ) +def init_db() -> None: + """Create the report tables if they don't exist yet (idempotent). + + The foreign keys into member.members / organization.teams use ON DELETE CASCADE, matching the rest + of the system: deleting a member/team removes their reports. The referenced tables must already + exist, so on a cold/fresh database this races against the member and organization services + creating them, raising UndefinedTable if we lose. Retried here rather than letting the process + crash: a crash falls back to the container's restart backoff, which is exponential and can push + the next attempt out far enough to blow past a deploy's overall timeout even though the actual + dependency only takes a couple of minutes to appear. The retry budget (2 minutes) stays well + within the startup probe's ~5.3 minute allowance, so losing this race no longer costs a container + restart at all, just a slower first boot. + """ + deadline = time.monotonic() + 120 + while True: + try: + _create_report_tables() + return + except ProgrammingError as e: + if not isinstance(e.orig, psycopg2.errors.UndefinedTable) or time.monotonic() >= deadline: + raise + time.sleep(2) + + def insert_member_report(member_id: str, report_text: str) -> str: report_id = str(uuid.uuid4()) with get_engine().begin() as conn: diff --git a/services/spring-letter/.env.example b/services/spring-letter/.env.example index 9c401745..5836843a 100644 --- a/services/spring-letter/.env.example +++ b/services/spring-letter/.env.example @@ -1,4 +1,4 @@ # Example environment file for spring-letter. # Copy to `.env` and fill in real values locally. Never commit secrets. -MAIL_USERNAME= -MAIL_PASSWORD= +MAIL_USERNAME=your_email@gmail.com +MAIL_PASSWORD=CHANGE_ME_MAIL_PASSWORD diff --git a/services/spring-member/src/main/resources/application.properties b/services/spring-member/src/main/resources/application.properties index d0416824..704b0a3c 100644 --- a/services/spring-member/src/main/resources/application.properties +++ b/services/spring-member/src/main/resources/application.properties @@ -19,4 +19,4 @@ spring.flyway.create-schemas=true keycloak.base-url=http://keycloak:8080/auth keycloak.realm=devops keycloak.service-account.client-id=${KEYCLOAK_SERVICE_ACCOUNT_CLIENT_ID:org-role-sync} -keycloak.service-account.client-secret=${KEYCLOAK_SERVICE_ACCOUNT_CLIENT_SECRET:org-role-sync-secret} +keycloak.service-account.client-secret=${KEYCLOAK_SERVICE_ACCOUNT_CLIENT_SECRET:CHANGE_ME_KEYCLOAK_ADMIN_CLIENT_SECRET} diff --git a/services/spring-organization/src/main/resources/application.properties b/services/spring-organization/src/main/resources/application.properties index 1140864d..a2e2706f 100644 --- a/services/spring-organization/src/main/resources/application.properties +++ b/services/spring-organization/src/main/resources/application.properties @@ -21,5 +21,5 @@ spring.flyway.create-schemas=true keycloak.base-url=http://keycloak:8080/auth keycloak.realm=devops keycloak.admin.client-id=org-role-sync -keycloak.admin.client-secret=org-role-sync-secret +keycloak.admin.client-secret=${KEYCLOAK_ADMIN_CLIENT_SECRET:CHANGE_ME_KEYCLOAK_ADMIN_CLIENT_SECRET} keycloak.roles-client=devops-client