feat: harden production readiness operations

.github/CODEOWNERS

@@ -1,9 +1,20 @@
-# Default code owners. Enforced via branch protection (required review).
+# Default code owners. This file routes review. Repository branch protection
+# must separately require a code-owner review before this becomes an enforced
+# merge gate.
 * @abrichr
 
 # Production backup and restore trust boundary.
 /.github/workflows/db-backup.yml @abrichr
+/.github/workflows/db-backup-freshness.yml @abrichr
+/.github/workflows/prod-health-alert.yml @abrichr
+/ops/PRODUCTION_OPERATIONS.md @abrichr
 /ops/backup/ @abrichr
 /scripts/database_backup_contract.py @abrichr
+/scripts/check_database_backup_freshness.py @abrichr
+/scripts/check_github_environment_gate.py @abrichr
+/scripts/check_production_readiness.py @abrichr
 /scripts/run_database_restore_drill.sh @abrichr
 /tests/test_database_backup_contract.py @abrichr
+/tests/test_database_backup_freshness.py @abrichr
+/tests/test_github_environment_gate.py @abrichr
+/tests/test_production_readiness.py @abrichr

.github/workflows/db-backup-freshness.yml

@@ -0,0 +1,164 @@
+name: Production DB backup freshness
+
+# Read-only check of the newest off-provider database recovery point. This job
+# cannot create, replace, or delete a backup. It inspects only the redacted
+# manifest and S3 metadata; it never downloads or decrypts database bytes.
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '43 * * * *'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: production-db-backup-freshness
+  cancel-in-progress: false
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment: production-backup-monitor
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
+    env:
+      AWS_REGION: us-east-1
+      BACKUP_BUCKET: ${{ vars.AWS_BACKUP_BUCKET }}
+      BACKUP_MONITOR_ROLE_ARN: ${{ vars.AWS_BACKUP_MONITOR_ROLE_ARN }}
+      EXPECTED_GITHUB_ENVIRONMENT: production-backup-monitor
+      GITHUB_REF_PROTECTED: ${{ github.ref_protected }}
+    steps:
+      - name: Validate the read-only monitor configuration
+        run: |
+          set -euo pipefail
+          missing=()
+          [ -n "${BACKUP_BUCKET}" ] || missing+=(AWS_BACKUP_BUCKET)
+          [ -n "${BACKUP_MONITOR_ROLE_ARN}" ] || missing+=(AWS_BACKUP_MONITOR_ROLE_ARN)
+          if [ "${#missing[@]}" -ne 0 ]; then
+            echo "::error::The production-backup-monitor environment is missing: ${missing[*]}"
+            exit 1
+          fi
+          if [ "${GITHUB_REF}" != 'refs/heads/main' ]; then
+            echo "::error::The backup monitor must run from refs/heads/main, not ${GITHUB_REF}."
+            exit 1
+          fi
+          if [ "${GITHUB_REF_PROTECTED}" != 'true' ]; then
+            echo '::error::The backup monitor refuses an unprotected main branch.'
+            exit 1
+          fi
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Verify the exact GitHub environment gate
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}" \
+            > /tmp/database-backup-environment.json
+          gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}/deployment-branch-policies?per_page=100" \
+            > /tmp/database-backup-environment-policies.json
+          python scripts/check_github_environment_gate.py \
+            --environment-json /tmp/database-backup-environment.json \
+            --policies-json /tmp/database-backup-environment-policies.json \
+            --expected-environment "${EXPECTED_GITHUB_ENVIRONMENT}" \
+            --expected-branch main \
+            --actual-ref "${GITHUB_REF}" \
+            --ref-protected "${GITHUB_REF_PROTECTED}"
+
+      - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ env.BACKUP_MONITOR_ROLE_ARN }}
+          allowed-account-ids: '992382684924'
+          role-session-name: openadapt-db-backup-monitor-${{ github.run_id }}
+
+      - name: Verify the private bucket and select the newest recovery point
+        id: select
+        run: |
+          set -euo pipefail
+          test "$(aws sts get-caller-identity --query Account --output text)" = '992382684924'
+          aws s3api get-public-access-block --bucket "$BACKUP_BUCKET" \
+            --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,IgnorePublicAcls,BlockPublicPolicy,RestrictPublicBuckets]' \
+            --output text | grep -q $'True\tTrue\tTrue\tTrue'
+          test "$(aws s3api get-bucket-encryption --bucket "$BACKUP_BUCKET" \
+            --query 'ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault.SSEAlgorithm' \
+            --output text)" = 'AES256'
+          aws s3api list-objects-v2 --bucket "$BACKUP_BUCKET" --prefix daily/ \
+            > /tmp/database-backup-inventory.json
+          python scripts/check_database_backup_freshness.py select \
+            --inventory /tmp/database-backup-inventory.json \
+            --output /tmp/database-backup-selection.json \
+            --github-output "$GITHUB_OUTPUT" \
+            --maximum-age-seconds 86400
+
+      - name: Verify the redacted manifest and the encrypted object
+        run: |
+          set -euo pipefail
+          aws s3api get-object --bucket "$BACKUP_BUCKET" \
+            --key '${{ steps.select.outputs.manifest_key }}' \
+            /tmp/database-backup-manifest.json > /dev/null
+          aws s3api get-object-attributes --bucket "$BACKUP_BUCKET" \
+            --key '${{ steps.select.outputs.ciphertext_key }}' \
+            --object-attributes Checksum,ObjectSize,StorageClass \
+            > /tmp/database-backup-attributes.json
+          python scripts/check_database_backup_freshness.py verify \
+            --selection /tmp/database-backup-selection.json \
+            --manifest /tmp/database-backup-manifest.json \
+            --attributes /tmp/database-backup-attributes.json
+
+  record-alert:
+    name: Keep one durable backup freshness alert
+    needs: verify
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      TITLE: Production database recovery point is stale or unverified
+    steps:
+      - name: Open, reopen, update, or close the freshness issue
+        env:
+          VERIFY_RESULT: ${{ needs.verify.result }}
+        run: |
+          set -euo pipefail
+          existing_json=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \
+            --limit 1000 --json number,title,state \
+            --jq '[.[] | select(.title == env.TITLE)][0] // {}')
+          existing=$(jq -r '.number // empty' <<< "${existing_json}")
+          existing_state=$(jq -r '.state // empty' <<< "${existing_json}")
+          if [ "${VERIFY_RESULT}" = 'success' ]; then
+            if [ -n "${existing}" ] && [ "${existing_state}" = 'OPEN' ]; then
+              gh issue close "${existing}" --repo "${GITHUB_REPOSITORY}" \
+                --comment "The newest encrypted recovery point is complete and less than 24 hours old in ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}."
+            else
+              echo 'The recovery point is current and no freshness issue is open.'
+            fi
+            exit 0
+          fi
+
+          printf '%s\n' \
+            'The read-only monitor did not prove a complete encrypted database recovery point from the last 24 hours.' \
+            '' \
+            "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+            '' \
+            'The check reads only the redacted manifest and S3 metadata. It does not read or decrypt database bytes.' \
+            '' \
+            'Do not claim the 24-hour database RPO until a later run passes and closes this issue.' \
+            > database-backup-freshness-alert.md
+          if [ -n "${existing}" ]; then
+            if [ "${existing_state}" = 'CLOSED' ]; then
+              gh issue reopen "${existing}" --repo "${GITHUB_REPOSITORY}"
+            fi
+            gh issue edit "${existing}" --repo "${GITHUB_REPOSITORY}" \
+              --body-file database-backup-freshness-alert.md
+          else
+            gh issue create --repo "${GITHUB_REPOSITORY}" \
+              --title "${TITLE}" --body-file database-backup-freshness-alert.md
+          fi

.github/workflows/db-backup.yml

@@ -4,7 +4,9 @@ name: Production DB logical backup
 # production-backup environment secret. Only age ciphertext and a redacted
 # integrity manifest enter the private, public-access-blocked S3 bucket.
 # Maximum RPO: 24 hours. Retention: 90 days. This does not cover Storage
-# objects and does not replace provider PITR.
+# objects and does not replace provider PITR. The launch path uses one
+# S3 PutObject with a service-validated full-object SHA-256 and refuses an
+# encrypted archive above the 5 GiB PutObject limit before upload.
 
 on:
   workflow_dispatch:
@@ -13,7 +15,6 @@ on:
 
 permissions:
   contents: read
-  id-token: write
 
 concurrency:
   group: production-db-backup
@@ -24,18 +25,64 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     environment: production-backup
+    permissions:
+      actions: read
+      contents: read
+      id-token: write
     env:
       AWS_REGION: us-east-1
+      BACKUP_ROLE_ARN: ${{ vars.AWS_BACKUP_ROLE_ARN }}
       BACKUP_BUCKET: ${{ vars.AWS_BACKUP_BUCKET }}
       SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }}
       SUPABASE_PROJECT_REF: ${{ secrets.SUPABASE_PROJECT_REF }}
+      EXPECTED_GITHUB_ENVIRONMENT: production-backup
+      GITHUB_REF_PROTECTED: ${{ github.ref_protected }}
     steps:
+      - name: Validate the protected environment configuration
+        run: |
+          set -euo pipefail
+          missing=()
+          [ -n "${BACKUP_ROLE_ARN}" ] || missing+=(AWS_BACKUP_ROLE_ARN)
+          [ -n "${BACKUP_BUCKET}" ] || missing+=(AWS_BACKUP_BUCKET)
+          [ -n "${SUPABASE_DB_URL}" ] || missing+=(SUPABASE_DB_URL)
+          [ -n "${SUPABASE_PROJECT_REF}" ] || missing+=(SUPABASE_PROJECT_REF)
+          if [ "${#missing[@]}" -ne 0 ]; then
+            echo "::error::The production-backup environment is missing: ${missing[*]}"
+            exit 1
+          fi
+          if [ "${GITHUB_REF}" != 'refs/heads/main' ]; then
+            echo "::error::The database backup must run from refs/heads/main, not ${GITHUB_REF}."
+            exit 1
+          fi
+          if [ "${GITHUB_REF_PROTECTED}" != 'true' ]; then
+            echo '::error::The database backup refuses an unprotected main branch.'
+            exit 1
+          fi
+          echo 'The four required production-backup settings are present.'
+
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      - name: Verify the exact GitHub environment gate
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}" \
+            > /tmp/database-backup-environment.json
+          gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}/deployment-branch-policies?per_page=100" \
+            > /tmp/database-backup-environment-policies.json
+          python scripts/check_github_environment_gate.py \
+            --environment-json /tmp/database-backup-environment.json \
+            --policies-json /tmp/database-backup-environment-policies.json \
+            --expected-environment "${EXPECTED_GITHUB_ENVIRONMENT}" \
+            --expected-branch main \
+            --actual-ref "${GITHUB_REF}" \
+            --ref-protected "${GITHUB_REF_PROTECTED}"
+
       - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3
         with:
           aws-region: ${{ env.AWS_REGION }}
-          role-to-assume: ${{ vars.AWS_BACKUP_ROLE_ARN }}
+          role-to-assume: ${{ env.BACKUP_ROLE_ARN }}
           allowed-account-ids: '992382684924'
           role-session-name: openadapt-db-backup-${{ github.run_id }}
 
@@ -80,6 +127,8 @@ jobs:
             if [ -n "$plain" ]; then rm -f "$plain"; fi
             if [ -n "$cipher" ]; then rm -f "$cipher"; fi
             rm -f artifact-manifest.json
+            rm -f s3-upload-contract.json /tmp/database-backup-object-attributes.json
+            rm -f /tmp/database-backup-put-response.json
           }
           trap cleanup EXIT
 
@@ -122,19 +171,81 @@ jobs:
             --manifest artifact-manifest.json \
             --ciphertext-archive "$cipher"
 
-          local_sha=$(sha256sum "$cipher" | cut -d' ' -f1)
-          local_checksum=$(openssl dgst -sha256 -binary "$cipher" | base64)
-          aws s3 cp "$cipher" "s3://${BACKUP_BUCKET}/${prefix}/${cipher}" \
-            --only-show-errors --sse AES256 --metadata "sha256=${local_sha}" \
-            --checksum-algorithm SHA256
+          python scripts/database_backup_contract.py prepare-single-put \
+            --manifest artifact-manifest.json \
+            --ciphertext-archive "$cipher" \
+            --output s3-upload-contract.json
+          cipher_bytes=$(jq -r '.bytes' s3-upload-contract.json)
+          local_sha=$(jq -r '.sha256' s3-upload-contract.json)
+          local_checksum=$(jq -r '.checksum_sha256' s3-upload-contract.json)
+          aws s3api put-object \
+            --bucket "$BACKUP_BUCKET" --key "${prefix}/${cipher}" \
+            --body "$cipher" --content-length "$cipher_bytes" \
+            --server-side-encryption AES256 --metadata "sha256=${local_sha}" \
+            --checksum-algorithm SHA256 --checksum-sha256 "$local_checksum" \
+            --expected-bucket-owner 992382684924 \
+            > /tmp/database-backup-put-response.json
           aws s3 cp artifact-manifest.json \
             "s3://${BACKUP_BUCKET}/${prefix}/artifact-manifest.json" \
             --only-show-errors --sse AES256 \
             --content-type application/json --checksum-algorithm SHA256
 
-          remote_checksum=$(aws s3api get-object-attributes \
+          aws s3api get-object-attributes \
             --bucket "$BACKUP_BUCKET" --key "${prefix}/${cipher}" \
-            --object-attributes Checksum \
-            --query 'Checksum.ChecksumSHA256' --output text)
-          test "$remote_checksum" = "$local_checksum"
+            --object-attributes Checksum,ObjectSize \
+            --expected-bucket-owner 992382684924 \
+            > /tmp/database-backup-object-attributes.json
+          python scripts/database_backup_contract.py verify-single-put \
+            --upload-contract s3-upload-contract.json \
+            --attributes /tmp/database-backup-object-attributes.json
           echo "Encrypted database backup stored at s3://${BACKUP_BUCKET}/${prefix}/"
+
+  record-alert:
+    name: Keep one durable backup alert
+    needs: dump
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      TITLE: Production database backup is not current
+    steps:
+      - name: Open, reopen, update, or close the backup issue
+        env:
+          BACKUP_RESULT: ${{ needs.dump.result }}
+        run: |
+          set -euo pipefail
+          existing_json=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \
+            --limit 1000 --json number,title,state \
+            --jq '[.[] | select(.title == env.TITLE)][0] // {}')
+          existing=$(jq -r '.number // empty' <<< "${existing_json}")
+          existing_state=$(jq -r '.state // empty' <<< "${existing_json}")
+          if [ "${BACKUP_RESULT}" = 'success' ]; then
+            if [ -n "${existing}" ] && [ "${existing_state}" = 'OPEN' ]; then
+              gh issue close "${existing}" --repo "${GITHUB_REPOSITORY}" \
+                --comment "The encrypted database backup passed in ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}. The independent freshness check remains authoritative for the current recovery point."
+            else
+              echo 'The backup passed and no workflow-failure issue is open.'
+            fi
+            exit 0
+          fi
+
+          printf '%s\n' \
+            'The scheduled production database backup did not complete.' \
+            '' \
+            "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
+            '' \
+            'No new recovery point is claimed. Review the failed step before any retry.' \
+            > database-backup-alert.md
+          if [ -n "${existing}" ]; then
+            if [ "${existing_state}" = 'CLOSED' ]; then
+              gh issue reopen "${existing}" --repo "${GITHUB_REPOSITORY}"
+            fi
+            gh issue edit "${existing}" --repo "${GITHUB_REPOSITORY}" \
+              --body-file database-backup-alert.md
+          else
+            gh issue create --repo "${GITHUB_REPOSITORY}" \
+              --title "${TITLE}" --body-file database-backup-alert.md
+          fi