fix: harden production database recovery path

.github/workflows/db-backup-freshness.yml

@@ -81,14 +81,9 @@ jobs:
         id: select
         run: |
           set -euo pipefail
-          test "$(aws sts get-caller-identity --query Account --output text)" = '992382684924'
-          aws s3api get-public-access-block --bucket "$BACKUP_BUCKET" \
-            --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,IgnorePublicAcls,BlockPublicPolicy,RestrictPublicBuckets]' \
-            --output text | grep -q $'True\tTrue\tTrue\tTrue'
-          test "$(aws s3api get-bucket-encryption --bucket "$BACKUP_BUCKET" \
-            --query 'ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault.SSEAlgorithm' \
-            --output text)" = 'AES256'
+          scripts/check_live_database_backup_target.sh "$BACKUP_BUCKET"
           aws s3api list-objects-v2 --bucket "$BACKUP_BUCKET" --prefix daily/ \
+            --expected-bucket-owner 992382684924 \
             > /tmp/database-backup-inventory.json
           python scripts/check_database_backup_freshness.py select \
             --inventory /tmp/database-backup-inventory.json \
@@ -101,10 +96,12 @@ jobs:
           set -euo pipefail
           aws s3api get-object --bucket "$BACKUP_BUCKET" \
             --key '${{ steps.select.outputs.manifest_key }}' \
+            --expected-bucket-owner 992382684924 \
             /tmp/database-backup-manifest.json > /dev/null
           aws s3api get-object-attributes --bucket "$BACKUP_BUCKET" \
             --key '${{ steps.select.outputs.ciphertext_key }}' \
             --object-attributes Checksum,ObjectSize,StorageClass \
+            --expected-bucket-owner 992382684924 \
             > /tmp/database-backup-attributes.json
           python scripts/check_database_backup_freshness.py verify \
             --selection /tmp/database-backup-selection.json \

.github/workflows/db-backup.yml

@@ -22,7 +22,12 @@ concurrency:
 
 jobs:
   dump:
-    runs-on: ubuntu-latest
+    # The complete production database is plaintext until the local age step.
+    # Route this job only to the workflow-restricted production backup group.
+    # GitHub-hosted runners and the default self-hosted group are not permitted.
+    runs-on:
+      group: production-backup
+      labels: self-hosted
     timeout-minutes: 30
     environment: production-backup
     permissions:
@@ -39,6 +44,8 @@ jobs:
       GITHUB_REF_PROTECTED: ${{ github.ref_protected }}
     steps:
       - name: Validate the protected environment configuration
+        env:
+          RUNNER_BOUNDARY: ${{ runner.environment }}
         run: |
           set -euo pipefail
           missing=()
@@ -58,6 +65,10 @@ jobs:
             echo '::error::The database backup refuses an unprotected main branch.'
             exit 1
           fi
+          if [ "${RUNNER_BOUNDARY}" != 'self-hosted' ]; then
+            echo '::error::The database backup requires its isolated self-hosted production boundary.'
+            exit 1
+          fi
           echo 'The four required production-backup settings are present.'
 
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -139,10 +150,7 @@ jobs:
             echo '::error::The production-backup environment needs AWS_BACKUP_BUCKET.'
             exit 1
           fi
-          test "$(aws sts get-caller-identity --query Account --output text)" = '992382684924'
-          aws s3api get-public-access-block --bucket "$BACKUP_BUCKET" \
-            --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,IgnorePublicAcls,BlockPublicPolicy,RestrictPublicBuckets]' \
-            --output text | grep -q $'True\tTrue\tTrue\tTrue'
+          scripts/check_live_database_backup_target.sh "$BACKUP_BUCKET"
           python scripts/database_backup_contract.py validate-source \
             --db-url "$SUPABASE_DB_URL" \
             --project-ref "$SUPABASE_PROJECT_REF" \
@@ -221,7 +229,8 @@ jobs:
           aws s3 cp artifact-manifest.json \
             "s3://${BACKUP_BUCKET}/${prefix}/artifact-manifest.json" \
             --only-show-errors --sse AES256 \
-            --content-type application/json --checksum-algorithm SHA256
+            --content-type application/json --checksum-algorithm SHA256 \
+            --expected-bucket-owner 992382684924
 
           aws s3api get-object-attributes \
             --bucket "$BACKUP_BUCKET" --key "${prefix}/${cipher}" \

ops/PRODUCTION_OPERATIONS.md

@@ -104,17 +104,20 @@ this order:
 2. Create `production-backup` and `production-backup-monitor`. Give each
    environment one exact custom `main` branch policy and no other deployment
    policy.
-3. Deploy the reviewed backup CloudFormation stack in AWS account
+3. Create the workflow-restricted `production-backup` runner group and an
+   ephemeral runner inside the production data boundary. Put the runner only in
+   that group. Permit only the exact backup workflow on `refs/heads/main`.
+4. Deploy the reviewed backup CloudFormation stack in AWS account
    `992382684924`.
-4. Configure the four settings in the `production-backup` GitHub environment.
-5. Configure the two variables in the `production-backup-monitor` environment.
-6. Store a second copy of the private `age` key in a team vault or offline
+5. Configure the four settings in the `production-backup` GitHub environment.
+6. Configure the two variables in the `production-backup-monitor` environment.
+7. Store a second copy of the private `age` key in a team vault or offline
    medium.
-7. Create an isolated scratch Supabase project and complete the first recovery
+8. Create an isolated scratch Supabase project and complete the first recovery
    drill.
-8. Select the primary operator, the secondary operator, the support hours, and
+9. Select the primary operator, the secondary operator, the support hours, and
    the response targets for human halts.
-9. Configure an external monitor for the production health and backup freshness
+10. Configure an external monitor for the production health and backup freshness
    schedules.
-10. Complete the first genuine customer transaction when an authorized customer
+11. Complete the first genuine customer transaction when an authorized customer
    is available. Do not create a founder self-charge as evidence.

ops/backup/RESTORE_DRILL.md

@@ -30,6 +30,7 @@ the matching protected-environment configuration failure. As of that check:
 - provider PITR is not enabled;
 - the AWS stack is not deployed;
 - `main` has no repository ruleset or branch protection;
+- no workflow-restricted production backup runner group or runner is confirmed;
 - the `production-backup` GitHub environment has none of its four required
   settings and has no deployment-branch restriction;
 - the `production-backup-monitor` GitHub environment is not configured;
@@ -77,15 +78,28 @@ deployment policy is the exact branch gate.
    select every protected branch. Do not add a tag or wildcard policy.
 4. Create the `production-backup-monitor` GitHub environment.
 5. Give it the same single custom `main` branch policy.
-6. Do not require a manual environment approval. An approval wait would prevent
+6. Create an organization runner group named `production-backup`. Permit only
+   this repository. Restrict it to
+   `OpenAdaptAI/openadapt-ops/.github/workflows/db-backup.yml@refs/heads/main`.
+   Do not put the runner in the default group and do not permit another
+   workflow to use the group.
+7. Register an ephemeral Linux runner inside the declared OpenAdapt production
+   boundary and put it only in the `production-backup` group. Use a clean
+   encrypted work volume for each job and remove the runner after the job.
+   Permit network access only to GitHub Actions, the exact production Supabase
+   database endpoints, and the private AWS backup target.
+8. Do not require a manual environment approval. An approval wait would prevent
    the scheduled jobs.
 
 Verify that `main` reports as protected. Verify that each environment reports
 `custom_branch_policies: true`, `protected_branches: false`, and one policy with
 the exact name `main`. Both workflows repeat this check after the environment
 admits the job and before they request AWS credentials. The job has read-only
 Actions permission for this API check. Do not deploy the AWS stack until this
-gate passes.
+gate passes. Also verify that the runner group is restricted to the exact
+workflow and that the selected runner reports `self-hosted`. A self-hosted
+runner attached to this public repository without the exact workflow
+restriction is unsafe because pull-request code can target it.
 
 ## One-time AWS setup
 
@@ -111,6 +125,13 @@ SHA-256. S3 validates that checksum before it accepts the object. This launch
 path refuses an encrypted archive above 5 GiB before upload. Build and qualify a
 multipart contract before a production database can exceed that limit.
 
+The writer, monitor, and restore paths read the live bucket controls before
+they use an object. They require AWS account `992382684924`, region
+`us-east-1`, complete public-access blocking, SSE-S3, versioning,
+bucket-owner-enforced ownership, the exact 90-day and 365-day lifecycle rules,
+TLS-only transport, and the exact encryption policy. A drifted target stops the
+operation before database access or recovery-point selection.
+
 An AWS principal with CloudFormation, IAM, and S3 administration rights must
 run:
 
@@ -162,6 +183,12 @@ The workflow validates that the URL belongs to the declared Supabase project.
 It also checks AWS account `992382684924`, complete S3 public-access blocking,
 and the committed `age` recipient before it reads the database.
 
+The dump job does not use a GitHub-hosted runner. The complete database exists
+in plaintext on the isolated backup runner until local `age` encryption. The
+cleanup trap removes the plaintext before the runner is destroyed. The
+freshness monitor can use a GitHub-hosted runner because it reads only the
+redacted manifest and S3 object metadata. It cannot read the ciphertext.
+
 The public recipient is in `ops/backup/age-recipients.txt`. Store its private
 key with mode `0600` on an encrypted trusted device. Make a second copy in a
 team vault or an offline medium before the first backup. Without a second copy,
@@ -231,7 +258,9 @@ The script:
 6. decrypts into a private temporary directory;
 7. extracts only four exact regular files and rejects unsafe archive members;
 8. restores with `ON_ERROR_STOP` in one transaction;
-9. dumps the scratch schema and data again and compares their digests;
+9. dumps the scratch schema and data again, normalizes only the matched random
+   PostgreSQL restriction guard outside `COPY` data, and compares their
+   digests;
 10. writes a new database-only evidence file without overwriting old evidence;
 11. uploads that metadata-only evidence below `drills/database-only/`; and
 12. removes all temporary plaintext.
@@ -240,6 +269,13 @@ RTO starts before AWS role assumption and download. It ends after the scratch
 database redump and validation. RPO is measured from the backup recovery point
 to the same start time.
 
+Patched PostgreSQL clients create a new `\restrict` key for each plain-text
+dump. Supabase CLI comments that key, so two correct dumps have different raw
+digests. The verifier keeps the raw source digests in the evidence and uses a
+second comparison digest that replaces only one matched `restrict` and
+`unrestrict` pair outside `COPY` blocks. A guard-shaped database value remains
+data and a changed value still fails the drill.
+
 The script does not delete the scratch project. Review the evidence first.
 Then decommission the project through its authorized owner process.
 

ops/backup/aws-backup-target.yml

@@ -121,7 +121,14 @@ Resources:
             Version: '2012-10-17'
             Statement:
               - Effect: Allow
-                Action: s3:GetBucketPublicAccessBlock
+                Action:
+                  - s3:GetBucketPublicAccessBlock
+                  - s3:GetEncryptionConfiguration
+                  - s3:GetBucketVersioning
+                  - s3:GetBucketOwnershipControls
+                  - s3:GetBucketLocation
+                  - s3:GetLifecycleConfiguration
+                  - s3:GetBucketPolicy
                 Resource: !GetAtt BackupBucket.Arn
               - Effect: Allow
                 Action:
@@ -154,6 +161,11 @@ Resources:
                 Action:
                   - s3:GetBucketPublicAccessBlock
                   - s3:GetEncryptionConfiguration
+                  - s3:GetBucketVersioning
+                  - s3:GetBucketOwnershipControls
+                  - s3:GetBucketLocation
+                  - s3:GetLifecycleConfiguration
+                  - s3:GetBucketPolicy
                 Resource: !GetAtt BackupBucket.Arn
               - Effect: Allow
                 Action: s3:ListBucket
@@ -185,7 +197,14 @@ Resources:
             Version: '2012-10-17'
             Statement:
               - Effect: Allow
-                Action: s3:GetBucketPublicAccessBlock
+                Action:
+                  - s3:GetBucketPublicAccessBlock
+                  - s3:GetEncryptionConfiguration
+                  - s3:GetBucketVersioning
+                  - s3:GetBucketOwnershipControls
+                  - s3:GetBucketLocation
+                  - s3:GetLifecycleConfiguration
+                  - s3:GetBucketPolicy
                 Resource: !GetAtt BackupBucket.Arn
               - Effect: Allow
                 Action: s3:GetObject