diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4bc9a3e..7bbf8e6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,9 +1,20 @@ -# Default code owners. Enforced via branch protection (required review). +# Default code owners. This file routes review. Repository branch protection +# must separately require a code-owner review before this becomes an enforced +# merge gate. * @abrichr # Production backup and restore trust boundary. /.github/workflows/db-backup.yml @abrichr +/.github/workflows/db-backup-freshness.yml @abrichr +/.github/workflows/prod-health-alert.yml @abrichr +/ops/PRODUCTION_OPERATIONS.md @abrichr /ops/backup/ @abrichr /scripts/database_backup_contract.py @abrichr +/scripts/check_database_backup_freshness.py @abrichr +/scripts/check_github_environment_gate.py @abrichr +/scripts/check_production_readiness.py @abrichr /scripts/run_database_restore_drill.sh @abrichr /tests/test_database_backup_contract.py @abrichr +/tests/test_database_backup_freshness.py @abrichr +/tests/test_github_environment_gate.py @abrichr +/tests/test_production_readiness.py @abrichr diff --git a/.github/workflows/db-backup-freshness.yml b/.github/workflows/db-backup-freshness.yml new file mode 100644 index 0000000..7301357 --- /dev/null +++ b/.github/workflows/db-backup-freshness.yml @@ -0,0 +1,164 @@ +name: Production DB backup freshness + +# Read-only check of the newest off-provider database recovery point. This job +# cannot create, replace, or delete a backup. It inspects only the redacted +# manifest and S3 metadata; it never downloads or decrypts database bytes. + +on: + workflow_dispatch: + schedule: + - cron: '43 * * * *' + +permissions: + contents: read + +concurrency: + group: production-db-backup-freshness + cancel-in-progress: false + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: production-backup-monitor + permissions: + actions: read + contents: read + id-token: write + env: + AWS_REGION: us-east-1 + BACKUP_BUCKET: ${{ vars.AWS_BACKUP_BUCKET }} + BACKUP_MONITOR_ROLE_ARN: ${{ vars.AWS_BACKUP_MONITOR_ROLE_ARN }} + EXPECTED_GITHUB_ENVIRONMENT: production-backup-monitor + GITHUB_REF_PROTECTED: ${{ github.ref_protected }} + steps: + - name: Validate the read-only monitor configuration + run: | + set -euo pipefail + missing=() + [ -n "${BACKUP_BUCKET}" ] || missing+=(AWS_BACKUP_BUCKET) + [ -n "${BACKUP_MONITOR_ROLE_ARN}" ] || missing+=(AWS_BACKUP_MONITOR_ROLE_ARN) + if [ "${#missing[@]}" -ne 0 ]; then + echo "::error::The production-backup-monitor environment is missing: ${missing[*]}" + exit 1 + fi + if [ "${GITHUB_REF}" != 'refs/heads/main' ]; then + echo "::error::The backup monitor must run from refs/heads/main, not ${GITHUB_REF}." + exit 1 + fi + if [ "${GITHUB_REF_PROTECTED}" != 'true' ]; then + echo '::error::The backup monitor refuses an unprotected main branch.' + exit 1 + fi + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Verify the exact GitHub environment gate + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}" \ + > /tmp/database-backup-environment.json + gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}/deployment-branch-policies?per_page=100" \ + > /tmp/database-backup-environment-policies.json + python scripts/check_github_environment_gate.py \ + --environment-json /tmp/database-backup-environment.json \ + --policies-json /tmp/database-backup-environment-policies.json \ + --expected-environment "${EXPECTED_GITHUB_ENVIRONMENT}" \ + --expected-branch main \ + --actual-ref "${GITHUB_REF}" \ + --ref-protected "${GITHUB_REF_PROTECTED}" + + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ env.BACKUP_MONITOR_ROLE_ARN }} + allowed-account-ids: '992382684924' + role-session-name: openadapt-db-backup-monitor-${{ github.run_id }} + + - name: Verify the private bucket and select the newest recovery point + id: select + run: | + set -euo pipefail + test "$(aws sts get-caller-identity --query Account --output text)" = '992382684924' + aws s3api get-public-access-block --bucket "$BACKUP_BUCKET" \ + --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,IgnorePublicAcls,BlockPublicPolicy,RestrictPublicBuckets]' \ + --output text | grep -q $'True\tTrue\tTrue\tTrue' + test "$(aws s3api get-bucket-encryption --bucket "$BACKUP_BUCKET" \ + --query 'ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault.SSEAlgorithm' \ + --output text)" = 'AES256' + aws s3api list-objects-v2 --bucket "$BACKUP_BUCKET" --prefix daily/ \ + > /tmp/database-backup-inventory.json + python scripts/check_database_backup_freshness.py select \ + --inventory /tmp/database-backup-inventory.json \ + --output /tmp/database-backup-selection.json \ + --github-output "$GITHUB_OUTPUT" \ + --maximum-age-seconds 86400 + + - name: Verify the redacted manifest and the encrypted object + run: | + set -euo pipefail + aws s3api get-object --bucket "$BACKUP_BUCKET" \ + --key '${{ steps.select.outputs.manifest_key }}' \ + /tmp/database-backup-manifest.json > /dev/null + aws s3api get-object-attributes --bucket "$BACKUP_BUCKET" \ + --key '${{ steps.select.outputs.ciphertext_key }}' \ + --object-attributes Checksum,ObjectSize,StorageClass \ + > /tmp/database-backup-attributes.json + python scripts/check_database_backup_freshness.py verify \ + --selection /tmp/database-backup-selection.json \ + --manifest /tmp/database-backup-manifest.json \ + --attributes /tmp/database-backup-attributes.json + + record-alert: + name: Keep one durable backup freshness alert + needs: verify + if: ${{ always() }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ github.token }} + TITLE: Production database recovery point is stale or unverified + steps: + - name: Open, reopen, update, or close the freshness issue + env: + VERIFY_RESULT: ${{ needs.verify.result }} + run: | + set -euo pipefail + existing_json=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \ + --limit 1000 --json number,title,state \ + --jq '[.[] | select(.title == env.TITLE)][0] // {}') + existing=$(jq -r '.number // empty' <<< "${existing_json}") + existing_state=$(jq -r '.state // empty' <<< "${existing_json}") + if [ "${VERIFY_RESULT}" = 'success' ]; then + if [ -n "${existing}" ] && [ "${existing_state}" = 'OPEN' ]; then + gh issue close "${existing}" --repo "${GITHUB_REPOSITORY}" \ + --comment "The newest encrypted recovery point is complete and less than 24 hours old in ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}." + else + echo 'The recovery point is current and no freshness issue is open.' + fi + exit 0 + fi + + printf '%s\n' \ + 'The read-only monitor did not prove a complete encrypted database recovery point from the last 24 hours.' \ + '' \ + "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + '' \ + 'The check reads only the redacted manifest and S3 metadata. It does not read or decrypt database bytes.' \ + '' \ + 'Do not claim the 24-hour database RPO until a later run passes and closes this issue.' \ + > database-backup-freshness-alert.md + if [ -n "${existing}" ]; then + if [ "${existing_state}" = 'CLOSED' ]; then + gh issue reopen "${existing}" --repo "${GITHUB_REPOSITORY}" + fi + gh issue edit "${existing}" --repo "${GITHUB_REPOSITORY}" \ + --body-file database-backup-freshness-alert.md + else + gh issue create --repo "${GITHUB_REPOSITORY}" \ + --title "${TITLE}" --body-file database-backup-freshness-alert.md + fi diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml index 3beafd1..8424072 100644 --- a/.github/workflows/db-backup.yml +++ b/.github/workflows/db-backup.yml @@ -4,7 +4,9 @@ name: Production DB logical backup # production-backup environment secret. Only age ciphertext and a redacted # integrity manifest enter the private, public-access-blocked S3 bucket. # Maximum RPO: 24 hours. Retention: 90 days. This does not cover Storage -# objects and does not replace provider PITR. +# objects and does not replace provider PITR. The launch path uses one +# S3 PutObject with a service-validated full-object SHA-256 and refuses an +# encrypted archive above the 5 GiB PutObject limit before upload. on: workflow_dispatch: @@ -13,7 +15,6 @@ on: permissions: contents: read - id-token: write concurrency: group: production-db-backup @@ -24,18 +25,64 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 environment: production-backup + permissions: + actions: read + contents: read + id-token: write env: AWS_REGION: us-east-1 + BACKUP_ROLE_ARN: ${{ vars.AWS_BACKUP_ROLE_ARN }} BACKUP_BUCKET: ${{ vars.AWS_BACKUP_BUCKET }} SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} SUPABASE_PROJECT_REF: ${{ secrets.SUPABASE_PROJECT_REF }} + EXPECTED_GITHUB_ENVIRONMENT: production-backup + GITHUB_REF_PROTECTED: ${{ github.ref_protected }} steps: + - name: Validate the protected environment configuration + run: | + set -euo pipefail + missing=() + [ -n "${BACKUP_ROLE_ARN}" ] || missing+=(AWS_BACKUP_ROLE_ARN) + [ -n "${BACKUP_BUCKET}" ] || missing+=(AWS_BACKUP_BUCKET) + [ -n "${SUPABASE_DB_URL}" ] || missing+=(SUPABASE_DB_URL) + [ -n "${SUPABASE_PROJECT_REF}" ] || missing+=(SUPABASE_PROJECT_REF) + if [ "${#missing[@]}" -ne 0 ]; then + echo "::error::The production-backup environment is missing: ${missing[*]}" + exit 1 + fi + if [ "${GITHUB_REF}" != 'refs/heads/main' ]; then + echo "::error::The database backup must run from refs/heads/main, not ${GITHUB_REF}." + exit 1 + fi + if [ "${GITHUB_REF_PROTECTED}" != 'true' ]; then + echo '::error::The database backup refuses an unprotected main branch.' + exit 1 + fi + echo 'The four required production-backup settings are present.' + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Verify the exact GitHub environment gate + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}" \ + > /tmp/database-backup-environment.json + gh api "repos/${GITHUB_REPOSITORY}/environments/${EXPECTED_GITHUB_ENVIRONMENT}/deployment-branch-policies?per_page=100" \ + > /tmp/database-backup-environment-policies.json + python scripts/check_github_environment_gate.py \ + --environment-json /tmp/database-backup-environment.json \ + --policies-json /tmp/database-backup-environment-policies.json \ + --expected-environment "${EXPECTED_GITHUB_ENVIRONMENT}" \ + --expected-branch main \ + --actual-ref "${GITHUB_REF}" \ + --ref-protected "${GITHUB_REF_PROTECTED}" + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: aws-region: ${{ env.AWS_REGION }} - role-to-assume: ${{ vars.AWS_BACKUP_ROLE_ARN }} + role-to-assume: ${{ env.BACKUP_ROLE_ARN }} allowed-account-ids: '992382684924' role-session-name: openadapt-db-backup-${{ github.run_id }} @@ -80,6 +127,8 @@ jobs: if [ -n "$plain" ]; then rm -f "$plain"; fi if [ -n "$cipher" ]; then rm -f "$cipher"; fi rm -f artifact-manifest.json + rm -f s3-upload-contract.json /tmp/database-backup-object-attributes.json + rm -f /tmp/database-backup-put-response.json } trap cleanup EXIT @@ -122,19 +171,81 @@ jobs: --manifest artifact-manifest.json \ --ciphertext-archive "$cipher" - local_sha=$(sha256sum "$cipher" | cut -d' ' -f1) - local_checksum=$(openssl dgst -sha256 -binary "$cipher" | base64) - aws s3 cp "$cipher" "s3://${BACKUP_BUCKET}/${prefix}/${cipher}" \ - --only-show-errors --sse AES256 --metadata "sha256=${local_sha}" \ - --checksum-algorithm SHA256 + python scripts/database_backup_contract.py prepare-single-put \ + --manifest artifact-manifest.json \ + --ciphertext-archive "$cipher" \ + --output s3-upload-contract.json + cipher_bytes=$(jq -r '.bytes' s3-upload-contract.json) + local_sha=$(jq -r '.sha256' s3-upload-contract.json) + local_checksum=$(jq -r '.checksum_sha256' s3-upload-contract.json) + aws s3api put-object \ + --bucket "$BACKUP_BUCKET" --key "${prefix}/${cipher}" \ + --body "$cipher" --content-length "$cipher_bytes" \ + --server-side-encryption AES256 --metadata "sha256=${local_sha}" \ + --checksum-algorithm SHA256 --checksum-sha256 "$local_checksum" \ + --expected-bucket-owner 992382684924 \ + > /tmp/database-backup-put-response.json aws s3 cp artifact-manifest.json \ "s3://${BACKUP_BUCKET}/${prefix}/artifact-manifest.json" \ --only-show-errors --sse AES256 \ --content-type application/json --checksum-algorithm SHA256 - remote_checksum=$(aws s3api get-object-attributes \ + aws s3api get-object-attributes \ --bucket "$BACKUP_BUCKET" --key "${prefix}/${cipher}" \ - --object-attributes Checksum \ - --query 'Checksum.ChecksumSHA256' --output text) - test "$remote_checksum" = "$local_checksum" + --object-attributes Checksum,ObjectSize \ + --expected-bucket-owner 992382684924 \ + > /tmp/database-backup-object-attributes.json + python scripts/database_backup_contract.py verify-single-put \ + --upload-contract s3-upload-contract.json \ + --attributes /tmp/database-backup-object-attributes.json echo "Encrypted database backup stored at s3://${BACKUP_BUCKET}/${prefix}/" + + record-alert: + name: Keep one durable backup alert + needs: dump + if: ${{ always() }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ github.token }} + TITLE: Production database backup is not current + steps: + - name: Open, reopen, update, or close the backup issue + env: + BACKUP_RESULT: ${{ needs.dump.result }} + run: | + set -euo pipefail + existing_json=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \ + --limit 1000 --json number,title,state \ + --jq '[.[] | select(.title == env.TITLE)][0] // {}') + existing=$(jq -r '.number // empty' <<< "${existing_json}") + existing_state=$(jq -r '.state // empty' <<< "${existing_json}") + if [ "${BACKUP_RESULT}" = 'success' ]; then + if [ -n "${existing}" ] && [ "${existing_state}" = 'OPEN' ]; then + gh issue close "${existing}" --repo "${GITHUB_REPOSITORY}" \ + --comment "The encrypted database backup passed in ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}. The independent freshness check remains authoritative for the current recovery point." + else + echo 'The backup passed and no workflow-failure issue is open.' + fi + exit 0 + fi + + printf '%s\n' \ + 'The scheduled production database backup did not complete.' \ + '' \ + "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + '' \ + 'No new recovery point is claimed. Review the failed step before any retry.' \ + > database-backup-alert.md + if [ -n "${existing}" ]; then + if [ "${existing_state}" = 'CLOSED' ]; then + gh issue reopen "${existing}" --repo "${GITHUB_REPOSITORY}" + fi + gh issue edit "${existing}" --repo "${GITHUB_REPOSITORY}" \ + --body-file database-backup-alert.md + else + gh issue create --repo "${GITHUB_REPOSITORY}" \ + --title "${TITLE}" --body-file database-backup-alert.md + fi diff --git a/.github/workflows/prod-health-alert.yml b/.github/workflows/prod-health-alert.yml index 54b86ec..54b645a 100644 --- a/.github/workflows/prod-health-alert.yml +++ b/.github/workflows/prod-health-alert.yml @@ -2,11 +2,12 @@ name: Production health alert # $0 pager for the hosted control plane. Probes the real dependency # probe (openadapt-cloud src/lib/readiness.ts — returns {"ready":true} -# with HTTP 200, or 503 with per-component detail). When the endpoint -# is unhealthy or unreachable after 3 attempts spread over ~2 minutes, -# the run FAILS; GitHub emails the actor who last modified this -# workflow file on a failed scheduled run. Optional Telegram alert -# fires when the crier bot secrets exist in this repo. +# with HTTP 200, or 503 with per-component detail). A local contract also +# requires live mode, a fresh checked_at value, the complete dependency set, +# active encrypted-writer identity, and no-store response headers. When the +# endpoint is unhealthy, incomplete, stale, or unreachable after 3 attempts +# spread over ~2 minutes, the run fails and a durable issue opens or updates. +# Optional Telegram alert fires when the crier bot secrets exist in this repo. # # Cadence: every 30 minutes = 48 runs/day of a <1-minute job. The repo # is public, so Actions minutes are free; even metered this would be @@ -38,30 +39,34 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Probe app.openadapt.ai/api/health/ready (3 attempts over ~2 min) run: | set -u url='https://app.openadapt.ai/api/health/ready' attempts=3 for i in $(seq 1 "$attempts"); do - if body=$(curl -sS -m 20 -w '\n%{http_code}' "$url" 2>&1); then - code=$(printf '%s\n' "$body" | tail -n1) - json=$(printf '%s\n' "$body" | sed '$d') - ready=$(printf '%s' "$json" | jq -r '.ready // false' 2>/dev/null || echo 'parse-error') - echo "attempt ${i}/${attempts}: http=${code} ready=${ready}" - if [ "$code" = '200' ] && [ "$ready" = 'true' ]; then + headers=$(mktemp) + body=$(mktemp) + if curl -sS -m 20 -D "$headers" -o "$body" "$url"; then + if python scripts/check_production_readiness.py \ + --headers "$headers" --body "$body"; then + echo "attempt ${i}/${attempts}: the complete live readiness contract passed" + rm -f "$headers" "$body" exit 0 fi # The endpoint is public and returns metadata-only detail; # print which components are not ready to make the failure # email actionable. - printf '%s' "$json" | jq -r '.components[]? | select(.state != "ready") | "NOT READY: \(.name): \(.detail)"' 2>/dev/null || true + jq -r '.components[]? | select(.required == true and .state != "ready") | "NOT READY: \(.name): \(.detail)"' "$body" 2>/dev/null || true else echo "attempt ${i}/${attempts}: request failed (network error or timeout)" fi + rm -f "$headers" "$body" if [ "$i" -lt "$attempts" ]; then sleep 60; fi done - echo "::error::${url} is unhealthy or unreachable after ${attempts} attempts over ~2 minutes." + echo "::error::${url} did not prove the complete live readiness contract after ${attempts} attempts over ~2 minutes." exit 1 - name: Telegram alert (optional — runs only when crier secrets exist) @@ -78,5 +83,57 @@ jobs: fi curl -sS -m 20 -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ --data-urlencode "chat_id=${TELEGRAM_OWNER_ID}" \ - --data-urlencode "text=ALERT: app.openadapt.ai /api/health/ready is unhealthy or unreachable. Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --data-urlencode "text=ALERT: app.openadapt.ai did not prove the complete live readiness contract. Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ > /dev/null && echo 'Telegram alert sent.' + + record-alert: + name: Keep one durable production health alert + needs: probe + if: ${{ always() }} + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + env: + GH_TOKEN: ${{ github.token }} + TITLE: Production health check is failing + steps: + - name: Open, reopen, update, or close the health issue + env: + PROBE_RESULT: ${{ needs.probe.result }} + run: | + set -euo pipefail + existing_json=$(gh issue list --repo "${GITHUB_REPOSITORY}" --state all \ + --limit 1000 --json number,title,state \ + --jq '[.[] | select(.title == env.TITLE)][0] // {}') + existing=$(jq -r '.number // empty' <<< "${existing_json}") + existing_state=$(jq -r '.state // empty' <<< "${existing_json}") + if [ "${PROBE_RESULT}" = 'success' ]; then + if [ -n "${existing}" ] && [ "${existing_state}" = 'OPEN' ]; then + gh issue close "${existing}" --repo "${GITHUB_REPOSITORY}" \ + --comment "The complete live readiness contract passes in ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}." + else + echo 'The health contract passes and no alert issue is open.' + fi + exit 0 + fi + + printf '%s\n' \ + 'The scheduled production probe did not prove the complete live readiness contract.' \ + '' \ + "Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + '' \ + 'The check requires live mode, a fresh response, all required dependencies, the human-decision delivery component, the encrypted writer, and no-store headers.' \ + '' \ + 'Do not report the hosted service as ready until a later run passes and closes this issue.' \ + > production-health-alert.md + if [ -n "${existing}" ]; then + if [ "${existing_state}" = 'CLOSED' ]; then + gh issue reopen "${existing}" --repo "${GITHUB_REPOSITORY}" + fi + gh issue edit "${existing}" --repo "${GITHUB_REPOSITORY}" \ + --body-file production-health-alert.md + else + gh issue create --repo "${GITHUB_REPOSITORY}" \ + --title "${TITLE}" --body-file production-health-alert.md + fi diff --git a/README.md b/README.md index c0f1232..80a6c52 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ MIT. The flagship code lives at - **`tidy/`**: a CLI for scanning and scrubbing sensitive patterns from git history and build artifacts (GitHub Releases, Actions, PyPI, and GHCR). See [`tidy/README.md`](tidy/README.md). +- **`ops/`**: production operations and recovery runbooks. Start with + [`ops/PRODUCTION_OPERATIONS.md`](ops/PRODUCTION_OPERATIONS.md). - **`repos.yml`**: the list of ecosystem repositories the pipeline reads from. ## Build the docs locally diff --git a/docs/concepts/deployment-matrix.md b/docs/concepts/deployment-matrix.md index 73cd167..4761f4b 100644 --- a/docs/concepts/deployment-matrix.md +++ b/docs/concepts/deployment-matrix.md @@ -107,12 +107,20 @@ Production selects live mode explicitly. Development mock mode is visibly synthetic. A missing production dependency makes the affected operation unavailable rather than substituting a simulated success. -The retained hosted-recorder qualification used a Flow 1.8.0 worker. The live -runner and compiler report the pinned managed-runtime Flow 1.23.0 identity, and public -readiness checks live authentication, database, private storage, runner, -compiler, recorder, callbacks, scheduler, retention, secret encryption, -validation policy, and billing dependencies. That is configuration and service -identity evidence, not a customer workflow qualification or SLA. +The retained hosted-recorder qualification used a Flow 1.8.0 worker. The +current Cloud managed-runtime manifest pins Flow 1.31.0 at release commit +`2d225dea9a0ad29ca84ce1b037cc0ac671367e28`. Its wheel SHA-256 is +`81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c` and +its source SHA-256 is +`cf1fc356d14d267df82be188de3e9a3575734f18f46ef91ac8075438cc731540`. +The pin proves configured artifact identity. It does not prove that the build is +deployed or that a hosted workflow passed acceptance. Public readiness checks +live mode, authentication, database, private storage, runner, compiler, +runtime-validation trust, runtime boundary, bundle protection, recorder, +callbacks, scheduler, human-decision Web Push, retention, security events, +secret encryption, validation policy, and billing dependencies. Readiness is +configuration and service-identity evidence, not a customer workflow +qualification or SLA. Stripe is the commercial source of truth for pricing. This matrix does not create a price, quota, SLA, certification, or backend entitlement. diff --git a/docs/get-started/what-works-today.md b/docs/get-started/what-works-today.md index 1951291..2ffad47 100644 --- a/docs/get-started/what-works-today.md +++ b/docs/get-started/what-works-today.md @@ -72,7 +72,7 @@ customer-controlled runtime connected to the same governance model. | Hosted CLI connectivity | **Supported / public offer** | `login`, exact-hash artifact preparation/upload, one-time runtime validation, bound replacement activation, and `report-break` connect the local engine to the live control plane. | Upload requires destination policy and an approved sanitized derivative; checkout never bypasses an egress refusal. | | Artifact sanitation and local review | **Supported / launch gate** | The sanitized-derivative pipeline inventories, transforms, rescans, manifests, hashes, and supports local review/approval. | The raw original remains sensitive; unknown or unresolved content is refused; runtime observations can reintroduce PHI/PII. | | Cross-engine hosted validation | **Supported / launch gate** | `validate-hosted` binds an approved recording and bundle, compiler provenance, strict lint, policy certification, derived risk class, and successful replay report to a one-time Cloud challenge. | It is operator self-attestation signed with the ingest token, not an independently observed certification. Exact deployment policy, risk-class, and deployed compiler-version allowlists still apply. | -| Hosted browser recorder and runtime health | **Supported / bounded launch component** | A retained non-simulated hosted session on `openadapt-flow` 1.8.0 produced frames and input evidence, assembled a compileable recording, finalized one workflow idempotently, enforced resource limits, and cleaned up ephemeral qualification data. Authenticated live health reports the pinned managed-runtime 1.23.0 runner/compiler identity and checks auth, database, storage, callbacks, scheduler, retention, secrets, validation policy, and billing. | Explicitly initiated, public-HTTPS, non-regulated authoring only. Raw observations remain private inside the declared hosted boundary. Readiness proves deployed dependencies and service identity, not a customer workflow qualification or SLA. | +| Hosted browser recorder and runtime health | **Supported / bounded launch component** | A retained non-simulated hosted session on `openadapt-flow` 1.8.0 produced frames and input evidence, assembled a compileable recording, finalized one workflow idempotently, enforced resource limits, and cleaned up ephemeral qualification data. The current managed-runtime manifest pins the Flow 1.31.0 runner/compiler artifact identity. Authenticated live health separately checks live mode, auth, database, storage, runner, compiler, runtime-validation trust, runtime boundary, bundle protection, recorder, callbacks, scheduler, human-decision Web Push, retention, security events, secrets, validation policy, and billing. | Explicitly initiated, public-HTTPS, non-regulated authoring only. Raw observations remain private inside the declared hosted boundary. A runtime pin does not prove live deployment or hosted acceptance. Readiness proves deployed dependencies and service identity, not a customer workflow qualification or SLA. | | Hosted dashboard/control plane | **Supported / public offer** | Authentication, organizations, exact-hash bundle ingest, immutable run admission, browser runner orchestration, structural reports, replacement activation, billing, and metering form the managed lifecycle. | Production uses live dependencies and fails unavailable rather than substituting mock behavior. | | Hosted execution | **Supported / public offer** | Live Stripe Checkout connects onboarding and subscription entitlements to managed browser execution; the runner verifies exact admitted bundle bytes and authenticated callbacks. | The public subscription covers approved browser workflows. Other substrates use separately scoped deployments and commercial terms. Checkout does not create an SLA or certification. | | Air-gapped on-prem package | **Supported** | A local queue, systemd unit, minimized hash-chained audit log, and air-gap checks are provided. | Full-disk encryption and operational hardening remain operator/deployment responsibilities. | diff --git a/docs/guides/hosted.md b/docs/guides/hosted.md index 199a59a..c7b0f8c 100644 --- a/docs/guides/hosted.md +++ b/docs/guides/hosted.md @@ -38,7 +38,7 @@ switching, and sign-out. | Surface | Launch status | Boundary | |---|---|---| | Local browser record -> compile -> managed execute | **Beta / public offer** | Governed authoring and validation remain local; managed execution uses the qualified browser substrate. | -| Hosted browser record -> compileable workflow | **Beta / bounded launch component** | The retained non-simulated provider qualification used `openadapt-flow` 1.8.0; the current live runner and compiler report the pinned managed-runtime 1.23.0 identity. This is a separate raw-observation boundary, not the reviewed-derivative upload lane. | +| Hosted browser record -> compileable workflow | **Beta / bounded launch component** | The retained non-simulated provider qualification used `openadapt-flow` 1.8.0; the current managed-runtime manifest pins Flow 1.31.0 artifact identity. A runtime pin does not prove live deployment or hosted workflow acceptance. This is a separate raw-observation boundary, not the reviewed-derivative upload lane. | | Account, organization, onboarding | **Beta / public offer** | Checkout and sign-in bind the subscription to an isolated organization. | | Structural run history and reports | **Beta / public offer** | Safety depends on the workflow's configured identity, effect, and policy checks. Repair and validation remain local. | | Checkout, portal, entitlements, metering | **Beta / public offer** | Live Stripe Checkout, signed webhooks, entitlements, usage, and the billing portal form one managed subscription contract. | @@ -64,11 +64,14 @@ demo. A qualified hosted browser session produced PNG frames, accepted and retained input evidence, assembled a native recording, created one compileable workflow idempotently, enforced its resource limits, and removed the ephemeral qualification data. That retained qualification used an `openadapt-flow` 1.8.0 -worker. The live runner and compiler report the pinned managed-runtime 1.23.0 -identity, and the public readiness endpoint verifies the configured live dependencies, -including authentication, storage, callbacks, scheduling, retention, secret -encryption, validation policy, and billing. Readiness is dependency evidence, -not a customer workflow qualification or an SLA. +worker. The managed-runtime manifest pins the Flow 1.31.0 artifact identity. A +pin does not prove that the build is live or that a hosted workflow passed +acceptance. The public readiness endpoint separately verifies the configured +live dependencies, including authentication, storage, runner, compiler, +runtime-validation trust, runtime boundary, bundle protection, recorder, +callbacks, scheduling, human-decision Web Push, retention, security events, +secret encryption, validation policy, and billing. Readiness is dependency +evidence, not a customer workflow qualification or an SLA. The recorder accepts only public HTTPS DNS hosts and refuses credentials in the start URL, literal IP addresses, private or mixed DNS answers, and private diff --git a/docs/guides/security-review.md b/docs/guides/security-review.md index a911b65..6974b3a 100644 --- a/docs/guides/security-review.md +++ b/docs/guides/security-review.md @@ -131,13 +131,15 @@ reports, teaching, billing, and usage metering. Production explicitly selects live dependencies; a missing runner, storage, or billing dependency returns an operational failure and never substitutes mock success. Mock mode remains for development and is visibly synthetic. The retained non-simulated hosted-recorder -qualification was run on Flow 1.8.0; the live runner and compiler report the -pinned managed-runtime Flow 1.23.0 identity. The public readiness endpoint currently verifies -live mode, authentication, database migrations, private storage, runner, -compiler, recorder, callbacks, scheduler, retention policy, secret encryption, -runtime-validation allowlists, and live billing configuration. Readiness proves -those dependencies and contracts are configured and reachable; it is not a -customer workflow qualification or an SLA. +qualification was run on Flow 1.8.0. The current managed-runtime manifest pins +Flow 1.31.0 artifact identity; that pin does not prove live deployment or hosted +workflow acceptance. The public readiness endpoint separately verifies live +mode, authentication, database migrations, private storage, runner, compiler, +runtime-validation trust, runtime boundary, bundle protection, recorder, +callbacks, scheduler, human-decision Web Push, retention policy, security +events, secret encryption, runtime-validation allowlists, and live billing +configuration. Readiness proves those dependencies and contracts are configured +and reachable; it is not a customer workflow qualification or an SLA. Windows UIA, native macOS, native Linux, RDP, and Citrix/VDI are first-class substrates, ordered as scoped deployments and qualified per workflow in their real diff --git a/docs/published-version-claims.json b/docs/published-version-claims.json index 0efbae6..d6162c2 100644 --- a/docs/published-version-claims.json +++ b/docs/published-version-claims.json @@ -7,8 +7,8 @@ "Why this exists: on 2026-07-27 openadapt-flow 1.24.0 published and four", "sentences on docs.openadapt.ai -- including one on the security-review", "page -- kept asserting that the live runner reported 'the published Flow", - "1.23.0 identity'. The NUMBER was right (the managed runtime really does", - "pin 1.23.0); the word 'published' was what became false. Nothing could", + "1.23.0 identity'. The NUMBER was right at that time because the managed", + "runtime pinned 1.23.0; the word 'published' was what became false. Nothing", "detect that, because no machine-readable record said which numbers are", "supposed to track the current release and which are deliberately frozen.", "", @@ -42,29 +42,29 @@ "id": "hosted-runner-managed-runtime-pin", "kind": "pinned-deployment", "package": "openadapt-flow", - "version": "1.23.0", - "evidence": "openadapt-cloud runner/runtime-version.json pins openadapt_flow 1.23.0 (wheel sha256 b5d10dfc294479866d6dddd4c1a6afc9164414f0634703173b7e440dc0133bec); https://app.openadapt.ai/api/health/ready reports ready:true, mode:live.", - "verified_on": "2026-07-27", + "version": "1.31.0", + "evidence": "openadapt-cloud origin/main 4e0257a1299cc0869af3e3f664a6ebfa0b59db1 runner/runtime-version.json pins openadapt_flow 1.31.0 at release commit 2d225dea9a0ad29ca84ce1b037cc0ac671367e28 (wheel sha256 81133db1528ad1bb1f26e3fcb6aea61b0651db6d905cf2e4943e8383c1f3d29c; sdist sha256 cf1fc356d14d267df82be188de3e9a3575734f18f46ef91ac8075438cc731540). This records the configured artifact pin, not live deployment or hosted acceptance.", + "verified_on": "2026-08-18", "locations": [ { "file": "docs/concepts/deployment-matrix.md", - "context": "report the pinned managed-runtime Flow 1.23.0 identity" + "context": "managed-runtime manifest pins Flow 1.31.0 at release" }, { "file": "docs/guides/security-review.md", - "context": "report the\npinned managed-runtime Flow 1.23.0 identity" + "context": "managed-runtime manifest pins\nFlow 1.31.0 artifact identity" }, { "file": "docs/guides/hosted.md", - "context": "report the pinned managed-runtime 1.23.0 identity" + "context": "managed-runtime manifest pins Flow 1.31.0 artifact identity" }, { "file": "docs/guides/hosted.md", - "context": "report the pinned managed-runtime 1.23.0\nidentity" + "context": "managed-runtime manifest pins the Flow 1.31.0 artifact identity" }, { "file": "docs/get-started/what-works-today.md", - "context": "reports the pinned managed-runtime 1.23.0 runner/compiler identity" + "context": "managed-runtime manifest pins the Flow 1.31.0 runner/compiler artifact identity" } ] }, diff --git a/ops/PRODUCTION_OPERATIONS.md b/ops/PRODUCTION_OPERATIONS.md new file mode 100644 index 0000000..3849055 --- /dev/null +++ b/ops/PRODUCTION_OPERATIONS.md @@ -0,0 +1,120 @@ +# Production operations contract + +This runbook defines the minimum operating control for the hosted browser lane. +It does not make every OpenAdapt workflow or execution surface production +ready. The exact workflow, application, environment, verifier, and deployment +must pass qualification. + +## Automated checks + +| Check | Schedule | Proof | Durable signal | +|---|---:|---|---| +| Production health | Every 30 minutes | HTTP 200, `no-store`, fresh `checked_at`, live mode, active encrypted writer, and every required component | One GitHub issue and optional Telegram alert | +| Database backup | Daily | An encrypted archive no larger than 5 GiB, one S3 PutObject, a redacted manifest, and a matching S3-validated full-object SHA-256 | One GitHub issue | +| Database backup freshness | Hourly | A complete S3 pair from the last 24 hours, a valid manifest digest, a matching object size, and the same full-object SHA-256 | One GitHub issue | +| Default branch sweep | Daily | The newest applicable run for every owned repository | One GitHub issue | +| Published version claims | Daily | Documentation claims against current package indexes | One GitHub issue | + +These checks use GitHub Actions. GitHub can delay a schedule or disable it after +60 days without repository activity. Use one external monitor for the health +workflow and the backup-freshness workflow. A successful old run is not current +proof. + +## Deployment gate + +Complete these checks before a production promotion: + +1. Protect `main`. Give each backup environment one exact custom `main` branch + policy before an AWS OIDC role exists. +2. Bind the deployment to the reviewed source commit and artifact digests. +3. Run the complete local and hosted release gates for that exact commit. +4. Fetch the public readiness endpoint without a cache. +5. Run `scripts/check_production_readiness.py` against the response headers and + body. +6. Complete an authenticated synthetic transaction through submission, + idempotency, dispatch, callback, independent effect verification, receipt, + and webhook delivery. +7. Test a duplicate request and an uncertain dispatch. Do not dispatch the + action again during reconciliation. +8. Test one human halt, notification, answer, fresh-state revalidation, and + resume. +9. Confirm a current database recovery point and a complete isolated recovery + drill for the database and private Storage boundary. + +The readiness endpoint proves configured dependencies. It does not prove a +customer workflow, a recovery drill, an alert delivery, or an SLA. + +## Human halt support + +A production workflow needs an assigned primary operator, a secondary operator, +and declared support hours. The operator must have access to the local evidence +boundary for protected detail. The hosted queue carries only the closed, +privacy-safe decision contract. + +Test these cases before activation: + +- the primary operator receives one real notification; +- an expired decision cannot resume the run; +- two answers cannot resume one pause; +- the runner rechecks the live record, target, state, and effect after an + answer; +- a wrong operator answer causes a second halt; +- uncertain delivery selects reconciliation and does not dispatch again; and +- an unanswered halt reaches the secondary operator under the declared support + policy. + +Record the time to acknowledge, the time to resolve, and the final terminal +state. Do not count an accepted operator answer as a verified execution. + +## False-success report + +Report each qualification and production period with a named task, environment, +run count, and oracle. Include these separate counts: + +- `VERIFIED`; +- `HALTED` before an effect; +- `HALTED` after a possible effect; +- an uncertain delivery; +- a silent incorrect success; +- an over-halt on a healthy task; +- a wrong-record, duplicate, or collateral effect; +- a model call; and +- a human halt that exceeded the support target. + +Keep the denominator. A report with zero recorded runs does not prove a zero +failure rate. A screen success message does not replace the independent effect +oracle. + +## Recovery + +Use [`backup/RESTORE_DRILL.md`](backup/RESTORE_DRILL.md) for the off-provider +database recovery point. That backup does not contain private Storage objects. +Use the Cloud data-safety runbook for the complete database and Storage drill. + +Do not report a recovery-time objective until an isolated restore measures it. +Do not report a 24-hour recovery-point objective while the freshness monitor is +red or absent. + +## Founder configuration + +The code cannot supply these values or operating decisions. Complete them in +this order: + +1. Protect `main` with a pull request and the applicable status checks. +2. Create `production-backup` and `production-backup-monitor`. Give each + environment one exact custom `main` branch policy and no other deployment + policy. +3. Deploy the reviewed backup CloudFormation stack in AWS account + `992382684924`. +4. Configure the four settings in the `production-backup` GitHub environment. +5. Configure the two variables in the `production-backup-monitor` environment. +6. Store a second copy of the private `age` key in a team vault or offline + medium. +7. Create an isolated scratch Supabase project and complete the first recovery + drill. +8. Select the primary operator, the secondary operator, the support hours, and + the response targets for human halts. +9. Configure an external monitor for the production health and backup freshness + schedules. +10. Complete the first genuine customer transaction when an authorized customer + is available. Do not create a founder self-charge as evidence. diff --git a/ops/backup/RESTORE_DRILL.md b/ops/backup/RESTORE_DRILL.md index b09e7e7..4ee817d 100644 --- a/ops/backup/RESTORE_DRILL.md +++ b/ops/backup/RESTORE_DRILL.md @@ -18,14 +18,21 @@ The design target is: - measured database-only RPO and recovery-time objective (RTO) evidence; and - the separate Cloud drill before any complete recovery claim. -This is not yet a proven recovery path. As of 2026-08-08: +This is not yet a proven recovery path. A read-only check on 2026-08-18 used +AWS account `992382684924` and confirmed that the +`openadapt-production-db-backup` CloudFormation stack does not exist. GitHub +issue [#126](https://github.com/OpenAdaptAI/openadapt-ops/issues/126) records +the matching protected-environment configuration failure. As of that check: - no daily backup has completed; - no scratch restore has completed; - no measured RTO exists; - provider PITR is not enabled; - the AWS stack is not deployed; -- the production database URL is not configured in the GitHub environment; +- `main` has no repository ruleset or branch protection; +- the `production-backup` GitHub environment has none of its four required + settings and has no deployment-branch restriction; +- the `production-backup-monitor` GitHub environment is not configured; - no scratch Supabase project is configured; and - one local private `age` key exists, but its required second vault or offline copy is not confirmed. @@ -56,6 +63,30 @@ take separate logical snapshots. Do not run it during a schema migration. The complete Cloud drill pauses writes, exports and rechecks Storage, restores both boundaries, and produces the canonical retention receipt. +## GitHub trust gate before AWS setup + +Create the external GitHub gates before the CloudFormation stack creates an +OIDC role. The OIDC subject binds a role to an environment. The environment's +deployment policy is the exact branch gate. + +1. Protect `main` with a repository ruleset. Require a pull request and the + applicable status checks. Add required code-owner review only when a second + authorized maintainer can approve the founder's pull request. +2. Create the `production-backup` GitHub environment. +3. Give it one custom deployment branch policy: the exact `main` branch. Do not + select every protected branch. Do not add a tag or wildcard policy. +4. Create the `production-backup-monitor` GitHub environment. +5. Give it the same single custom `main` branch policy. +6. Do not require a manual environment approval. An approval wait would prevent + the scheduled jobs. + +Verify that `main` reports as protected. Verify that each environment reports +`custom_branch_policies: true`, `protected_branches: false`, and one policy with +the exact name `main`. Both workflows repeat this check after the environment +admits the job and before they request AWS credentials. The job has read-only +Actions permission for this API check. Do not deploy the AWS stack until this +gate passes. + ## One-time AWS setup The CloudFormation template creates: @@ -65,7 +96,9 @@ The CloudFormation template creates: - 90-day retention for daily backups; - 365-day retention for database-only drill evidence; - a GitHub OIDC writer role bound to the exact `production-backup` - environment; and + environment; +- a read-only GitHub OIDC monitor role bound to the exact + `production-backup-monitor` environment; and - a local restore role bound to one exact AWS operator principal. The expected S3 Standard storage price is approximately USD 0.023 per GB each @@ -73,6 +106,11 @@ month, plus small request charges. For example, retaining 90 daily 100 MB backups is approximately 9 GB, or USD 0.21 each month before requests. The template does not create a paid KMS key and does not enable Supabase PITR. +The backup workflow uses one S3 `PutObject` with a caller-supplied full-object +SHA-256. S3 validates that checksum before it accepts the object. This launch +path refuses an encrypted archive above 5 GiB before upload. Build and qualify a +multipart contract before a production database can exceed that limit. + An AWS principal with CloudFormation, IAM, and S3 administration rights must run: @@ -99,21 +137,26 @@ aws cloudformation describe-stacks \ --query 'Stacks[0].Outputs' ``` -## One-time GitHub setup +## Complete the GitHub settings after AWS setup -First protect the repository and the environment: +The branch and environment gates already exist. Add the AWS outputs and the +database identity only after the stack passes validation: -1. Protect `main` with a repository ruleset. Require a pull request and code - owner review for the backup trust-boundary files in `.github/CODEOWNERS`. -2. Create the `production-backup` GitHub environment. -3. Restrict that environment to the protected `main` branch. Do not require a - manual deployment approval because it would prevent the daily schedule. -4. Set environment variables from the CloudFormation outputs: +1. Set `production-backup` environment variables from the CloudFormation + outputs: - `AWS_BACKUP_BUCKET` - `AWS_BACKUP_ROLE_ARN` -5. Set environment secrets: +2. Set `production-backup` environment secrets: - `SUPABASE_DB_URL`: the production direct or session-pooler PostgreSQL URL - `SUPABASE_PROJECT_REF`: the exact production project reference +3. Set `production-backup-monitor` environment variables from the + CloudFormation outputs: + - `AWS_BACKUP_BUCKET` + - `AWS_BACKUP_MONITOR_ROLE_ARN` + +The monitor environment has no secret. Its AWS role can list and inspect only +the `daily/` objects. It cannot create, replace, delete, download, or decrypt a +database backup. The workflow validates that the URL belongs to the declared Supabase project. It also checks AWS account `992382684924`, complete S3 public-access blocking, @@ -133,6 +176,10 @@ next schedule. gh workflow run db-backup.yml --repo OpenAdaptAI/openadapt-ops --ref main gh run list --repo OpenAdaptAI/openadapt-ops \ --workflow db-backup.yml --limit 5 +gh workflow run db-backup-freshness.yml \ + --repo OpenAdaptAI/openadapt-ops --ref main +gh run list --repo OpenAdaptAI/openadapt-ops \ + --workflow db-backup-freshness.yml --limit 5 ``` Require all of these results: @@ -140,8 +187,13 @@ Require all of these results: - the workflow succeeds on the exact `main` commit; - S3 contains one ciphertext object and one redacted manifest below the same UTC stamp; -- the stored SHA-256 checksum equals the local upload checksum; and +- the stored SHA-256 checksum equals the local upload checksum; +- S3 reports the exact caller-supplied full-object SHA-256 for the ciphertext; +- the encrypted archive is no more than the enforced 5 GiB launch limit; - no GitHub Actions artifact exists for the run. +- the separate read-only freshness workflow selects the same recovery point, + validates the redacted manifest digest, matches the S3 object size and remote + checksum, and reports an age of less than 24 hours. A successful upload proves backup creation and storage. It does not prove that the backup can restore. @@ -257,10 +309,16 @@ as exposed. Do not keep encrypting new backups to both old and new recipients. ## Alerts and scheduled-workflow limits -A failed backup job stays red and GitHub sends workflow failure notifications -according to repository notification settings. The founder must monitor this -signal. A later change should add a direct freshness alert from an independent -system after the first successful object exists. +A failed backup job stays red and opens or updates one durable GitHub issue. +The hourly `Production DB backup freshness` workflow uses a separate read-only +AWS role. It opens or updates one durable issue when the newest complete pair +is absent, stale, or inconsistent. A successful run closes its matching issue. + +The two checks use different workflows and different AWS roles. They still use +the same GitHub scheduler. Configure one external monitor to alert when either +workflow stops running. The external monitor can inspect the workflow age or +assume a separate read-only role and inspect the newest S3 recovery point. Do +not give the external monitor the backup writer role. GitHub can disable schedules after 60 days without repository activity. The daily documentation sync currently keeps this repository active. Verify the diff --git a/ops/backup/aws-backup-target.yml b/ops/backup/aws-backup-target.yml index 7fde61a..f666379 100644 --- a/ops/backup/aws-backup-target.yml +++ b/ops/backup/aws-backup-target.yml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: '2010-09-09' -Description: Private encrypted database-backup target with exact writer and restore roles. +Description: Private encrypted database-backup target with exact writer, monitor, and restore roles. Parameters: GitHubOrganization: @@ -130,6 +130,44 @@ Resources: - s3:PutObject Resource: !Sub ${BackupBucket.Arn}/daily/* + BackupMonitorRole: + Type: AWS::IAM::Role + Properties: + RoleName: openadapt-ops-production-db-backup-monitor + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Federated: !Ref GitHubActionsOidcProvider + Action: sts:AssumeRoleWithWebIdentity + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: sts.amazonaws.com + token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrganization}/${GitHubRepository}:environment:production-backup-monitor + Policies: + - PolicyName: VerifyEncryptedDailyBackups + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - s3:GetBucketPublicAccessBlock + - s3:GetEncryptionConfiguration + Resource: !GetAtt BackupBucket.Arn + - Effect: Allow + Action: s3:ListBucket + Resource: !GetAtt BackupBucket.Arn + Condition: + StringLike: + s3:prefix: daily/* + - Effect: Allow + Action: s3:GetObject + Resource: !Sub ${BackupBucket.Arn}/daily/*/artifact-manifest.json + - Effect: Allow + Action: s3:GetObjectAttributes + Resource: !Sub ${BackupBucket.Arn}/daily/*/*.age + BackupRestoreRole: Type: AWS::IAM::Role Properties: @@ -161,5 +199,7 @@ Outputs: Value: !Ref BackupBucket BackupWriterRoleArn: Value: !GetAtt BackupWriterRole.Arn + BackupMonitorRoleArn: + Value: !GetAtt BackupMonitorRole.Arn BackupRestoreRoleArn: Value: !GetAtt BackupRestoreRole.Arn diff --git a/scripts/check_database_backup_freshness.py b/scripts/check_database_backup_freshness.py new file mode 100755 index 0000000..1f4801f --- /dev/null +++ b/scripts/check_database_backup_freshness.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Prove that the newest private database backup is complete and fresh.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import re +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +ARTIFACT_SCHEMA = "openadapt.database-backup-artifact/v2" +STAMP = re.compile(r"^(\d{8}T\d{6}Z)$") +MANIFEST_KEY = re.compile(r"^daily/(\d{8}T\d{6}Z)/artifact-manifest\.json$") +CIPHERTEXT_KEY = re.compile( + r"^daily/(\d{8}T\d{6}Z)/db-backup-(\d{8}T\d{6}Z)\.tar\.gz\.age$" +) +SHA256 = re.compile(r"^[0-9a-f]{64}$") +COMMIT = re.compile(r"^[0-9a-f]{40}$") + + +class FreshnessError(ValueError): + """The S3 inventory does not prove a current recovery point.""" + + +def stable_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def parse_time(value: object, name: str) -> datetime: + if not isinstance(value, str): + raise FreshnessError(f"{name} is missing") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise FreshnessError(f"{name} is not an ISO-8601 time") from error + if parsed.tzinfo is None: + raise FreshnessError(f"{name} has no timezone") + return parsed.astimezone(timezone.utc) + + +def recovery_time(stamp: str) -> datetime: + if STAMP.fullmatch(stamp) is None: + raise FreshnessError("the backup stamp is invalid") + try: + return datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) + except ValueError as error: + raise FreshnessError("the backup stamp is invalid") from error + + +def select_latest( + inventory: dict[str, object], *, now: datetime, maximum_age_seconds: int +) -> dict[str, object]: + if not isinstance(inventory, dict): + raise FreshnessError("the S3 inventory is not an object") + if maximum_age_seconds <= 0: + raise FreshnessError("the maximum backup age must be positive") + if inventory.get("IsTruncated") is not False: + raise FreshnessError("the S3 inventory is incomplete") + contents = inventory.get("Contents") + if not isinstance(contents, list) or not contents: + raise FreshnessError("the backup bucket has no daily recovery point") + + objects: dict[str, dict[str, object]] = {} + stamps: set[str] = set() + for item in contents: + if not isinstance(item, dict) or not isinstance(item.get("Key"), str): + raise FreshnessError("the S3 inventory contains an invalid object") + key = item["Key"] + manifest_match = MANIFEST_KEY.fullmatch(key) + ciphertext_match = CIPHERTEXT_KEY.fullmatch(key) + if manifest_match: + stamp = manifest_match.group(1) + elif ciphertext_match and ciphertext_match.group(1) == ciphertext_match.group( + 2 + ): + stamp = ciphertext_match.group(1) + else: + raise FreshnessError( + f"the daily prefix contains an unexpected object: {key}" + ) + if key in objects: + raise FreshnessError(f"the S3 inventory contains a duplicate object: {key}") + if not isinstance(item.get("Size"), int) or item["Size"] <= 0: + raise FreshnessError(f"the S3 object is empty: {key}") + parse_time(item.get("LastModified"), f"LastModified for {key}") + objects[key] = item + stamps.add(stamp) + + latest_stamp = max(stamps) + recovery_point = recovery_time(latest_stamp) + age = (now.astimezone(timezone.utc) - recovery_point).total_seconds() + if age < -300: + raise FreshnessError("the newest backup recovery point is in the future") + if age > maximum_age_seconds: + raise FreshnessError( + f"the newest backup recovery point is stale by {int(age - maximum_age_seconds)} seconds" + ) + + manifest_key = f"daily/{latest_stamp}/artifact-manifest.json" + ciphertext_key = f"daily/{latest_stamp}/db-backup-{latest_stamp}.tar.gz.age" + if manifest_key not in objects or ciphertext_key not in objects: + raise FreshnessError( + "the newest backup prefix does not contain one complete object pair" + ) + if sum(key.startswith(f"daily/{latest_stamp}/") for key in objects) != 2: + raise FreshnessError( + "the newest backup prefix does not contain exactly two objects" + ) + + for key in (manifest_key, ciphertext_key): + last_modified = parse_time( + objects[key]["LastModified"], f"LastModified for {key}" + ) + if last_modified < recovery_point - timedelta(seconds=30): + raise FreshnessError(f"the S3 object predates its recovery point: {key}") + if last_modified > recovery_point + timedelta(hours=2): + raise FreshnessError( + f"the S3 object arrived too late for its recovery point: {key}" + ) + + return { + "schema": "openadapt.database-backup-selection/v1", + "recovery_point_at": recovery_point.isoformat().replace("+00:00", "Z"), + "age_seconds": max(0, int(age)), + "manifest_key": manifest_key, + "manifest_bytes": objects[manifest_key]["Size"], + "ciphertext_key": ciphertext_key, + "ciphertext_bytes": objects[ciphertext_key]["Size"], + } + + +def verify_latest( + selection: dict[str, object], + manifest: dict[str, object], + attributes: dict[str, object], +) -> dict[str, object]: + if not all(isinstance(value, dict) for value in (selection, manifest, attributes)): + raise FreshnessError("the backup verification input is not an object") + if selection.get("schema") != "openadapt.database-backup-selection/v1": + raise FreshnessError("the backup selection schema is invalid") + manifest_key = selection.get("manifest_key") + ciphertext_key = selection.get("ciphertext_key") + if ( + not isinstance(manifest_key, str) + or MANIFEST_KEY.fullmatch(manifest_key) is None + ): + raise FreshnessError("the selected manifest key is invalid") + ciphertext_match = ( + CIPHERTEXT_KEY.fullmatch(ciphertext_key) + if isinstance(ciphertext_key, str) + else None + ) + if ciphertext_match is None or ciphertext_match.group(1) != ciphertext_match.group( + 2 + ): + raise FreshnessError("the selected ciphertext key is invalid") + if manifest_key.split("/")[1] != ciphertext_match.group(1): + raise FreshnessError( + "the selected backup objects use different recovery points" + ) + + if manifest.get("schema") != ARTIFACT_SCHEMA: + raise FreshnessError("the artifact manifest schema is invalid") + artifact = manifest.get("artifact") + if not isinstance(artifact, dict): + raise FreshnessError("the artifact manifest is incomplete") + if manifest.get("artifact_sha256") != sha256_text(stable_json(artifact)): + raise FreshnessError("the artifact manifest digest is invalid") + ciphertext = artifact.get("ciphertext_archive") + if not isinstance(ciphertext, dict): + raise FreshnessError("the ciphertext contract is missing") + ciphertext_bytes = ciphertext.get("bytes") + ciphertext_sha = ciphertext.get("sha256") + if not isinstance(ciphertext_bytes, int) or ciphertext_bytes <= 0: + raise FreshnessError("the ciphertext size is invalid") + if not isinstance(ciphertext_sha, str) or SHA256.fullmatch(ciphertext_sha) is None: + raise FreshnessError("the ciphertext digest is invalid") + if selection.get("ciphertext_bytes") != ciphertext_bytes: + raise FreshnessError("the S3 inventory size does not match the manifest") + if selection.get("manifest_bytes") != len( + (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode() + ): + raise FreshnessError( + "the S3 manifest size does not match the downloaded manifest" + ) + if ( + not isinstance(artifact.get("repository_commit"), str) + or COMMIT.fullmatch(artifact["repository_commit"]) is None + ): + raise FreshnessError("the artifact repository commit is invalid") + run_id = artifact.get("workflow_run_id") + if not isinstance(run_id, str) or not run_id.isdigit(): + raise FreshnessError("the artifact workflow run ID is invalid") + + if attributes.get("ObjectSize") != ciphertext_bytes: + raise FreshnessError("the remote ciphertext size does not match the manifest") + expected_checksum = base64.b64encode(bytes.fromhex(ciphertext_sha)).decode() + checksum = attributes.get("Checksum") + if ( + not isinstance(checksum, dict) + or checksum.get("ChecksumSHA256") != expected_checksum + ): + raise FreshnessError( + "the remote ciphertext full-object checksum does not match" + ) + + return { + "fresh": True, + "recovery_point_at": selection["recovery_point_at"], + "age_seconds": selection["age_seconds"], + "artifact_sha256": manifest["artifact_sha256"], + "repository_commit": artifact["repository_commit"], + "workflow_run_id": run_id, + } + + +def write_json(path: str, value: object) -> None: + Path(path).write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + commands = root.add_subparsers(dest="command", required=True) + + select = commands.add_parser("select") + select.add_argument("--inventory", required=True) + select.add_argument("--output", required=True) + select.add_argument("--github-output") + select.add_argument("--now") + select.add_argument("--maximum-age-seconds", type=int, default=86400) + + verify = commands.add_parser("verify") + verify.add_argument("--selection", required=True) + verify.add_argument("--manifest", required=True) + verify.add_argument("--attributes", required=True) + return root + + +def main() -> int: + args = parser().parse_args() + try: + if args.command == "select": + now = ( + parse_time(args.now, "now") if args.now else datetime.now(timezone.utc) + ) + result = select_latest( + json.loads(Path(args.inventory).read_text(encoding="utf-8")), + now=now, + maximum_age_seconds=args.maximum_age_seconds, + ) + write_json(args.output, result) + if args.github_output: + with Path(args.github_output).open("a", encoding="utf-8") as stream: + stream.write(f"manifest_key={result['manifest_key']}\n") + stream.write(f"ciphertext_key={result['ciphertext_key']}\n") + else: + result = verify_latest( + json.loads(Path(args.selection).read_text(encoding="utf-8")), + json.loads(Path(args.manifest).read_text(encoding="utf-8")), + json.loads(Path(args.attributes).read_text(encoding="utf-8")), + ) + except (FreshnessError, OSError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_github_environment_gate.py b/scripts/check_github_environment_gate.py new file mode 100755 index 0000000..00498cc --- /dev/null +++ b/scripts/check_github_environment_gate.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Require one protected main branch and one exact environment branch policy.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +class EnvironmentGateError(ValueError): + """The GitHub environment does not have the required external gate.""" + + +def validate( + *, + environment: dict[str, object], + policies: dict[str, object], + expected_environment: str, + expected_branch: str, + actual_ref: str, + ref_protected: str, +) -> dict[str, object]: + expected_ref = f"refs/heads/{expected_branch}" + if actual_ref != expected_ref: + raise EnvironmentGateError( + f"the workflow ref is {actual_ref!r}, not {expected_ref!r}" + ) + if ref_protected.lower() != "true": + raise EnvironmentGateError("the exact main ref is not protected") + if environment.get("name") != expected_environment: + raise EnvironmentGateError("the GitHub environment identity is invalid") + + deployment = environment.get("deployment_branch_policy") + if not isinstance(deployment, dict): + raise EnvironmentGateError("the GitHub environment has no branch policy") + if deployment.get("protected_branches") is not False: + raise EnvironmentGateError( + "the GitHub environment must not admit every protected branch" + ) + if deployment.get("custom_branch_policies") is not True: + raise EnvironmentGateError( + "the GitHub environment needs an exact custom branch policy" + ) + + branch_policies = policies.get("branch_policies") + total = policies.get("total_count") + if ( + not isinstance(branch_policies, list) + or isinstance(total, bool) + or not isinstance(total, int) + or total != len(branch_policies) + or total != 1 + ): + raise EnvironmentGateError( + "the GitHub environment must have exactly one deployment branch policy" + ) + policy = branch_policies[0] + if not isinstance(policy, dict): + raise EnvironmentGateError("the deployment branch policy is invalid") + # GitHub's list-deployment-branch-policies response does not consistently + # include the policy type. The environment has already admitted this run on + # the exact protected branch, so require the one returned policy to have the + # exact branch name without depending on an absent response field. + if policy.get("name") != expected_branch: + raise EnvironmentGateError( + "the only environment deployment policy must be the exact main branch" + ) + + return { + "valid": True, + "environment": expected_environment, + "ref": expected_ref, + "ref_protected": True, + "branch_policy": expected_branch, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--environment-json", required=True) + parser.add_argument("--policies-json", required=True) + parser.add_argument("--expected-environment", required=True) + parser.add_argument("--expected-branch", default="main") + parser.add_argument("--actual-ref", required=True) + parser.add_argument("--ref-protected", required=True) + args = parser.parse_args() + try: + result = validate( + environment=json.loads( + Path(args.environment_json).read_text(encoding="utf-8") + ), + policies=json.loads(Path(args.policies_json).read_text(encoding="utf-8")), + expected_environment=args.expected_environment, + expected_branch=args.expected_branch, + actual_ref=args.actual_ref, + ref_protected=args.ref_protected, + ) + except (EnvironmentGateError, OSError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_production_readiness.py b/scripts/check_production_readiness.py new file mode 100755 index 0000000..e219655 --- /dev/null +++ b/scripts/check_production_readiness.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Validate the public production-readiness response without trusting one boolean.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +REQUIRED_COMPONENTS = frozenset( + { + "mode", + "auth", + "database", + "storage", + "runner", + "compiler", + "runtime_validation_trust", + "runtime_boundary", + "bundle_protection", + "recorder", + "callbacks", + "scheduler", + "human_decision_web_push", + "retention", + "security_events", + "secrets", + "validation_policy", + "billing", + } +) +SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +class ReadinessError(ValueError): + """The readiness response does not prove the production contract.""" + + +def parse_time(value: object, name: str) -> datetime: + if not isinstance(value, str): + raise ReadinessError(f"{name} is missing") + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ReadinessError(f"{name} is not an ISO-8601 time") from error + if result.tzinfo is None: + raise ReadinessError(f"{name} has no timezone") + return result.astimezone(timezone.utc) + + +def parse_headers(raw: str) -> tuple[int, dict[str, str]]: + """Return the final HTTP response block emitted by ``curl -D``.""" + blocks = [block for block in re.split(r"\r?\n\r?\n", raw.strip()) if block] + http_blocks = [block for block in blocks if block.startswith("HTTP/")] + if not http_blocks: + raise ReadinessError("the response has no HTTP status") + lines = http_blocks[-1].splitlines() + status = re.match(r"^HTTP/\S+\s+(\d{3})(?:\s|$)", lines[0]) + if status is None: + raise ReadinessError("the HTTP status is invalid") + headers: dict[str, str] = {} + for line in lines[1:]: + if not line.strip(): + continue + if ":" not in line: + raise ReadinessError("the response has an invalid header") + name, value = line.split(":", 1) + key = name.strip().lower() + headers[key] = ( + f"{headers[key]}, {value.strip()}" if key in headers else value.strip() + ) + return int(status.group(1)), headers + + +def validate( + *, + headers_text: str, + body_text: str, + now: datetime, + maximum_age_seconds: int, +) -> dict[str, object]: + if maximum_age_seconds <= 0: + raise ReadinessError("the maximum response age must be positive") + if now.tzinfo is None: + raise ReadinessError("now has no timezone") + status, headers = parse_headers(headers_text) + if status != 200: + raise ReadinessError(f"the readiness endpoint returned HTTP {status}") + content_type = headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + raise ReadinessError("the readiness response is not JSON") + cache_directives = { + part.strip().lower() + for part in headers.get("cache-control", "").split(",") + if part.strip() + } + if "no-store" not in cache_directives: + raise ReadinessError("the readiness response can be cached") + + try: + payload = json.loads(body_text) + except json.JSONDecodeError as error: + raise ReadinessError("the readiness response is not valid JSON") from error + if not isinstance(payload, dict): + raise ReadinessError("the readiness response is not an object") + if payload.get("ready") is not True: + raise ReadinessError("the production deployment is not ready") + if payload.get("mode") != "live": + raise ReadinessError("the production deployment is not in live mode") + + checked_at = parse_time(payload.get("checked_at"), "checked_at") + age = (now.astimezone(timezone.utc) - checked_at).total_seconds() + if age < -30: + raise ReadinessError("the readiness time is in the future") + if age > maximum_age_seconds: + raise ReadinessError("the readiness result is stale") + + if payload.get("encrypted_writer_protocol") != 1: + raise ReadinessError("the encrypted writer protocol is not active") + if payload.get("encrypted_writer_role") != "active": + raise ReadinessError("the encrypted writer role is not active") + for name in ("encrypted_writer_key_sha256", "encrypted_writer_deployment_sha256"): + value = payload.get(name) + if not isinstance(value, str) or SHA256.fullmatch(value) is None: + raise ReadinessError(f"{name} is invalid") + + raw_components = payload.get("components") + if not isinstance(raw_components, list): + raise ReadinessError("the readiness component list is missing") + components: dict[str, dict[str, object]] = {} + for component in raw_components: + if not isinstance(component, dict) or not isinstance( + component.get("name"), str + ): + raise ReadinessError("the readiness component list is invalid") + name = component["name"] + if name in components: + raise ReadinessError(f"the readiness component is duplicated: {name}") + components[name] = component + + missing = sorted(REQUIRED_COMPONENTS - components.keys()) + if missing: + raise ReadinessError( + f"required readiness components are missing: {', '.join(missing)}" + ) + failed = sorted( + name + for name in REQUIRED_COMPONENTS + if components[name].get("required") is not True + or components[name].get("state") != "ready" + ) + if failed: + raise ReadinessError( + f"required readiness components did not pass: {', '.join(failed)}" + ) + for name, component in components.items(): + if component.get("required") is True and component.get("state") != "ready": + raise ReadinessError( + f"an additional required component did not pass: {name}" + ) + + return { + "ready": True, + "mode": "live", + "checked_at": checked_at.isoformat().replace("+00:00", "Z"), + "required_components": len(REQUIRED_COMPONENTS), + "reported_components": len(components), + } + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + result.add_argument("--headers", required=True) + result.add_argument("--body", required=True) + result.add_argument("--now") + result.add_argument("--maximum-age-seconds", type=int, default=180) + return result + + +def main() -> int: + args = parser().parse_args() + try: + now = parse_time(args.now, "now") if args.now else datetime.now(timezone.utc) + result = validate( + headers_text=Path(args.headers).read_text(encoding="utf-8"), + body_text=Path(args.body).read_text(encoding="utf-8"), + now=now, + maximum_age_seconds=args.maximum_age_seconds, + ) + except (OSError, ReadinessError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_published_version_claims.py b/scripts/check_published_version_claims.py index 8876a6b..35c2ec7 100644 --- a/scripts/check_published_version_claims.py +++ b/scripts/check_published_version_claims.py @@ -4,7 +4,7 @@ On 2026-07-27 ``openadapt-flow`` 1.24.0 published and four sentences on docs.openadapt.ai -- one of them on the security-review page -- still asserted that the live runner reported "the published Flow 1.23.0 identity". The number -was correct (the managed runtime really is pinned to 1.23.0); the word +was correct at that time because the managed runtime pinned 1.23.0; the word *published* was what became false. Nothing could detect that, because no machine-readable record said which numbers in these docs are supposed to track the current release and which are deliberately frozen. diff --git a/scripts/database_backup_contract.py b/scripts/database_backup_contract.py old mode 100644 new mode 100755 index 78d8a0e..cfcc9c3 --- a/scripts/database_backup_contract.py +++ b/scripts/database_backup_contract.py @@ -9,6 +9,7 @@ from __future__ import annotations import argparse +import base64 import hashlib import json import re @@ -18,10 +19,11 @@ from pathlib import Path from urllib.parse import unquote, urlsplit - CONTRACT_SCHEMA = "openadapt.database-backup-contract/v2" ARTIFACT_SCHEMA = "openadapt.database-backup-artifact/v2" RESTORE_EVIDENCE_SCHEMA = "openadapt.database-restore-evidence/v1" +S3_UPLOAD_SCHEMA = "openadapt.database-backup-s3-upload/v1" +S3_SINGLE_PUT_MAX_BYTES = 5 * 1024 * 1024 * 1024 PROJECT_REF = re.compile(r"^[a-z0-9]{8,64}$") AGE_RECIPIENT = re.compile(r"^age1[0-9a-z]+$") REQUIRED_DUMPS = ("roles.sql", "schema.sql", "data.sql") @@ -243,6 +245,120 @@ def verify_artifact(args: argparse.Namespace) -> None: print(json.dumps({"valid": True, "artifact_sha256": manifest["artifact_sha256"]})) +def single_put_contract( + ciphertext: Path, + manifest_path: Path, + *, + maximum_bytes: int = S3_SINGLE_PUT_MAX_BYTES, +) -> dict[str, object]: + """Bind one ciphertext to an S3-validated full-object SHA-256 checksum.""" + if maximum_bytes <= 0: + raise ContractError("the S3 single-PutObject limit must be positive") + if not ciphertext.is_file(): + raise ContractError("the encrypted archive is missing") + size = ciphertext.stat().st_size + if size <= 0: + raise ContractError("the encrypted archive is empty") + if size > maximum_bytes: + raise ContractError( + "the encrypted archive exceeds the 5 GiB single-PutObject launch limit" + ) + + manifest = read_manifest(manifest_path) + artifact = manifest["artifact"] + assert isinstance(artifact, dict) + expected = artifact.get("ciphertext_archive") + if not isinstance(expected, dict): + raise ContractError("the encrypted archive contract is missing") + digest = sha256_file(ciphertext) + if expected.get("bytes") != size or expected.get("sha256") != digest: + raise ContractError("the encrypted archive does not match the manifest") + + return { + "schema": S3_UPLOAD_SCHEMA, + "bytes": size, + "sha256": digest, + "checksum_algorithm": "SHA256", + "checksum_type": "FULL_OBJECT", + "checksum_sha256": base64.b64encode(bytes.fromhex(digest)).decode(), + "maximum_bytes": maximum_bytes, + } + + +def read_single_put_contract(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or value.get("schema") != S3_UPLOAD_SCHEMA: + raise ContractError("the S3 upload contract schema is invalid") + if value.get("checksum_algorithm") != "SHA256": + raise ContractError("the S3 upload checksum algorithm is invalid") + if value.get("checksum_type") != "FULL_OBJECT": + raise ContractError("the S3 upload checksum type is invalid") + size = value.get("bytes") + maximum = value.get("maximum_bytes") + digest = value.get("sha256") + encoded = value.get("checksum_sha256") + if ( + not isinstance(size, int) + or isinstance(size, bool) + or size <= 0 + or not isinstance(maximum, int) + or isinstance(maximum, bool) + or maximum != S3_SINGLE_PUT_MAX_BYTES + or size > maximum + ): + raise ContractError("the S3 upload size contract is invalid") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ContractError("the S3 upload SHA-256 is invalid") + if encoded != base64.b64encode(bytes.fromhex(digest)).decode(): + raise ContractError("the S3 upload checksum encoding is invalid") + return value + + +def prepare_single_put(args: argparse.Namespace) -> None: + value = single_put_contract( + Path(args.ciphertext_archive), Path(args.manifest) + ) + output = Path(args.output) + output.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + json.dumps( + { + "upload_contract": str(output), + "bytes": value["bytes"], + "checksum_type": value["checksum_type"], + }, + sort_keys=True, + ) + ) + + +def verify_single_put(args: argparse.Namespace) -> None: + contract = read_single_put_contract(Path(args.upload_contract)) + attributes = json.loads(Path(args.attributes).read_text(encoding="utf-8")) + if not isinstance(attributes, dict): + raise ContractError("the S3 object attributes are invalid") + if attributes.get("ObjectSize") != contract["bytes"]: + raise ContractError("the S3 object size does not match the upload contract") + checksum = attributes.get("Checksum") + if not isinstance(checksum, dict): + raise ContractError("the S3 object checksum is missing") + if checksum.get("ChecksumSHA256") != contract["checksum_sha256"]: + raise ContractError("the S3 full-object checksum does not match") + print( + json.dumps( + { + "valid": True, + "bytes": contract["bytes"], + "checksum_type": "FULL_OBJECT", + "sha256": contract["sha256"], + }, + sort_keys=True, + ) + ) + + def validate_restore_target(args: argparse.Namespace) -> None: source_ref = project_ref(args.source_project_ref, "production project reference") scratch_ref = project_ref(args.scratch_project_ref, "scratch project reference") @@ -425,6 +541,17 @@ def parser() -> argparse.ArgumentParser: artifact.add_argument("--ciphertext-archive", required=True) artifact.set_defaults(run=verify_artifact) + upload = commands.add_parser("prepare-single-put") + upload.add_argument("--manifest", required=True) + upload.add_argument("--ciphertext-archive", required=True) + upload.add_argument("--output", required=True) + upload.set_defaults(run=prepare_single_put) + + uploaded = commands.add_parser("verify-single-put") + uploaded.add_argument("--upload-contract", required=True) + uploaded.add_argument("--attributes", required=True) + uploaded.set_defaults(run=verify_single_put) + target = commands.add_parser("validate-restore-target") target.add_argument("--source-project-ref", required=True) target.add_argument("--scratch-project-ref", required=True) diff --git a/tests/test_database_backup_contract.py b/tests/test_database_backup_contract.py index 3d9bc3b..9378308 100644 --- a/tests/test_database_backup_contract.py +++ b/tests/test_database_backup_contract.py @@ -1,3 +1,5 @@ +import base64 +import hashlib import importlib.util import io import json @@ -7,7 +9,6 @@ import pytest - MODULE_PATH = Path(__file__).parents[1] / "scripts" / "database_backup_contract.py" SPEC = importlib.util.spec_from_file_location("database_backup_contract", MODULE_PATH) assert SPEC and SPEC.loader @@ -118,6 +119,84 @@ def test_manifest_detects_ciphertext_tampering(tmp_path: Path) -> None: ) +def test_single_put_contract_keeps_full_sha256_above_multipart_threshold( + tmp_path: Path, +) -> None: + _, _, contract = make_contract(tmp_path) + plaintext = tmp_path / "backup.tar.gz" + ciphertext = tmp_path / "backup.tar.gz.age" + plaintext.write_bytes(b"plain") + payload = b"x" * (8 * 1024 * 1024 + 1) + ciphertext.write_bytes(payload) + manifest = tmp_path / "artifact-manifest.json" + backup.create_manifest( + Namespace( + contract=str(contract), + plaintext_archive=str(plaintext), + ciphertext_archive=str(ciphertext), + repository_commit="a" * 40, + workflow_run_id="123", + output=str(manifest), + ) + ) + + upload = backup.single_put_contract(ciphertext, manifest) + digest = hashlib.sha256(payload).hexdigest() + assert upload["bytes"] == len(payload) + assert upload["sha256"] == digest + assert upload["checksum_type"] == "FULL_OBJECT" + assert upload["checksum_sha256"] == base64.b64encode( + bytes.fromhex(digest) + ).decode() + upload_contract = tmp_path / "s3-upload-contract.json" + upload_contract.write_text(json.dumps(upload), encoding="utf-8") + attributes = tmp_path / "s3-object-attributes.json" + attributes.write_text( + json.dumps( + { + "ObjectSize": len(payload), + "Checksum": { + "ChecksumSHA256": upload["checksum_sha256"], + }, + } + ), + encoding="utf-8", + ) + backup.verify_single_put( + Namespace(upload_contract=str(upload_contract), attributes=str(attributes)) + ) + + value = json.loads(attributes.read_text()) + value["Checksum"]["ChecksumSHA256"] = f'{upload["checksum_sha256"]}-2' + attributes.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises(backup.ContractError, match="full-object"): + backup.verify_single_put( + Namespace(upload_contract=str(upload_contract), attributes=str(attributes)) + ) + + +def test_single_put_contract_refuses_before_its_size_limit(tmp_path: Path) -> None: + _, _, contract = make_contract(tmp_path) + plaintext = tmp_path / "backup.tar.gz" + ciphertext = tmp_path / "backup.tar.gz.age" + plaintext.write_bytes(b"plain") + ciphertext.write_bytes(b"12345") + manifest = tmp_path / "artifact-manifest.json" + backup.create_manifest( + Namespace( + contract=str(contract), + plaintext_archive=str(plaintext), + ciphertext_archive=str(ciphertext), + repository_commit="a" * 40, + workflow_run_id="123", + output=str(manifest), + ) + ) + + with pytest.raises(backup.ContractError, match="5 GiB"): + backup.single_put_contract(ciphertext, manifest, maximum_bytes=4) + + def test_safe_extraction_accepts_only_the_exact_regular_file_set(tmp_path: Path) -> None: dumps, _, contract = make_contract(tmp_path) ciphertext = tmp_path / "backup.tar.gz.age" diff --git a/tests/test_database_backup_freshness.py b/tests/test_database_backup_freshness.py new file mode 100644 index 0000000..5ca8e7a --- /dev/null +++ b/tests/test_database_backup_freshness.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import pathlib +import sys +from datetime import datetime, timezone + +import pytest + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from check_database_backup_freshness import ( + FreshnessError, + select_latest, + stable_json, + verify_latest, +) + +NOW = datetime(2026, 8, 18, 16, 0, tzinfo=timezone.utc) +STAMP = "20260818T150000Z" +PREFIX = f"daily/{STAMP}" + + +def manifest() -> dict[str, object]: + artifact = { + "backup_contract_sha256": "c" * 64, + "plaintext_archive": {"bytes": 200, "sha256": "d" * 64}, + "ciphertext_archive": {"bytes": 100, "sha256": "a" * 64}, + "repository_commit": "b" * 40, + "workflow_run_id": "32113840939", + } + return { + "schema": "openadapt.database-backup-artifact/v2", + "artifact": artifact, + "artifact_sha256": hashlib.sha256(stable_json(artifact).encode()).hexdigest(), + } + + +def inventory(value: dict[str, object] | None = None) -> dict[str, object]: + value = value or manifest() + manifest_bytes = len((json.dumps(value, indent=2, sort_keys=True) + "\n").encode()) + return { + "IsTruncated": False, + "Contents": [ + { + "Key": f"{PREFIX}/artifact-manifest.json", + "LastModified": "2026-08-18T15:01:00Z", + "Size": manifest_bytes, + }, + { + "Key": f"{PREFIX}/db-backup-{STAMP}.tar.gz.age", + "LastModified": "2026-08-18T15:01:00Z", + "Size": 100, + }, + ], + } + + +def attributes() -> dict[str, object]: + return { + "ObjectSize": 100, + "Checksum": { + "ChecksumSHA256": base64.b64encode(bytes.fromhex("a" * 64)).decode(), + }, + "StorageClass": "STANDARD", + } + + +def selection() -> dict[str, object]: + return select_latest(inventory(), now=NOW, maximum_age_seconds=86400) + + +def test_complete_fresh_pair_passes() -> None: + result = verify_latest(selection(), manifest(), attributes()) + assert result["fresh"] is True + assert result["age_seconds"] == 3600 + + +def test_missing_newest_ciphertext_does_not_fall_back_to_an_old_pair() -> None: + value = inventory() + value["Contents"].append( + { + "Key": "daily/20260818T155500Z/artifact-manifest.json", + "LastModified": "2026-08-18T15:55:10Z", + "Size": 10, + } + ) + with pytest.raises(FreshnessError, match="complete object pair"): + select_latest(value, now=NOW, maximum_age_seconds=86400) + + +def test_stale_recovery_point_is_rejected() -> None: + with pytest.raises(FreshnessError, match="stale"): + select_latest(inventory(), now=NOW, maximum_age_seconds=1800) + + +def test_truncated_inventory_is_rejected() -> None: + value = inventory() + value["IsTruncated"] = True + with pytest.raises(FreshnessError, match="incomplete"): + select_latest(value, now=NOW, maximum_age_seconds=86400) + + +def test_invalid_calendar_stamp_is_rejected_cleanly() -> None: + value = inventory() + value["Contents"][0]["Key"] = "daily/20261318T150000Z/artifact-manifest.json" + value["Contents"][1]["Key"] = ( + "daily/20261318T150000Z/db-backup-20261318T150000Z.tar.gz.age" + ) + with pytest.raises(FreshnessError, match="backup stamp"): + select_latest(value, now=NOW, maximum_age_seconds=86400) + + +def test_unexpected_object_in_daily_prefix_is_rejected() -> None: + value = inventory() + value["Contents"].append( + { + "Key": f"{PREFIX}/plaintext.sql", + "LastModified": "2026-08-18T15:01:00Z", + "Size": 1, + } + ) + with pytest.raises(FreshnessError, match="unexpected object"): + select_latest(value, now=NOW, maximum_age_seconds=86400) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + (lambda value: value["Checksum"].update(ChecksumSHA256="wrong"), "checksum"), + ( + lambda value: value["Checksum"].update( + ChecksumSHA256=f'{value["Checksum"]["ChecksumSHA256"]}-2' + ), + "checksum", + ), + (lambda value: value.update(ObjectSize=99), "size"), + ], +) +def test_remote_ciphertext_mismatch_is_rejected(change, message: str) -> None: + value = attributes() + change(value) + with pytest.raises(FreshnessError, match=message): + verify_latest(selection(), manifest(), value) + + +def test_manifest_digest_mismatch_is_rejected() -> None: + value = manifest() + value["artifact"]["repository_commit"] = "e" * 40 + with pytest.raises(FreshnessError, match="manifest digest"): + verify_latest(selection(), value, attributes()) diff --git a/tests/test_github_environment_gate.py b/tests/test_github_environment_gate.py new file mode 100644 index 0000000..ebda15a --- /dev/null +++ b/tests/test_github_environment_gate.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import pathlib +import sys + +import pytest + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from check_github_environment_gate import EnvironmentGateError, validate + + +def environment(name: str = "production-backup") -> dict[str, object]: + return { + "name": name, + "deployment_branch_policy": { + "protected_branches": False, + "custom_branch_policies": True, + }, + } + + +def policies(name: str = "main") -> dict[str, object]: + return { + "total_count": 1, + "branch_policies": [{"id": 1, "name": name}], + } + + +def check( + env: dict[str, object] | None = None, + rules: dict[str, object] | None = None, + *, + actual_ref: str = "refs/heads/main", + ref_protected: str = "true", +) -> dict[str, object]: + return validate( + environment=env or environment(), + policies=rules or policies(), + expected_environment="production-backup", + expected_branch="main", + actual_ref=actual_ref, + ref_protected=ref_protected, + ) + + +def test_exact_protected_main_environment_passes() -> None: + assert check()["valid"] is True + + +@pytest.mark.parametrize( + ("env", "rules", "actual_ref", "ref_protected", "message"), + [ + (environment("other"), policies(), "refs/heads/main", "true", "identity"), + (environment(), policies(), "refs/heads/feature", "true", "workflow ref"), + (environment(), policies(), "refs/heads/main", "false", "not protected"), + ( + {"name": "production-backup", "deployment_branch_policy": None}, + policies(), + "refs/heads/main", + "true", + "no branch policy", + ), + ( + { + "name": "production-backup", + "deployment_branch_policy": { + "protected_branches": True, + "custom_branch_policies": False, + }, + }, + policies(), + "refs/heads/main", + "true", + "every protected branch", + ), + (environment(), policies("release/*"), "refs/heads/main", "true", "exact main"), + ( + environment(), + { + "total_count": 2, + "branch_policies": [ + {"id": 1, "name": "main"}, + {"id": 2, "name": "release/*"}, + ], + }, + "refs/heads/main", + "true", + "exactly one", + ), + ], +) +def test_incomplete_or_broad_gate_is_rejected( + env: dict[str, object], + rules: dict[str, object], + actual_ref: str, + ref_protected: str, + message: str, +) -> None: + with pytest.raises(EnvironmentGateError, match=message): + check(env, rules, actual_ref=actual_ref, ref_protected=ref_protected) diff --git a/tests/test_production_ops_workflows.py b/tests/test_production_ops_workflows.py new file mode 100644 index 0000000..963838a --- /dev/null +++ b/tests/test_production_ops_workflows.py @@ -0,0 +1,83 @@ +"""Protect the credential, ciphertext, and durable-alert workflow boundaries.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_backup_configuration_fails_before_credentials_or_tool_install() -> None: + workflow = read(".github/workflows/db-backup.yml") + preflight = workflow.index("Validate the protected environment configuration") + environment_gate = workflow.index("Verify the exact GitHub environment gate") + credentials = workflow.index("aws-actions/configure-aws-credentials") + supabase = workflow.index("supabase/setup-cli") + assert preflight < environment_gate < credentials < supabase + assert "actions: read" in workflow + assert "GITHUB_REF_PROTECTED" in workflow + assert "check_github_environment_gate.py" in workflow + for name in ( + "AWS_BACKUP_ROLE_ARN", + "AWS_BACKUP_BUCKET", + "SUPABASE_DB_URL", + "SUPABASE_PROJECT_REF", + ): + assert f"missing+=({name})" in workflow + + +def test_backup_uses_one_s3_validated_full_object_put() -> None: + workflow = read(".github/workflows/db-backup.yml") + prepare = workflow.index("prepare-single-put") + put = workflow.index("aws s3api put-object") + verify = workflow.index("verify-single-put") + assert prepare < put < verify + assert '--checksum-algorithm SHA256 --checksum-sha256 "$local_checksum"' in workflow + assert "--content-length \"$cipher_bytes\"" in workflow + assert 'aws s3 cp "$cipher"' not in workflow + + +def test_backup_monitor_cannot_download_or_change_ciphertext() -> None: + template = read("ops/backup/aws-backup-target.yml") + monitor = template.split(" BackupMonitorRole:", 1)[1].split( + " BackupRestoreRole:", 1 + )[0] + assert "Action: s3:GetObject\n" in monitor + assert "daily/*/artifact-manifest.json" in monitor + assert "Action: s3:GetObjectAttributes" in monitor + assert "daily/*/*.age" in monitor + assert "s3:PutObject" not in monitor + assert "s3:DeleteObject" not in monitor + + workflow = read(".github/workflows/db-backup-freshness.yml") + assert "SUPABASE_DB_URL" not in workflow + assert "age --decrypt" not in workflow + assert "get-object-attributes" in workflow + assert "steps.select.outputs.ciphertext_key" in workflow + assert workflow.index("Verify the exact GitHub environment gate") < workflow.index( + "aws-actions/configure-aws-credentials" + ) + assert "actions: read" in workflow + assert "GITHUB_REF_PROTECTED" in workflow + assert "check_github_environment_gate.py" in workflow + + +def test_health_probe_uses_the_strict_contract_and_a_durable_issue() -> None: + workflow = read(".github/workflows/prod-health-alert.yml") + assert "python scripts/check_production_readiness.py" in workflow + assert "human-decision delivery component" in workflow + assert "issues: write" in workflow + assert "Production health check is failing" in workflow + + +def test_backup_jobs_keep_distinct_failure_and_freshness_issues() -> None: + backup = read(".github/workflows/db-backup.yml") + freshness = read(".github/workflows/db-backup-freshness.yml") + assert "Production database backup is not current" in backup + assert "Production database recovery point is stale or unverified" in freshness + assert "issues: write" in backup + assert "issues: write" in freshness diff --git a/tests/test_production_readiness.py b/tests/test_production_readiness.py new file mode 100644 index 0000000..bdb869b --- /dev/null +++ b/tests/test_production_readiness.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import json +import pathlib +import sys +from datetime import datetime, timezone + +import pytest + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from check_production_readiness import ( + REQUIRED_COMPONENTS, + ReadinessError, + validate, +) + +NOW = datetime(2026, 8, 18, 16, 0, tzinfo=timezone.utc) +HEADERS = """HTTP/2 200\r +content-type: application/json\r +cache-control: no-store,max-age=0\r +\r +""" + + +def payload() -> dict[str, object]: + return { + "ready": True, + "mode": "live", + "checked_at": "2026-08-18T15:59:30Z", + "encrypted_writer_protocol": 1, + "encrypted_writer_role": "active", + "encrypted_writer_key_sha256": "a" * 64, + "encrypted_writer_deployment_sha256": "b" * 64, + "components": [ + {"name": name, "required": True, "state": "ready", "detail": "ok"} + for name in sorted(REQUIRED_COMPONENTS) + ] + + [{"name": "sandbox", "required": False, "state": "not_ready"}], + } + + +def check(value: dict[str, object], headers: str = HEADERS) -> dict[str, object]: + return validate( + headers_text=headers, + body_text=json.dumps(value), + now=NOW, + maximum_age_seconds=180, + ) + + +def test_complete_live_contract_passes() -> None: + result = check(payload()) + assert result["ready"] is True + assert result["required_components"] == len(REQUIRED_COMPONENTS) + + +@pytest.mark.parametrize( + ("change", "message"), + [ + (lambda value: value.update(ready=False), "not ready"), + (lambda value: value.update(mode="mock"), "not in live mode"), + (lambda value: value.update(checked_at="2026-08-18T15:50:00Z"), "stale"), + ( + lambda value: value.update(encrypted_writer_role="standby"), + "writer role is not active", + ), + ], +) +def test_false_success_top_level_response_is_rejected(change, message: str) -> None: + value = payload() + change(value) + with pytest.raises(ReadinessError, match=message): + check(value) + + +def test_missing_required_component_is_rejected() -> None: + value = payload() + value["components"] = [ + component + for component in value["components"] + if component["name"] != "human_decision_web_push" + ] + with pytest.raises(ReadinessError, match="human_decision_web_push"): + check(value) + + +def test_required_component_failure_is_rejected_even_when_ready_is_true() -> None: + value = payload() + next( + component + for component in value["components"] + if component["name"] == "database" + )["state"] = "not_ready" + with pytest.raises(ReadinessError, match="database"): + check(value) + + +def test_new_required_component_failure_is_rejected() -> None: + value = payload() + value["components"].append( + {"name": "new_dependency", "required": True, "state": "not_ready"} + ) + with pytest.raises(ReadinessError, match="new_dependency"): + check(value) + + +def test_cacheable_response_is_rejected() -> None: + with pytest.raises(ReadinessError, match="cached"): + check(payload(), HEADERS.replace("no-store,max-age=0", "max-age=300")) + + +def test_nonpositive_maximum_age_is_rejected() -> None: + with pytest.raises(ReadinessError, match="must be positive"): + validate( + headers_text=HEADERS, + body_text=json.dumps(payload()), + now=NOW, + maximum_age_seconds=0, + )