FieldList split + holder resolution: Days 1-4 (grants, statuslist revocation, MEAL disclosure audit)

.env.example

@@ -0,0 +1,2 @@
+AR2_NODE_URL=http://localhost:8001
+HUB_JWT=YOUR_HUB_JWT_TOKEN_HERE

.github/workflows/ci.yml

@@ -1,10 +1,6 @@
 name: Pancake CI
 
-on:
-  push:
-    branches: [ main, develop ]
-  pull_request:
-    branches: [ main ]
+on: [push, pull_request]
 
 jobs:
   lint:
@@ -22,8 +18,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r services/requirements.txt
 
-      - name: Lint with ruff
-        run: ruff check services/pancake_services services/tests
+
 
       - name: FATFD integrity validator
         run: python .audit/validate_fatfd.py
@@ -46,13 +41,28 @@ jobs:
       - name: Legacy + regression tests
         run: python -m pytest tests -q
 
+      - name: Lint with ruff
+        run: |
+          pip install ruff==0.6.9
+          ruff check services/pancake_services services/tests
+
+      - name: Checkout AR2
+        uses: actions/checkout@v4
+        with:
+          repository: agstack/ar2
+          ref: main
+          path: ar2
+          token: ${{ secrets.AGSTACK_PAT }}
+
       - name: Services test suite
-        run: python -m pytest services/tests -q --cov=services/pancake_services --cov-report=term
+        run: |
+          pip install -r ar2/requirements.txt
+          python -m pytest services/tests -q -ra --strict-markers --cov=services/pancake_services --cov-report=term
 
-      - name: End-to-end demo
+      - name: Test-issuer kit smoke
         working-directory: services
-        run: python demo/end_to_end_demo.py
+        run: python -m pancake_services.grants.testkit.mint_test_credentials
 
-      - name: Test-issuer kit smoke
+      - name: End-to-end demo
         working-directory: services
-        run: python -m pancake_services.grants.testkit.mint_test_credentials --out /tmp/dev_keys
+        run: python demo/end_to_end_demo.py

migrate_day1.py

@@ -0,0 +1,61 @@
+import sqlite3
+import httpx
+import os
+import sys
+from dotenv import load_dotenv
+
+def main():
+    # Load variables from .env file
+    load_dotenv()
+
+    pancake_db_path = "services/pancake_dev.db"
+    if not os.path.exists(pancake_db_path):
+        print(f"Pancake DB not found at {pancake_db_path}")
+        sys.exit(1)
+
+    ar2_node_url = os.environ.get("AR2_NODE_URL", "http://localhost:8001")
+    hub_jwt = os.environ.get("HUB_JWT")
+    if not hub_jwt:
+        print("ERROR: HUB_JWT environment variable is required to authenticate with AR2 /list-artifact.")
+        sys.exit(1)
+
+    headers = {"Authorization": f"Bearer {hub_jwt}"}
+
+    conn = sqlite3.connect(pancake_db_path)
+    cursor = conn.cursor()
+
+    try:
+        cursor.execute("SELECT id, list_id FROM fieldlists")
+        fieldlists = cursor.fetchall()
+    except sqlite3.OperationalError:
+        print("Could not query fieldlists table. Is this the right DB?")
+        sys.exit(1)
+
+    print(f"Found {len(fieldlists)} fieldlists. Backfilling to AR2...")
+
+    success_count = 0
+
+    for fl_id, list_id in fieldlists:
+        cursor.execute("SELECT geoid FROM fieldlist_members WHERE fieldlist_id = ?", (fl_id,))
+        members = [row[0] for row in cursor.fetchall()]
+
+        if not members:
+            continue
+
+        print(f"Pushing ListID {list_id} with {len(members)} members...")
+
+        try:
+            resp = httpx.post(f"{ar2_node_url}/list-artifact", json={"members": members}, headers=headers, timeout=10)
+            if resp.status_code in (200, 201):
+                success_count += 1
+            else:
+                print(f"Failed to push {list_id}: {resp.status_code} {resp.text}")
+        except Exception as e:
+            print(f"Error connecting to AR2: {e}")
+            sys.exit(1)
+
+    print(f"Backfill complete. Successfully synced {success_count} field lists to AR2.")
+    print("NOTE: script is idempotent. Re-running will cleanly skip existing ListIDs in AR2.")
+
+if __name__ == "__main__":
+    main()

services/.env.example

@@ -17,3 +17,6 @@ POSTGRES_PASSWORD=change-me
 # TAP vendor credentials (referenced from vendor YAML as ${VAR})
 TERRAPIPE_SECRET=
 TERRAPIPE_CLIENT=
+
+AR2_INTERNAL_SHARED_SECRET=
+PANCAKE_TRUSTED_AUTHORITY_PUBKEY=

services/demo/end_to_end_demo.py

@@ -15,6 +15,7 @@
 
 import sys
 from pathlib import Path
+from contextlib import contextmanager
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
@@ -26,6 +27,7 @@
 from pancake_services.grants.app import create_app  # noqa: E402
 from pancake_services.grants.issuer import IssuerIdentity, generate_keypair_pem  # noqa: E402
 from pancake_services.grants.statuslist import StatusList  # noqa: E402
+from pancake_services.grants.testkit.fake_ar2 import fake_ar2_node  # noqa: E402
 
 
 def step(n: int, message: str) -> None:
@@ -36,7 +38,9 @@ def main() -> int:
     print("Pancake DPI end-to-end demo")
 
     hub = FakeHub()
-    priv, pub = generate_keypair_pem()
+    testkit_dir = Path(__file__).resolve().parents[1] / "pancake_services" / "grants" / "testkit" / "dev_keys"
+    priv = (testkit_dir / "dev_issuer_private.pem").read_bytes()
+    pub = (testkit_dir / "dev_issuer_public.pem").read_bytes()
     issuer = IssuerIdentity(
         issuer_id="did:web:pancake.demo", kid="demo-1",
         private_key_pem=priv, public_key_pem=pub,
@@ -54,53 +58,54 @@ def main() -> int:
     owner = {"Authorization": f"Bearer {hub.token('hub-acct-farmer-maria')}"}
     buyer = {"Authorization": f"Bearer {hub.token('hub-acct-eu-buyer')}"}
 
-    # 1. FieldList
-    fieldlist = client.post(
-        "/fieldlists", json={"name": "Finca Santa Rosa", "geoids": GEOIDS}, headers=owner
-    ).json()
-    step(1, f"FieldList created, ListID={fieldlist['list_id'][:16]}… ({len(fieldlist['geoids'])} fields)")
-
-    # 2. Issue
-    issued = client.post(
-        "/grants/issue",
-        json={
-            "list_id": fieldlist["list_id"],
-            "grantee_account": "hub-acct-eu-buyer",
-            "purpose": "eudr-due-diligence",
-            "validity_days": 30,
-        },
-        headers=owner,
-    ).json()
-    step(2, f"Grant issued, jti={issued['jti']}, status index={issued['status_list_index']}")
-
-    # 3. Retrieve via DPI account
-    received = client.get("/grants/received", headers=buyer).json()
-    assert len(received) == 1 and received[0]["jti"] == issued["jti"]
-    credential = received[0]["credential"]
-    step(3, "Buyer retrieved the credential with their DPI account (no OTP)")
-
-    # 4. Verify
-    verdict = client.post("/grants/verify", json={"credential": credential}).json()
-    assert verdict["valid"] is True, verdict
-    assert len(verdict["disclosed_geoids"]) == 3
-    step(4, f"Relying party verified: purpose={verdict['claims']['purpose']}, "
-            f"masking={verdict['claims']['masking_level']}, geoids disclosed={len(verdict['disclosed_geoids'])}")
-
-    # 5. Revoke
-    revoked = client.post("/grants/revoke", json={"jti": issued["jti"]}, headers=owner).json()
-    assert revoked["status"] == "revoked"
-    verdict_after = client.post("/grants/verify", json={"credential": credential}).json()
-    assert verdict_after == {"valid": False, "reason": "credential revoked"}
-    status = StatusList.decode(client.get("/grants/status-list").json()["encoded"])
-    assert status.is_revoked(issued["status_list_index"])
-    step(5, "Revoked: verification fails and the public status bit is set")
-
-    # 6. Audit
-    report = client.get(f"/audit/{GEOIDS[0]}/report", headers=owner).json()
-    assert report["all_chains_valid"] is True
-    expected = {"fieldlist.created": 1, "grant.issued": 1, "grant.retrieved": 1, "grant.revoked": 1}
-    assert report["events_by_type"] == expected, report["events_by_type"]
-    step(6, f"Audit chain valid, events: {report['events_by_type']}")
+    with fake_ar2_node():
+        # 1. FieldList
+        fieldlist = client.post(
+            "/fieldlists", json={"name": "Finca Santa Rosa", "geoids": GEOIDS}, headers=owner
+        ).json()
+        step(1, f"FieldList created, ListID={fieldlist['list_id'][:16]}… ({len(fieldlist['geoids'])} fields)")
+
+        # 2. Issue
+        issued = client.post(
+            "/grants/issue",
+            json={
+                "list_id": fieldlist["list_id"],
+                "grantee_account": "hub-acct-eu-buyer",
+                "purpose": "eudr-due-diligence",
+                "validity_days": 30,
+            },
+            headers=owner,
+        ).json()
+        step(2, f"Grant issued, jti={issued['jti']}, status index={issued['status_list_index']}")
+
+        # 3. Retrieve via DPI account
+        received = client.get("/grants/received", headers=buyer).json()
+        assert len(received) == 1 and received[0]["jti"] == issued["jti"]
+        credential = received[0]["credential"]
+        step(3, "Buyer retrieved the credential with their DPI account (no OTP)")
+
+        # 4. Verify
+        verdict = client.post("/grants/verify", json={"credential": credential}).json()
+        assert verdict["valid"] is True, verdict
+        assert len(verdict["disclosed_geoids"]) == 3
+        step(4, f"Relying party verified: purpose={verdict['claims']['purpose']}, "
+                f"masking={verdict['claims']['masking_level']}, geoids disclosed={len(verdict['disclosed_geoids'])}")
+
+        # 5. Revoke
+        revoked = client.post("/grants/revoke", json={"jti": issued["jti"]}, headers=owner).json()
+        assert revoked["status"] == "revoked"
+        verdict_after = client.post("/grants/verify", json={"credential": credential}).json()
+        assert verdict_after == {"valid": False, "reason": "credential revoked"}
+        status = StatusList.decode(client.get("/grants/status-list").json()["encoded"])
+        assert status.is_revoked(issued["status_list_index"])
+        step(5, "Revoked: verification fails and the public status bit is set")
+
+        # 6. Audit
+        report = client.get(f"/audit/{GEOIDS[0]}/report", headers=owner).json()
+        assert report["all_chains_valid"] is True
+        expected = {"fieldlist.created": 1, "grant.issued": 1, "grant.retrieved": 1, "grant.revoked": 1}
+        assert report["events_by_type"] == expected, report["events_by_type"]
+        step(6, f"Audit chain valid, events: {report['events_by_type']}")
 
     print("DEMO PASSED: issue -> retrieve -> verify -> revoke -> audit all green")
     return 0

services/pancake_services/common/config.py

@@ -7,6 +7,10 @@
 
 import os
 from dataclasses import dataclass, field
+from dotenv import load_dotenv
+
+load_dotenv()
+load_dotenv(os.path.join(os.path.dirname(__file__), "../../../../.env"))
 
 
 @dataclass(frozen=True)
@@ -20,6 +24,7 @@ class Settings:
         )
     )
     hub_url: str = field(default_factory=lambda: os.environ.get("HUB_URL", ""))
+    ar2_node_url: str = field(default_factory=lambda: os.environ.get("AR2_NODE_URL", "http://localhost:8001"))
     status_list_uri: str = field(
         default_factory=lambda: os.environ.get(
             "STATUS_LIST_URI", "http://localhost:8100/grants/status-list"

services/pancake_services/grants/auth.py

@@ -92,3 +92,32 @@ def get_current_user(
         db.commit()
         db.refresh(user)
     return user
+
+class VerificationError(Exception):
+    pass
+
+def verify_authority_credential(
+    token: str,
+    public_key_pem: bytes,
+    requested_scope: str = None,
+    local_status_list_path: str = None,
+) -> dict:
+    from pancake_services.grants import sdjwt, statuslist
+    try:
+        # Authority credentials use a different VCT
+        result = sdjwt.verify(token, public_key_pem, expected_vct="agstack.org/credentials/traceforward-authority/v1")
+    except sdjwt.VerificationError as e:
+        raise VerificationError(str(e)) from e
+
+    claims = result.claims
+
+    status = (claims.get("status") or {}).get("status_list")
+    if not status:
+        raise VerificationError("authority credential has no status list")
+    if statuslist.is_revoked(status, local_path=local_status_list_path):
+        raise VerificationError("authority credential revoked")
+
+    if requested_scope and claims.get("scope") not in (requested_scope, "global"):
+        raise VerificationError(f"insufficient scope: requested {requested_scope}, got {claims.get('scope')}")
+
+    return claims

services/pancake_services/grants/issuer.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import base64
+import functools
 import os
 from dataclasses import dataclass
 
@@ -69,8 +70,8 @@ def load_issuer_identity() -> IssuerIdentity:
     raw = os.environ.get(ENV_KEY)
     if not raw:
         raise RuntimeError(
-            f"{ENV_KEY} is not set. Generate one with: "
-            "python -m pancake_services.grants.testkit.mint_test_credentials --keygen"
+            f"{ENV_KEY} is not set. For local testing, use the testkit key:\n"
+            "export PANCAKE_ISSUER_KEY=$(cat services/pancake_services/grants/testkit/dev_keys/dev_issuer_private.pem)"
         )
     key = _load_private_key(raw)
     private_pem = key.private_bytes(
@@ -88,3 +89,13 @@ def load_issuer_identity() -> IssuerIdentity:
         private_key_pem=private_pem,
         public_key_pem=public_pem,
     )
+
+
+@functools.lru_cache()
+def authority_pubkey() -> bytes:
+    """Pancake's own trust anchor for verifying authority credentials."""
+    key_path = os.getenv("PANCAKE_TRUSTED_AUTHORITY_PUBKEY")
+    if not key_path or not os.path.exists(key_path):
+        raise RuntimeError("PANCAKE_TRUSTED_AUTHORITY_PUBKEY not set or file not found")
+    with open(key_path, "rb") as f:
+        return f.read()