|
| 1 | +#!/usr/bin/env tsx |
| 2 | +/** |
| 3 | + * Run the PROSE platform-overview integration tests and log the results |
| 4 | + * into a Provenant knowledge graph as SOP-compliance decisions. |
| 5 | + * |
| 6 | + * Usage: |
| 7 | + * npx tsx scripts/log-prose-results.ts [path/to/provenant.db] |
| 8 | + * |
| 9 | + * Requires PROVENANT_DB env var or a database path argument. |
| 10 | + */ |
| 11 | + |
| 12 | +import { execSync } from 'node:child_process'; |
| 13 | +import { mkdirSync } from 'node:fs'; |
| 14 | +import { resolve, dirname } from 'node:path'; |
| 15 | +import { fileURLToPath } from 'node:url'; |
| 16 | +import { Database } from '../packages/provenant/src/schema/database.js'; |
| 17 | + |
| 18 | +const __dirname = dirname(fileURLToPath(import.meta.url)); |
| 19 | + |
| 20 | +interface VitestResult { |
| 21 | + testResults?: Array<{ |
| 22 | + assertionResults: Array<{ |
| 23 | + title: string; |
| 24 | + fullName: string; |
| 25 | + status: 'passed' | 'failed'; |
| 26 | + duration: number; |
| 27 | + }>; |
| 28 | + }>; |
| 29 | + numPassedTests?: number; |
| 30 | + numFailedTests?: number; |
| 31 | +} |
| 32 | + |
| 33 | +function runTests(): VitestResult { |
| 34 | + const testFile = resolve( |
| 35 | + __dirname, |
| 36 | + '../src/__tests__/integration/platform-overview.test.ts' |
| 37 | + ); |
| 38 | + const raw = execSync( |
| 39 | + `npx vitest run "${testFile}" --reporter=json --silent`, |
| 40 | + { |
| 41 | + cwd: resolve(__dirname, '..'), |
| 42 | + encoding: 'utf8', |
| 43 | + timeout: 300_000, |
| 44 | + env: { |
| 45 | + ...process.env, |
| 46 | + VITEST: undefined, |
| 47 | + NODE_ENV: undefined, |
| 48 | + }, |
| 49 | + } |
| 50 | + ); |
| 51 | + // vitest --reporter=json outputs one JSON object at the end; find the first '{'. |
| 52 | + const start = raw.indexOf('{'); |
| 53 | + if (start === -1) { |
| 54 | + throw new Error('No JSON output from vitest'); |
| 55 | + } |
| 56 | + return JSON.parse(raw.slice(start)) as VitestResult; |
| 57 | +} |
| 58 | + |
| 59 | +function mapTestToSop(testName: string, fullName: string): { sop?: string; proseId?: string } { |
| 60 | + const mapping: Record<string, { sop: string; proseId: string }> = { |
| 61 | + 'active frame stack remains consistent': { |
| 62 | + sop: 'SOP-101 Frame Lifecycle', |
| 63 | + proseId: 'E.1', |
| 64 | + }, |
| 65 | + 'recorded decisions are immutable': { |
| 66 | + sop: 'SOP-102 Decision Record Keeping', |
| 67 | + proseId: 'E.2', |
| 68 | + }, |
| 69 | + 'projects in different directories are isolated': { |
| 70 | + sop: 'SOP-103 Project Boundary Enforcement', |
| 71 | + proseId: 'E.3', |
| 72 | + }, |
| 73 | + 'CLI commands return correct exit codes': { |
| 74 | + sop: 'SOP-201 CLI Exit-Code Compliance', |
| 75 | + proseId: 'E.4', |
| 76 | + }, |
| 77 | + 'SQLite database is self-contained in .stackmemory': { |
| 78 | + sop: 'SOP-202 Data Portability', |
| 79 | + proseId: 'E.5', |
| 80 | + }, |
| 81 | + }; |
| 82 | + |
| 83 | + for (const [key, value] of Object.entries(mapping)) { |
| 84 | + if (testName.includes(key) || fullName.includes(key)) { |
| 85 | + return value; |
| 86 | + } |
| 87 | + } |
| 88 | + return {}; |
| 89 | +} |
| 90 | + |
| 91 | +function main(): void { |
| 92 | + const dbPath = |
| 93 | + process.argv[2] ?? process.env['PROVENANT_DB'] ?? '.provenant/graph.db'; |
| 94 | + if (!dbPath) { |
| 95 | + console.error( |
| 96 | + 'Provide a Provenant database path as an argument or set PROVENANT_DB' |
| 97 | + ); |
| 98 | + process.exit(1); |
| 99 | + } |
| 100 | + |
| 101 | + mkdirSync(dirname(dbPath), { recursive: true }); |
| 102 | + |
| 103 | + console.log('Running PROSE integration tests...'); |
| 104 | + const result = runTests(); |
| 105 | + const tests = result.testResults ?? []; |
| 106 | + |
| 107 | + if (tests.length === 0) { |
| 108 | + console.error('No test results found'); |
| 109 | + process.exit(1); |
| 110 | + } |
| 111 | + |
| 112 | + console.log(`Test run complete: ${result.numPassedTests ?? 0} passed, ${result.numFailedTests ?? 0} failed`); |
| 113 | + |
| 114 | + const db = new Database(dbPath); |
| 115 | + try { |
| 116 | + const runId = `prose-run-${Date.now()}`; |
| 117 | + let logged = 0; |
| 118 | + |
| 119 | + const allTests = tests.flatMap((file) => file.assertionResults ?? []); |
| 120 | + |
| 121 | + for (const test of allTests) { |
| 122 | + const { sop, proseId } = mapTestToSop(test.title, test.fullName); |
| 123 | + if (!sop || !proseId) continue; // Only log SOP-backed Expectation tests |
| 124 | + |
| 125 | + const passed = test.status === 'passed'; |
| 126 | + const content = passed |
| 127 | + ? `${proseId} / ${sop}: compliance verified by integration test` |
| 128 | + : `${proseId} / ${sop}: compliance NOT verified by integration test`; |
| 129 | + |
| 130 | + const node = db.insertNode({ |
| 131 | + type: 'decision', |
| 132 | + content, |
| 133 | + embedding: null, |
| 134 | + actor: 'prose-harness', |
| 135 | + confidence: passed ? 0.95 : 0.4, |
| 136 | + }); |
| 137 | + |
| 138 | + const source = db.insertSource({ |
| 139 | + system: 'prose-test-run', |
| 140 | + external_id: `${runId}:${proseId}`, |
| 141 | + raw_payload: JSON.stringify({ |
| 142 | + testName: test.title, |
| 143 | + fullName: test.fullName, |
| 144 | + status: test.status, |
| 145 | + duration: test.duration, |
| 146 | + proseId, |
| 147 | + sop, |
| 148 | + }), |
| 149 | + hash: `${runId}:${proseId}`, |
| 150 | + }); |
| 151 | + |
| 152 | + db.linkNodeToSource( |
| 153 | + node.id, |
| 154 | + source.id, |
| 155 | + 'prose-test-run', |
| 156 | + `${runId}:${proseId}` |
| 157 | + ); |
| 158 | + logged++; |
| 159 | + } |
| 160 | + |
| 161 | + console.log(`Logged ${logged} SOP-compliance decisions to ${dbPath}`); |
| 162 | + } finally { |
| 163 | + db.close(); |
| 164 | + } |
| 165 | +} |
| 166 | + |
| 167 | +main(); |
0 commit comments