Skip to content

Commit 41c7e61

Browse files
author
StackMemory Bot (CLI)
committed
feat(scripts): log PROSE test results to Provenant
Add scripts/log-prose-results.ts that runs the PROSE platform-overview integration tests and logs SOP-compliance decisions into a Provenant database. - Runs tests via vitest --reporter=json - Extracts the 5 SOP-backed Expectation tests (E.1-E.5) - Inserts decision nodes with source attribution for provenance - Usage: npx tsx scripts/log-prose-results.ts [path/to/provenant.db]
1 parent 7be1970 commit 41c7e61

1 file changed

Lines changed: 167 additions & 0 deletions

File tree

scripts/log-prose-results.ts

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
#!/usr/bin/env tsx
2+
/**
3+
* Run the PROSE platform-overview integration tests and log the results
4+
* into a Provenant knowledge graph as SOP-compliance decisions.
5+
*
6+
* Usage:
7+
* npx tsx scripts/log-prose-results.ts [path/to/provenant.db]
8+
*
9+
* Requires PROVENANT_DB env var or a database path argument.
10+
*/
11+
12+
import { execSync } from 'node:child_process';
13+
import { mkdirSync } from 'node:fs';
14+
import { resolve, dirname } from 'node:path';
15+
import { fileURLToPath } from 'node:url';
16+
import { Database } from '../packages/provenant/src/schema/database.js';
17+
18+
const __dirname = dirname(fileURLToPath(import.meta.url));
19+
20+
interface VitestResult {
21+
testResults?: Array<{
22+
assertionResults: Array<{
23+
title: string;
24+
fullName: string;
25+
status: 'passed' | 'failed';
26+
duration: number;
27+
}>;
28+
}>;
29+
numPassedTests?: number;
30+
numFailedTests?: number;
31+
}
32+
33+
function runTests(): VitestResult {
34+
const testFile = resolve(
35+
__dirname,
36+
'../src/__tests__/integration/platform-overview.test.ts'
37+
);
38+
const raw = execSync(
39+
`npx vitest run "${testFile}" --reporter=json --silent`,
40+
{
41+
cwd: resolve(__dirname, '..'),
42+
encoding: 'utf8',
43+
timeout: 300_000,
44+
env: {
45+
...process.env,
46+
VITEST: undefined,
47+
NODE_ENV: undefined,
48+
},
49+
}
50+
);
51+
// vitest --reporter=json outputs one JSON object at the end; find the first '{'.
52+
const start = raw.indexOf('{');
53+
if (start === -1) {
54+
throw new Error('No JSON output from vitest');
55+
}
56+
return JSON.parse(raw.slice(start)) as VitestResult;
57+
}
58+
59+
function mapTestToSop(testName: string, fullName: string): { sop?: string; proseId?: string } {
60+
const mapping: Record<string, { sop: string; proseId: string }> = {
61+
'active frame stack remains consistent': {
62+
sop: 'SOP-101 Frame Lifecycle',
63+
proseId: 'E.1',
64+
},
65+
'recorded decisions are immutable': {
66+
sop: 'SOP-102 Decision Record Keeping',
67+
proseId: 'E.2',
68+
},
69+
'projects in different directories are isolated': {
70+
sop: 'SOP-103 Project Boundary Enforcement',
71+
proseId: 'E.3',
72+
},
73+
'CLI commands return correct exit codes': {
74+
sop: 'SOP-201 CLI Exit-Code Compliance',
75+
proseId: 'E.4',
76+
},
77+
'SQLite database is self-contained in .stackmemory': {
78+
sop: 'SOP-202 Data Portability',
79+
proseId: 'E.5',
80+
},
81+
};
82+
83+
for (const [key, value] of Object.entries(mapping)) {
84+
if (testName.includes(key) || fullName.includes(key)) {
85+
return value;
86+
}
87+
}
88+
return {};
89+
}
90+
91+
function main(): void {
92+
const dbPath =
93+
process.argv[2] ?? process.env['PROVENANT_DB'] ?? '.provenant/graph.db';
94+
if (!dbPath) {
95+
console.error(
96+
'Provide a Provenant database path as an argument or set PROVENANT_DB'
97+
);
98+
process.exit(1);
99+
}
100+
101+
mkdirSync(dirname(dbPath), { recursive: true });
102+
103+
console.log('Running PROSE integration tests...');
104+
const result = runTests();
105+
const tests = result.testResults ?? [];
106+
107+
if (tests.length === 0) {
108+
console.error('No test results found');
109+
process.exit(1);
110+
}
111+
112+
console.log(`Test run complete: ${result.numPassedTests ?? 0} passed, ${result.numFailedTests ?? 0} failed`);
113+
114+
const db = new Database(dbPath);
115+
try {
116+
const runId = `prose-run-${Date.now()}`;
117+
let logged = 0;
118+
119+
const allTests = tests.flatMap((file) => file.assertionResults ?? []);
120+
121+
for (const test of allTests) {
122+
const { sop, proseId } = mapTestToSop(test.title, test.fullName);
123+
if (!sop || !proseId) continue; // Only log SOP-backed Expectation tests
124+
125+
const passed = test.status === 'passed';
126+
const content = passed
127+
? `${proseId} / ${sop}: compliance verified by integration test`
128+
: `${proseId} / ${sop}: compliance NOT verified by integration test`;
129+
130+
const node = db.insertNode({
131+
type: 'decision',
132+
content,
133+
embedding: null,
134+
actor: 'prose-harness',
135+
confidence: passed ? 0.95 : 0.4,
136+
});
137+
138+
const source = db.insertSource({
139+
system: 'prose-test-run',
140+
external_id: `${runId}:${proseId}`,
141+
raw_payload: JSON.stringify({
142+
testName: test.title,
143+
fullName: test.fullName,
144+
status: test.status,
145+
duration: test.duration,
146+
proseId,
147+
sop,
148+
}),
149+
hash: `${runId}:${proseId}`,
150+
});
151+
152+
db.linkNodeToSource(
153+
node.id,
154+
source.id,
155+
'prose-test-run',
156+
`${runId}:${proseId}`
157+
);
158+
logged++;
159+
}
160+
161+
console.log(`Logged ${logged} SOP-compliance decisions to ${dbPath}`);
162+
} finally {
163+
db.close();
164+
}
165+
}
166+
167+
main();

0 commit comments

Comments
 (0)