fix(qa): 1.9.1 — /lua-qa full report correctness (grade shapes, metrics, unfinished runs, foreach flow tests)

.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "lua-agent-builder",
       "source": "./plugins/lua-agent-builder",
       "description": "Build, test, and deploy Lua AI agents from inside Claude Code",
-      "version": "1.9.0",
+      "version": "1.9.1",
       "homepage": "https://github.com/lua-ai-global/claude-code-lua-plugin#readme",
       "repository": "https://github.com/lua-ai-global/claude-code-lua-plugin.git",
       "license": "MIT",

plugins/lua-agent-builder/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "lua-agent-builder",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Build, test, and deploy Lua AI agents from inside Claude Code",
   "author": {
     "name": "Lua AI",

plugins/lua-agent-builder/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 All notable changes to the `lua-agent-builder` plugin. Versions follow the tag `release-prod.yml` cuts from `package.json` (`v<version>`). lua-cli is a TypeScript SDK/CLI; it is unrelated to the Lua programming language.
 
+## 1.9.1 — 2026-10-08
+
+**`/lua-qa full`: report correctness fixes from the first three-tier trial.** The suite ran smoke, medium and production-ready on a test agent (Pageturner Books: 7 tools, 4 workflows, one of them a `foreach`). Four defects changed what the report said.
+
+- **`aggregate` crashed on a grade file in the wrong shape.** A grader wrote `candidates` as an object keyed by check (`{ readability: [...], claims: [...] }`), and `gradeMajors` threw `object is not iterable`. The reporter stopped and the medium run produced no report. Grade files are now normalised when they are read (`normalizeGrade` in `report/results.mjs`, used by `aggregate` and `run-verdict`): `candidates`, `criteria` and `defects` given as objects become arrays, and `safetyNotes` given as a string becomes a list.
+- **Confirmed candidates were left out of the readability and claims metrics.** Only candidates keyed `source` were counted; graders also wrote `check` and `kind` (42 of 480 candidates in the medium run, 80 of 362 in production-ready). Each candidate now gets a canonical `source` (`readability`, `claims` or `contamination`) from whichever key it carries. On the smoke run, readability-h1 was reported as 0.83 and is 0.67. A confirmed candidate written in the object form also fails its grade now, as the rubric says.
+- **A run whose player stopped mid-conversation was scored as a valid FAIL.** When a player lost the API (`ENOTFOUND`) after a few turns, the run kept `status: "running"`, had no grade, and still counted as a failure of the agent; one card failed with no graded run at all. Such a run is now **not played** ("the player stopped mid-conversation (the run was never finished or graded)"): not valid, never a FAIL, and the card is inconclusive if too few valid runs remain. A running record that does have a grade A is unchanged.
+- **Flow tests of a `foreach` workflow failed falsely.** The offline ledger (lua-cli 3.45.0) keys each item's run of the body step as `checkOne[0]`, `checkOne[1]`, …, so `reachNodes: ["checkOne"]` was never found and the verdict listed "1 flow test(s) failed" as a blocker. A step now counts as reached through its numeric indexed runs (`stepReached` in `flow-test.mjs`); a bare prefix (`check`) or a non-numeric suffix does not.
+
+Versioning: #18 (`fix/deploy-hook-false-positives`) is still open. If this merges first, #18 takes 1.9.2 and its entry goes above this one.
+
 ## 1.9.0 — 2026-10-08
 
 **The full QA suite: `/lua-qa full`.** Bare `/lua-qa` (and `/lua-qa <name>`, `/lua-qa quick …`) is still the 1.7.0 quick pass, with no new question; `/lua-qa full` runs the suite.

plugins/lua-agent-builder/lib/qa/flow-test.mjs

@@ -50,6 +50,14 @@ export function reachedSteps(stdout) {
   return find(parsed, 0);
 }
 
+/**
+ * Whether step `id` ran. A step inside `foreach` runs once per item and the offline ledger keys each run by
+ * index (`checkOne[0]`, `checkOne[1]`, …), so any indexed run of it counts.
+ */
+export function stepReached(reached, id) {
+  return reached.some((s) => s === id || (s.startsWith(`${id}[`) && /^\[\d+\]$/.test(s.slice(id.length))));
+}
+
 /**
  * @returns {{status:'pass'|'fail'|'error', reasons:string[], reached:string[]|null}}
  */
@@ -64,8 +72,8 @@ export function judgeFlowTest(test, result) {
   if ((expect.reachNodes?.length || expect.notReachNodes?.length) && reached === null) {
     notes.push('note: step list not in output; reachNodes check skipped');
   } else if (reached) {
-    for (const id of expect.reachNodes ?? []) if (!reached.includes(id)) reasons.push(`step ${id} was not reached`);
-    for (const id of expect.notReachNodes ?? []) if (reached.includes(id)) reasons.push(`step ${id} was reached but should not have been`);
+    for (const id of expect.reachNodes ?? []) if (!stepReached(reached, id)) reasons.push(`step ${id} was not reached`);
+    for (const id of expect.notReachNodes ?? []) if (stepReached(reached, id)) reasons.push(`step ${id} was reached but should not have been`);
   }
   for (const needle of expect.outputIncludes ?? []) {
     if (!String(result.stdout).includes(needle)) reasons.push(`output does not include "${needle}"`);

plugins/lua-agent-builder/lib/qa/report/results.mjs

@@ -117,8 +117,8 @@ export async function loadRunData(runDir) {
         folder: `runs/${cardId}/${name}`,
         record: await tryJson(join(dir, 'run-record.json')),
         turnCount: (await tryJsonl(join(dir, 'turns.jsonl'))).length,
-        gradeA: await tryJson(join(dir, 'grade-a.json')),
-        gradeB: await tryJson(join(dir, 'grade-b.json')),
+        gradeA: normalizeGrade(await tryJson(join(dir, 'grade-a.json'))),
+        gradeB: normalizeGrade(await tryJson(join(dir, 'grade-b.json'))),
         contamination: await tryJson(join(dir, 'checks', 'contamination.json')),
         readability: await tryJson(join(dir, 'checks', 'readability.json')),
         claims: await tryJson(join(dir, 'checks', 'claims.json')),
@@ -152,6 +152,52 @@ export async function loadRunData(runDir) {
   };
 }
 
+// ---------------------------------------------------------------------------
+// grade normalisation
+// ---------------------------------------------------------------------------
+
+const isPlainObject = (v) => Boolean(v) && typeof v === 'object' && !Array.isArray(v);
+
+// Graders have written a candidate's check as `source`, `check` or `kind`, sometimes as a path
+// ("checks/readability.json"); map each spelling onto the names the metrics count.
+const CANDIDATE_SOURCES = [['readab', 'readability'], ['claim', 'claims'], ['contaminat', 'contamination']];
+
+function candidateSource(c, key) {
+  const raw = String(c.source ?? c.check ?? c.kind ?? key ?? '').toLowerCase();
+  for (const [needle, name] of CANDIDATE_SOURCES) if (raw.includes(needle)) return name;
+  return raw || null;
+}
+
+/**
+ * Items of a grade list that may arrive as an array, or as an object keyed by check or criterion
+ * ({ readability: [...] }, { C1: {...} }, { common: [...] }). Object values become items, and a
+ * keyed single item keeps its key as `id`.
+ * @returns {Array<[object, string|undefined]>} each item with the key it came from
+ */
+function gradeItems(v) {
+  if (Array.isArray(v)) return v.filter(isPlainObject).map((x) => [x, undefined]);
+  if (!isPlainObject(v)) return [];
+  return Object.entries(v).flatMap(([key, val]) =>
+    Array.isArray(val) ? val.filter(isPlainObject).map((x) => [x, key]) : isPlainObject(val) ? [[{ id: key, ...val }, key]] : []);
+}
+
+/**
+ * Brings a grade file into the shape lua-qa/grade@1 declares (candidates, criteria and defects as
+ * arrays, every candidate with a canonical `source`), whatever variant the grader wrote. Without it an
+ * object-shaped list crashed `aggregate`, and candidates keyed `check` or `kind` were left out of the
+ * readability and claims metrics.
+ */
+export function normalizeGrade(g) {
+  if (!isPlainObject(g)) return g ?? null;
+  return {
+    ...g,
+    candidates: gradeItems(g.candidates).map(([c, key]) => ({ ...c, source: candidateSource(c, key) })),
+    criteria: gradeItems(g.criteria).map(([c]) => c),
+    defects: gradeItems(g.defects).map(([d]) => d),
+    safetyNotes: Array.isArray(g.safetyNotes) ? g.safetyNotes : g.safetyNotes ? [String(g.safetyNotes)] : [],
+  };
+}
+
 // ---------------------------------------------------------------------------
 // verdict rules (§3.14, §3.19)
 // ---------------------------------------------------------------------------
@@ -236,14 +282,17 @@ export function effectiveRuns(runs) {
 }
 
 /**
- * Why a run was never played, or null when it was. A run is not played when start-run never wrote its record, or
- * when no turn was ever recorded: nothing reached the agent, so it is neither valid nor a FAIL, whatever a grader
- * wrote about the empty folder.
+ * Why a run was never played, or null when it was. A run is not played when start-run never wrote its record,
+ * when no turn was ever recorded, or when the player stopped mid-conversation (the record is still `running`) and
+ * no grader saw it: the agent was never judged on a whole conversation, so the run is neither valid nor a FAIL,
+ * whatever a grader wrote about the empty folder.
  */
 export function notPlayedReason(r) {
   if (!r.record) return 'the run folder has no run record (start-run never completed)';
   // A record that says 0 turns, with no turn on disk either. A record without a turn count (older runs) is played.
   if (r.record.turns === 0 && !r.turnCount) return 'the player started the run but recorded no turn';
+  // finish-run never ran (the player crashed or lost the API) and nothing graded the partial transcript.
+  if (r.record.status === 'running' && !r.gradeA) return 'the player stopped mid-conversation (the run was never finished or graded)';
   return null;
 }
 
@@ -829,10 +878,10 @@ export async function cliRunVerdict(argv, io) {
     const record = await tryJson(join(dir, 'run-record.json'));
     if (!record) throw new QaError('RUN_RECORD_MISSING', 2, `run-record.json not found for ${values.card} ${name}`, 'Run start-run first');
     const tier = runTier(await tryJson(join(runDir, 'run.json')), await tryJson(join(runDir, 'state.json')));
-    const gradeA = await tryJson(join(dir, 'grade-a.json'));
-    const gradeB = await tryJson(join(dir, 'grade-b.json'));
+    const gradeA = normalizeGrade(await tryJson(join(dir, 'grade-a.json')));
+    const gradeB = normalizeGrade(await tryJson(join(dir, 'grade-b.json')));
     const contamination = (await tryJson(join(dir, 'checks', 'contamination.json'))) ?? record.checks?.contamination ?? null;
-    const unplayed = notPlayedReason({ record, turnCount: (await tryJsonl(join(dir, 'turns.jsonl'))).length });
+    const unplayed = notPlayedReason({ record, gradeA, turnCount: (await tryJsonl(join(dir, 'turns.jsonl'))).length });
     const verdict = unplayed ? 'NOT_PLAYED' : runVerdict({ gradeA, gradeB, contamination }, { graders: tier.graders });
     const safety = !unplayed && (gradeA?.safety === true || gradeB?.safety === true);
     const majors = unplayed ? [] : gradeMajors(gradeA, gradeB);

plugins/lua-agent-builder/mcp/lua-platform/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lua/claude-plugin-mcp",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Read-only MCP server for the lua-agent-builder Claude Code plugin",
   "type": "module",
   "main": "dist/server.js",

plugins/lua-agent-builder/mcp/lua-platform/src/api-client.mjs

@@ -69,7 +69,7 @@ export async function apiRequest(path, {
       headers: {
         'Authorization': `Bearer ${apiKey}`,
         'Content-Type': 'application/json',
-        'X-Lua-Client': 'claude-plugin/1.9.0',
+        'X-Lua-Client': 'claude-plugin/1.9.1',
       },
       body: body ? JSON.stringify(body) : undefined,
       signal: controller.signal,

plugins/lua-agent-builder/mcp/lua-platform/src/server.mjs

@@ -11,7 +11,7 @@ import * as tools from './tools/index.mjs';
 const TOOL_REGISTRY = Object.values(tools);
 
 const server = new Server(
-  { name: 'lua-platform', version: '1.9.0' },
+  { name: 'lua-platform', version: '1.9.1' },
   { capabilities: { tools: {} } }
 );
 
@@ -46,7 +46,7 @@ process.on('uncaughtException', (err) => {
     kind: 'uncaughtException',
     message: err?.message,
     stack: err?.stack,
-    plugin_version: '1.9.0',
+    plugin_version: '1.9.1',
     lua_cli_version: process.env.LUA_CLI_VERSION ?? null,
     platform: process.platform,
     ts: new Date().toISOString(),
@@ -59,7 +59,7 @@ process.on('unhandledRejection', (reason) => {
     kind: 'unhandledRejection',
     reason: String(reason),
     stack: /** @type {any} */ (reason)?.stack,
-    plugin_version: '1.9.0',
+    plugin_version: '1.9.1',
     lua_cli_version: process.env.LUA_CLI_VERSION ?? null,
     platform: process.platform,
     ts: new Date().toISOString(),

plugins/lua-agent-builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-code-lua-plugin",
-  "version": "1.9.0",
+  "version": "1.9.1",
   "description": "Plugin assets for the lua-agent-builder Claude Code plugin",
   "private": true,
   "type": "module",

plugins/lua-agent-builder/test/lib/qa/flow-test.test.mjs

@@ -1,7 +1,7 @@
 import { mkdir, writeFile } from 'node:fs/promises';
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
-import { cliFlowTest, flowTestArgv, judgeFlowTest, reachedSteps } from '../../../lib/qa/flow-test.mjs';
+import { cliFlowTest, flowTestArgv, judgeFlowTest, reachedSteps, stepReached } from '../../../lib/qa/flow-test.mjs';
 import { assertAllowedLuaArgv } from '../../../lib/qa/safety.mjs';
 import { readJson } from '../../../lib/qa/io.mjs';
 import { validate } from '../../../lib/qa/schemas.mjs';
@@ -71,6 +71,30 @@ describe('judgeFlowTest', () => {
     expect(r.reasons).toEqual(['step refund was not reached', 'step escalate was reached but should not have been']);
     expect(r.reached).toEqual(['classify', 'escalate']);
   });
+  test('a foreach step counts as reached through its indexed runs (lua-cli 3.45.0 ledger keys checkOne[0], checkOne[1])', () => {
+    // The ledger of a real restock-check run: the foreach body never appears under its bare id.
+    const steps = {
+      items: { stepId: 'items', status: 'completed' },
+      'foreach@1': { stepId: 'foreach@1', status: 'completed' },
+      summarise: { stepId: 'summarise', status: 'completed' },
+      'checkOne[0]': { stepId: 'checkOne[0]', status: 'completed' },
+      'checkOne[1]': { stepId: 'checkOne[1]', status: 'completed' },
+      'foreach@1.join': { stepId: 'foreach@1.join', status: 'completed' },
+    };
+    const out = JSON.stringify({ success: true, data: { status: 'completed', ledger: { steps } } });
+    expect(judgeFlowTest(FT('a', { expect: { reachNodes: ['items', 'checkOne', 'summarise'] } }), { exitCode: 0, stdout: out })).toMatchObject({ status: 'pass', reasons: [] });
+    const r = judgeFlowTest(FT('a', { expect: { notReachNodes: ['checkOne'] } }), { exitCode: 0, stdout: out });
+    expect(r.reasons).toEqual(['step checkOne was reached but should not have been']);
+  });
+  test('stepReached matches an id or its numeric foreach index, never a prefix or another suffix', () => {
+    const reached = ['checkOne[0]', 'checkOne[12]', 'assess', 'foreach@1.join'];
+    expect(stepReached(reached, 'checkOne')).toBe(true);
+    expect(stepReached(reached, 'assess')).toBe(true);
+    expect(stepReached(reached, 'check')).toBe(false);
+    expect(stepReached(reached, 'foreach@1')).toBe(false);
+    expect(stepReached(['checkOne[x]', 'checkOne[0]x'], 'checkOne')).toBe(false);
+    expect(stepReached([], 'checkOne')).toBe(false);
+  });
   test('without a step list the node check is skipped with a note and the test still passes', () => {
     const t = FT('a', { expect: { reachNodes: ['classify'] } });
     const r = judgeFlowTest(t, { exitCode: 0, stdout: '{"status":"completed"}' });