PDX-486: feat(mcp) — strict validator unknown-key detection (Thread A, PR 1/3)

docs/mcp.md

@@ -78,6 +78,7 @@ The Provar DX CLI ships with a built-in **Model Context Protocol (MCP) server**
 - [Quality scores explained](#quality-scores-explained)
 - [API compatibility — `xml` vs `xml_content`](#api-compatibility--xml-vs-xml_content)
 - [Performance Tuning](#performance-tuning)
+- [Warning codes](#warning-codes)
 
 ---
 
@@ -531,6 +532,23 @@ On **Windows**, path comparisons are performed case-insensitively to account for
 
 ---
 
+## Warning codes
+
+Cross-cutting warning codes surfaced by validation, configuration, and run tooling. These complement the per-tool `rule_id` codes (e.g. `TC_001`, `VAR-REF-001`) documented under [Available tools](#available-tools). Subsequent revisions will refine the meanings as the relevant tool surfaces stabilise.
+
+| Code             | Surfaced by                             | Meaning                                                                   |
+| ---------------- | --------------------------------------- | ------------------------------------------------------------------------- |
+| `PROVARHOME-001` | properties / automation tooling         | `provarHome` is missing, blank, or does not point to a Provar install     |
+| `DATA-001`       | `provar_testcase_validate`              | `<dataTable>` iteration is silently ignored in CLI standalone execution   |
+| `PARALLEL-001`   | automation / run tooling                | Parallel-mode cache mismatch between properties and active runtime config |
+| `SCHEMA-001`     | strict properties / config validators   | Unknown or misspelled key in a JSON / properties schema (typo guard)      |
+| `RUN-001`        | `provar_automation_testrun` and friends | Test run produced no executable results — check input selection           |
+| `JUNIT-001`      | report / RCA tooling                    | JUnit results file is missing, empty, or not parseable                    |
+
+Warning-code messages emitted via `formatWarning()` follow the shape `WARNING [<CODE>]: <message>` (optionally suffixed with ` Did you mean '<suggestion>'?` when a typo is detected). Other free-form warnings without a structured code — such as the placeholder warnings emitted by `provar_properties_validate` — remain plain strings. See `src/mcp/utils/warningCodes.ts` for the canonical enum.
+
+---
+
 ## Available tools
 
 ### `provardx_ping`
@@ -1086,7 +1104,7 @@ Updates one or more fields in a `provardx-properties.json` file. Only the suppli
 
 ### `provar_properties_validate`
 
-Validates a `provardx-properties.json` file against the ProvarDX schema. Checks required fields, valid enum values, and warns about unfilled `${PLACEHOLDER}` values. Accepts either a file path or inline JSON content.
+Validates a `provardx-properties.json` file against the ProvarDX schema. Checks required fields, valid enum values, and warns about unfilled `${PLACEHOLDER}` values. Also surfaces a `SCHEMA-001` warning for any unknown top-level, `metadata.*`, or `environment.*` key, with a "Did you mean ..." suggestion when a canonical key is within Levenshtein distance 2. Accepts either a file path or inline JSON content.
 
 **Input**
 
@@ -1097,12 +1115,17 @@ Validates a `provardx-properties.json` file against the ProvarDX schema. Checks
 
 **Output**
 
-| Field           | Description                                     |
-| --------------- | ----------------------------------------------- |
-| `is_valid`      | `true` if no errors                             |
-| `error_count`   | Number of validation errors                     |
-| `warning_count` | Number of warnings (e.g. unfilled placeholders) |
-| `issues`        | Array of `{ field, severity, message }`         |
+| Field           | Description                                                 |
+| --------------- | ----------------------------------------------------------- |
+| `is_valid`      | `true` if no errors (warnings alone do not flip `is_valid`) |
+| `error_count`   | Number of validation errors                                 |
+| `warning_count` | Number of warnings (placeholders, unknown keys, etc.)       |
+| `errors`        | Array of `{ field, severity: 'error', message }`            |
+| `warnings`      | Array of `{ field, severity: 'warning', message }`          |
+
+**Warning codes (`warnings` array):**
+
+- `SCHEMA-001` — unknown key at top-level / `metadata.*` / `environment.*`. Example: `WARNING [SCHEMA-001]: Unknown field 'testCases' at top-level. Did you mean 'testCase'?` Unknown keys are **warnings, not errors**, so additive Provar versions do not break older MCP clients. The classic instance is the `testCases` (plural) typo for the canonical `testCase` (singular) — if you see SCHEMA-001 on `testCases`, fix the spelling before running any tests.
 
 **Error codes:** `MISSING_INPUT`, `PROPERTIES_FILE_NOT_FOUND`, `MALFORMED_JSON`, `PATH_NOT_ALLOWED`
 

src/mcp/tools/propertiesTools.ts

@@ -16,6 +16,7 @@ import type { ServerConfig } from '../server.js';
 import { assertPathAllowed, PathPolicyError } from '../security/pathPolicy.js';
 import { makeError, makeRequestId } from '../schemas/common.js';
 import { log } from '../logging/logger.js';
+import { WARNING_CODES, formatWarning } from '../utils/warningCodes.js';
 import { desc } from './descHelper.js';
 
 // ── Validation helpers ────────────────────────────────────────────────────────
@@ -30,12 +31,118 @@ const TOP_REQUIRED = ['provarHome', 'projectPath', 'resultsPath', 'metadata', 'e
 const METADATA_REQUIRED = ['metadataLevel', 'cachePath'] as const;
 const ENV_REQUIRED = ['webBrowser', 'webBrowserConfig', 'webBrowserProviderName', 'webBrowserDeviceName'] as const;
 
+/**
+ * Canonical key sets for provardx-properties.json. Sourced from the documented schema in
+ * `docs/mcp.md` (provar_properties_set updates schema) and the `propertyFileContent` template
+ * from `@provartesting/provardx-plugins-utils`. Sibling PRs (PDX-488, PDX-494) may extend
+ * these sets — keep them exported and additive.
+ *
+ * NOTE: This set is intentionally lenient on "future Provar additions" — unknown keys emit a
+ * SCHEMA-001 warning (not a hard error) so older MCP clients keep working when new keys ship.
+ */
+export const CANONICAL_TOP_LEVEL_KEYS: readonly string[] = [
+  // Required
+  'provarHome',
+  'projectPath',
+  'resultsPath',
+  'metadata',
+  'environment',
+  // Optional — documented in docs/mcp.md properties_set schema
+  'resultsPathDisposition',
+  'testOutputLevel',
+  'pluginOutputlevel',
+  'stopOnError',
+  'excludeCallable',
+  'testprojectSecrets',
+  'testCase',
+  'testPlan',
+  'connectionOverride',
+  // Optional — present in the standard template (propertyFileContent.js)
+  'smtpPath',
+  'lightningMode',
+  'connectionRefreshType',
+  'testplanFeatures',
+];
+
+export const CANONICAL_METADATA_KEYS: readonly string[] = ['metadataLevel', 'cachePath'];
+
+export const CANONICAL_ENVIRONMENT_KEYS: readonly string[] = [
+  'testEnvironment',
+  'webBrowser',
+  'webBrowserConfig',
+  'webBrowserProviderName',
+  'webBrowserDeviceName',
+];
+
 const VALID_RESULTS_DISPOSITION = ['Increment', 'Replace', 'Fail'];
 const VALID_OUTPUT_LEVELS = ['BASIC', 'DETAILED', 'DIAGNOSTIC'];
 const VALID_PLUGIN_LEVELS = ['SEVERE', 'WARNING', 'INFO', 'FINE', 'FINER', 'FINEST'];
 const VALID_BROWSERS = ['Chrome', 'Safari', 'Edge', 'Edge_Legacy', 'Firefox', 'IE', 'Chrome_Headless'];
 const VALID_METADATA_LEVELS = ['Reuse', 'Reload', 'Refresh'];
 
+/**
+ * Iterative Levenshtein distance — small, no-dependency implementation used solely for
+ * "did you mean ..." suggestions in SCHEMA-001 warnings. Returns the edit distance between
+ * `a` and `b`. Case-sensitive.
+ */
+function levenshtein(a: string, b: string): number {
+  if (a === b) return 0;
+  if (a.length === 0) return b.length;
+  if (b.length === 0) return a.length;
+  let prev: number[] = Array.from({ length: b.length + 1 }, (_, i) => i);
+  for (let i = 1; i <= a.length; i++) {
+    const curr: number[] = [i];
+    for (let j = 1; j <= b.length; j++) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      curr.push(Math.min(curr[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost));
+    }
+    prev = curr;
+  }
+  return prev[b.length];
+}
+
+/**
+ * Returns the canonical key whose Levenshtein distance from `key` is <= 2, or `undefined`
+ * if no key is within that threshold. Picks the closest match (smallest distance, then
+ * lexicographic) to keep suggestions stable across runs.
+ */
+function closestCanonicalKey(key: string, canonical: readonly string[]): string | undefined {
+  let best: { key: string; dist: number } | undefined;
+  for (const candidate of canonical) {
+    const dist = levenshtein(key, candidate);
+    if (dist > 2) continue;
+    if (!best || dist < best.dist || (dist === best.dist && candidate < best.key)) {
+      best = { key: candidate, dist };
+    }
+  }
+  return best?.key;
+}
+
+/**
+ * Emit SCHEMA-001 warnings for any keys in `actual` that are not in `canonical`. The
+ * `pathLabel` is the human-readable scope (`top-level`, `metadata`, `environment`).
+ */
+function findUnknownKeys(
+  actual: Record<string, unknown>,
+  canonical: readonly string[],
+  pathLabel: string,
+  fieldPrefix: string
+): ValidationError[] {
+  const results: ValidationError[] = [];
+  const canonicalSet = new Set(canonical);
+  for (const key of Object.keys(actual)) {
+    if (canonicalSet.has(key)) continue;
+    const suggestion = closestCanonicalKey(key, canonical);
+    const message = formatWarning(WARNING_CODES.SCHEMA_001, `Unknown field '${key}' at ${pathLabel}.`, suggestion);
+    results.push({
+      field: fieldPrefix ? `${fieldPrefix}.${key}` : key,
+      message,
+      severity: 'warning',
+    });
+  }
+  return results;
+}
+
 // eslint-disable-next-line complexity
 function validateProperties(props: Record<string, unknown>): ValidationError[] {
   const errors: ValidationError[] = [];
@@ -134,6 +241,15 @@ function validateProperties(props: Record<string, unknown>): ValidationError[] {
     }
   }
 
+  // SCHEMA-001: unknown keys at any level — warning only so additive Provar versions don't break old clients
+  errors.push(...findUnknownKeys(props, CANONICAL_TOP_LEVEL_KEYS, 'top-level', ''));
+  if (meta && typeof meta === 'object') {
+    errors.push(...findUnknownKeys(meta, CANONICAL_METADATA_KEYS, 'metadata', 'metadata'));
+  }
+  if (env && typeof env === 'object') {
+    errors.push(...findUnknownKeys(env, CANONICAL_ENVIRONMENT_KEYS, 'environment', 'environment'));
+  }
+
   return errors;
 }
 
@@ -624,9 +740,12 @@ export function registerPropertiesValidate(server: McpServer, config: ServerConf
         [
           'Validate a provardx-properties.json file against the ProvarDX schema.',
           'Checks required fields, valid enum values, and warns about unfilled placeholder values.',
+          'Also emits a SCHEMA-001 warning for any unknown top-level, metadata.*, or environment.* key',
+          "(e.g. 'testCases' → did you mean 'testCase'?). Unknown keys are warnings, not errors,",
+          'so additive keys in future Provar versions do not break older MCP clients.',
           'Accepts either a file path or inline JSON content.',
         ].join(' '),
-        'Validate a provardx-properties.json against required fields and enum values.'
+        'Validate a provardx-properties.json; warns on unknown keys (SCHEMA-001) like testCases vs testCase.'
       ),
       inputSchema: {
         file_path: z

src/mcp/utils/warningCodes.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2024 Provar Limited.
+ * All rights reserved.
+ * Licensed under the BSD 3-Clause license.
+ * For full license text, see LICENSE.md file in the repo root or https://opensource.org/licenses/BSD-3-Clause
+ */
+
+export const WARNING_CODES = {
+  PROVARHOME_001: 'PROVARHOME-001',
+  DATA_001: 'DATA-001',
+  PARALLEL_001: 'PARALLEL-001',
+  SCHEMA_001: 'SCHEMA-001',
+  RUN_001: 'RUN-001',
+  JUNIT_001: 'JUNIT-001',
+} as const;
+
+export type WarningCode = (typeof WARNING_CODES)[keyof typeof WARNING_CODES];
+
+export function formatWarning(code: WarningCode, message: string, suggestion?: string): string {
+  const base = `WARNING [${code}]: ${message}`;
+  return suggestion ? `${base} Did you mean '${suggestion}'?` : base;
+}

test/fixtures/properties/testcases-typo.json

@@ -0,0 +1,21 @@
+{
+  "provarHome": "/opt/provar",
+  "projectPath": "/projects/sample-project",
+  "resultsPath": "/projects/sample-project/ANT/Results",
+  "resultsPathDisposition": "Replace",
+  "testOutputLevel": "BASIC",
+  "pluginOutputlevel": "WARNING",
+  "stopOnError": false,
+  "metadata": {
+    "metadataLevel": "Reuse",
+    "cachePath": "/projects/sample-project/ANT/.provarCaches"
+  },
+  "environment": {
+    "testEnvironment": "default",
+    "webBrowser": "Chrome_Headless",
+    "webBrowserConfig": "Full Screen",
+    "webBrowserProviderName": "Desktop",
+    "webBrowserDeviceName": "Full Screen"
+  },
+  "testCases": ["tests/SmokeFlow.testcase"]
+}

test/unit/mcp/propertiesTools.test.ts

@@ -535,4 +535,107 @@ describe('provar_properties_validate', () => {
       'Expected error on metadata.metadataLevel'
     );
   });
+
+  // ── SCHEMA-001: unknown-key detection (PDX-486 VALIDATE-TYPO-A) ──────────────
+
+  describe('SCHEMA-001 unknown-key detection', () => {
+    it('fires for unknown top-level key with did-you-mean suggestion within distance 2', () => {
+      const props = validProps();
+      // Classic typo: testCases (plural) vs canonical testCase
+      props['testCases'] = ['tests/foo.testcase'];
+
+      const result = server.call('provar_properties_validate', { content: JSON.stringify(props) });
+
+      const body = parseText(result);
+      const warnings = body['warnings'] as Array<{ field: string; message: string; severity: string }>;
+      const w = warnings.find((x) => x.field === 'testCases');
+      assert.ok(w, 'Expected SCHEMA-001 warning for testCases');
+      assert.ok(w.message.includes('SCHEMA-001'), 'Warning should reference SCHEMA-001');
+      assert.ok(w.message.includes("Unknown field 'testCases'"), 'Warning should name the offending key');
+      assert.ok(w.message.includes('top-level'), 'Warning should label the scope as top-level');
+      assert.ok(w.message.includes("'testCase'"), `Expected "did you mean 'testCase'" suggestion, got: ${w.message}`);
+    });
+
+    it("fires for unknown metadata.* key (e.g. 'metadataLvel') with suggestion", () => {
+      const props = validProps();
+      (props['metadata'] as Record<string, unknown>)['metadataLvel'] = 'Reuse';
+
+      const result = server.call('provar_properties_validate', { content: JSON.stringify(props) });
+
+      const body = parseText(result);
+      const warnings = body['warnings'] as Array<{ field: string; message: string }>;
+      const w = warnings.find((x) => x.field === 'metadata.metadataLvel');
+      assert.ok(w, 'Expected SCHEMA-001 warning for metadata.metadataLvel');
+      assert.ok(w.message.includes('SCHEMA-001'));
+      assert.ok(w.message.includes('metadata'), 'Warning should label the scope as metadata');
+      assert.ok(w.message.includes("'metadataLevel'"), `Expected suggestion, got: ${w.message}`);
+    });
+
+    it("fires for unknown environment.* key (e.g. 'testEnvironments' → testEnvironment)", () => {
+      const props = validProps();
+      (props['environment'] as Record<string, unknown>)['testEnvironments'] = 'QA';
+
+      const result = server.call('provar_properties_validate', { content: JSON.stringify(props) });
+
+      const body = parseText(result);
+      const warnings = body['warnings'] as Array<{ field: string; message: string }>;
+      const w = warnings.find((x) => x.field === 'environment.testEnvironments');
+      assert.ok(w, 'Expected SCHEMA-001 warning for environment.testEnvironments');
+      assert.ok(w.message.includes('SCHEMA-001'));
+      assert.ok(w.message.includes('environment'));
+      assert.ok(w.message.includes("'testEnvironment'"), `Expected testEnvironment suggestion, got: ${w.message}`);
+    });
+
+    it('does NOT fire SCHEMA-001 for known top-level / metadata / environment keys', () => {
+      const result = server.call('provar_properties_validate', { content: JSON.stringify(validProps()) });
+
+      const body = parseText(result);
+      const warnings = body['warnings'] as Array<{ message: string }>;
+      assert.ok(
+        warnings.every((w) => !w.message.includes('SCHEMA-001')),
+        `Expected zero SCHEMA-001 warnings on a valid file, got: ${JSON.stringify(warnings)}`
+      );
+    });
+
+    it('emits SCHEMA-001 with no "Did you mean" when no canonical key is within distance 2', () => {
+      const props = validProps();
+      props['totallyUnrelatedKey'] = 'x';
+
+      const result = server.call('provar_properties_validate', { content: JSON.stringify(props) });
+
+      const body = parseText(result);
+      const warnings = body['warnings'] as Array<{ field: string; message: string }>;
+      const w = warnings.find((x) => x.field === 'totallyUnrelatedKey');
+      assert.ok(w, 'Expected SCHEMA-001 warning for totallyUnrelatedKey');
+      assert.ok(w.message.includes('SCHEMA-001'));
+      assert.ok(!w.message.includes('Did you mean'), `Should not include suggestion, got: ${w.message}`);
+    });
+
+    it('is_valid stays true when the only issues are SCHEMA-001 warnings (no errors)', () => {
+      const props = validProps();
+      props['testCases'] = ['x'];
+
+      const result = server.call('provar_properties_validate', { content: JSON.stringify(props) });
+
+      const body = parseText(result);
+      assert.equal(body['is_valid'], true, 'Unknown keys are warnings only — is_valid must remain true');
+      assert.equal(body['error_count'], 0);
+      assert.ok((body['warning_count'] as number) >= 1);
+    });
+
+    it("loads test/fixtures/properties/testcases-typo.json and reports SCHEMA-001 with 'testCase' suggestion", () => {
+      // Tests run from the repo root via wireit/yarn; resolve relative to cwd to avoid ESM __dirname.
+      const fixturePath = path.resolve(process.cwd(), 'test', 'fixtures', 'properties', 'testcases-typo.json');
+      const content = fs.readFileSync(fixturePath, 'utf-8');
+
+      const result = server.call('provar_properties_validate', { content });
+
+      const body = parseText(result);
+      assert.equal(body['is_valid'], true, 'Fixture should pass structural validation (warnings only)');
+      const warnings = body['warnings'] as Array<{ field: string; message: string }>;
+      const w = warnings.find((x) => x.field === 'testCases');
+      assert.ok(w, 'Expected SCHEMA-001 warning for the testCases typo in the fixture');
+      assert.ok(w.message.includes("Did you mean 'testCase'?"), `Expected suggestion, got: ${w.message}`);
+    });
+  });
 });