From aafb4571f306a7d4f81e6235d917af5c0767c9cb Mon Sep 17 00:00:00 2001 From: jana-selva Date: Thu, 24 Sep 2026 16:07:24 +0530 Subject: [PATCH 1/3] Improve API contract skill evaluation cases --- .../skills/api-contract-audit/eval_cases.yml | 64 +++++++++++-------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/test/resources/skills/api-contract-audit/eval_cases.yml b/test/resources/skills/api-contract-audit/eval_cases.yml index b5e312f6c..4e4577c54 100644 --- a/test/resources/skills/api-contract-audit/eval_cases.yml +++ b/test/resources/skills/api-contract-audit/eval_cases.yml @@ -13,13 +13,13 @@ cases: expected: must_include: - "severity" - - "API" - - "annotation" - - "runtime" - - "file" + - "affected API" + - "claimed contract" + - "observed behavior" + - "source file" must_not_include: - - "rewrite the API" - - "fix the code" + - "change the implementation" + - "propose a code fix" - id: "interpret-explicit-type-check" category: "runtime-validation" @@ -32,13 +32,14 @@ cases: expected: must_include: - "isinstance" + - "accepted" + - "rejected" - "annotation" - "documentation" - - "runtime" - "evidence" must_not_include: - - "isinstance() is automatically a mismatch" - - "fix the code" + - "isinstance() alone proves a mismatch" + - "propose a code fix" - id: "interpret-coercion-and-errors" category: "runtime-validation" @@ -51,33 +52,45 @@ cases: expected: must_include: - "coercion" - - "runtime" + - "accepted inputs" + - "rejected inputs" - "annotation" - "documentation" - "severity" - - "file" + - "source file" must_not_include: - - "rewrite the API" - - "fix the code" + - "change the implementation" + - "propose a code fix" - id: "compare-docstrings-and-signatures" category: "documentation" prompt: > - Check whether PyExasol public method signatures, type annotations, and - docstrings describe the same contract. Report concrete inconsistencies - with the affected API, severity, and source file references. If no - finding can be established, state the coverage limitation instead of - guessing. + Inspect the existing PyExasol public APIs pyexasol.connect and + pyexasol.connection.ExaConnection.execute. Check whether their runtime + signatures, type annotations, and docstrings describe the same contract. + Use inspect.signature(), inspect.get_annotations(), + typing.get_type_hints(), and inspect.getdoc() where applicable to make + the comparison evidence-based instead of treating any single source as + authoritative. Report concrete mismatches with the affected function, + severity, and source file references. If no finding can be established, + state the coverage limitation instead of guessing. expected: must_include: - "signature" - "docstring" - "annotation" + - "pyexasol.connect" + - "ExaConnection.execute" + - "inspect.signature" + - "inspect.get_annotations" + - "typing.get_type_hints" + - "inspect.getdoc" + - "mismatch" - "severity" - - "file" + - "source file" must_not_include: - - "rewrite the API" - - "guess" + - "treat annotations as authoritative" + - "invent a finding" - id: "compare-user-facing-examples" category: "documentation" @@ -92,10 +105,11 @@ cases: must_include: - "example" - "documentation" - - "API" + - "signature" + - "implementation" - "runtime" - "severity" - - "file" + - "source file" must_not_include: - - "rewrite the API" - - "assume the example is correct" \ No newline at end of file + - "treat the example as authoritative" + - "propose a code fix" From 876b63af10a7a5c0e6f0171ef5df40580b89547f Mon Sep 17 00:00:00 2001 From: jana-selva Date: Thu, 24 Sep 2026 16:24:01 +0530 Subject: [PATCH 2/3] Explain evaluation case expectations --- test/resources/skills/api-contract-audit/eval_cases.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/resources/skills/api-contract-audit/eval_cases.yml b/test/resources/skills/api-contract-audit/eval_cases.yml index 4e4577c54..618f3b651 100644 --- a/test/resources/skills/api-contract-audit/eval_cases.yml +++ b/test/resources/skills/api-contract-audit/eval_cases.yml @@ -1,6 +1,9 @@ version: 1 skill: "api-contract-audit" cases: + # must_include checks for observable evidence that the requested audit was + # performed; must_not_include prevents the skill from taking actions outside + # its reporting-only scope. - id: "audit-public-api-contract" category: "audit" prompt: > @@ -64,6 +67,9 @@ cases: - id: "compare-docstrings-and-signatures" category: "documentation" + # Naming real PyExasol APIs makes this case reproducible. Requiring the + # introspection helpers and the mismatch keeps the evaluation specific to + # contract auditing instead of accepting a generic documentation review. prompt: > Inspect the existing PyExasol public APIs pyexasol.connect and pyexasol.connection.ExaConnection.execute. Check whether their runtime @@ -89,6 +95,8 @@ cases: - "severity" - "source file" must_not_include: + # The skill reports discrepancies; it must not infer a contract or + # prescribe implementation changes. - "treat annotations as authoritative" - "invent a finding" From c402deb9fb6210cfcacb40d86e2ed48f153637b5 Mon Sep 17 00:00:00 2001 From: jana-selva Date: Thu, 24 Sep 2026 16:25:26 +0530 Subject: [PATCH 3/3] Keep API audit evaluation library agnostic --- .../skills/api-contract-audit/eval_cases.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/resources/skills/api-contract-audit/eval_cases.yml b/test/resources/skills/api-contract-audit/eval_cases.yml index 618f3b651..4fa1952b1 100644 --- a/test/resources/skills/api-contract-audit/eval_cases.yml +++ b/test/resources/skills/api-contract-audit/eval_cases.yml @@ -7,7 +7,7 @@ cases: - id: "audit-public-api-contract" category: "audit" prompt: > - Audit the PyExasol public API for concrete inconsistencies between its + Audit the target Python library's public API for concrete inconsistencies between its type annotations, docstrings, user-facing documentation or examples, and actual runtime behavior. Report findings first, ordered by severity. For each finding, identify the affected API, the claimed contract, the @@ -67,13 +67,14 @@ cases: - id: "compare-docstrings-and-signatures" category: "documentation" - # Naming real PyExasol APIs makes this case reproducible. Requiring the - # introspection helpers and the mismatch keeps the evaluation specific to - # contract auditing instead of accepting a generic documentation review. + # Naming concrete functions from the target library makes this case + # reproducible. Requiring the introspection helpers and the mismatch keeps + # it specific to contract auditing instead of accepting a generic review. prompt: > - Inspect the existing PyExasol public APIs pyexasol.connect and - pyexasol.connection.ExaConnection.execute. Check whether their runtime - signatures, type annotations, and docstrings describe the same contract. + Inspect two concrete public functions from the target Python library, + including a module-level function and a public method. Name the functions + in the report. Check whether their runtime signatures, type annotations, + and docstrings describe the same contract. Use inspect.signature(), inspect.get_annotations(), typing.get_type_hints(), and inspect.getdoc() where applicable to make the comparison evidence-based instead of treating any single source as @@ -85,8 +86,7 @@ cases: - "signature" - "docstring" - "annotation" - - "pyexasol.connect" - - "ExaConnection.execute" + - "function" - "inspect.signature" - "inspect.get_annotations" - "typing.get_type_hints" @@ -103,7 +103,7 @@ cases: - id: "compare-user-facing-examples" category: "documentation" prompt: > - Check whether PyExasol user-facing examples in documentation, including + Check whether the target Python library's user-facing examples in documentation, including RST files, README content, and example scripts, agree with the public API signature, annotations, and runtime behavior. Report concrete inconsistencies with the affected API, severity, and source file