From 6e111cc2ad6eecc899488b73cdda63762860cc26 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Fri, 12 Jun 2026 13:25:55 +0000 Subject: [PATCH] test(conformance): pin the conformance suite to 0.2.0-alpha.2 and reconcile the baseline Bumps the published conformance pin from 0.2.0-alpha.1 to 0.2.0-alpha.2 and reconciles expected-failures.yaml against it: 45 -> 33 entries. The twelve removed client auth entries failed only on the SEP-837 application_type check, which 0.2.0-alpha.2 re-scopes to draft-version runs; the dated-version scenarios now pass. The auth/iss-* scenarios do not exist in this release yet (reported unknown/failed; covered by their baseline entries). Disputed draft error-ladder cells stay listed pending conformance #336. All three CI legs verified green with zero stale entries. --- pnpm-lock.yaml | 10 +++--- test/conformance/expected-failures.yaml | 41 +++++++++++++------------ test/conformance/package.json | 2 +- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c13f1ebcde..28ea326060 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1099,8 +1099,8 @@ importers: specifier: workspace:^ version: link:../../packages/client '@modelcontextprotocol/conformance': - specifier: 0.2.0-alpha.1 - version: 0.2.0-alpha.1(@cfworker/json-schema@4.1.1) + specifier: 0.2.0-alpha.2 + version: 0.2.0-alpha.2(@cfworker/json-schema@4.1.1) '@modelcontextprotocol/core': specifier: workspace:^ version: link:../../packages/core @@ -2108,8 +2108,8 @@ packages: '@manypkg/get-packages@1.1.3': resolution: {integrity: sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A==} - '@modelcontextprotocol/conformance@0.2.0-alpha.1': - resolution: {integrity: sha512-w5YpjJ7uNXGIWs2r+gMwowNgREykYvDlrFS1Ckl47lVbkZJn2CAytmuV/zCrsCuZkeE7djK1tJkuZ0oa72BhGA==} + '@modelcontextprotocol/conformance@0.2.0-alpha.2': + resolution: {integrity: sha512-/8bde9d0mfsvgd9IwQgNIl1AS9uNOp/+ZG+2nNRWXtPs6xrz/cNp4ObBMmGY9kP8dkDaF3bvjtC/2Hj8TStMRg==} hasBin: true '@modelcontextprotocol/sdk@1.29.0': @@ -5998,7 +5998,7 @@ snapshots: globby: 11.1.0 read-yaml-file: 1.1.0 - '@modelcontextprotocol/conformance@0.2.0-alpha.1(@cfworker/json-schema@4.1.1)': + '@modelcontextprotocol/conformance@0.2.0-alpha.2(@cfworker/json-schema@4.1.1)': dependencies: '@modelcontextprotocol/sdk': 1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6) '@octokit/rest': 22.0.1 diff --git a/test/conformance/expected-failures.yaml b/test/conformance/expected-failures.yaml index f4b7ce4213..87bd2e9af8 100644 --- a/test/conformance/expected-failures.yaml +++ b/test/conformance/expected-failures.yaml @@ -1,10 +1,21 @@ # Conformance scenarios not yet implemented in the v2 TypeScript SDK. # CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. # -# Baseline established against @modelcontextprotocol/conformance 0.2.0-alpha.1, -# which adds the draft-spec scenario suite (SEP-2575, SEP-2322, SEP-2243, SEP-2549, -# SEP-2468, ...) plus new checks on existing scenarios (SEP-837 application_type -# during DCR). +# Baseline established against the published @modelcontextprotocol/conformance +# release pinned in package.json (0.2.0-alpha.2). Relative to 0.2.0-alpha.1, +# that release re-scoped the SEP-837 application_type check to draft-version +# runs only, so the dated-version auth scenarios that failed only on that +# check were removed from this baseline. Newer conformance releases are +# adopted by deliberately bumping the package.json pin and reconciling this +# file in the same change. (The auth/iss-* scenarios are not present in +# 0.2.0-alpha.2; the runner reports them as unknown/failed, covered by their +# entries below until a release ships them.) +# +# NOTE: the draft error-code assignments exercised by the SEP-2243 server +# scenarios (-32001 HeaderMismatch) and their neighbours (-32602, -32004) are +# still under discussion upstream (pending conformance #336). Those cells are +# treated as parameterized, not settled: the entries below record today's +# referee behavior and are re-derived when a #336-containing referee is pinned. # # Entries are grouped by SEP. As each SEP/milestone is implemented in the SDK the # corresponding scenarios start passing and MUST be removed from this list (the @@ -34,26 +45,15 @@ client: # SEP-2352 (authorization server migration): client does not re-register when # PRM authorization_servers changes. - auth/authorization-server-migration - # SEP-2207 (offline_access scope) scenario, currently failing only on the new - # SEP-837 application_type check (see the SEP-837 group below). + # SEP-837 (application_type during DCR): the check only fires on draft-version + # runs; this draft scenario is the one place the client still hits it. - auth/offline-access-not-supported # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 --- - # SEP-837: client MUST send application_type during Dynamic Client Registration. - # Single new check; everything else in these scenarios passes. - - auth/metadata-default - - auth/metadata-var1 - - auth/metadata-var2 - - auth/metadata-var3 - - auth/scope-from-www-authenticate - - auth/scope-from-scopes-supported - - auth/scope-omitted-when-undefined + # SEP-2350 (scope step-up): WARNING-only — client does not compute the union of + # previously requested and newly challenged scopes on re-authorization; the + # expected-failures evaluator counts WARNINGs as failures. - auth/scope-step-up - - auth/scope-retry-limit - - auth/token-endpoint-auth-basic - - auth/token-endpoint-auth-post - - auth/token-endpoint-auth-none - - auth/2025-03-26-oauth-metadata-backcompat # SEP-990 (enterprise-managed authorization extension): no fixture handler / # client support for the token-exchange + JWT bearer flow. - auth/enterprise-managed-authorization @@ -81,6 +81,7 @@ server: - caching # SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and # case-insensitive/whitespace-trimmed header validation not implemented. + # (Error-code cells parameterized pending conformance #336 — see header note.) - http-header-validation - http-custom-header-server-validation # WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level diff --git a/test/conformance/package.json b/test/conformance/package.json index 6ce6307f5e..214351c90f 100644 --- a/test/conformance/package.json +++ b/test/conformance/package.json @@ -38,7 +38,7 @@ "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all" }, "devDependencies": { - "@modelcontextprotocol/conformance": "0.2.0-alpha.1", + "@modelcontextprotocol/conformance": "0.2.0-alpha.2", "@modelcontextprotocol/client": "workspace:^", "@modelcontextprotocol/server": "workspace:^", "@modelcontextprotocol/core": "workspace:^",