diff --git a/src/index.ts b/src/index.ts index 38f7701..8b4b959 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,6 +11,7 @@ import { runInteractiveMode } from './runner'; import { + printAuthorizationServerResults, printAuthorizationServerSummary, runAuthorizationServerConformanceTest } from './runner/authorization-server'; @@ -461,20 +462,40 @@ program 'Run conformance tests against an authorization server implementation' ) .requiredOption('--url ', 'URL of the authorization server issuer') + .option('--scenario ', 'Test scenario to run') .option('-o, --output-dir ', 'Save results to this directory') .option( '--spec-version ', 'Filter scenarios by spec version (cumulative for date versions)' ) + .option('--verbose', 'Show verbose output (JSON instead of pretty print)') .action(async (options) => { try { // Validate options with Zod const validated = AuthorizationServerOptionsSchema.parse(options); + const verbose = options.verbose ?? false; const outputDir = options.outputDir; const specVersionFilter = options.specVersion ? resolveSpecVersion(options.specVersion) : undefined; + // If a single scenario is specified, run just that one + if (validated.scenario) { + const result = await runAuthorizationServerConformanceTest( + validated.url, + validated.scenario, + outputDir + ); + + const { failed } = printAuthorizationServerResults( + result.checks, + result.scenarioDescription, + verbose + ); + + process.exit(failed > 0 ? 1 : 0); + } + let scenarios: string[]; scenarios = listClientScenariosForAuthorizationServer(); if (specVersionFilter) { diff --git a/src/runner/authorization-server.ts b/src/runner/authorization-server.ts index 5b4094b..ba9d535 100644 --- a/src/runner/authorization-server.ts +++ b/src/runner/authorization-server.ts @@ -2,7 +2,7 @@ import { promises as fs } from 'fs'; import path from 'path'; import { ConformanceCheck } from '../types'; import { getClientScenarioForAuthorizationServer } from '../scenarios'; -import { createResultDir } from './utils'; +import { createResultDir, formatPrettyChecks } from './utils'; export async function runAuthorizationServerConformanceTest( serverUrl: string, @@ -49,6 +49,49 @@ export async function runAuthorizationServerConformanceTest( }; } +export function printAuthorizationServerResults( + checks: ConformanceCheck[], + scenarioDescription: string, + verbose: boolean = false +): { + passed: number; + failed: number; + denominator: number; + warnings: number; +} { + const denominator = checks.filter( + (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' + ).length; + const passed = checks.filter((c) => c.status === 'SUCCESS').length; + const failed = checks.filter((c) => c.status === 'FAILURE').length; + const warnings = checks.filter((c) => c.status === 'WARNING').length; + + if (verbose) { + console.log(JSON.stringify(checks, null, 2)); + } else { + console.log(`Checks:\n${formatPrettyChecks(checks)}`); + } + + console.log(`\nTest Results:`); + console.log( + `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` + ); + + if (failed > 0) { + console.log('\n=== Failed Checks ==='); + checks + .filter((c) => c.status === 'FAILURE') + .forEach((c) => { + console.log(`\n - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.log(` Error: ${c.errorMessage}`); + } + }); + } + + return { passed, failed, denominator, warnings }; +} + export function printAuthorizationServerSummary( allResults: { scenario: string; checks: ConformanceCheck[] }[] ): { totalPassed: number; totalFailed: number } { diff --git a/src/scenarios/authorization-server/authorization-server-metadata.test.ts b/src/scenarios/authorization-server/authorization-server-metadata.test.ts index 1cf862b..1c1d6bb 100644 --- a/src/scenarios/authorization-server/authorization-server-metadata.test.ts +++ b/src/scenarios/authorization-server/authorization-server-metadata.test.ts @@ -89,3 +89,138 @@ describe('AuthorizationServerMetadataEndpointScenario', () => { expect(check.errorMessage).toContain('code_challenge_methods_supported'); }); }); + +describe('AuthorizationServerOptionsSchema', () => { + // Dynamic import to avoid circular dependency issues at module level + async function getSchema() { + const { AuthorizationServerOptionsSchema } = + await import('../../schemas.js'); + return AuthorizationServerOptionsSchema; + } + + it('accepts a valid scenario name', async () => { + const schema = await getSchema(); + const result = schema.safeParse({ + url: 'https://example.com', + scenario: 'authorization-server-metadata-endpoint' + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.scenario).toBe( + 'authorization-server-metadata-endpoint' + ); + } + }); + + it('rejects an unknown scenario name', async () => { + const schema = await getSchema(); + const result = schema.safeParse({ + url: 'https://example.com', + scenario: 'nonexistent-scenario' + }); + expect(result.success).toBe(false); + }); + + it('accepts without scenario (optional)', async () => { + const schema = await getSchema(); + const result = schema.safeParse({ + url: 'https://example.com' + }); + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.scenario).toBeUndefined(); + } + }); +}); + +describe('printAuthorizationServerResults', () => { + async function getPrintFn() { + const { printAuthorizationServerResults } = + await import('../../runner/authorization-server.js'); + return printAuthorizationServerResults; + } + + const successCheck = { + id: 'test-check', + name: 'TestCheck', + description: 'A test check', + status: 'SUCCESS' as const, + timestamp: new Date().toISOString() + }; + + const failureCheck = { + id: 'test-check-fail', + name: 'TestCheckFail', + description: 'A failing check', + status: 'FAILURE' as const, + timestamp: new Date().toISOString(), + errorMessage: 'Something went wrong' + }; + + it('prints pretty-printed output when verbose is false', async () => { + const printFn = await getPrintFn(); + const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + const result = printFn([successCheck], 'Test scenario description', false); + + expect(result).toEqual({ + passed: 1, + failed: 0, + denominator: 1, + warnings: 0 + }); + + // Should print "Checks:" header (pretty format), not raw JSON + const allOutput = consoleSpy.mock.calls.map((c) => c[0]).join('\n'); + expect(allOutput).toContain('Checks:'); + expect(allOutput).not.toContain('"id"'); + + consoleSpy.mockRestore(); + }); + + it('prints JSON output when verbose is true', async () => { + const printFn = await getPrintFn(); + const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + const result = printFn([successCheck], 'Test scenario description', true); + + expect(result).toEqual({ + passed: 1, + failed: 0, + denominator: 1, + warnings: 0 + }); + + // Should print raw JSON, not "Checks:" header + const allOutput = consoleSpy.mock.calls.map((c) => c[0]).join('\n'); + expect(allOutput).toContain('"id"'); + expect(allOutput).toContain('"test-check"'); + expect(allOutput).not.toMatch(/^Checks:/m); + + consoleSpy.mockRestore(); + }); + + it('reports failed checks with error messages', async () => { + const printFn = await getPrintFn(); + const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + + const result = printFn( + [successCheck, failureCheck], + 'Test scenario description', + false + ); + + expect(result).toEqual({ + passed: 1, + failed: 1, + denominator: 2, + warnings: 0 + }); + + const allOutput = consoleSpy.mock.calls.map((c) => c[0]).join('\n'); + expect(allOutput).toContain('Failed Checks'); + expect(allOutput).toContain('Something went wrong'); + + consoleSpy.mockRestore(); + }); +}); diff --git a/src/schemas.ts b/src/schemas.ts index e0df8ce..4888dd5 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -1,5 +1,9 @@ import { z } from 'zod'; -import { getScenario, getClientScenario } from './scenarios'; +import { + getScenario, + getClientScenario, + getClientScenarioForAuthorizationServer +} from './scenarios'; // Client command options schema export const ClientOptionsSchema = z.object({ @@ -40,7 +44,17 @@ export type ServerOptions = z.infer; // Authorization server command options schema export const AuthorizationServerOptionsSchema = z.object({ - url: z.string().url('Invalid authorization server URL') + url: z.string().url('Invalid authorization server URL'), + scenario: z + .string() + .refine( + (scenario) => + getClientScenarioForAuthorizationServer(scenario) !== undefined, + { + error: (iss) => `Unknown scenario '${iss.input}'` + } + ) + .optional() }); export type AuthorizationServerOptions = z.infer<