diff --git a/packages/ai-bot/lib/load-skill-loop.ts b/packages/ai-bot/lib/load-skill-loop.ts new file mode 100644 index 0000000000..a7d70e1ac3 --- /dev/null +++ b/packages/ai-bot/lib/load-skill-loop.ts @@ -0,0 +1,77 @@ +import type { + ChatCompletion, + ChatCompletionMessageParam, +} from 'openai/resources'; +import { + executeLoadSkill, + LOAD_SKILL_TOOL_NAME, + type LoadSkillArgs, +} from './load-skill.ts'; +import type { DelegatedRealmSessionManager } from './user-delegated-realm-server-session.ts'; + +// Cap on how many loadSkill rounds a single user turn may drive, so a model +// that keeps asking for more skills can't loop the bot indefinitely. +export const LOAD_SKILL_MAX_ROUNDS = 4; + +export interface LoadSkillLoopDeps { + onBehalfOf: string; + delegatedRealmSessions: Pick; + fetch?: typeof globalThis.fetch; +} + +// Decides what happens after a generation round given the assistant message it +// produced. When that message's tool calls are exclusively `loadSkill`, runs +// them and returns the messages to append before generating again — the +// assistant turn followed by one tool result per call. Returns an empty array +// in every other case (no tool calls, or a mix that includes host-dispatched +// commands), which tells the caller to stop looping and let the answer stand. +export async function buildLoadSkillFollowup( + assistantMessage: ChatCompletion.Choice['message'], + deps: LoadSkillLoopDeps, +): Promise { + let toolCalls = assistantMessage.tool_calls ?? []; + if (toolCalls.length === 0) { + return []; + } + + let loadSkillCalls = toolCalls.filter( + (call) => + call.type === 'function' && call.function.name === LOAD_SKILL_TOOL_NAME, + ); + // Only the bot's own tool was called this round — anything else (a + // host-dispatched command) means the turn is doing more than loading skills, + // so leave it to the normal command-request path rather than re-prompting. + if ( + loadSkillCalls.length === 0 || + loadSkillCalls.length !== toolCalls.length + ) { + return []; + } + + let toolMessages: ChatCompletionMessageParam[] = []; + for (let call of loadSkillCalls) { + if (call.type !== 'function') { + continue; + } + let content: string; + let args: LoadSkillArgs | undefined; + try { + args = JSON.parse(call.function.arguments) as LoadSkillArgs; + } catch { + args = undefined; + } + if (!args || !args.realm || !args.name) { + content = 'Error: loadSkill needs a realm and a skill name.'; + } else { + let result = await executeLoadSkill(args, deps); + content = result.ok ? result.content : `Error: ${result.error}`; + } + toolMessages.push({ + role: 'tool', + tool_call_id: call.id, + content, + }); + } + + return [assistantMessage as ChatCompletionMessageParam, ...toolMessages]; +} diff --git a/packages/ai-bot/lib/load-skill.ts b/packages/ai-bot/lib/load-skill.ts new file mode 100644 index 0000000000..b7098ed7c4 --- /dev/null +++ b/packages/ai-bot/lib/load-skill.ts @@ -0,0 +1,140 @@ +import { logger, SupportedMimeType } from '@cardstack/runtime-common'; +import { ensureTrailingSlash } from '@cardstack/runtime-common/paths'; +import { DelegatedRealmSessionError } from '@cardstack/runtime-common/user-delegated-realm-server-session'; +import type { Tool } from 'https://cardstack.com/base/matrix-event'; +import type { DelegatedRealmSessionManager } from './user-delegated-realm-server-session.ts'; + +let log = logger('ai-bot:load-skill'); + +export const LOAD_SKILL_TOOL_NAME = 'loadSkill'; + +// On-demand skill loading. The model calls `loadSkill` to pull a skill's full +// instructions only when it needs them, instead of having every skill body +// pushed into the prompt up front. ai-bot executes it in-process: it mints a +// delegated, user-scoped realm token and fetches the file over HTTP, so the bot +// can only read what the requesting human can already read, and the content is +// always live (no Matrix snapshots, no host round-trip). +export const loadSkillTool: Tool = { + type: 'function', + function: { + name: LOAD_SKILL_TOOL_NAME, + description: + "Load a skill's instructions on demand. Returns the SKILL.md body for a " + + "skill in a realm, or — with `path` — a single file under the skill's " + + 'references/ directory. Use this to get the full instructions for a skill ' + + 'you have only seen listed by name, or a reference file it cites.', + parameters: { + type: 'object', + properties: { + realm: { + type: 'string', + description: + 'Realm URL the skill lives in, e.g. https://app.boxel.ai/user/jane/. ' + + 'Use a realm advertised in the room or referenced in the conversation.', + }, + name: { + type: 'string', + description: + 'The skill directory name under skills/, e.g. trip-planner.', + }, + path: { + type: 'string', + description: + "Optional file under the skill's references/ directory, e.g. " + + 'api-notes.md. Omit to load SKILL.md.', + }, + }, + required: ['realm', 'name'], + }, + }, +}; + +export interface LoadSkillArgs { + realm: string; + name: string; + path?: string; +} + +export type LoadSkillResult = + | { ok: true; url: string; content: string } + | { ok: false; error: string }; + +// The realm file a loadSkill call resolves to: SKILL.md by default, or a single +// file under references/ when `path` is given. +export function skillFileUrl({ realm, name, path }: LoadSkillArgs): string { + let rel = path + ? `skills/${name}/references/${path}` + : `skills/${name}/SKILL.md`; + return new URL(rel, ensureTrailingSlash(realm)).href; +} + +// Executes a loadSkill tool call inside the bot process: mints a delegated, +// read-only token for `onBehalfOf` scoped to `realm`, then GETs the skill file +// as raw source. Never throws — returns a result the caller hands back to the +// model as the tool result, so a missing skill or a permission failure becomes +// information the model can act on rather than a crashed turn. +export async function executeLoadSkill( + args: LoadSkillArgs, + { + onBehalfOf, + delegatedRealmSessions, + fetch = globalThis.fetch, + }: { + onBehalfOf: string; + delegatedRealmSessions: Pick; + fetch?: typeof globalThis.fetch; + }, +): Promise { + let url = skillFileUrl(args); + + let token: string; + try { + token = await delegatedRealmSessions.getToken({ + onBehalfOf, + realm: args.realm, + }); + } catch (e: any) { + if (e instanceof DelegatedRealmSessionError) { + if (e.kind === 'disabled') { + return { + ok: false, + error: 'skill loading is unavailable (delegation is not configured)', + }; + } + if (e.kind === 'forbidden') { + return { ok: false, error: `no read access to ${args.realm}` }; + } + } + log.error( + `loadSkill: could not obtain a delegated token for ${args.realm}: ${ + e?.message ?? e + }`, + ); + return { + ok: false, + error: `could not obtain realm access for ${args.realm}`, + }; + } + + let response: Response; + try { + response = await fetch(url, { + headers: { + Accept: SupportedMimeType.CardSource, + Authorization: `Bearer ${token}`, + }, + }); + } catch (e: any) { + log.error(`loadSkill: fetch failed for ${url}: ${e?.message ?? e}`); + return { ok: false, error: `could not fetch ${url}` }; + } + + if (!response.ok) { + return { + ok: false, + error: `could not load ${url} (HTTP ${response.status})`, + }; + } + + return { ok: true, url, content: await response.text() }; +} diff --git a/packages/ai-bot/lib/response-state.ts b/packages/ai-bot/lib/response-state.ts index 86cd467e1f..5cc4eaf4b1 100644 --- a/packages/ai-bot/lib/response-state.ts +++ b/packages/ai-bot/lib/response-state.ts @@ -1,6 +1,7 @@ import { thinkingMessage } from '../constants.ts'; import type { ChatCompletionSnapshot } from 'openai/lib/ChatCompletionStream'; import { cleanContent } from '@cardstack/runtime-common/ai'; +import { LOAD_SKILL_TOOL_NAME } from './load-skill.ts'; export default class ResponseState { latestReasoning: string = ''; @@ -52,6 +53,11 @@ export default class ResponseState { if (name === 'checkCorrectness') { return false; } + // loadSkill runs inside the bot, not on the host — never surface it as + // a command request for the user to execute. + if (name === LOAD_SKILL_TOOL_NAME) { + return false; + } if (this.allowedToolNames && !this.allowedToolNames.has(name)) { return false; } diff --git a/packages/ai-bot/main.ts b/packages/ai-bot/main.ts index cd48cd089e..91097246c5 100644 --- a/packages/ai-bot/main.ts +++ b/packages/ai-bot/main.ts @@ -35,6 +35,11 @@ import { import { handleDebugCommands } from './lib/debug.ts'; import { DelegatedRealmSessionManager } from './lib/user-delegated-realm-server-session.ts'; +import { loadSkillTool } from './lib/load-skill.ts'; +import { + buildLoadSkillFollowup, + LOAD_SKILL_MAX_ROUNDS, +} from './lib/load-skill-loop.ts'; import { Responder } from './lib/responder.ts'; import { shouldSetRoomTitle, @@ -104,14 +109,21 @@ class Assistant { ); } - getResponse(prompt: PromptParts, senderMatrixUserId?: string) { + getResponse( + prompt: PromptParts, + senderMatrixUserId?: string, + // Lets the loadSkill loop re-run a turn with tool results appended without + // rebuilding the rest of the prompt. + messagesOverride?: ChatCompletionMessageParam[], + ) { if (!prompt.model) { throw new Error('Model is required'); } let request: Parameters[0] = { model: this.getModel(prompt), - messages: prompt.messages as ChatCompletionMessageParam[], + messages: + messagesOverride ?? (prompt.messages as ChatCompletionMessageParam[]), }; if (prompt.reasoningEffort !== undefined) { @@ -127,6 +139,13 @@ class Assistant { request.tool_choice = prompt.toolChoice; } + // Offer the bot-executed loadSkill tool whenever delegation is configured, + // even in rooms that carry no other tools. Inert otherwise: with no secret + // the manager is disabled and the tool is never advertised. + if (prompt.toolsSupported === true && this.delegatedRealmSessions.enabled) { + request.tools = [...(request.tools ?? []), loadSkillTool]; + } + if (senderMatrixUserId) { request.user = senderMatrixUserId; } @@ -485,68 +504,113 @@ Common issues are: model: promptParts.model, }); } - const runner = assistant - .getResponse(promptParts, senderMatrixUserId) - .on('chunk', async (chunk, snapshot) => { - log.info(`[${eventId}] Received chunk %s`, chunk.id); - if (profEnabled() && firstChunkAt == null) { - firstChunkAt = Date.now(); - profNote(eventId, 'llm:ttft', { - ms: firstChunkAt - requestStart, - model: promptParts.model, + // Messages for the current round. The loadSkill loop appends the + // assistant's tool call and the fetched skill content here, then + // generates again. + let workingMessages: ChatCompletionMessageParam[] = + promptParts.messages as ChatCompletionMessageParam[]; + let loadSkillRounds = 0; + + try { + // Usually one pass. When the model calls only the bot-executed + // loadSkill tool, run those calls and generate again with the + // results in context, up to LOAD_SKILL_MAX_ROUNDS. + for (;;) { + let roundCostInUsd: number | undefined; + const runner = assistant + .getResponse( + promptParts, + senderMatrixUserId, + workingMessages, + ) + .on('chunk', async (chunk, snapshot) => { + log.info(`[${eventId}] Received chunk %s`, chunk.id); + if (profEnabled() && firstChunkAt == null) { + firstChunkAt = Date.now(); + profNote(eventId, 'llm:ttft', { + ms: firstChunkAt - requestStart, + model: promptParts.model, + }); + } + generationId = chunk.id; + if (chunk.usage && (chunk.usage as any).cost != null) { + roundCostInUsd = (chunk.usage as any).cost; + } + let activeGeneration = activeGenerations.get(room.roomId); + if (activeGeneration) { + activeGeneration.lastGeneratedChunkId = generationId; + } + + let chunkProcessingResult = await profTime( + eventId, + 'llm:chunk:onChunk', + async () => responder.onChunk(chunk, snapshot), + ); + let chunkProcessingResultError = + chunkProcessingResult.find( + (promiseResult) => + promiseResult && + 'errorMessage' in promiseResult && + promiseResult.errorMessage != null, + ) as { errorMessage: string } | undefined; + + if (chunkProcessingResultError) { + chunkHandlingError = + chunkProcessingResultError.errorMessage; + + // If there was an error processing the chunk, e.g. matrix sending error (e.g. event too large), + // then we want to stop accepting more chunks by aborting the runner. This will throw an error + // where the await responder.finalize() is called (the catch block below will handle this) + runner.abort(); + } + }) + .on('error', async (error) => { + await responder.onError(error); }); - } - generationId = chunk.id; - if (chunk.usage && (chunk.usage as any).cost != null) { - costInUsd = (chunk.usage as any).cost; - } - let activeGeneration = activeGenerations.get(room.roomId); - if (activeGeneration) { - activeGeneration.lastGeneratedChunkId = generationId; - } - let chunkProcessingResult = await profTime( + activeGenerations.set(room.roomId, { + responder, + runner, + lastGeneratedChunkId: generationId, + completionPromise: generationCompletionPromise, + }); + + let completion = await profTime( eventId, - 'llm:chunk:onChunk', - async () => responder.onChunk(chunk, snapshot), + 'llm:finalChatCompletion', + async () => runner.finalChatCompletion(), ); - let chunkProcessingResultError = chunkProcessingResult.find( - (promiseResult) => - promiseResult && - 'errorMessage' in promiseResult && - promiseResult.errorMessage != null, - ) as { errorMessage: string } | undefined; - - if (chunkProcessingResultError) { - chunkHandlingError = - chunkProcessingResultError.errorMessage; - - // If there was an error processing the chunk, e.g. matrix sending error (e.g. event too large), - // then we want to stop accepting more chunks by aborting the runner. This will throw an error - // where the await responder.finalize() is called (the catch block below will handle this) - runner.abort(); + // Each round's chunk usage reports that round's running total; + // sum them so billing sees the whole turn. + if (typeof roundCostInUsd === 'number') { + costInUsd = (costInUsd ?? 0) + roundCostInUsd; } - }) - .on('error', async (error) => { - await responder.onError(error); - }); - activeGenerations.set(room.roomId, { - responder, - runner, - lastGeneratedChunkId: generationId, - completionPromise: generationCompletionPromise, - }); + let message = completion.choices?.[0]?.message; + if ( + message && + senderMatrixUserId && + assistant.delegatedRealmSessions.enabled && + loadSkillRounds < LOAD_SKILL_MAX_ROUNDS + ) { + let followup = await buildLoadSkillFollowup(message, { + onBehalfOf: senderMatrixUserId, + delegatedRealmSessions: assistant.delegatedRealmSessions, + }); + if (followup.length > 0) { + workingMessages = [...workingMessages, ...followup]; + loadSkillRounds++; + continue; + } + } - try { - await profTime(eventId, 'llm:finalChatCompletion', async () => - runner.finalChatCompletion(), - ); - log.info(`[${eventId}] Generation complete`); - await profTime(eventId, 'response:finalize', async () => - responder.finalize(), - ); - log.info(`[${eventId}] Response finalized`); + log.info(`[${eventId}] Generation complete`); + await profTime(eventId, 'response:finalize', async () => + responder.finalize(), + ); + log.info(`[${eventId}] Response finalized`); + break; + } } catch (error) { // When the cancel handler aborts the runner, // finalChatCompletion() throws APIUserAbortError. diff --git a/packages/ai-bot/tests/index.ts b/packages/ai-bot/tests/index.ts index d078081a26..07dabf8595 100644 --- a/packages/ai-bot/tests/index.ts +++ b/packages/ai-bot/tests/index.ts @@ -13,5 +13,7 @@ import './locking-test.ts'; import './interrupt-test.ts'; import './credit-tracking-test.ts'; import './user-delegated-realm-server-session-test.ts'; +import './load-skill-test.ts'; +import './load-skill-loop-test.ts'; QUnit.start(); diff --git a/packages/ai-bot/tests/load-skill-loop-test.ts b/packages/ai-bot/tests/load-skill-loop-test.ts new file mode 100644 index 0000000000..8e1c9ca4cf --- /dev/null +++ b/packages/ai-bot/tests/load-skill-loop-test.ts @@ -0,0 +1,106 @@ +import QUnit from 'qunit'; +const { module, test, assert } = QUnit; + +import { buildLoadSkillFollowup } from '../lib/load-skill-loop.ts'; +import { LOAD_SKILL_TOOL_NAME } from '../lib/load-skill.ts'; + +const ON_BEHALF_OF = '@user:localhost'; +const REALM = 'https://localhost:4201/user/jane/'; + +function assistantMessage(toolCalls: any[]): any { + return { role: 'assistant', content: null, tool_calls: toolCalls }; +} + +function loadSkillCall(id: string, args: object) { + return { + id, + type: 'function', + function: { name: LOAD_SKILL_TOOL_NAME, arguments: JSON.stringify(args) }, + }; +} + +function deps(body = 'SKILL BODY') { + let calls: { onBehalfOf: string; realm: string }[] = []; + return { + calls, + onBehalfOf: ON_BEHALF_OF, + delegatedRealmSessions: { + getToken: async (a: { onBehalfOf: string; realm: string }) => { + calls.push(a); + return 'tok'; + }, + }, + fetch: (async () => + new Response(body, { + status: 200, + })) as unknown as typeof globalThis.fetch, + }; +} + +module('buildLoadSkillFollowup', () => { + test('returns [] when the message made no tool calls', async () => { + let d = deps(); + let out = await buildLoadSkillFollowup(assistantMessage([]), d); + assert.deepEqual(out, []); + }); + + test('runs loadSkill calls and returns assistant turn + one tool result each', async () => { + let d = deps('# Trip Planner'); + let msg = assistantMessage([ + loadSkillCall('c1', { realm: REALM, name: 'trip-planner' }), + ]); + let out = await buildLoadSkillFollowup(msg, d); + + assert.strictEqual(out.length, 2, 'assistant message + one tool result'); + assert.strictEqual(out[0], msg, 'assistant turn is preserved first'); + assert.strictEqual((out[1] as any).role, 'tool'); + assert.strictEqual((out[1] as any).tool_call_id, 'c1'); + assert.strictEqual((out[1] as any).content, '# Trip Planner'); + assert.deepEqual(d.calls, [{ onBehalfOf: ON_BEHALF_OF, realm: REALM }]); + }); + + test('does not loop when host-dispatched tool calls are mixed in', async () => { + let d = deps(); + let msg = assistantMessage([ + loadSkillCall('c1', { realm: REALM, name: 'trip-planner' }), + { + id: 'c2', + type: 'function', + function: { name: 'SomeHostCommand', arguments: '{}' }, + }, + ]); + let out = await buildLoadSkillFollowup(msg, d); + assert.deepEqual(out, [], 'leaves the turn to the normal command path'); + assert.strictEqual(d.calls.length, 0, 'no skill fetched'); + }); + + test('reports an error result for malformed arguments', async () => { + let d = deps(); + let msg = assistantMessage([ + { + id: 'c1', + type: 'function', + function: { name: LOAD_SKILL_TOOL_NAME, arguments: '{not json' }, + }, + ]); + let out = await buildLoadSkillFollowup(msg, d); + assert.strictEqual(out.length, 2); + assert.true( + (out[1] as any).content.startsWith('Error:'), + 'tool result carries an error the model can read', + ); + }); + + test('feeds a fetch failure back as an error tool result', async () => { + let d = deps(); + d.fetch = (async () => + new Response('nope', { + status: 404, + })) as unknown as typeof globalThis.fetch; + let msg = assistantMessage([ + loadSkillCall('c1', { realm: REALM, name: 'missing' }), + ]); + let out = await buildLoadSkillFollowup(msg, d); + assert.true((out[1] as any).content.includes('404')); + }); +}); diff --git a/packages/ai-bot/tests/load-skill-test.ts b/packages/ai-bot/tests/load-skill-test.ts new file mode 100644 index 0000000000..390ce952a8 --- /dev/null +++ b/packages/ai-bot/tests/load-skill-test.ts @@ -0,0 +1,184 @@ +import QUnit from 'qunit'; +const { module, test, assert } = QUnit; + +import { SupportedMimeType } from '@cardstack/runtime-common'; +import { DelegatedRealmSessionError } from '@cardstack/runtime-common/user-delegated-realm-server-session'; +import { + executeLoadSkill, + skillFileUrl, + loadSkillTool, + LOAD_SKILL_TOOL_NAME, +} from '../lib/load-skill.ts'; + +const ON_BEHALF_OF = '@user:localhost'; +const REALM = 'https://localhost:4201/user/jane/'; + +// A fake fetch that records each call and returns a scripted Response. +function recordingFetch( + handler: (url: string, init: RequestInit) => Response, +): { + fetch: typeof globalThis.fetch; + calls: { url: string; init: RequestInit }[]; +} { + let calls: { url: string; init: RequestInit }[] = []; + let fetch = (async (input: any, init: any) => { + let url = typeof input === 'string' ? input : input.url; + calls.push({ url, init }); + return handler(url, init); + }) as unknown as typeof globalThis.fetch; + return { fetch, calls }; +} + +// A stand-in for DelegatedRealmSessionManager that records getToken calls and +// either returns a token or throws a scripted error. +function stubSessions(result: { token: string } | { throws: unknown }): { + getToken: (args: { onBehalfOf: string; realm: string }) => Promise; + calls: { onBehalfOf: string; realm: string }[]; +} { + let calls: { onBehalfOf: string; realm: string }[] = []; + return { + calls, + getToken: async (args) => { + calls.push(args); + if ('throws' in result) { + throw result.throws; + } + return result.token; + }, + }; +} + +module('loadSkill tool definition', () => { + test('advertises name + required args', () => { + assert.strictEqual(loadSkillTool.function.name, LOAD_SKILL_TOOL_NAME); + assert.strictEqual(loadSkillTool.type, 'function'); + let required = (loadSkillTool.function.parameters as any) + .required as string[]; + assert.true(required.includes('realm'), 'realm is required'); + assert.true(required.includes('name'), 'name is required'); + assert.false(required.includes('path'), 'path is optional'); + }); +}); + +module('skillFileUrl', () => { + test('resolves SKILL.md when no path is given', () => { + assert.strictEqual( + skillFileUrl({ realm: REALM, name: 'trip-planner' }), + 'https://localhost:4201/user/jane/skills/trip-planner/SKILL.md', + ); + }); + + test('resolves a references/ file when path is given', () => { + assert.strictEqual( + skillFileUrl({ + realm: REALM, + name: 'trip-planner', + path: 'api-notes.md', + }), + 'https://localhost:4201/user/jane/skills/trip-planner/references/api-notes.md', + ); + }); + + test('tolerates a realm URL without a trailing slash', () => { + assert.strictEqual( + skillFileUrl({ realm: 'https://localhost:4201/user/jane', name: 's' }), + 'https://localhost:4201/user/jane/skills/s/SKILL.md', + ); + }); +}); + +module('executeLoadSkill', () => { + test('mints a token for the user/realm and returns the SKILL.md source', async () => { + let sessions = stubSessions({ token: 'tok-123' }); + let { fetch, calls } = recordingFetch( + () => new Response('# Trip Planner\n\ninstructions', { status: 200 }), + ); + + let result = await executeLoadSkill( + { realm: REALM, name: 'trip-planner' }, + { onBehalfOf: ON_BEHALF_OF, delegatedRealmSessions: sessions, fetch }, + ); + + assert.deepEqual(sessions.calls, [ + { onBehalfOf: ON_BEHALF_OF, realm: REALM }, + ]); + assert.strictEqual( + calls[0].url, + 'https://localhost:4201/user/jane/skills/trip-planner/SKILL.md', + ); + let headers = calls[0].init.headers as Record; + assert.strictEqual(headers['Authorization'], 'Bearer tok-123'); + assert.strictEqual(headers['Accept'], SupportedMimeType.CardSource); + assert.true(result.ok, 'result ok'); + assert.strictEqual( + (result as { ok: true; content: string }).content, + '# Trip Planner\n\ninstructions', + ); + }); + + test('loads a references/ file when path is given', async () => { + let sessions = stubSessions({ token: 'tok' }); + let { fetch, calls } = recordingFetch( + () => new Response('ref', { status: 200 }), + ); + await executeLoadSkill( + { realm: REALM, name: 'trip-planner', path: 'api-notes.md' }, + { onBehalfOf: ON_BEHALF_OF, delegatedRealmSessions: sessions, fetch }, + ); + assert.strictEqual( + calls[0].url, + 'https://localhost:4201/user/jane/skills/trip-planner/references/api-notes.md', + ); + }); + + test('returns an error result when the file is missing (404)', async () => { + let sessions = stubSessions({ token: 'tok' }); + let { fetch } = recordingFetch( + () => new Response('not found', { status: 404 }), + ); + let result = await executeLoadSkill( + { realm: REALM, name: 'nope' }, + { onBehalfOf: ON_BEHALF_OF, delegatedRealmSessions: sessions, fetch }, + ); + assert.false(result.ok, 'result not ok'); + assert.true( + (result as { ok: false; error: string }).error.includes('404'), + 'error mentions the status', + ); + }); + + test('reports a clear message when delegation is disabled', async () => { + let sessions = stubSessions({ + throws: new DelegatedRealmSessionError('disabled', 'off'), + }); + let { fetch, calls } = recordingFetch( + () => new Response('', { status: 200 }), + ); + let result = await executeLoadSkill( + { realm: REALM, name: 'trip-planner' }, + { onBehalfOf: ON_BEHALF_OF, delegatedRealmSessions: sessions, fetch }, + ); + assert.false(result.ok); + assert.true( + (result as { ok: false; error: string }).error.includes('unavailable'), + 'error explains the feature is off', + ); + assert.strictEqual(calls.length, 0, 'never fetched without a token'); + }); + + test('reports no-access when the user lacks read on the realm', async () => { + let sessions = stubSessions({ + throws: new DelegatedRealmSessionError('forbidden', 'nope', 403), + }); + let { fetch } = recordingFetch(() => new Response('', { status: 200 })); + let result = await executeLoadSkill( + { realm: REALM, name: 'trip-planner' }, + { onBehalfOf: ON_BEHALF_OF, delegatedRealmSessions: sessions, fetch }, + ); + assert.false(result.ok); + assert.true( + (result as { ok: false; error: string }).error.includes('no read access'), + 'error explains the access problem', + ); + }); +});