From c9c8d93aaac0cc61f7d91555c23b8f73c8b49611 Mon Sep 17 00:00:00 2001 From: LIlGG <1103069291@qq.com> Date: Fri, 15 May 2026 18:24:59 +0800 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20=E5=AF=B9=E6=8E=A5=20Halo=20AI=20?= =?UTF-8?q?=E5=B9=B6=E4=BC=98=E5=8C=96=20Live2D=20=E8=81=8A=E5=A4=A9?= =?UTF-8?q?=E4=BD=93=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用 Halo AI foundation 替换插件内置 AI 后端实现\n- 增加聊天即时反馈、输入框回焦和上下文轮数配置\n- 同步归档 OpenSpec 变更并新增主规范 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build.gradle | 3 +- .../.openspec.yaml | 2 + .../design.md | 80 +++++++++++ .../proposal.md | 28 ++++ .../specs/halo-ai-chat-integration/spec.md | 46 +++++++ .../tasks.md | 21 +++ .../specs/halo-ai-chat-integration/spec.md | 46 +++++++ packages/live2d/src/api/chat-api.ts | 34 ++++- .../src/components/Live2dChatWindow.tsx | 11 +- packages/live2d/src/config/default-config.ts | 2 + .../live2d/src/config/normalize-config.ts | 14 ++ packages/live2d/src/context/config-context.ts | 6 + packages/live2d/src/halo.ts | 4 +- .../run/halo/live2d/Live2dSettingProcess.java | 5 +- .../halo/live2d/chat/AIChatServiceImpl.java | 129 +++++++++++------ .../run/halo/live2d/chat/AiChatEndpoint.java | 70 ++++++---- .../run/halo/live2d/chat/AiChatService.java | 5 +- .../run/halo/live2d/chat/ChatRequest.java | 28 +++- .../halo/live2d/chat/WebClientFactory.java | 48 ------- .../halo/live2d/chat/client/ChatClient.java | 14 -- .../live2d/chat/client/DefaultChatClient.java | 32 ----- .../chat/client/openai/OpenAiChatClient.java | 130 ------------------ src/main/resources/extensions/settings.yaml | 80 +++-------- src/main/resources/plugin.yaml | 4 +- 24 files changed, 475 insertions(+), 367 deletions(-) create mode 100644 openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/.openspec.yaml create mode 100644 openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/design.md create mode 100644 openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/proposal.md create mode 100644 openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/specs/halo-ai-chat-integration/spec.md create mode 100644 openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/tasks.md create mode 100644 openspec/specs/halo-ai-chat-integration/spec.md delete mode 100644 src/main/java/run/halo/live2d/chat/WebClientFactory.java delete mode 100644 src/main/java/run/halo/live2d/chat/client/ChatClient.java delete mode 100644 src/main/java/run/halo/live2d/chat/client/DefaultChatClient.java delete mode 100644 src/main/java/run/halo/live2d/chat/client/openai/OpenAiChatClient.java diff --git a/build.gradle b/build.gradle index 83e8aad..bed0868 100644 --- a/build.gradle +++ b/build.gradle @@ -16,11 +16,12 @@ configurations.runtimeClasspath { dependencies { implementation platform('run.halo.tools.platform:plugin:2.24.0') - implementation 'com.theokanning.openai-gpt3-java:api:0.17.0' compileOnly 'run.halo.app:api' + compileOnly files('api-1.0.0-SNAPSHOT.jar') testImplementation 'run.halo.app:api' + testImplementation files('api-1.0.0-SNAPSHOT.jar') testImplementation 'org.springframework.boot:spring-boot-starter-test' } diff --git a/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/.openspec.yaml b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/.openspec.yaml new file mode 100644 index 0000000..9f70866 --- /dev/null +++ b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-05-15 diff --git a/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/design.md b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/design.md new file mode 100644 index 0000000..a313b9c --- /dev/null +++ b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/design.md @@ -0,0 +1,80 @@ +## Context + +Live2D chat currently sends the frontend's message history to `/live2d/ai/chat-process`, where the plugin prepends a system prompt and then delegates generation to plugin-owned `ChatClient` implementations. The only concrete client today is an OpenAI-specific streaming client backed by plugin settings for proxy, token, base URL, and model selection. + +The target state is to keep the Live2D chat UX and endpoint shape stable while delegating model discovery and invocation to Halo AI 基础设施. The upstream SDK is not yet available from a published repository, so this change must integrate through the provided `api-1.0.0-SNAPSHOT.jar` and the documented `AiServices` / `AiModelService` APIs. + +## Goals / Non-Goals + +**Goals:** +- Route Live2D chat generation through Halo AI foundation instead of the plugin's custom OpenAI client stack. +- Preserve the existing frontend chat contract so the widget keeps working with minimal behavioral change. +- Reduce plugin-owned AI configuration to chat-specific controls only: enablement, anonymous access, role prompt, model selection, and frontend timing values. +- Remove obsolete backend classes, settings, and dependencies once Halo AI integration is in place. + +**Non-Goals:** +- Redesign the Live2D chat UI or change the existing streaming presentation model. +- Add provider management, credential storage, or model provisioning inside the Live2D plugin. +- Generalize this change into a reusable abstraction for non-chat AI features. + +## Decisions + +### Use Halo AI foundation as the only chat backend +The backend will replace `ChatClient`-based provider dispatch with a single service path built on `AiServices.getModelService()`. The endpoint will obtain the configured Halo model name, resolve a `LanguageModel`, build a foundation `ChatRequest`, and stream foundation `ChatChunk` events back into the existing SSE response format consumed by the frontend. + +**Why this approach** +- It removes duplicated provider integration code from the plugin. +- It aligns provider credentials and model lifecycle with Halo's centralized AI management. +- It minimizes frontend churn because the SSE endpoint can still emit the current `ChatResult` payload shape. + +**Alternatives considered** +- Keep the `ChatClient` abstraction and add a Halo-backed implementation: rejected because the abstraction exists only to multiplex plugin-owned providers and would preserve unnecessary complexity. +- Proxy raw Halo AI responses directly to the frontend: rejected because the frontend already depends on `ChatResult` chunks and changing that contract would widen the migration surface. + +### Keep a plugin-level model selector, but not provider credentials +The plugin will retain a single AI-model identifier field that stores the Halo AI model resource name (for example `openai/gpt-4o`) alongside the existing persona prompt and access-control settings. Provider token, base URL, proxy, and provider enablement will move fully out of this plugin. + +**Why this approach** +- Halo AI foundation still requires the caller to request a concrete model. +- Selecting a model per plugin preserves expected plugin autonomy without reintroducing provider-specific configuration. +- It avoids brittle "first configured model wins" behavior. + +**Alternatives considered** +- Always use the first available Halo model: rejected because availability order is undefined and can change across environments. +- Add no plugin-level model choice and wait for a Halo-wide default model API: rejected because the current SDK and documentation do not define that capability. + +### Depend on the local SDK jar until the upstream artifact is published +The Gradle build will reference the provided `api-1.0.0-SNAPSHOT.jar` as a compile-time dependency and remove the old OpenAI SDK dependency that is no longer needed. + +**Why this approach** +- It matches the currently available distribution mechanism. +- It keeps the migration unblockable without inventing a separate publishing pipeline. + +**Alternatives considered** +- Depend on a Maven coordinate that is not yet published: rejected because builds would not be reproducible in the current repository state. +- Vendor copied source or decompiled classes: rejected because it would increase maintenance cost and drift from the upstream SDK. + +### Preserve backend-to-frontend error semantics at the plugin boundary +The endpoint will translate Halo AI foundation failures into readable plugin responses consistent with the current chat UX: unauthorized access remains an HTTP 401, while model/plugin/provider failures stream a user-facing error message through the existing SSE contract and terminate cleanly. + +**Why this approach** +- The current frontend already handles 401 and streamed error text. +- It avoids exposing raw provider internals in the browser while still giving administrators actionable logs on the server side. + +## Risks / Trade-offs + +- **[SDK distribution friction]** Local jar dependency is less ergonomic than a published artifact → Document the dependency strategy in the change and isolate it so it can be swapped to a Maven coordinate later. +- **[Model selection discoverability]** A free-form model name field is easier to misconfigure than a populated dropdown → Validate the field on use, surface a clear error when the model cannot be resolved, and consider a follow-up enhancement for dynamic options if Halo exposes them. +- **[Streaming contract translation]** Halo AI chunk types may not map 1:1 to the plugin's current `[DONE]`-terminated SSE stream → Normalize chunk types in one backend adapter layer and keep the frontend contract unchanged. +- **[Plugin dependency coupling]** Live2D chat will now require the ai-foundation plugin at runtime when AI chat is enabled → Fail fast with a clear administrative message when the dependency is missing or disabled. + +## Migration Plan + +1. Add the Halo AI SDK dependency from the provided jar and remove the obsolete OpenAI SDK usage. +2. Replace the custom backend generation path with a Halo AI-backed service that adapts foundation chunks into existing `ChatResult` SSE events. +3. Simplify `settings.yaml` and runtime config shaping to remove provider/proxy fields while keeping chat-specific controls. +4. Verify the frontend works unchanged against the preserved endpoint contract, then remove dead backend classes and configuration paths. + +## Open Questions + +- Whether Halo will soon expose a canonical default chat model API; if it does, the plugin-level model name field could be simplified in a follow-up change. diff --git a/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/proposal.md b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/proposal.md new file mode 100644 index 0000000..3a25613 --- /dev/null +++ b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/proposal.md @@ -0,0 +1,28 @@ +## Why + +Live2D chat currently depends on plugin-owned AI settings, custom OpenAI client logic, and proxy/token management that duplicate capabilities Halo is beginning to provide centrally. Migrating to Halo AI 基础设施 will reduce duplicated backend code, align chat behavior with Halo-wide model management, and let administrators configure AI once instead of per plugin. + +## What Changes + +- Replace the Live2D chat backend integration with Halo AI 基础设施 instead of the plugin's custom OpenAI client pipeline. +- Use the AI foundation SDK from the provided `api-1.0.0-SNAPSHOT.jar` until the upstream repository artifact is publicly available. +- Keep the existing Live2D chat entry point and streaming user experience, but source model access and streaming responses from Halo AI services. +- Simplify plugin settings so AI chat keeps only widget-specific behavior controls and persona prompt configuration. +- **BREAKING** Remove plugin-specific provider settings such as OpenAI token/base URL/model and proxy configuration from the Live2D plugin. +- Remove unused custom backend AI classes and dependency wiring after Halo AI integration is in place. + +## Capabilities + +### New Capabilities +- `halo-ai-chat-integration`: Live2D chat streams replies through Halo AI foundation services while preserving the widget-facing chat API and persona behavior. + +### Modified Capabilities +- None. + +## Impact + +- Backend chat flow under `src/main/java/run/halo/live2d/chat/**` +- Plugin settings schema in `src/main/resources/extensions/settings.yaml` +- Runtime config shaping in `src/main/java/run/halo/live2d/Live2dSettingProcess.java` +- Frontend chat client and widget integration in `packages/live2d/src/**` +- Build dependency management in `build.gradle` diff --git a/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/specs/halo-ai-chat-integration/spec.md b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/specs/halo-ai-chat-integration/spec.md new file mode 100644 index 0000000..1a0fab3 --- /dev/null +++ b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/specs/halo-ai-chat-integration/spec.md @@ -0,0 +1,46 @@ +## ADDED Requirements + +### Requirement: Live2D chat SHALL stream replies through Halo AI foundation +When AI chat is enabled, the Live2D backend SHALL invoke Halo AI foundation for chat generation instead of any plugin-owned provider client, while preserving the existing widget-facing streaming endpoint contract. + +#### Scenario: Chat requests are fulfilled by Halo AI foundation +- **WHEN** a user sends a chat message to `/live2d/ai/chat-process` +- **THEN** the backend MUST resolve the configured Halo AI language model through `AiServices.getModelService()` +- **AND** it MUST submit the system prompt plus conversation history to that model +- **AND** it MUST stream assistant text chunks back through the existing SSE response format + +#### Scenario: Stream completion preserves the current frontend terminator +- **WHEN** Halo AI foundation reports a successful end of generation +- **THEN** the backend MUST emit the plugin's existing completion marker payload for the frontend consumer +- **AND** it MUST complete the SSE response without requiring frontend protocol changes + +### Requirement: Live2D AI chat settings SHALL only keep plugin-specific chat controls +The plugin SHALL configure Live2D chat through Halo-integrated settings that keep widget-specific behavior in this plugin and remove provider-specific connection settings from the plugin schema. + +#### Scenario: Plugin settings retain persona and model selection +- **WHEN** an administrator enables AI chat in the Live2D plugin settings +- **THEN** the plugin MUST allow configuration of anonymous-access behavior, persona/system prompt, Halo AI model identifier, and frontend timing controls +- **AND** those settings MUST be used to build chat requests and widget behavior + +#### Scenario: Provider-specific settings are removed from the plugin +- **WHEN** an administrator opens the Live2D AI chat settings after this change +- **THEN** OpenAI token, base URL, provider toggle, proxy host, and proxy port settings MUST NOT be present in the plugin configuration form +- **AND** provider credentials and provider enablement MUST be managed through Halo AI infrastructure instead + +#### Scenario: Public runtime config excludes backend-only AI foundation settings +- **WHEN** the plugin exposes public runtime configuration to the frontend +- **THEN** it MUST continue exposing AI chat enablement and widget timing fields needed by the browser +- **AND** it MUST NOT expose backend-only Halo model identifiers or provider configuration details to the public config payload + +### Requirement: Live2D chat SHALL surface Halo AI dependency failures clearly +The Live2D chat integration SHALL translate Halo AI foundation availability and model-resolution failures into stable plugin behavior for both users and administrators. + +#### Scenario: Anonymous-disabled chat still enforces login +- **WHEN** AI chat is enabled but anonymous chat is disabled and an unauthenticated user sends a message +- **THEN** the endpoint MUST reject the request with HTTP 401 +- **AND** it MUST NOT invoke Halo AI foundation for that request + +#### Scenario: Missing AI foundation dependency or model configuration is reported cleanly +- **WHEN** the ai-foundation plugin is unavailable, disabled, or the configured Halo model cannot be resolved +- **THEN** the backend MUST log the failure for administrators +- **AND** it MUST return a user-facing chat failure message through the existing plugin response flow instead of an unhandled server error diff --git a/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/tasks.md b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/tasks.md new file mode 100644 index 0000000..5770881 --- /dev/null +++ b/openspec/changes/archive/2026-05-15-migrate-chat-to-halo-ai/tasks.md @@ -0,0 +1,21 @@ +## 1. Dependency and integration setup + +- [x] 1.1 Add the Halo AI foundation SDK from `api-1.0.0-SNAPSHOT.jar` to the Gradle build and remove the obsolete OpenAI SDK dependency. +- [x] 1.2 Introduce a Halo AI-backed chat service path that obtains `AiModelService` via `AiServices` and resolves the configured language model. + +## 2. Backend chat migration + +- [x] 2.1 Replace the current `ChatClient`-based backend flow with a single Halo AI streaming adapter that converts foundation chunks into existing `ChatResult` SSE events. +- [x] 2.2 Preserve current access-control behavior in `AiChatEndpoint`, including anonymous toggle handling and HTTP 401 responses for unauthenticated requests. +- [x] 2.3 Add backend error handling for missing ai-foundation availability, disabled providers, and unknown model names so failures become readable plugin responses and server logs. + +## 3. Settings and runtime config cleanup + +- [x] 3.1 Simplify `settings.yaml` to remove OpenAI and proxy settings while adding or retaining only chat-specific controls needed after the Halo migration. +- [x] 3.2 Update `Live2dSettingProcess` so public runtime config still exposes frontend timing fields but does not leak backend-only Halo model configuration. +- [x] 3.3 Remove dead custom AI backend classes, configuration records, and other unused code paths after the Halo AI integration is wired. + +## 4. Frontend compatibility verification + +- [x] 4.1 Confirm the frontend chat client continues to work against the preserved `/live2d/ai/chat-process` SSE contract without protocol changes. +- [x] 4.2 Update any frontend typing or config handling needed to stay compatible with the simplified backend settings shape. diff --git a/openspec/specs/halo-ai-chat-integration/spec.md b/openspec/specs/halo-ai-chat-integration/spec.md new file mode 100644 index 0000000..1a0fab3 --- /dev/null +++ b/openspec/specs/halo-ai-chat-integration/spec.md @@ -0,0 +1,46 @@ +## ADDED Requirements + +### Requirement: Live2D chat SHALL stream replies through Halo AI foundation +When AI chat is enabled, the Live2D backend SHALL invoke Halo AI foundation for chat generation instead of any plugin-owned provider client, while preserving the existing widget-facing streaming endpoint contract. + +#### Scenario: Chat requests are fulfilled by Halo AI foundation +- **WHEN** a user sends a chat message to `/live2d/ai/chat-process` +- **THEN** the backend MUST resolve the configured Halo AI language model through `AiServices.getModelService()` +- **AND** it MUST submit the system prompt plus conversation history to that model +- **AND** it MUST stream assistant text chunks back through the existing SSE response format + +#### Scenario: Stream completion preserves the current frontend terminator +- **WHEN** Halo AI foundation reports a successful end of generation +- **THEN** the backend MUST emit the plugin's existing completion marker payload for the frontend consumer +- **AND** it MUST complete the SSE response without requiring frontend protocol changes + +### Requirement: Live2D AI chat settings SHALL only keep plugin-specific chat controls +The plugin SHALL configure Live2D chat through Halo-integrated settings that keep widget-specific behavior in this plugin and remove provider-specific connection settings from the plugin schema. + +#### Scenario: Plugin settings retain persona and model selection +- **WHEN** an administrator enables AI chat in the Live2D plugin settings +- **THEN** the plugin MUST allow configuration of anonymous-access behavior, persona/system prompt, Halo AI model identifier, and frontend timing controls +- **AND** those settings MUST be used to build chat requests and widget behavior + +#### Scenario: Provider-specific settings are removed from the plugin +- **WHEN** an administrator opens the Live2D AI chat settings after this change +- **THEN** OpenAI token, base URL, provider toggle, proxy host, and proxy port settings MUST NOT be present in the plugin configuration form +- **AND** provider credentials and provider enablement MUST be managed through Halo AI infrastructure instead + +#### Scenario: Public runtime config excludes backend-only AI foundation settings +- **WHEN** the plugin exposes public runtime configuration to the frontend +- **THEN** it MUST continue exposing AI chat enablement and widget timing fields needed by the browser +- **AND** it MUST NOT expose backend-only Halo model identifiers or provider configuration details to the public config payload + +### Requirement: Live2D chat SHALL surface Halo AI dependency failures clearly +The Live2D chat integration SHALL translate Halo AI foundation availability and model-resolution failures into stable plugin behavior for both users and administrators. + +#### Scenario: Anonymous-disabled chat still enforces login +- **WHEN** AI chat is enabled but anonymous chat is disabled and an unauthenticated user sends a message +- **THEN** the endpoint MUST reject the request with HTTP 401 +- **AND** it MUST NOT invoke Halo AI foundation for that request + +#### Scenario: Missing AI foundation dependency or model configuration is reported cleanly +- **WHEN** the ai-foundation plugin is unavailable, disabled, or the configured Halo model cannot be resolved +- **THEN** the backend MUST log the failure for administrators +- **AND** it MUST return a user-facing chat failure message through the existing plugin response flow instead of an unhandled server error diff --git a/packages/live2d/src/api/chat-api.ts b/packages/live2d/src/api/chat-api.ts index 4dcfc86..d5909aa 100644 --- a/packages/live2d/src/api/chat-api.ts +++ b/packages/live2d/src/api/chat-api.ts @@ -24,6 +24,10 @@ export interface ChatApiConfig { chunkTimeout?: number; // 消息显示时间（秒） showChatMessageTimeout?: number; + // 请求已收到时的即时提示语 + requestAcceptedMessage?: string; + // 保留上下文轮数 + chatContextRounds?: number; } /** @@ -50,6 +54,9 @@ export class ChatApi { "/apis/api.live2d.halo.run/v1alpha1/live2d/ai/chat-process", chunkTimeout: config.chunkTimeout || 60, showChatMessageTimeout: config.showChatMessageTimeout || 10, + requestAcceptedMessage: + config.requestAcceptedMessage || "收到啦，马上就来陪你啦～", + chatContextRounds: this.normalizeContextRounds(config.chatContextRounds), }; } @@ -63,12 +70,14 @@ export class ChatApi { message: string, historyMessages: ChatMessage[], ): Promise { + const trimmedHistory = this.trimHistory(historyMessages); + // 添加用户消息到历史 const userMessage: ChatMessage = { role: "user", content: message, }; - historyMessages.push(userMessage); + trimmedHistory.push(userMessage); // 创建 AbortController 用于取消请求 this.controller = new AbortController(); @@ -79,6 +88,8 @@ export class ChatApi { this.abort(); }, timeoutMs) as unknown as number; + sendMessage(this.config.requestAcceptedMessage, 2000, 2); + // 显示等待消息 if (this.messageTimer) { clearTimeout(this.messageTimer); @@ -103,7 +114,7 @@ export class ChatApi { Accept: "text/event-stream", }, body: JSON.stringify({ - message: historyMessages, + message: trimmedHistory, }), signal: this.controller.signal, }); @@ -114,7 +125,7 @@ export class ChatApi { } // 处理流式响应 - await this.handleStreamResponse(response, historyMessages); + await this.handleStreamResponse(response, trimmedHistory); } catch (error) { if ((error as Error).name !== "AbortError") { console.error("[Chat API] Request failed:", error); @@ -257,4 +268,21 @@ export class ChatApi { isLoading(): boolean { return this.controller !== null; } + + private normalizeContextRounds(rounds: number | undefined): number { + if (!Number.isFinite(rounds) || !rounds || rounds < 1) { + return 20; + } + return Math.floor(rounds); + } + + private trimHistory(historyMessages: ChatMessage[]): ChatMessage[] { + const maxMessages = this.config.chatContextRounds + ? this.config.chatContextRounds * 2 + : 40; + if (historyMessages.length <= maxMessages) { + return [...historyMessages]; + } + return historyMessages.slice(-maxMessages); + } } diff --git a/packages/live2d/src/components/Live2dChatWindow.tsx b/packages/live2d/src/components/Live2dChatWindow.tsx index 26c547e..ed019ab 100644 --- a/packages/live2d/src/components/Live2dChatWindow.tsx +++ b/packages/live2d/src/components/Live2dChatWindow.tsx @@ -90,7 +90,6 @@ export class Live2dChatWindow extends UnoLitElement { class="h-8.5 w-full appearance-none rounded-full border border-solid border-[#eadbc5] bg-white/98 px-3.5 py-0.5 text-3.25 text-slate-700 shadow-none outline-none transition-colors placeholder:text-slate-400 focus:border-[#ffbb72] focus:ring-1 focus:ring-[#ffd8ac] focus:shadow-none" @input=${this.handleInput} @keydown=${this.handleKeydown} - ?disabled=${this._isLoading} /> + + `; + const runtime = new AgentToolRuntime({ config: baseConfig() }); + + const result = await runtime.execute(toolPart("get_current_page_context")); + + expect(result.ok).toBe(true); + expect(result.capabilities).toMatchObject({ + comment: { + hasArea: false, + hasInput: false, + hasSubmitButton: false, + }, + }); + expect(result.headings).toEqual([{ level: 1, text: "首页" }]); + }); + + it("executes default page context tool when runtime config is not provided", async () => { + document.title = "默认配置页面"; + document.body.innerHTML = "

默认配置页面

"; + const runtime = new AgentToolRuntime(); + + expect(runtime.canExecute("get_current_page_context")).toBe(true); + + const result = await runtime.execute(toolPart("get_current_page_context")); + + expect(result).toMatchObject({ + ok: true, + title: "默认配置页面", + capabilities: { + comment: { + hasArea: false, + }, + }, + }); + }); + + it("reports writable comment controls when present", async () => { + document.body.innerHTML = ` +
+ + +
+ `; + const runtime = new AgentToolRuntime({ config: baseConfig() }); + + const result = await runtime.execute(toolPart("get_current_page_context")); + + expect(result.capabilities).toMatchObject({ + comment: { + hasArea: true, + hasInput: true, + hasSubmitButton: true, + }, + }); + }); + + it("does not draft comments when current page has no comment area", async () => { + document.body.innerHTML = "

首页

"; + const runtime = new AgentToolRuntime({ config: baseConfig() }); + + const result = await runtime.execute( + toolPart("draft_comment", { content: "我想留言" }), + ); + + expect(result.ok).toBe(false); + expect(result.errorCode).toBe("COMMENT_INPUT_NOT_FOUND"); + expect(result.message).toBe("当前页面没有检测到评论区，无法填写评论"); + }); + + it("fills comment draft in assist mode without submitting", async () => { + document.body.innerHTML = ` +
+ + +
+ `; + const submit = document.querySelector("button"); + const click = vi.spyOn(submit as HTMLButtonElement, "click"); + const runtime = new AgentToolRuntime({ config: baseConfig() }); + + const result = await runtime.execute( + toolPart("draft_comment", { content: "这篇文章很有帮助" }), + ); + + expect(result.ok).toBe(true); + expect(document.querySelector("textarea")?.value).toBe( + "这篇文章很有帮助", + ); + expect(click).not.toHaveBeenCalled(); + }); + + it("submits comment only when submit capability is enabled", async () => { + document.body.innerHTML = ` +
+ + +
+ `; + const config = baseConfig(); + config.builtIn.commentCapability = "submit"; + const submit = document.querySelector("button"); + const click = vi.spyOn(submit as HTMLButtonElement, "click"); + const runtime = new AgentToolRuntime({ config }); + + const result = await runtime.execute( + toolPart("submit_comment", { content: "请帮我提交这条评论" }), + ); + + expect(result.ok).toBe(true); + expect(document.querySelector("textarea")?.value).toBe( + "请帮我提交这条评论", + ); + expect(click).toHaveBeenCalledOnce(); + }); + + it("remembers chat recovery intent before opening trusted resources", async () => { + vi.useFakeTimers(); + const assign = vi.fn(); + const originalLocation = window.location; + Object.defineProperty(window, "location", { + configurable: true, + value: { ...originalLocation, assign }, + }); + const runtime = new AgentToolRuntime({ config: baseConfig() }); + runtime.ingestMessages([ + { + id: "assistant-1", + role: "assistant", + parts: [ + { + type: "tool-search_halo_contents", + toolCallId: "call-search", + toolName: "search_halo_contents", + state: "output-available", + input: {}, + output: { + resources: [ + { + resourceId: "post.content.halo.run:demo", + permalink: "/archives/demo", + title: "Demo", + }, + ], + }, + }, + ], + }, + ]); + + const result = await runtime.execute( + toolPart("open_halo_resource", { + resourceId: "post.content.halo.run:demo", + }), + ); + vi.runAllTimers(); + + expect(result.ok).toBe(true); + expect(assign).toHaveBeenCalledWith("/archives/demo"); + expect(sessionStorage.getItem("live2d:agent-after-navigation")).toContain( + '"openChat":true', + ); + Object.defineProperty(window, "location", { + configurable: true, + value: originalLocation, + }); + vi.useRealTimers(); + }); + + it("resolves approval overlay from visitor choice", async () => { + const runtime = new AgentToolRuntime({ config: baseConfig() }); + + const approval = runtime.requestApproval("要执行这个动作吗？"); + document + .querySelector("#live2d-agent-approval button") + ?.click(); + + await expect(approval).resolves.toBe(true); + }); +}); diff --git a/packages/live2d/src/api/__tests__/chat-api.test.ts b/packages/live2d/src/api/__tests__/chat-api.test.ts new file mode 100644 index 0000000..518a547 --- /dev/null +++ b/packages/live2d/src/api/__tests__/chat-api.test.ts @@ -0,0 +1,647 @@ +import { + STREAM_MESSAGE_EVENT_NAME, + type StreamMessageEventDetail, +} from "@/live2d/events/stream-message"; +import { + HALO_UI_MESSAGE_STREAM_HEADER, + HALO_UI_MESSAGE_STREAM_VERSION, + type UIMessageChunk, +} from "@halo-dev/ai-foundation-sdk"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ChatApi } from "../chat-api"; + +const streamResponse = (chunks: UIMessageChunk[]) => { + const encoder = new TextEncoder(); + return new Response( + new ReadableStream({ + start(controller) { + for (const chunk of chunks) { + controller.enqueue( + encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`), + ); + } + controller.enqueue(encoder.encode("data: [DONE]\n\n")); + controller.close(); + }, + }), + { + headers: { + "Content-Type": "text/event-stream", + [HALO_UI_MESSAGE_STREAM_HEADER]: HALO_UI_MESSAGE_STREAM_VERSION, + }, + }, + ); +}; + +describe("ChatApi", () => { + beforeEach(() => { + document.title = "首页"; + document.body.innerHTML = "

首页

"; + localStorage.clear(); + vi.restoreAllMocks(); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("continues the SDK UI message stream after a browser tool call", async () => { + const streamMessages: StreamMessageEventDetail[] = []; + window.addEventListener(STREAM_MESSAGE_EVENT_NAME, (event) => { + streamMessages.push( + (event as CustomEvent).detail, + ); + }); + const requests: unknown[] = []; + const fetchMock = vi.fn(async (_url: string | URL | Request, init) => { + requests.push(JSON.parse(String(init?.body))); + if (requests.length === 1) { + return streamResponse([ + { type: "text-start", id: "txt-1" }, + { type: "text-delta", id: "txt-1", delta: "我先看一下页面。" }, + { type: "text-end", id: "txt-1" }, + { + type: "tool-input-available", + toolCallId: "call-ctx", + toolName: "get_current_page_context", + input: {}, + }, + { + type: "finish-step", + stepIndex: 0, + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + return streamResponse([ + { type: "text-start", id: "txt-2" }, + { + type: "text-delta", + id: "txt-2", + delta: "当前页面没有检测到评论区，不能直接留言。", + }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + }); + + await chatApi.sendMessage("帮我给站长留个言", []); + + expect(fetchMock).toHaveBeenCalledTimes(2); + const secondRequest = requests[1] as { + messages: Array<{ + role: string; + parts: Array>; + }>; + }; + const assistant = secondRequest.messages.find( + (message) => message.role === "assistant", + ); + expect(assistant?.parts).toContainEqual( + expect.objectContaining({ + toolCallId: "call-ctx", + toolName: "get_current_page_context", + state: "output-available", + output: expect.objectContaining({ + ok: true, + capabilities: expect.objectContaining({ + comment: expect.objectContaining({ + hasArea: false, + }), + }), + }), + }), + ); + const storedHistory = JSON.parse( + localStorage.getItem("historyMessages") ?? "[]", + ) as Array<{ + role: string; + parts: Array>; + }>; + const storedToolParts = storedHistory + .flatMap((message) => message.parts) + .filter((part) => part.toolCallId === "call-ctx"); + expect(storedToolParts).toHaveLength(1); + expect(storedToolParts[0]).toEqual( + expect.objectContaining({ + state: "output-available", + }), + ); + expect(streamMessages).not.toContainEqual( + expect.objectContaining({ + mode: "replace", + text: "", + }), + ); + expect(replayStreamMessage(streamMessages)).toBe( + "当前页面没有检测到评论区，不能直接留言。", + ); + }); + + it("keeps the pre-tool text visible before replacing it with automatic continuation text", async () => { + const streamMessages: StreamMessageEventDetail[] = []; + const listener = (event: Event) => { + streamMessages.push( + (event as CustomEvent).detail, + ); + }; + window.addEventListener(STREAM_MESSAGE_EVENT_NAME, listener); + const fetchMock = vi.fn(async () => { + if (fetchMock.mock.calls.length === 1) { + return streamResponse([ + { type: "text-start", id: "txt-before-tool" }, + { + type: "text-delta", + id: "txt-before-tool", + delta: "我先看一下页面。", + }, + { + type: "tool-input-available", + toolCallId: "call-delayed-replace", + toolName: "get_current_page_context", + input: {}, + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + return streamResponse([ + { type: "text-start", id: "txt-after-tool" }, + { + type: "text-delta", + id: "txt-after-tool", + delta: "首页没有评论框，不能直接留言。", + }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + autoContinuationMessageMinVisibleMs: 250, + }); + + try { + const sendPromise = chatApi.sendMessage("帮我给站长留个言", []); + await waitForCondition(() => fetchMock.mock.calls.length === 2); + + expect(replayStreamMessage(streamMessages)).toContain("我先看一下页面。"); + expect(replayStreamMessage(streamMessages)).not.toContain( + "首页没有评论框，不能直接留言。", + ); + + await waitForCondition(() => + replayStreamMessage(streamMessages).includes( + "首页没有评论框，不能直接留言。", + ), + ); + await sendPromise; + + expect(replayStreamMessage(streamMessages)).toBe( + "首页没有评论框，不能直接留言。", + ); + } finally { + window.removeEventListener(STREAM_MESSAGE_EVENT_NAME, listener); + } + }); + + it("does not keep auto-submitting after the completed tool output has already continued once", async () => { + const requests: unknown[] = []; + const fetchMock = vi.fn(async (_url: string | URL | Request, init) => { + requests.push(JSON.parse(String(init?.body))); + if (requests.length === 1) { + return streamResponse([ + { type: "text-start", id: "txt-1" }, + { type: "text-delta", id: "txt-1", delta: "我先看看当前页面。" }, + { + type: "tool-input-available", + toolCallId: "call-page-context", + toolName: "get_current_page_context", + input: {}, + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + if (requests.length === 2) { + return streamResponse([ + { type: "start", messageId: "assistant-after-tool" }, + { type: "text-start", id: "txt-2" }, + { + type: "text-delta", + id: "txt-2", + delta: "首页没有评论框，不能直接给站长留言。", + }, + ]); + } + throw new Error("Unexpected automatic continuation without a new tool."); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + }); + + await chatApi.sendMessage("帮我给站长留个言", []); + + expect(fetchMock).toHaveBeenCalledTimes(2); + const secondRequest = requests[1] as { + messages: Array<{ + role: string; + parts: Array>; + }>; + }; + const assistant = secondRequest.messages.find( + (message) => message.role === "assistant", + ); + expect(assistant?.parts).toContainEqual( + expect.objectContaining({ + toolCallId: "call-page-context", + state: "output-available", + }), + ); + }); + + it("continues after an async browser tool resolves later than the first stream", async () => { + window.Live2DAI?.registerTool("slow_page_context", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + return { page: "home" }; + }); + + const requests: unknown[] = []; + const fetchMock = vi.fn(async (_url: string | URL | Request, init) => { + requests.push(JSON.parse(String(init?.body))); + if (requests.length === 1) { + return streamResponse([ + { type: "text-start", id: "txt-1" }, + { type: "text-delta", id: "txt-1", delta: "我先看一下。" }, + { + type: "tool-input-available", + toolCallId: "call-slow", + toolName: "slow_page_context", + input: {}, + }, + { + type: "finish-step", + stepIndex: 0, + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + return streamResponse([ + { type: "text-start", id: "txt-2" }, + { type: "text-delta", id: "txt-2", delta: "已经看完了。" }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + agent: { + builtIn: { + pageContext: false, + haloNavigation: false, + haloContentSearch: false, + networkAccess: false, + commentCapability: "off", + }, + aiTools: [ + { + name: "slow_page_context", + description: "读取页面上下文", + inputSchema: {}, + approval: "never", + requiredAuth: "none", + actionType: "registered", + action: { type: "registered" }, + }, + ], + toolSecurity: { + allowedExternalOrigins: [], + allowNewTab: false, + }, + haloSearch: { + allowedTypes: [], + defaultLimit: 5, + }, + haloResourceDetail: { + maxContentChars: 1000, + }, + }, + }); + + await chatApi.sendMessage("帮我看看页面", []); + + expect(fetchMock).toHaveBeenCalledTimes(2); + const secondRequest = requests[1] as { + messages: Array<{ + role: string; + parts: Array>; + }>; + }; + const assistant = secondRequest.messages.find( + (message) => message.role === "assistant", + ); + expect(assistant?.parts).toContainEqual( + expect.objectContaining({ + toolCallId: "call-slow", + toolName: "slow_page_context", + state: "output-available", + output: expect.objectContaining({ + ok: true, + output: { page: "home" }, + }), + }), + ); + }); + + it("continues even when the first assistant step only contains a tool call", async () => { + const fetchMock = vi.fn(async () => { + if (fetchMock.mock.calls.length === 1) { + return streamResponse([ + { + type: "tool-input-available", + toolCallId: "call-ctx-only", + toolName: "get_current_page_context", + input: {}, + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + return streamResponse([ + { type: "text-start", id: "txt-final" }, + { type: "text-delta", id: "txt-final", delta: "页面看完了。" }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + }); + + await chatApi.sendMessage("查看当前页面", []); + + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("shows automatic text when the second stream reuses the previous text part id", async () => { + const streamMessages: StreamMessageEventDetail[] = []; + window.addEventListener(STREAM_MESSAGE_EVENT_NAME, (event) => { + streamMessages.push( + (event as CustomEvent).detail, + ); + }); + const fetchMock = vi.fn(async () => { + if (fetchMock.mock.calls.length === 1) { + return streamResponse([ + { type: "text-start", id: "txt-shared" }, + { type: "text-delta", id: "txt-shared", delta: "我先看一下页面。" }, + { + type: "tool-input-available", + toolCallId: "call-reused-text", + toolName: "get_current_page_context", + input: {}, + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + return streamResponse([ + { type: "reasoning-start", id: "rsn-1" }, + { type: "reasoning-delta", id: "rsn-1", delta: "正在分析页面" }, + { type: "reasoning-end", id: "rsn-1" }, + { + type: "text-delta", + id: "txt-shared", + delta: "当前页面可以继续处理。", + }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + }); + + await chatApi.sendMessage("查看当前页面", []); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(replayStreamMessage(streamMessages)).toBe("当前页面可以继续处理。"); + }); + + it("compacts duplicate final tool results before sending history", async () => { + const requests: unknown[] = []; + const fetchMock = vi.fn(async (_url: string | URL | Request, init) => { + requests.push(JSON.parse(String(init?.body))); + return streamResponse([ + { type: "text-start", id: "txt-reply" }, + { type: "text-delta", id: "txt-reply", delta: "可以继续聊天。" }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + }); + + await chatApi.sendMessage("继续", [ + { + id: "msg-user", + role: "user", + parts: [{ type: "text", id: "txt-user", text: "查看页面" }], + }, + { + id: "msg-assistant", + role: "assistant", + parts: [ + { + type: "tool-get_current_page_context", + toolCallId: "call-duplicated", + toolName: "get_current_page_context", + state: "output-available", + input: {}, + output: { ok: true, stale: true }, + }, + { + type: "tool-get_current_page_context", + toolCallId: "call-duplicated", + toolName: "get_current_page_context", + state: "output-available", + input: {}, + output: { ok: true, fresh: true }, + }, + ], + }, + ]); + + const firstRequest = requests[0] as { + messages: Array<{ parts: Array> }>; + }; + const duplicatedToolParts = firstRequest.messages + .flatMap((message) => message.parts) + .filter((part) => part.toolCallId === "call-duplicated"); + expect(duplicatedToolParts).toHaveLength(1); + expect(duplicatedToolParts[0]).toEqual( + expect.objectContaining({ + output: { ok: true, fresh: true }, + }), + ); + }); + + it("stores an agent resume intent when a tool navigates with page reload", async () => { + window.Live2DAI?.registerTool("reload_navigation", () => ({ + ok: true, + navigating: true, + pageReload: true, + })); + + const fetchMock = vi.fn(async () => { + if (fetchMock.mock.calls.length === 1) { + return streamResponse([ + { + type: "tool-input-available", + toolCallId: "call-reload", + toolName: "reload_navigation", + input: {}, + }, + { + type: "finish", + finishReason: "tool-calls", + rawFinishReason: "TOOL_CALLS", + }, + ]); + } + return streamResponse([ + { type: "text-start", id: "txt-after-reload" }, + { type: "text-delta", id: "txt-after-reload", delta: "继续处理。" }, + ]); + }); + vi.stubGlobal("fetch", fetchMock); + + const chatApi = new ChatApi({ + apiEndpoint: "/api/chat", + chunkTimeout: 1, + showChatMessageTimeout: 1, + agent: { + builtIn: { + pageContext: false, + haloNavigation: false, + haloContentSearch: false, + networkAccess: false, + commentCapability: "off", + }, + aiTools: [ + { + name: "reload_navigation", + description: "刷新当前页面", + inputSchema: {}, + approval: "never", + requiredAuth: "none", + actionType: "registered", + action: { type: "registered" }, + }, + ], + toolSecurity: { + allowedExternalOrigins: [], + allowNewTab: false, + }, + haloSearch: { + allowedTypes: [], + defaultLimit: 5, + }, + haloResourceDetail: { + maxContentChars: 1000, + }, + }, + }); + + await chatApi.sendMessage("打开一篇文章后继续留言", []); + + const intent = JSON.parse( + sessionStorage.getItem("live2d:agent-after-navigation") ?? "{}", + ) as { + openChat?: boolean; + resume?: { + message?: string; + historyMessages?: Array<{ parts: Array> }>; + }; + }; + expect(intent.openChat).toBe(true); + expect(intent.resume?.message).toContain("继续完成上一条用户请求"); + const resumeToolParts = intent.resume?.historyMessages + ?.flatMap((message) => message.parts) + .filter((part) => part.toolCallId === "call-reload"); + expect(resumeToolParts).toHaveLength(1); + expect(resumeToolParts?.[0]).toEqual( + expect.objectContaining({ + state: "output-available", + output: expect.objectContaining({ + output: expect.objectContaining({ + navigating: true, + pageReload: true, + }), + }), + }), + ); + }); +}); + +const replayStreamMessage = (messages: StreamMessageEventDetail[]) => + messages.reduce((current, message) => { + if (message.mode === "replace") { + return message.text; + } + return `${current}${message.text}`; + }, ""); + +const waitForCondition = async (condition: () => boolean, timeoutMs = 1000) => { + const startedAt = Date.now(); + while (!condition()) { + if (Date.now() - startedAt > timeoutMs) { + throw new Error("Timed out waiting for condition"); + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } +}; diff --git a/packages/live2d/src/api/agent-navigation-intent.ts b/packages/live2d/src/api/agent-navigation-intent.ts new file mode 100644 index 0000000..0016a2b --- /dev/null +++ b/packages/live2d/src/api/agent-navigation-intent.ts @@ -0,0 +1,88 @@ +import type { UIMessage } from "@halo-dev/ai-foundation-sdk"; + +export interface AgentAfterNavigationResume { + message: string; + historyMessages: UIMessage[]; +} + +export interface AgentAfterNavigationIntent { + openChat: boolean; + focusChatInput: boolean; + resume?: AgentAfterNavigationResume; + expiresAt: number; +} + +const STORAGE_KEY = "live2d:agent-after-navigation"; +const DEFAULT_TTL_MS = 15_000; +const DEFAULT_RESUME_MESSAGE = + "我已经打开了新的页面，请基于当前页面继续完成上一条用户请求。必要时先查看当前页面上下文。"; + +export interface RememberAgentAfterNavigationOptions { + resume?: { + message?: string; + historyMessages?: UIMessage[]; + }; +} + +export const rememberAgentAfterNavigationIntent = ( + options: RememberAgentAfterNavigationOptions = {}, + storage: Storage = window.sessionStorage, +): void => { + const intent: AgentAfterNavigationIntent = { + openChat: true, + focusChatInput: true, + expiresAt: Date.now() + DEFAULT_TTL_MS, + }; + if (options.resume?.historyMessages?.length) { + intent.resume = { + message: options.resume.message?.trim() || DEFAULT_RESUME_MESSAGE, + historyMessages: options.resume.historyMessages, + }; + } + storage.setItem(STORAGE_KEY, JSON.stringify(intent)); +}; + +export const consumeAgentAfterNavigationIntent = ( + storage: Storage = window.sessionStorage, +): AgentAfterNavigationIntent | undefined => { + const raw = storage.getItem(STORAGE_KEY); + storage.removeItem(STORAGE_KEY); + if (!raw) { + return undefined; + } + + try { + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed.expiresAt !== "number" || parsed.expiresAt < Date.now()) { + return undefined; + } + return { + openChat: parsed.openChat === true, + focusChatInput: parsed.focusChatInput !== false, + resume: normalizeResume(parsed.resume), + expiresAt: parsed.expiresAt, + }; + } catch { + return undefined; + } +}; + +const normalizeResume = ( + resume: unknown, +): AgentAfterNavigationResume | undefined => { + if (typeof resume !== "object" || resume === null) { + return undefined; + } + const candidate = resume as Partial; + if ( + typeof candidate.message !== "string" || + !candidate.message.trim() || + !Array.isArray(candidate.historyMessages) + ) { + return undefined; + } + return { + message: candidate.message.trim(), + historyMessages: candidate.historyMessages, + }; +}; diff --git a/packages/live2d/src/api/agent-tool-runtime.ts b/packages/live2d/src/api/agent-tool-runtime.ts new file mode 100644 index 0000000..3c44f0a --- /dev/null +++ b/packages/live2d/src/api/agent-tool-runtime.ts @@ -0,0 +1,695 @@ +import { rememberAgentAfterNavigationIntent } from "@/live2d/api/agent-navigation-intent"; +import type { AgentRuntimeConfig } from "@/live2d/config/agent-tools/agent-tool-config"; +import { normalizeAgentRuntimeConfig } from "@/live2d/config/agent-tools/normalize-agent-tools"; +import { sendMessage } from "@/live2d/helpers/sendMessage"; +import type { ToolPart, UIMessage } from "@halo-dev/ai-foundation-sdk"; + +export interface AgentToolRuntimeOptions { + config?: AgentRuntimeConfig; +} + +export interface AgentToolResult { + ok: boolean; + [key: string]: unknown; +} + +type RegisteredExecutor = (context: { + input: Record; + toolName: string; +}) => Promise | unknown; + +interface TrustedResource { + resourceId: string; + permalink: string; + title?: string; +} + +const registeredExecutors = new Map(); + +declare global { + interface Window { + Live2DAI?: { + registerTool: (name: string, executor: RegisteredExecutor) => void; + }; + } +} + +const ensureGlobalRegistry = () => { + if (window.Live2DAI) { + return; + } + window.Live2DAI = { + registerTool(name, executor) { + if (!/^[a-z][a-z0-9_]{2,63}$/.test(name)) { + return; + } + registeredExecutors.set(name, executor); + }, + }; +}; + +ensureGlobalRegistry(); + +export class AgentToolRuntime { + private readonly config: AgentRuntimeConfig; + private readonly trustedResources = new Map(); + + constructor(options: AgentToolRuntimeOptions = {}) { + this.config = normalizeAgentRuntimeConfig(options.config); + } + + canExecute(toolName: string): boolean { + return ( + this.builtInToolNames().has(toolName) || + this.config.aiTools.some((tool) => tool.name === toolName) + ); + } + + ingestMessages(messages: UIMessage[]): void { + for (const message of messages) { + for (const part of message.parts) { + if (!isToolPart(part) || part.state !== "output-available") { + continue; + } + this.ingestToolOutput(part.output); + } + } + } + + async execute(part: ToolPart): Promise { + const input = part.input ?? {}; + try { + if (part.toolName === "get_current_page_context") { + return this.getCurrentPageContext(); + } + if (part.toolName === "open_halo_resource") { + return this.openHaloResource(input); + } + if (part.toolName === "open_comment_area") { + return this.scrollToCommentArea(); + } + if (part.toolName === "draft_comment") { + if (this.config.builtIn.commentCapability === "off") { + return failure("TOOL_NOT_ALLOWED", "评论辅助能力未启用"); + } + return this.fillCommentDraft(input); + } + if (part.toolName === "submit_comment") { + if (this.config.builtIn.commentCapability !== "submit") { + return failure("TOOL_NOT_ALLOWED", "评论提交能力未启用"); + } + return this.submitComment(input); + } + const customTool = this.config.aiTools.find( + (tool) => tool.name === part.toolName, + ); + if (!customTool) { + return failure("TOOL_NOT_FOUND", "这个功能还没有配置好"); + } + const action = customTool.action; + if (this.requiresApproval(customTool.approval, action)) { + const approved = await this.requestApproval( + `要我帮你执行「${customTool.description}」吗？`, + ); + if (!approved) { + return failure("TOOL_APPROVAL_DENIED", "访客取消了这次操作"); + } + } + this.showStatus(action.pendingMessage ?? "我来帮你处理一下～"); + let result: AgentToolResult; + switch (action.type) { + case "navigate": + result = this.navigate(action.url, action.target); + break; + case "scroll-to": + result = this.scrollToSelector(action.selector, action.behavior); + break; + case "highlight": + result = this.highlight(action.selector, action.duration); + break; + case "dispatch-event": + window.dispatchEvent( + new CustomEvent(action.event, { + detail: input, + }), + ); + result = { ok: true }; + break; + case "registered": { + const executor = registeredExecutors.get(part.toolName); + if (!executor) { + return failure( + "TOOL_EXECUTOR_NOT_FOUND", + "这个站点还没有启用对应能力", + ); + } + result = { + ok: true, + output: await executor({ input, toolName: part.toolName }), + }; + break; + } + } + this.showStatus( + result.ok + ? (action.successMessage ?? "已经处理好啦～") + : (action.errorMessage ?? "这个功能暂时没处理成功～"), + ); + return result; + } catch (error) { + return failure( + "TOOL_EXECUTION_FAILED", + error instanceof Error ? error.message : "工具执行失败", + ); + } + } + + private builtInToolNames(): Set { + const names = new Set(); + const builtIn = this.config.builtIn; + if (builtIn.pageContext) { + names.add("get_current_page_context"); + } + if (builtIn.haloNavigation) { + names.add("open_halo_resource"); + } + if (builtIn.commentCapability !== "off") { + names.add("open_comment_area"); + names.add("draft_comment"); + } + if (builtIn.commentCapability === "submit") { + names.add("submit_comment"); + } + return names; + } + + private getCurrentPageContext(): AgentToolResult { + const selectedText = window.getSelection()?.toString().trim() ?? ""; + const commentInput = this.findCommentInput(); + const commentArea = this.findCommentArea(); + const commentSubmit = commentInput?.container + ? this.findCommentSubmitButton(commentInput.container) + : undefined; + return { + ok: true, + title: document.title, + url: window.location.href, + path: window.location.pathname, + description: + document + .querySelector("meta[name='description']") + ?.content.trim() ?? "", + headings: this.collectHeadings(), + selectedText: selectedText.slice(0, 1000), + capabilities: { + comment: { + hasArea: !!commentArea, + hasInput: !!commentInput, + hasSubmitButton: !!commentSubmit, + reason: !commentArea + ? "当前页面没有检测到评论区" + : !commentInput + ? "当前页面有评论区，但没有检测到可写评论输入框" + : !commentSubmit + ? "当前页面有可写评论输入框，但没有检测到提交按钮" + : "当前页面支持填写评论", + }, + forms: this.collectFormSummaries(), + links: this.collectLinkSummaries(), + }, + }; + } + + private openHaloResource(input: Record): AgentToolResult { + const resourceId = stringInput(input.resourceId); + if (!resourceId) { + return failure("INVALID_INPUT", "resourceId is required"); + } + const resource = this.trustedResources.get(resourceId); + if (!resource) { + return failure("RESOURCE_NOT_TRUSTED", "没有找到可信资源"); + } + this.showStatus("我带你过去看看～"); + window.setTimeout(() => { + rememberAgentAfterNavigationIntent(); + window.location.assign(resource.permalink); + }, 50); + return { ok: true, navigating: true, pageReload: true, resourceId }; + } + + private scrollToCommentArea(): AgentToolResult { + const commentArea = this.findCommentArea(); + if (!commentArea) { + this.showStatus("当前页面没有检测到评论区"); + return failure("COMMENT_AREA_NOT_FOUND", "当前页面没有检测到评论区"); + } + commentArea.scrollIntoView?.({ behavior: "smooth", block: "center" }); + this.showStatus("已经帮你定位到评论区啦～"); + return { ok: true }; + } + + private fillCommentDraft(input: Record): AgentToolResult { + const content = stringInput(input.content); + if (!content) { + return failure("INVALID_INPUT", "content is required"); + } + const target = this.findCommentInput(); + if (!target) { + this.scrollToCommentArea(); + return failure( + "COMMENT_INPUT_NOT_FOUND", + this.findCommentArea() + ? "当前页面有评论区，但没有找到可写评论输入框" + : "当前页面没有检测到评论区，无法填写评论", + ); + } + target.container?.scrollIntoView?.({ behavior: "smooth", block: "center" }); + this.writeCommentInput(target.input, content); + target.input.focus(); + this.showStatus("评论草稿已经帮你填好了～"); + return { ok: true, drafted: true }; + } + + private submitComment(input: Record): AgentToolResult { + const draftResult = this.fillCommentDraft(input); + if (!draftResult.ok) { + return draftResult; + } + const target = this.findCommentInput(); + const submitButton = target?.container + ? this.findCommentSubmitButton(target.container) + : undefined; + if (!submitButton) { + return failure("COMMENT_SUBMIT_NOT_FOUND", "没有找到评论提交按钮"); + } + submitButton.click(); + this.showStatus("已经尝试提交评论啦～"); + return { ok: true, submitted: true }; + } + + private navigate( + url: string, + target: "_self" | "_blank" = "_self", + ): AgentToolResult { + const destination = new URL(url, window.location.origin); + if (!this.isAllowedUrl(destination)) { + return failure("URL_NOT_ALLOWED", "这个链接不在允许范围内"); + } + const useBlank = + target === "_blank" && this.config.toolSecurity.allowNewTab; + window.setTimeout(() => { + if (useBlank) { + window.open(destination.href, "_blank", "noopener,noreferrer"); + } else { + rememberAgentAfterNavigationIntent(); + window.location.assign(destination.href); + } + }, 50); + return { + ok: true, + navigating: true, + pageReload: !useBlank, + url: destination.href, + }; + } + + private scrollToSelector( + selector: string, + behavior: ScrollBehavior = "smooth", + ): AgentToolResult { + const element = document.querySelector(selector); + if (!element) { + return failure("ELEMENT_NOT_FOUND", "没有找到对应的位置"); + } + element.scrollIntoView?.({ behavior, block: "center" }); + return { ok: true }; + } + + private highlight(selector: string, duration = 1600): AgentToolResult { + const element = document.querySelector(selector); + if (!element) { + return failure("ELEMENT_NOT_FOUND", "没有找到对应的位置"); + } + const previousOutline = element.style.outline; + const previousOutlineOffset = element.style.outlineOffset; + element.style.outline = "2px solid #ffab5c"; + element.style.outlineOffset = "3px"; + window.setTimeout(() => { + element.style.outline = previousOutline; + element.style.outlineOffset = previousOutlineOffset; + }, duration); + return { ok: true }; + } + + private isAllowedUrl(url: URL): boolean { + if (url.protocol !== "http:" && url.protocol !== "https:") { + return false; + } + if (url.origin === window.location.origin) { + return true; + } + return this.config.toolSecurity.allowedExternalOrigins.includes(url.origin); + } + + private requiresApproval( + approval: "default" | "never" | "always", + action: { type: string; url?: string }, + ): boolean { + if (approval === "always") { + return true; + } + if (approval === "never") { + return false; + } + if (action.type === "registered" || action.type === "dispatch-event") { + return true; + } + if (action.type === "navigate" && action.url) { + const destination = new URL(action.url, window.location.origin); + return destination.origin !== window.location.origin; + } + return false; + } + + requestApproval(message: string): Promise { + return new Promise((resolve) => { + const existing = document.getElementById("live2d-agent-approval"); + existing?.remove(); + const container = document.createElement("div"); + container.id = "live2d-agent-approval"; + container.style.cssText = [ + "position:fixed", + "left:50%", + "bottom:5.5rem", + "z-index:10001", + "transform:translateX(-50%)", + "display:flex", + "align-items:center", + "gap:.5rem", + "max-width:min(28rem,calc(100vw - 1rem))", + "padding:.5rem .625rem", + "border:1px solid #eadfce", + "border-radius:.75rem", + "background:rgba(255,250,244,.98)", + "box-shadow:0 10px 24px rgba(15,23,42,.12)", + "font-size:13px", + "color:#334155", + ].join(";"); + const text = document.createElement("span"); + text.textContent = message; + text.style.cssText = "line-height:1.4;min-width:0;flex:1;"; + const allow = document.createElement("button"); + allow.type = "button"; + allow.textContent = "允许"; + allow.style.cssText = buttonStyle("#ffab5c", "#fff"); + const deny = document.createElement("button"); + deny.type = "button"; + deny.textContent = "取消"; + deny.style.cssText = buttonStyle("#eee7de", "#475569"); + const cleanup = (approved: boolean) => { + container.remove(); + resolve(approved); + }; + allow.addEventListener("click", () => cleanup(true), { once: true }); + deny.addEventListener("click", () => cleanup(false), { once: true }); + container.append(text, allow, deny); + document.body.append(container); + }); + } + + private ingestToolOutput(output: unknown): void { + if (!output || typeof output !== "object") { + return; + } + const resources = (output as { resources?: unknown }).resources; + if (!Array.isArray(resources)) { + return; + } + for (const resource of resources) { + if (!resource || typeof resource !== "object") { + continue; + } + const item = resource as Partial; + if (item.resourceId && item.permalink) { + this.trustedResources.set(item.resourceId, { + resourceId: item.resourceId, + permalink: item.permalink, + title: item.title, + }); + } + } + } + + private showStatus(message?: string): void { + if (message) { + sendMessage(message, 3000, 3); + } + } + + private findCommentInput(): + | { container: Element | undefined; input: WritableCommentInput } + | undefined { + const containers = this.commentContainers(); + for (const container of containers) { + const input = this.findWritableCommentInput(container); + if (input) { + return { container, input }; + } + } + return this.findWritableCommentInput(document.body) + ? { + container: undefined, + input: this.findWritableCommentInput( + document.body, + ) as WritableCommentInput, + } + : undefined; + } + + private commentContainers(): Element[] { + const selectors = [ + "#comment", + "#comments", + "halo-comment", + "[data-comment]", + ".comment", + ".comments", + ".comment-form", + ].join(","); + return [...document.querySelectorAll(selectors)]; + } + + private findCommentArea(): Element | undefined { + return this.commentContainers()[0]; + } + + private collectHeadings(): Array<{ level: number; text: string }> { + return [...document.querySelectorAll("h1,h2,h3")] + .map((heading) => ({ + level: Number(heading.tagName.slice(1)), + text: (heading.textContent ?? "").trim(), + })) + .filter((heading) => heading.text) + .slice(0, 8); + } + + private collectFormSummaries(): Array<{ + id?: string; + name?: string; + fields: string[]; + submitLabels: string[]; + }> { + return [...document.querySelectorAll("form")] + .map((form) => ({ + id: form.id || undefined, + name: form.getAttribute("name") || undefined, + fields: [ + ...form.querySelectorAll< + HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement + >("input,textarea,select"), + ] + .map( + (field) => + field.getAttribute("name") || + field.getAttribute("aria-label") || + field.id, + ) + .filter((field): field is string => !!field) + .slice(0, 12), + submitLabels: [ + ...form.querySelectorAll( + "button,input[type='submit']", + ), + ] + .map((button) => + ( + button.textContent || + button.getAttribute("value") || + button.getAttribute("aria-label") || + "" + ).trim(), + ) + .filter((label) => label) + .slice(0, 6), + })) + .slice(0, 6); + } + + private collectLinkSummaries(): Array<{ text: string; href: string }> { + return [...document.querySelectorAll("a[href]")] + .map((link) => ({ + text: (link.textContent ?? "").trim().replace(/\s+/g, " "), + href: link.href, + })) + .filter( + (link) => link.text && link.href.startsWith(window.location.origin), + ) + .slice(0, 10); + } + + private findWritableCommentInput( + root: ParentNode, + ): WritableCommentInput | undefined { + const selectors = [ + "textarea:not([disabled]):not([readonly])", + "[contenteditable='true']", + "[contenteditable='']", + "input[name='content']:not([disabled]):not([readonly])", + "input[name='comment']:not([disabled]):not([readonly])", + ]; + for (const selector of selectors) { + const found = this.deepQuerySelector( + root, + selector, + ); + if (found && this.isWritableCommentInput(found)) { + return found; + } + } + return undefined; + } + + private findCommentSubmitButton(root: ParentNode): HTMLElement | undefined { + const selectors = [ + "button[type='submit']:not([disabled])", + "input[type='submit']:not([disabled])", + "button:not([disabled])", + "[role='button']:not([aria-disabled='true'])", + ]; + for (const selector of selectors) { + const elements = this.deepQuerySelectorAll(root, selector); + const submit = elements.find((element) => { + const text = `${element.textContent ?? ""} ${ + element.getAttribute("aria-label") ?? "" + } ${element.getAttribute("value") ?? ""}`.trim(); + return /提交|评论|发送|发布|回复|submit|send|post|reply/i.test(text); + }); + if (submit) { + return submit; + } + } + return undefined; + } + + private deepQuerySelector( + root: ParentNode, + selector: string, + ): T | undefined { + return this.deepQuerySelectorAll(root, selector)[0]; + } + + private deepQuerySelectorAll( + root: ParentNode, + selector: string, + ): T[] { + const results: T[] = []; + const visit = (node: ParentNode) => { + if ("querySelectorAll" in node) { + results.push( + ...[ + ...( + node as Document | DocumentFragment | Element + ).querySelectorAll(selector), + ], + ); + for (const element of [ + ...(node as Document | DocumentFragment | Element).querySelectorAll( + "*", + ), + ]) { + if (element.shadowRoot) { + visit(element.shadowRoot); + } + } + } + }; + visit(root); + return results; + } + + private isWritableCommentInput( + input: Element, + ): input is WritableCommentInput { + return ( + input instanceof HTMLTextAreaElement || + input instanceof HTMLInputElement || + (input instanceof HTMLElement && input.isContentEditable) + ); + } + + private writeCommentInput( + input: WritableCommentInput, + content: string, + ): void { + if ( + input instanceof HTMLTextAreaElement || + input instanceof HTMLInputElement + ) { + const descriptor = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(input), + "value", + ); + descriptor?.set?.call(input, content); + } else { + input.textContent = content; + } + const inputEvent = + typeof InputEvent === "function" + ? new InputEvent("input", { + bubbles: true, + inputType: "insertText", + data: content, + }) + : new Event("input", { bubbles: true }); + input.dispatchEvent(inputEvent); + input.dispatchEvent(new Event("change", { bubbles: true })); + } +} + +const stringInput = (value: unknown): string | undefined => + typeof value === "string" && value.trim() ? value.trim() : undefined; + +const failure = (errorCode: string, message: string): AgentToolResult => ({ + ok: false, + errorCode, + message, +}); + +const buttonStyle = (background: string, color: string): string => + [ + "border:none", + "border-radius:999px", + "padding:.25rem .625rem", + "font-size:12px", + "cursor:pointer", + `background:${background}`, + `color:${color}`, + ].join(";"); + +const isToolPart = (part: UIMessage["parts"][number]): part is ToolPart => + part.type.startsWith("tool-"); + +type WritableCommentInput = + | HTMLTextAreaElement + | HTMLInputElement + | HTMLElement; diff --git a/packages/live2d/src/api/chat-api.ts b/packages/live2d/src/api/chat-api.ts index 08de2bd..5b958f1 100644 --- a/packages/live2d/src/api/chat-api.ts +++ b/packages/live2d/src/api/chat-api.ts @@ -1,11 +1,15 @@ +import { rememberAgentAfterNavigationIntent } from "@/live2d/api/agent-navigation-intent"; +import { AgentToolRuntime } from "@/live2d/api/agent-tool-runtime"; +import type { AgentRuntimeConfig } from "@/live2d/config/agent-tools/agent-tool-config"; import { createStreamMessage } from "@/live2d/helpers/createStreamMessage"; import { sendMessage } from "@/live2d/helpers/sendMessage"; import { Chat, DefaultChatTransport, - messageText, - pruneMessages, + type ToolPart, type UIMessage, + lastAssistantMessageHasCompletedToolContinuations, + pruneMessages, validateUIMessages, } from "@halo-dev/ai-foundation-sdk"; @@ -34,6 +38,8 @@ export interface ChatApiConfig { chunkTimeout?: number; // 消息显示时间（秒） showChatMessageTimeout?: number; + // Agent 自动续写时，上一段助手消息最短可见时间（毫秒） + autoContinuationMessageMinVisibleMs?: number; // 请求已收到时的即时提示语 requestAcceptedMessage?: string; // 模型思考阶段的提示语 @@ -42,6 +48,8 @@ export interface ChatApiConfig { reasoningMessageInterval?: number; // 保留上下文轮数 chatContextRounds?: number; + // Agent 能力运行时配置 + agent?: AgentRuntimeConfig; } /** @@ -50,6 +58,7 @@ export interface ChatApiConfig { export class ChatApi { private config: ChatApiConfig; private chat: Chat | null = null; + private agentRuntime: AgentToolRuntime; private requestTimeoutId: number | null = null; private messageTimer: number | null = null; @@ -60,6 +69,10 @@ export class ChatApi { "/apis/api.live2d.halo.run/v1alpha1/live2d/ai/chat-process", chunkTimeout: config.chunkTimeout || 60, showChatMessageTimeout: config.showChatMessageTimeout || 10, + autoContinuationMessageMinVisibleMs: + this.normalizeAutoContinuationMessageMinVisibleMs( + config.autoContinuationMessageMinVisibleMs, + ), requestAcceptedMessage: config.requestAcceptedMessage || "收到啦，马上就来陪你啦～", reasoningMessages: this.normalizeReasoningMessages( @@ -69,7 +82,11 @@ export class ChatApi { config.reasoningMessageInterval, ), chatContextRounds: this.normalizeContextRounds(config.chatContextRounds), + agent: config.agent, }; + this.agentRuntime = new AgentToolRuntime({ + config: this.config.agent, + }); } /** @@ -176,45 +193,336 @@ export class ChatApi { (this.config.reasoningMessageInterval || 5) * 1000, ) as unknown as number; }; + let streamedText = ""; + let displayedTextAnchor = ""; + let displayedTextBaseline = ""; + let lastVisibleAssistantText = ""; + let lastVisibleAssistantTextAt = 0; + let automaticContinuationHoldUntil = 0; + let deferredReplaceText: string | null = null; + let deferredReplaceTimer: number | null = null; + const markVisibleAssistantText = (text: string) => { + lastVisibleAssistantText = text; + lastVisibleAssistantTextAt = Date.now(); + }; + const clearDeferredReplaceTimer = () => { + if (deferredReplaceTimer) { + clearTimeout(deferredReplaceTimer); + deferredReplaceTimer = null; + } + }; + const flushDeferredReplace = () => { + if (deferredReplaceText === null) { + return; + } + const text = deferredReplaceText; + deferredReplaceText = null; + clearDeferredReplaceTimer(); + stopWaitingMessage(); + stopReasoningMessages(); + hasVisibleAssistantContent = true; + chat.setMessage(text); + reasoningMessageVisible = false; + streamedText = text; + markVisibleAssistantText(text); + }; + const scheduleDeferredReplace = (text: string) => { + deferredReplaceText = text; + const remaining = automaticContinuationHoldUntil - Date.now(); + if (remaining <= 0) { + flushDeferredReplace(); + return false; + } + if (!deferredReplaceTimer) { + deferredReplaceTimer = setTimeout( + flushDeferredReplace, + remaining, + ) as unknown as number; + } + return true; + }; + const shouldHoldAutomaticContinuationReplace = ( + willReplaceIncomingText: boolean, + ) => + willReplaceIncomingText && + lastVisibleAssistantText.length > 0 && + automaticContinuationHoldUntil > Date.now(); + const shouldKeepVisibleAssistantTextDuringToolWait = () => + (this.config.autoContinuationMessageMinVisibleMs ?? 0) > 0 && + lastVisibleAssistantText.length > 0; + const waitForDeferredReplace = async () => { + if (deferredReplaceText === null) { + return; + } + const remaining = automaticContinuationHoldUntil - Date.now(); + if (remaining > 0) { + await new Promise((resolve) => setTimeout(resolve, remaining)); + } + flushDeferredReplace(); + }; try { - let streamedText = ""; - const sdkChat = new Chat({ + const pendingContinuations = new Set>(); + const queuedToolCalls = new Map(); + const submittedAutomaticContinuationKeys = new Set(); + const trackContinuation = (continuation: Promise) => { + const tracked = continuation.finally(() => + pendingContinuations.delete(tracked), + ); + pendingContinuations.add(tracked); + return tracked; + }; + const waitForContinuations = async () => { + while (pendingContinuations.size > 0 || queuedToolCalls.size > 0) { + flushQueuedToolCalls(); + if (pendingContinuations.size === 0) { + break; + } + await Promise.allSettled([...pendingContinuations]); + } + }; + let sdkChat: Chat; + const handledToolCalls = new Set(); + const shouldQueueToolCall = () => + sdkChat.status === "submitted" || sdkChat.status === "streaming"; + const executeToolCall = (part: ToolPart) => { + if (handledToolCalls.has(part.toolCallId)) { + return; + } + if (shouldQueueToolCall()) { + queuedToolCalls.set(part.toolCallId, part); + return; + } + handledToolCalls.add(part.toolCallId); + if (!this.agentRuntime.canExecute(part.toolName)) { + return trackContinuation( + sdkChat + .addToolOutput({ + toolCallId: part.toolCallId, + toolName: part.toolName, + state: "output-error", + errorText: "当前页面没有启用这个浏览器能力", + }) + .then(() => undefined), + ); + } + return trackContinuation( + this.agentRuntime + .execute(part) + .then((output) => { + if (this.shouldResumeAfterNavigation(output)) { + rememberAgentAfterNavigationIntent({ + resume: { + historyMessages: this.historyWithToolOutput( + sdkChat.messages, + part, + output, + ), + }, + }); + } + return sdkChat + .addToolOutput({ + toolCallId: part.toolCallId, + toolName: part.toolName, + output, + }) + .then(() => output); + }) + .then((output) => { + if (this.shouldResumeAfterNavigation(output)) { + rememberAgentAfterNavigationIntent({ + resume: { + historyMessages: this.stabilizeHistory(sdkChat.messages), + }, + }); + } + }) + .then(() => undefined), + ); + }; + const flushQueuedToolCalls = () => { + if (shouldQueueToolCall() || queuedToolCalls.size === 0) { + return; + } + const parts = [...queuedToolCalls.values()]; + queuedToolCalls.clear(); + for (const part of parts) { + executeToolCall(part); + } + }; + sdkChat = new Chat({ id: "live2d-chat", messages: historyMessages, transport: new DefaultChatTransport({ api: apiEndpoint, + prepareSendMessagesRequest: (request) => { + const continuationKeys = this.automaticContinuationKeys( + request.messages, + ); + if (continuationKeys.length > 0) { + for (const key of continuationKeys) { + submittedAutomaticContinuationKeys.add(key); + } + const minVisibleMs = + this.config.autoContinuationMessageMinVisibleMs ?? 0; + if ( + minVisibleMs > 0 && + lastVisibleAssistantText.length > 0 && + lastVisibleAssistantTextAt > 0 + ) { + automaticContinuationHoldUntil = Math.max( + automaticContinuationHoldUntil, + lastVisibleAssistantTextAt + minVisibleMs, + ); + } + } + return {}; + }, }), onError: (error) => { console.error("[Chat API] Stream error:", error); }, + onToolCall: (part) => { + return executeToolCall(part); + }, + sendAutomaticallyWhen: ({ messages }) => { + const continuationKeys = this.automaticContinuationKeys(messages); + const unsubmittedContinuationKeys = continuationKeys.filter( + (key) => !submittedAutomaticContinuationKeys.has(key), + ); + const shouldSend = + this.shouldAutoSend(messages) && + unsubmittedContinuationKeys.length > 0; + return shouldSend; + }, + maxAutomaticSteps: 5, + onAutomaticStepLimitExceeded: () => { + sendMessage( + "工具连续执行次数达到上限啦，请换个说法再试试～", + 5000, + 3, + ); + }, onFinish: ({ messages, isAbort, isError }) => { if (!isAbort && !isError) { - localStorage.setItem("historyMessages", JSON.stringify(messages)); + localStorage.setItem( + "historyMessages", + JSON.stringify(this.stabilizeHistory(messages)), + ); } }, }); this.chat = sdkChat; + const handledApprovals = new Set(); unsubscribe = sdkChat.subscribe(() => { const latest = sdkChat.messages[sdkChat.messages.length - 1]; if (!latest || latest.role !== "assistant") { + flushQueuedToolCalls(); + return; + } + this.agentRuntime.ingestMessages(sdkChat.messages); + for (const part of latest.parts) { + if (this.isToolPart(part) && part.state === "input-available") { + executeToolCall(part); + } + if ( + !this.isToolPart(part) || + part.state !== "approval-requested" || + !part.approval?.id || + handledApprovals.has(part.approval.id) + ) { + continue; + } + handledApprovals.add(part.approval.id); + trackContinuation( + this.agentRuntime + .requestApproval(`要我帮你执行「${part.toolName}」吗？`) + .then((approved) => { + return sdkChat.addToolApprovalResponse({ + id: part.approval?.id, + toolCallId: part.toolCallId, + toolName: part.toolName, + approved, + reason: approved + ? "Approved by visitor" + : "Denied by visitor", + }); + }) + .then(() => undefined), + ); + } + const textAnchor = this.displayTextAnchor(latest); + const allText = this.messageText(latest); + let shouldReplaceIncomingText = false; + if ( + textAnchor !== displayedTextAnchor || + this.displayText(latest, displayedTextBaseline).length < + streamedText.length + ) { + const hadDisplayedTextAnchor = displayedTextAnchor.length > 0; + const previousDisplayedTextBaseline = displayedTextBaseline; + displayedTextAnchor = textAnchor; + displayedTextBaseline = this.displayTextBaseline(latest); + streamedText = ""; + const text = this.displayText(latest, displayedTextBaseline); + if (text.length === 0 && this.hasPendingOrCompletedTool(latest)) { + if (!shouldKeepVisibleAssistantTextDuringToolWait()) { + startReasoningMessages(); + } + } else if (text.length === 0 && hadDisplayedTextAnchor) { + if (!shouldKeepVisibleAssistantTextDuringToolWait()) { + startReasoningMessages(); + } + } else if (text.length > 0) { + shouldReplaceIncomingText = + hadDisplayedTextAnchor || + reasoningMessageVisible || + allText.startsWith(previousDisplayedTextBaseline); + } + } + const text = this.displayText(latest, displayedTextBaseline); + if (deferredReplaceText !== null) { + scheduleDeferredReplace(text); + flushQueuedToolCalls(); return; } - const text = messageText(latest); if (text.length > streamedText.length) { stopWaitingMessage(); hasVisibleAssistantContent = true; stopReasoningMessages(); - if (reasoningMessageVisible && streamedText.length === 0) { - chat.setMessage(""); + const isFirstTextAfterTool = + displayedTextBaseline.length > 0 && + streamedText.length === 0 && + lastVisibleAssistantText.length > 0; + const willReplaceIncomingText = + shouldReplaceIncomingText || + isFirstTextAfterTool || + (reasoningMessageVisible && streamedText.length === 0); + if (willReplaceIncomingText) { + if ( + shouldHoldAutomaticContinuationReplace(willReplaceIncomingText) && + scheduleDeferredReplace(text) + ) { + flushQueuedToolCalls(); + return; + } + chat.setMessage(text); reasoningMessageVisible = false; + markVisibleAssistantText(text); + } else { + chat.sendMessage(text.slice(streamedText.length)); + markVisibleAssistantText(text); } - chat.sendMessage(text.slice(streamedText.length)); } else if (text.length === 0 && this.hasReasoningContent(latest)) { startReasoningMessages(); } streamedText = text; + flushQueuedToolCalls(); }); await sdkChat.sendMessage({ text: messageTextValue }); + flushQueuedToolCalls(); + await waitForContinuations(); + await waitForDeferredReplace(); if (sdkChat.error) { throw sdkChat.error; } @@ -233,6 +541,7 @@ export class ChatApi { } finally { stopWaitingMessage(); stopReasoningMessages(); + clearDeferredReplaceTimer(); unsubscribe?.(); } } @@ -278,7 +587,7 @@ export class ChatApi { const maxMessages = this.config.chatContextRounds ? this.config.chatContextRounds * 2 : 40; - return pruneMessages(historyMessages, { maxMessages }); + return this.stabilizeHistory(historyMessages, maxMessages); } private normalizeReasoningMessages(messages: unknown): string[] { @@ -308,13 +617,24 @@ export class ChatApi { return normalized.length > 0 ? normalized : [...DEFAULT_REASONING_MESSAGES]; } - private normalizeReasoningMessageInterval(interval: number | undefined): number { + private normalizeReasoningMessageInterval( + interval: number | undefined, + ): number { if (!Number.isFinite(interval) || !interval || interval < 1) { return 5; } return Math.floor(interval); } + private normalizeAutoContinuationMessageMinVisibleMs( + minVisibleMs: number | undefined, + ): number { + if (!Number.isFinite(minVisibleMs) || !minVisibleMs || minVisibleMs < 0) { + return 0; + } + return Math.floor(Math.min(minVisibleMs, 10_000)); + } + private pickReasoningMessage(): string | undefined { const messages = this.config.reasoningMessages; if (!Array.isArray(messages) || messages.length === 0) { @@ -328,21 +648,97 @@ export class ChatApi { } private hasReasoningContent(message: UIMessage): boolean { - return message.parts.some( + return this.partsAfterLastTool(message).some( (part) => part.type === "reasoning" && part.text.trim().length > 0, ); } + private hasPendingOrCompletedTool(message: UIMessage): boolean { + return message.parts.some((part) => this.isToolPart(part)); + } + + private messageText(message: UIMessage): string { + return message.parts + .filter( + (part): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + } + + private displayText(message: UIMessage, baseline = ""): string { + const textAfterLastTool = this.partsAfterLastTool(message) + .filter( + (part): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + if (textAfterLastTool.length > 0) { + return textAfterLastTool; + } + const text = this.messageText(message); + if (baseline.length > 0) { + return text.startsWith(baseline) ? text.slice(baseline.length) : text; + } + return this.hasPendingOrCompletedTool(message) ? "" : text; + } + + private displayTextBaseline(message: UIMessage): string { + const lastToolIndex = this.lastToolPartIndex(message); + if (lastToolIndex === -1) { + return ""; + } + return message.parts + .slice(0, lastToolIndex) + .filter( + (part): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + } + + private displayTextAnchor(message: UIMessage): string { + const lastToolIndex = this.lastToolPartIndex(message); + if (lastToolIndex === -1) { + return `${message.id}:initial`; + } + const part = message.parts[lastToolIndex]; + if (!this.isToolPart(part)) { + return `${message.id}:initial`; + } + return `${message.id}:${part.toolCallId}:${lastToolIndex}`; + } + + private partsAfterLastTool(message: UIMessage): UIMessage["parts"] { + const lastToolIndex = this.lastToolPartIndex(message); + return lastToolIndex === -1 + ? message.parts + : message.parts.slice(lastToolIndex + 1); + } + + private lastToolPartIndex(message: UIMessage): number { + for (let index = message.parts.length - 1; index >= 0; index -= 1) { + if (this.isToolPart(message.parts[index])) { + return index; + } + } + return -1; + } + private normalizeHistory(messages: unknown): ChatMessage[] { if (!Array.isArray(messages)) { return []; } - const issues = validateUIMessages(messages); + const normalizedMessages = this.stabilizeHistory(messages as ChatMessage[]); + const issues = validateUIMessages(normalizedMessages); if (issues.length > 0) { console.warn("[Chat API] Ignore invalid UI message history:", issues); return []; } - return messages as ChatMessage[]; + return normalizedMessages; } private resolveErrorMessage(error: unknown): string { @@ -354,4 +750,131 @@ export class ChatApi { } return "对话接口异常了哦～快去联系我的主人吧！"; } + + private isToolPart(part: UIMessage["parts"][number]): part is ToolPart { + return part.type.startsWith("tool-"); + } + + private shouldAutoSend(messages: UIMessage[]): boolean { + return lastAssistantMessageHasCompletedToolContinuations({ messages }); + } + + private automaticContinuationKeys(messages: UIMessage[]): string[] { + const assistant = [...messages] + .reverse() + .find((message) => message.role === "assistant"); + if (!assistant) { + return []; + } + const keys = assistant.parts + .filter((part): part is ToolPart => this.isToolPart(part)) + .filter((part) => this.isContinuableToolPart(part)) + .map((part) => this.automaticContinuationKey(part)); + return [...new Set(keys)]; + } + + private isContinuableToolPart(part: ToolPart): boolean { + return ( + this.hasFinalToolResultState(part) || part.state === "approval-responded" + ); + } + + private automaticContinuationKey(part: ToolPart): string { + return JSON.stringify({ + toolCallId: part.toolCallId, + toolName: part.toolName, + state: part.state, + approvalId: part.approval?.id, + approved: part.approval?.approved, + }); + } + + private hasFinalToolResultState(part: ToolPart): boolean { + return ( + part.state === "output-available" || + part.state === "output-error" || + part.state === "output-denied" + ); + } + + private shouldResumeAfterNavigation(output: unknown): boolean { + if (typeof output !== "object" || output === null) { + return false; + } + const record = output as Record; + if (record.navigating === true) { + return record.pageReload !== false; + } + return this.shouldResumeAfterNavigation(record.output); + } + + private historyWithToolOutput( + messages: UIMessage[], + part: ToolPart, + output: unknown, + ): ChatMessage[] { + return this.stabilizeHistory( + messages.map((message) => { + if (message.role !== "assistant") { + return message; + } + const parts = message.parts.map((messagePart) => { + if ( + !this.isToolPart(messagePart) || + messagePart.toolCallId !== part.toolCallId + ) { + return messagePart; + } + return { + ...messagePart, + type: `tool-${part.toolName}`, + toolName: part.toolName, + toolCallId: part.toolCallId, + state: "output-available", + output, + } satisfies ToolPart; + }); + return { ...message, parts }; + }), + ); + } + + private stabilizeHistory( + messages: ChatMessage[], + maxMessages?: number, + ): ChatMessage[] { + return this.sanitizeHistory(pruneMessages(messages, { maxMessages })); + } + + private sanitizeHistory(messages: ChatMessage[]): ChatMessage[] { + const seenFinalToolCallIds = new Set(); + return [...messages] + .reverse() + .map((message) => { + if (message.role !== "assistant") { + return message; + } + const parts = [...message.parts] + .reverse() + .filter((part) => { + if (!this.isToolPart(part)) { + return true; + } + if (!this.hasFinalToolResultState(part)) { + return true; + } + if (seenFinalToolCallIds.has(part.toolCallId)) { + return false; + } + seenFinalToolCallIds.add(part.toolCallId); + return true; + }) + .reverse(); + return parts.length === message.parts.length + ? message + : { ...message, parts }; + }) + .reverse() + .filter((message) => message.parts.length > 0); + } } diff --git a/packages/live2d/src/components/Live2dCanvas.tsx b/packages/live2d/src/components/Live2dCanvas.tsx index 44da9c3..7fc49bf 100644 --- a/packages/live2d/src/components/Live2dCanvas.tsx +++ b/packages/live2d/src/components/Live2dCanvas.tsx @@ -6,6 +6,10 @@ import { import { BeforeInitEvent } from "@/live2d/events/before-init.js"; import { ModelReadyEvent } from "@/live2d/events/model-ready"; import Model from "@/live2d/live2d/model"; +import { + clearCurrentLive2dModel, + setCurrentLive2dModel, +} from "@/live2d/live2d/model-store"; import { consume } from "@lit/context"; import { type PropertyValues, type TemplateResult, html } from "lit"; import { property, query, state } from "lit/decorators.js"; @@ -36,6 +40,7 @@ export class Live2dCanvas extends UnoLitElement { disconnectedCallback(): void { super.disconnectedCallback(); + clearCurrentLive2dModel(this.model); this.model?.destroy(); this.model = null; this._modelInitialized = false; @@ -62,6 +67,7 @@ export class Live2dCanvas extends UnoLitElement { try { this.model = await Model.create(this._live2d, this.config); + setCurrentLive2dModel(this.model); window.dispatchEvent(new ModelReadyEvent({ model: this.model })); } catch (error) { this._modelInitialized = false; diff --git a/packages/live2d/src/components/Live2dChatWindow.tsx b/packages/live2d/src/components/Live2dChatWindow.tsx index 91f5a1c..b7a50e3 100644 --- a/packages/live2d/src/components/Live2dChatWindow.tsx +++ b/packages/live2d/src/components/Live2dChatWindow.tsx @@ -1,3 +1,7 @@ +import { + type AgentAfterNavigationResume, + consumeAgentAfterNavigationIntent, +} from "@/live2d/api/agent-navigation-intent"; import { ChatApi, type ChatMessage } from "@/live2d/api/chat-api"; import { UnoLitElement } from "@/live2d/common/UnoLitElement"; import { @@ -138,7 +142,16 @@ export class Live2dChatWindow extends DraggableUnoLitElement { `; } - handleToggle = (): void => { + handleToggle = (event?: Event): void => { + const detail = event instanceof CustomEvent ? event.detail : undefined; + if (detail?.open === true) { + void this.showChat(detail.focus !== false); + return; + } + if (detail?.open === false) { + this.hideChat(); + return; + } if (this._isShow) { this.hideChat(); return; @@ -183,6 +196,35 @@ export class Live2dChatWindow extends DraggableUnoLitElement { this._isLoading = true; this.focusInput(); + try { + await this.sendChatMessage(message, this.readStoredHistoryMessages()); + } catch (error) { + console.error("[Live2dChatWindow] Send message error:", error); + } finally { + this._isLoading = false; + if (this._input) { + this._canSend = this._input.value.length > 0; + } + this.focusInput(); + } + } + + protected firstUpdated(_changedProperties: PropertyValues): void { + super.firstUpdated(_changedProperties); + this._input?.addEventListener("focus", () => { + sendMessage("按下回车键可以快速发送消息哦", 2000, 1); + }); + const intent = consumeAgentAfterNavigationIntent(); + if (intent?.openChat) { + void this.showChat(intent.focusChatInput).then(() => { + if (intent.resume) { + void this.resumeAgentAfterNavigation(intent.resume); + } + }); + } + } + + private async ensureChatApi(): Promise { if (!this.chatApi) { const mergedTips = this.config ? await loadMergedTips(this.config).catch(() => undefined) @@ -192,6 +234,9 @@ export class Live2dChatWindow extends DraggableUnoLitElement { showChatMessageTimeout: Number( this.config?.showChatMessageTimeout || 10, ), + autoContinuationMessageMinVisibleMs: Number( + this.config?.autoContinuationMessageMinVisibleMs ?? 1500, + ), requestAcceptedMessage: this.config?.requestAcceptedMessage, reasoningMessages: this.config?.reasoningMessages ?? mergedTips?.message.reasoning, @@ -199,16 +244,42 @@ export class Live2dChatWindow extends DraggableUnoLitElement { this.config?.reasoningMessageInterval || 5, ), chatContextRounds: Number(this.config?.chatContextRounds || 20), + agent: this.config?.agent, }); } + return this.chatApi; + } + private readStoredHistoryMessages(): ChatMessage[] { const historyJson = localStorage.getItem("historyMessages"); this.historyMessages = historyJson ? JSON.parse(historyJson) : []; + return this.historyMessages; + } + private async sendChatMessage( + message: string, + historyMessages: ChatMessage[], + ): Promise { + const chatApi = await this.ensureChatApi(); + await chatApi.sendMessage(message, historyMessages); + } + + private async resumeAgentAfterNavigation( + resume: AgentAfterNavigationResume, + ): Promise { + if (this._isLoading) { + return; + } + this._isLoading = true; + this._canSend = false; + this.focusInput(); try { - await this.chatApi.sendMessage(message, this.historyMessages); + await this.sendChatMessage(resume.message, resume.historyMessages); } catch (error) { - console.error("[Live2dChatWindow] Send message error:", error); + console.error( + "[Live2dChatWindow] Resume agent after navigation error:", + error, + ); } finally { this._isLoading = false; if (this._input) { @@ -218,14 +289,7 @@ export class Live2dChatWindow extends DraggableUnoLitElement { } } - protected firstUpdated(_changedProperties: PropertyValues): void { - super.firstUpdated(_changedProperties); - this._input?.addEventListener("focus", () => { - sendMessage("按下回车键可以快速发送消息哦", 2000, 1); - }); - } - - private async showChat(): Promise { + private async showChat(focus = true): Promise { this.clearHidePopoverTimer(); await this.updateComplete; if ( @@ -237,9 +301,11 @@ export class Live2dChatWindow extends DraggableUnoLitElement { requestAnimationFrame(() => { this._isShow = true; - void this.updateComplete.then(() => { - this._input?.focus(); - }); + if (focus) { + void this.updateComplete.then(() => { + this.focusInput(); + }); + } }); } diff --git a/packages/live2d/src/components/Live2dTips.tsx b/packages/live2d/src/components/Live2dTips.tsx index 4266164..6cddefe 100644 --- a/packages/live2d/src/components/Live2dTips.tsx +++ b/packages/live2d/src/components/Live2dTips.tsx @@ -34,6 +34,7 @@ export class Live2dTips extends UnoLitElement { private _bottomOffset = Live2dTips.DEFAULT_BOTTOM_OFFSET; private priority = -1; private messageTimer: number | null = null; + private streamInactivityTimeout = 60_000; // 流式消息模式标志 private isStreamMode = false; private readonly onMessage = (event: Event) => { @@ -108,6 +109,7 @@ export class Live2dTips extends UnoLitElement { window.removeEventListener("live2d:stream-message", this.onStreamMessage); window.removeEventListener("live2d:stream-message-stop", this.onStreamStop); window.removeEventListener("live2d:model-layout", this.onModelLayout); + this.clearMessageTimer(); } handleMessage(e: SendMessageEvent): void { @@ -122,10 +124,7 @@ export class Live2dTips extends UnoLitElement { if (priority < this.priority) { return; } - if (this.messageTimer) { - clearTimeout(this.messageTimer); - this.messageTimer = null; - } + this.clearMessageTimer(); const message = randomSelection(text); if (!isNotEmpty(message)) { return; @@ -147,23 +146,13 @@ export class Live2dTips extends UnoLitElement { const STREAM_PRIORITY = 99999; this.priority = STREAM_PRIORITY; this.isStreamMode = true; + this.streamInactivityTimeout = timeout; // 清空消息并显示 this._message = ""; this._isShow = true; - // 清除旧的定时器 - if (this.messageTimer) { - clearTimeout(this.messageTimer); - this.messageTimer = null; - } - - // 设置超时自动关闭 - this.messageTimer = setTimeout(() => { - this._isShow = false; - this.priority = -1; - this.isStreamMode = false; - }, timeout); + this.scheduleStreamInactivityTimeout(timeout); } /** @@ -174,6 +163,7 @@ export class Live2dTips extends UnoLitElement { return; } const { text } = e.detail; + this.scheduleStreamInactivityTimeout(this.streamInactivityTimeout); if (e.detail.mode === "replace") { this._message = text; return; @@ -190,11 +180,7 @@ export class Live2dTips extends UnoLitElement { } const { showTimeout } = e.detail; - // 清除旧的定时器 - if (this.messageTimer) { - clearTimeout(this.messageTimer); - this.messageTimer = null; - } + this.clearMessageTimer(); // 设置新的定时器，在指定时间后关闭 this.messageTimer = setTimeout(() => { @@ -215,6 +201,25 @@ export class Live2dTips extends UnoLitElement { private applyHostPosition(): void { this.style.bottom = `${this._bottomOffset}px`; } + + private clearMessageTimer(): void { + if (this.messageTimer) { + clearTimeout(this.messageTimer); + this.messageTimer = null; + } + } + + private scheduleStreamInactivityTimeout(timeout?: number): void { + this.clearMessageTimer(); + this.messageTimer = setTimeout( + () => { + this._isShow = false; + this.priority = -1; + this.isStreamMode = false; + }, + timeout ?? Number(this.config?.chunkTimeout || 60) * 1000, + ); + } } customElements.define("live2d-tips", Live2dTips); diff --git a/packages/live2d/src/config/agent-tools/agent-tool-config.ts b/packages/live2d/src/config/agent-tools/agent-tool-config.ts new file mode 100644 index 0000000..bdb03b7 --- /dev/null +++ b/packages/live2d/src/config/agent-tools/agent-tool-config.ts @@ -0,0 +1,81 @@ +export type AgentAccessMode = + | "anonymous_chat" + | "anonymous_chat_agent" + | "authenticated_chat" + | "authenticated_chat_agent"; + +export type AgentToolApproval = "default" | "never" | "always"; +export type AgentToolAuth = "none" | "authenticated"; +export type AgentCommentCapability = "off" | "assist" | "submit"; + +export type AgentBrowserAction = + | { + type: "navigate"; + url: string; + target?: "_self" | "_blank"; + pendingMessage?: string; + successMessage?: string; + errorMessage?: string; + } + | { + type: "scroll-to"; + selector: string; + behavior?: ScrollBehavior; + pendingMessage?: string; + successMessage?: string; + errorMessage?: string; + } + | { + type: "highlight"; + selector: string; + duration?: number; + pendingMessage?: string; + successMessage?: string; + errorMessage?: string; + } + | { + type: "dispatch-event"; + event: string; + pendingMessage?: string; + successMessage?: string; + errorMessage?: string; + } + | { + type: "registered"; + pendingMessage?: string; + successMessage?: string; + errorMessage?: string; + }; + +export interface AgentToolConfig { + name: string; + description: string; + inputSchema: Record; + approval: AgentToolApproval; + requiredAuth: AgentToolAuth; + actionType: AgentBrowserAction["type"]; + action: AgentBrowserAction; + testInput?: unknown; +} + +export interface AgentRuntimeConfig { + builtIn: { + pageContext: boolean; + haloNavigation: boolean; + haloContentSearch: boolean; + networkAccess: boolean; + commentCapability: AgentCommentCapability; + }; + aiTools: AgentToolConfig[]; + toolSecurity: { + allowedExternalOrigins: string[]; + allowNewTab: boolean; + }; + haloSearch: { + allowedTypes: string[]; + defaultLimit: number; + }; + haloResourceDetail: { + maxContentChars: number; + }; +} diff --git a/packages/live2d/src/config/agent-tools/normalize-agent-tools.ts b/packages/live2d/src/config/agent-tools/normalize-agent-tools.ts new file mode 100644 index 0000000..7aafac4 --- /dev/null +++ b/packages/live2d/src/config/agent-tools/normalize-agent-tools.ts @@ -0,0 +1,183 @@ +import type { + AgentBrowserAction, + AgentCommentCapability, + AgentRuntimeConfig, + AgentToolApproval, + AgentToolAuth, + AgentToolConfig, +} from "@/live2d/config/agent-tools/agent-tool-config"; +import { + isRecord, + pickBoolean, + pickNumber, + pickString, +} from "@/live2d/config/normalize-helpers"; +import { isNotEmptyString } from "@/live2d/utils/isString"; + +const ACTION_TYPES = new Set([ + "navigate", + "scroll-to", + "highlight", + "dispatch-event", + "registered", +]); + +const TOOL_NAME_PATTERN = /^[a-z][a-z0-9_]{2,63}$/; + +const pickApproval = (value: unknown): AgentToolApproval => + value === "never" || value === "always" ? value : "default"; + +const pickAuth = (value: unknown): AgentToolAuth => + value === "authenticated" ? "authenticated" : "none"; + +const pickCommentCapability = (value: unknown): AgentCommentCapability => + value === "off" || value === "submit" ? value : "assist"; + +const normalizeStringArray = (value: unknown): string[] => { + if (!Array.isArray(value)) { + return []; + } + return value.filter(isNotEmptyString); +}; + +const normalizeObjectStringArray = ( + value: unknown, + objectKey: string, +): string[] => { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((item) => + isRecord(item) && isNotEmptyString(item[objectKey]) + ? [item[objectKey]] + : [], + ); +}; + +const normalizeAction = (value: unknown): AgentBrowserAction | undefined => { + if (!isRecord(value)) { + return; + } + const type = pickString(value.type); + if (!type || !ACTION_TYPES.has(type)) { + return; + } + if (type === "navigate") { + const url = pickString(value.url); + if (!url) { + return; + } + return { + ...pickMessages(value), + type, + url, + target: value.target === "_blank" ? "_blank" : "_self", + }; + } + if (type === "scroll-to" || type === "highlight") { + const selector = pickString(value.selector); + if (!selector) { + return; + } + return { + ...pickMessages(value), + type, + selector, + ...(type === "scroll-to" + ? { behavior: value.behavior === "auto" ? "auto" : "smooth" } + : { duration: pickNumber(value.duration) }), + } as AgentBrowserAction; + } + if (type === "dispatch-event") { + const event = pickString(value.event); + if (!event) { + return; + } + return { ...pickMessages(value), type, event }; + } + if (type === "registered") { + return { ...pickMessages(value), type }; + } + return; +}; + +const pickMessages = (value: Record) => ({ + pendingMessage: pickString(value.pendingMessage), + successMessage: pickString(value.successMessage), + errorMessage: pickString(value.errorMessage), +}); + +const normalizeTool = (value: unknown): AgentToolConfig | undefined => { + if (!isRecord(value)) { + return; + } + const name = pickString(value.name); + const description = pickString(value.description); + const action = normalizeAction(value.action); + if (!name || !TOOL_NAME_PATTERN.test(name) || !description || !action) { + return; + } + const inputSchema = isRecord(value.inputSchema) + ? value.inputSchema + : { type: "object", properties: {} }; + return { + name, + description, + inputSchema, + approval: pickApproval(value.approval), + requiredAuth: pickAuth(value.requiredAuth), + actionType: action.type, + action, + testInput: value.testInput, + }; +}; + +export const normalizeAgentRuntimeConfig = ( + input: unknown, +): AgentRuntimeConfig => { + const source = isRecord(input) ? input : {}; + const builtIn = isRecord(source.builtIn) ? source.builtIn : {}; + const toolSecurity = isRecord(source.toolSecurity) ? source.toolSecurity : {}; + const haloSearch = isRecord(source.haloSearch) ? source.haloSearch : {}; + const haloResourceDetail = isRecord(source.haloResourceDetail) + ? source.haloResourceDetail + : {}; + const allowedTypes = normalizeStringArray(haloSearch.allowedTypes); + + return { + builtIn: { + pageContext: pickBoolean(builtIn.pageContext, true) ?? true, + haloNavigation: pickBoolean(builtIn.haloNavigation, true) ?? true, + haloContentSearch: pickBoolean(builtIn.haloContentSearch, true) ?? true, + networkAccess: pickBoolean(builtIn.networkAccess, false) ?? false, + commentCapability: pickCommentCapability(builtIn.commentCapability), + }, + aiTools: Array.isArray(source.aiTools) + ? source.aiTools.flatMap((tool) => { + const normalized = normalizeTool(tool); + return normalized ? [normalized] : []; + }) + : [], + toolSecurity: { + allowedExternalOrigins: normalizeStringArray( + toolSecurity.allowedExternalOrigins, + ).concat( + normalizeObjectStringArray( + toolSecurity.allowedExternalOrigins, + "origin", + ), + ), + allowNewTab: pickBoolean(toolSecurity.allowNewTab, false) ?? false, + }, + haloSearch: { + allowedTypes: + allowedTypes.length > 0 + ? allowedTypes + : ["post.content.halo.run", "singlepage.content.halo.run"], + defaultLimit: pickNumber(haloSearch.defaultLimit) ?? 5, + }, + haloResourceDetail: { + maxContentChars: pickNumber(haloResourceDetail.maxContentChars) ?? 3000, + }, + }; +}; diff --git a/packages/live2d/src/config/default-config.ts b/packages/live2d/src/config/default-config.ts index 1d961ba..c16abf4 100644 --- a/packages/live2d/src/config/default-config.ts +++ b/packages/live2d/src/config/default-config.ts @@ -29,8 +29,31 @@ export const createDefaultLive2dConfig = (): Live2dConfig => ({ isTools: true, tools: [...DEFAULT_TOOL_NAMES], isAiChat: true, + accessMode: "anonymous_chat", + agent: { + builtIn: { + pageContext: true, + haloNavigation: true, + haloContentSearch: true, + networkAccess: false, + commentCapability: "assist", + }, + aiTools: [], + toolSecurity: { + allowedExternalOrigins: [], + allowNewTab: false, + }, + haloSearch: { + allowedTypes: ["post.content.halo.run", "singlepage.content.halo.run"], + defaultLimit: 5, + }, + haloResourceDetail: { + maxContentChars: 3000, + }, + }, chunkTimeout: 10, showChatMessageTimeout: 10, + autoContinuationMessageMinVisibleMs: 1500, requestAcceptedMessage: "收到啦，马上就来陪你啦～", reasoningMessages: [ "我正在认真想一想～", diff --git a/packages/live2d/src/config/normalize-config.ts b/packages/live2d/src/config/normalize-config.ts index dea652c..09e5e2b 100644 --- a/packages/live2d/src/config/normalize-config.ts +++ b/packages/live2d/src/config/normalize-config.ts @@ -1,3 +1,4 @@ +import { normalizeAgentRuntimeConfig } from "@/live2d/config/agent-tools/normalize-agent-tools"; import { normalizeCustomTools } from "@/live2d/config/custom-tools/normalize-custom-tools"; import { createDefaultLive2dConfig } from "@/live2d/config/default-config"; import { @@ -13,10 +14,13 @@ export interface LegacyLive2dConfigInput extends Partial { aiChatBaseSetting?: { chunkTimeout?: number | string; showChatMessageTimeout?: number | string; + autoContinuationMessageMinVisibleMs?: number | string; requestAcceptedMessage?: string; reasoningMessages?: string[] | string | { message?: string }[]; reasoningMessageInterval?: number | string; chatContextRounds?: number | string; + accessMode?: Live2dConfig["accessMode"]; + isAnonymous?: boolean; }; consoleShowStatu?: boolean; photoName?: string; @@ -56,6 +60,21 @@ const normalizeMessages = (messages: unknown): string[] | undefined => { return normalized.length > 0 ? normalized : undefined; }; +const normalizeAccessMode = ( + accessMode: unknown, + legacyAnonymous: unknown, +): Live2dConfig["accessMode"] => { + if ( + accessMode === "anonymous_chat" || + accessMode === "anonymous_chat_agent" || + accessMode === "authenticated_chat" || + accessMode === "authenticated_chat_agent" + ) { + return accessMode; + } + return legacyAnonymous === false ? "authenticated_chat" : "anonymous_chat"; +}; + export const normalizeLive2dConfig = ( assetPath: string, input: LegacyLive2dConfigInput = {}, @@ -88,6 +107,11 @@ export const normalizeLive2dConfig = ( defaults.screenshotName, tools: normalizeTools(input.tools) ?? [...(defaults.tools ?? [])], customTools: normalizeCustomTools(input.customTools) ?? [], + accessMode: normalizeAccessMode( + input.accessMode ?? input.aiChatBaseSetting?.accessMode, + input.aiChatBaseSetting?.isAnonymous, + ), + agent: normalizeAgentRuntimeConfig(input.agent), chunkTimeout: pickNumber( input.chunkTimeout, @@ -100,6 +124,12 @@ export const normalizeLive2dConfig = ( input.aiChatBaseSetting?.showChatMessageTimeout, defaults.showChatMessageTimeout, ) ?? defaults.showChatMessageTimeout, + autoContinuationMessageMinVisibleMs: + pickNumber( + input.autoContinuationMessageMinVisibleMs, + input.aiChatBaseSetting?.autoContinuationMessageMinVisibleMs, + defaults.autoContinuationMessageMinVisibleMs, + ) ?? defaults.autoContinuationMessageMinVisibleMs, requestAcceptedMessage: pickString( input.requestAcceptedMessage, diff --git a/packages/live2d/src/context/config-context.ts b/packages/live2d/src/context/config-context.ts index d56293d..abfcb5c 100644 --- a/packages/live2d/src/context/config-context.ts +++ b/packages/live2d/src/context/config-context.ts @@ -1,6 +1,10 @@ +import type { + AgentAccessMode, + AgentRuntimeConfig, +} from "@/live2d/config/agent-tools/agent-tool-config"; import type { CustomToolConfig } from "@/live2d/live2d/tools/custom-tool-config"; -import type { ProceduralConfig } from "@/live2d/runtime/procedural"; import type { EmotionTimelineConfig } from "@/live2d/runtime/emotion"; +import type { ProceduralConfig } from "@/live2d/runtime/procedural"; import { createContext } from "@lit/context"; export interface ObjectAny extends Record {} @@ -51,12 +55,18 @@ export interface Live2dToolsConfig { customTools?: CustomToolConfig[]; // 是否启用 AI 聊天功能 isAiChat?: boolean; + // AI 聊天与 Agent 能力访问模式 + accessMode?: AgentAccessMode; + // Agent 能力运行时配置 + agent?: AgentRuntimeConfig; // openai 图标 openaiIcon?: string; // 聊天请求超时时间（秒） chunkTimeout?: number; // 聊天消息显示时间（秒） showChatMessageTimeout?: number; + // Agent 自动续写时，上一段助手消息最短可见时间（毫秒） + autoContinuationMessageMinVisibleMs?: number; // 用户请求已收到时的即时提示语 requestAcceptedMessage?: string; // 模型思考阶段的提示语 @@ -136,10 +146,13 @@ export interface Live2dConfig extends Live2dToolsConfig { aiChatBaseSetting?: { chunkTimeout?: number | string; showChatMessageTimeout?: number | string; + autoContinuationMessageMinVisibleMs?: number | string; requestAcceptedMessage?: string; reasoningMessages?: string[] | string | { message?: string }[]; reasoningMessageInterval?: number | string; chatContextRounds?: number | string; + accessMode?: AgentAccessMode; + isAnonymous?: boolean; }; // 滤镜质量等级 (low / medium / high) filterQuality?: "low" | "medium" | "high"; diff --git a/packages/live2d/src/events/toggle-chat-window.ts b/packages/live2d/src/events/toggle-chat-window.ts index f186016..4121bfc 100644 --- a/packages/live2d/src/events/toggle-chat-window.ts +++ b/packages/live2d/src/events/toggle-chat-window.ts @@ -3,6 +3,11 @@ import { Live2dEvent } from "@/live2d/events/types"; export const TOGGLE_CHAT_WINDOW_EVENT_NAME = "live2d:toggle-chat-window" as const; +export interface ToggleChatWindowEventDetail { + open?: boolean; + focus?: boolean; +} + declare global { interface GlobalEventHandlersEventMap { [TOGGLE_CHAT_WINDOW_EVENT_NAME]: ToggleChatWindowEvent; @@ -10,11 +15,11 @@ declare global { } /** - * 切换聊天窗口事件 - * 不需要传递参数，每次触发都会切换显示状态 + * 切换或设置聊天窗口显示状态。 + * 不传 open 时保持原有切换语义。 */ -export class ToggleChatWindowEvent extends Live2dEvent> { - constructor() { - super(TOGGLE_CHAT_WINDOW_EVENT_NAME, {}); +export class ToggleChatWindowEvent extends Live2dEvent { + constructor(detail: ToggleChatWindowEventDetail = {}) { + super(TOGGLE_CHAT_WINDOW_EVENT_NAME, detail); } } diff --git a/packages/live2d/src/live2d/model-store.ts b/packages/live2d/src/live2d/model-store.ts new file mode 100644 index 0000000..dfcd49b --- /dev/null +++ b/packages/live2d/src/live2d/model-store.ts @@ -0,0 +1,15 @@ +import type Model from "@/live2d/live2d/model"; + +let currentModel: Model | null = null; + +export const setCurrentLive2dModel = (model: Model): void => { + currentModel = model; +}; + +export const clearCurrentLive2dModel = (model?: Model | null): void => { + if (!model || currentModel === model) { + currentModel = null; + } +}; + +export const getCurrentLive2dModel = (): Model | null => currentModel; diff --git a/packages/live2d/vite.config.ts b/packages/live2d/vite.config.ts index 1ae787b..dcf8739 100644 --- a/packages/live2d/vite.config.ts +++ b/packages/live2d/vite.config.ts @@ -6,6 +6,11 @@ export default defineConfig({ test: { environment: "jsdom", globals: true, + server: { + deps: { + inline: ["@halo-dev/ai-foundation-sdk"], + }, + }, }, plugins: [ react({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2bdde6..c3578cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,8 +18,8 @@ importers: packages/live2d: dependencies: '@halo-dev/ai-foundation-sdk': - specifier: https://pkg.pr.new/halo-dev/plugin-ai-foundation/@halo-dev/ai-foundation-sdk@8200c2a - version: https://pkg.pr.new/halo-dev/plugin-ai-foundation/@halo-dev/ai-foundation-sdk@8200c2a(vue@3.5.38(typescript@5.7.3)) + specifier: 1.0.0-beta.1 + version: 1.0.0-beta.1(vue@3.5.38(typescript@5.7.3)) '@lit/context': specifier: ^1.1.6 version: 1.1.6 @@ -59,7 +59,7 @@ importers: version: 19.2.3(@types/react@19.2.14) '@unocss/postcss': specifier: ^66.6.8 - version: 66.6.8(postcss@8.5.14) + version: 66.6.8(postcss@8.5.15) '@vitejs/plugin-react': specifier: ^6.0.2 version: 6.0.2(vite@8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2)) @@ -71,7 +71,7 @@ importers: version: 2.0.2 unocss: specifier: ^66.6.8 - version: 66.6.8(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@unocss/postcss@66.6.8(postcss@8.5.14))(vite@8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2)) + version: 66.6.8(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@unocss/postcss@66.6.8(postcss@8.5.15))(vite@8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2)) vite: specifier: ^8.0.13 version: 8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2) @@ -379,9 +379,8 @@ packages: '@noble/hashes': optional: true - '@halo-dev/ai-foundation-sdk@https://pkg.pr.new/halo-dev/plugin-ai-foundation/@halo-dev/ai-foundation-sdk@8200c2a': - resolution: {integrity: sha512-83g/hZWKUyeoTj2Fp9yEzeAFFP/6OR5urXAhdbKokzWD2SgXYsRE7ZgTNjZdyXTa/aYmxdC5L6UAg1pV/MaPcQ==, tarball: https://pkg.pr.new/halo-dev/plugin-ai-foundation/@halo-dev/ai-foundation-sdk@8200c2a} - version: 0.1.0 + '@halo-dev/ai-foundation-sdk@1.0.0-beta.1': + resolution: {integrity: sha512-r9bOT0Azso7iJldN8k2AiSBJrTJjFQ9lmRegPxbA2hueZ9xdGnvYl6eM5M25sJvzA0yO1lpvZnji5v/WP/SJVQ==} peerDependencies: vue: ^3.5.x zod: '*' @@ -1908,7 +1907,7 @@ snapshots: '@exodus/bytes@1.15.0': {} - '@halo-dev/ai-foundation-sdk@https://pkg.pr.new/halo-dev/plugin-ai-foundation/@halo-dev/ai-foundation-sdk@8200c2a(vue@3.5.38(typescript@5.7.3))': + '@halo-dev/ai-foundation-sdk@1.0.0-beta.1(vue@3.5.38(typescript@5.7.3))': dependencies: vue: 3.5.38(typescript@5.7.3) @@ -2182,13 +2181,13 @@ snapshots: gzip-size: 6.0.0 sirv: 3.0.2 - '@unocss/postcss@66.6.8(postcss@8.5.14)': + '@unocss/postcss@66.6.8(postcss@8.5.15)': dependencies: '@unocss/config': 66.6.8 '@unocss/core': 66.6.8 '@unocss/rule-utils': 66.6.8 css-tree: 3.2.1 - postcss: 8.5.14 + postcss: 8.5.15 tinyglobby: 0.2.16 '@unocss/preset-attributify@66.6.8': @@ -3046,7 +3045,7 @@ snapshots: undici@7.25.0: {} - unocss@66.6.8(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@unocss/postcss@66.6.8(postcss@8.5.14))(vite@8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2)): + unocss@66.6.8(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@unocss/postcss@66.6.8(postcss@8.5.15))(vite@8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2)): dependencies: '@unocss/cli': 66.6.8 '@unocss/core': 66.6.8 @@ -3066,7 +3065,7 @@ snapshots: '@unocss/transformer-variant-group': 66.6.8 '@unocss/vite': 66.6.8(vite@8.0.13(@types/node@22.13.1)(jiti@2.7.0)(terser@5.38.0)(tsx@4.19.2)) optionalDependencies: - '@unocss/postcss': 66.6.8(postcss@8.5.14) + '@unocss/postcss': 66.6.8(postcss@8.5.15) transitivePeerDependencies: - '@emnapi/core' - '@emnapi/runtime' diff --git a/src/main/java/run/halo/live2d/Live2dSettingProcess.java b/src/main/java/run/halo/live2d/Live2dSettingProcess.java index 8e17faf..17ded8d 100644 --- a/src/main/java/run/halo/live2d/Live2dSettingProcess.java +++ b/src/main/java/run/halo/live2d/Live2dSettingProcess.java @@ -1,6 +1,7 @@ package run.halo.live2d; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -9,6 +10,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; +import run.halo.live2d.agent.AgentSettings; +import run.halo.live2d.agent.AgentToolNormalizer; import run.halo.app.plugin.ReactiveSettingFetcher; /** @@ -30,10 +33,13 @@ public class Live2dSettingProcess implements Live2dSetting { private static final List ADVANCED_FIELDS = List.of( "consoleShowStatu", "photoName", "live2dLocation"); private static final List AI_CHAT_PUBLIC_FIELDS = List.of( - "chunkTimeout", "showChatMessageTimeout", "requestAcceptedMessage", - "reasoningMessages", "reasoningMessageInterval", "chatContextRounds"); + "chunkTimeout", "showChatMessageTimeout", "autoContinuationMessageMinVisibleMs", + "requestAcceptedMessage", "reasoningMessages", "reasoningMessageInterval", + "chatContextRounds"); private final ReactiveSettingFetcher settingFetcher; + private final AgentToolNormalizer agentToolNormalizer; + private final ObjectMapper objectMapper = new ObjectMapper(); @Override public Mono getGroup(String groupName) { @@ -54,6 +60,7 @@ public Mono getPublicConfig(String themeTipsPath) { copyFields(objectNode, data.get("tips"), TIPS_FIELDS); copyFields(objectNode, data.get("advanced"), ADVANCED_FIELDS); copyAiChatFields(objectNode, data.get("aichat")); + copyAgentFields(objectNode, data.get("agent")); copyCustomTools(objectNode, data.get("customTools")); if (themeTipsPath != null && !themeTipsPath.isBlank()) { objectNode.put("themeTipsPath", themeTipsPath); @@ -92,6 +99,45 @@ private void copyAiChatFields(ObjectNode target, JsonNode source) { copyFields(target, aiChatBaseSetting, AI_CHAT_PUBLIC_FIELDS); } + private void copyAgentFields(ObjectNode target, JsonNode source) { + if (source == null || source.isNull()) { + return; + } + + AgentSettings settings = objectMapper.convertValue(source, AgentSettings.class); + ObjectNode agentNode = JsonNodeFactory.instance.objectNode(); + agentNode.set("builtIn", objectMapper.valueToTree(settings.builtIn())); + ObjectNode securityNode = JsonNodeFactory.instance.objectNode(); + securityNode.set("allowedExternalOrigins", + objectMapper.valueToTree(settings.toolSecurity().normalizedAllowedExternalOrigins())); + securityNode.put("allowNewTab", settings.toolSecurity().allowNewTab()); + agentNode.set("toolSecurity", securityNode); + ObjectNode searchNode = JsonNodeFactory.instance.objectNode(); + searchNode.set("allowedTypes", + objectMapper.valueToTree(settings.haloSearch().normalizedAllowedTypes())); + searchNode.put("defaultLimit", settings.haloSearch().normalizedDefaultLimit()); + agentNode.set("haloSearch", searchNode); + agentNode.set("haloResourceDetail", objectMapper.valueToTree(settings.haloResourceDetail())); + + ArrayNode tools = JsonNodeFactory.instance.arrayNode(); + agentToolNormalizer.normalizeCustomTools(settings).forEach(tool -> { + ObjectNode toolNode = JsonNodeFactory.instance.objectNode(); + toolNode.put("name", tool.name()); + toolNode.put("description", tool.description()); + toolNode.set("inputSchema", objectMapper.valueToTree(tool.inputSchema())); + toolNode.put("approval", tool.approval().value()); + toolNode.put("requiredAuth", tool.requiredAuth().value()); + toolNode.put("actionType", tool.actionType()); + toolNode.set("action", tool.action()); + if (tool.testInput() != null && !tool.testInput().isNull()) { + toolNode.set("testInput", tool.testInput()); + } + tools.add(toolNode); + }); + agentNode.set("aiTools", tools); + target.set("agent", agentNode); + } + private void copyCustomTools(ObjectNode target, JsonNode source) { if (source == null || source.isNull()) { return; diff --git a/src/main/java/run/halo/live2d/agent/AgentAccessMode.java b/src/main/java/run/halo/live2d/agent/AgentAccessMode.java new file mode 100644 index 0000000..f464dff --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentAccessMode.java @@ -0,0 +1,48 @@ +package run.halo.live2d.agent; + +public enum AgentAccessMode { + ANONYMOUS_CHAT("anonymous_chat", true, false, false), + ANONYMOUS_CHAT_AGENT("anonymous_chat_agent", true, true, false), + AUTHENTICATED_CHAT("authenticated_chat", false, false, true), + AUTHENTICATED_CHAT_AGENT("authenticated_chat_agent", false, true, true); + + private final String value; + private final boolean anonymousChatAllowed; + private final boolean agentAllowed; + private final boolean authenticationRequired; + + AgentAccessMode(String value, boolean anonymousChatAllowed, boolean agentAllowed, + boolean authenticationRequired) { + this.value = value; + this.anonymousChatAllowed = anonymousChatAllowed; + this.agentAllowed = agentAllowed; + this.authenticationRequired = authenticationRequired; + } + + public String value() { + return value; + } + + public boolean anonymousChatAllowed() { + return anonymousChatAllowed; + } + + public boolean agentAllowed() { + return agentAllowed; + } + + public boolean authenticationRequired() { + return authenticationRequired; + } + + public static AgentAccessMode from(String value, boolean legacyAnonymous) { + if (value != null) { + for (var mode : values()) { + if (mode.value.equals(value)) { + return mode; + } + } + } + return legacyAnonymous ? ANONYMOUS_CHAT : AUTHENTICATED_CHAT; + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentBrowserActionType.java b/src/main/java/run/halo/live2d/agent/AgentBrowserActionType.java new file mode 100644 index 0000000..6dcbe2e --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentBrowserActionType.java @@ -0,0 +1,22 @@ +package run.halo.live2d.agent; + +import java.util.Set; + +public final class AgentBrowserActionType { + public static final String NAVIGATE = "navigate"; + public static final String SCROLL_TO = "scroll-to"; + public static final String HIGHLIGHT = "highlight"; + public static final String DISPATCH_EVENT = "dispatch-event"; + public static final String REGISTERED = "registered"; + + public static final Set SUPPORTED = Set.of( + NAVIGATE, + SCROLL_TO, + HIGHLIGHT, + DISPATCH_EVENT, + REGISTERED + ); + + private AgentBrowserActionType() { + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentCommentCapability.java b/src/main/java/run/halo/live2d/agent/AgentCommentCapability.java new file mode 100644 index 0000000..a00928a --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentCommentCapability.java @@ -0,0 +1,33 @@ +package run.halo.live2d.agent; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum AgentCommentCapability { + OFF("off"), + ASSIST("assist"), + SUBMIT("submit"); + + private final String value; + + AgentCommentCapability(String value) { + this.value = value; + } + + @JsonValue + public String value() { + return value; + } + + @JsonCreator + public static AgentCommentCapability from(String value) { + if (value != null) { + for (var capability : values()) { + if (capability.value.equals(value)) { + return capability; + } + } + } + return ASSIST; + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentSettings.java b/src/main/java/run/halo/live2d/agent/AgentSettings.java new file mode 100644 index 0000000..d372294 --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentSettings.java @@ -0,0 +1,198 @@ +package run.halo.live2d.agent; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +public record AgentSettings( + AgentBuiltInCapabilities builtIn, + Object aiTools, + AgentToolSecurity toolSecurity, + AgentHaloSearchSettings haloSearch, + AgentHaloResourceDetailSettings haloResourceDetail, + AgentNetworkAccessSettings networkAccess +) { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public AgentSettings { + builtIn = builtIn == null ? AgentBuiltInCapabilities.defaults() : builtIn; + aiTools = aiTools == null ? List.of() : aiTools; + toolSecurity = toolSecurity == null ? AgentToolSecurity.defaults() : toolSecurity; + haloSearch = haloSearch == null ? AgentHaloSearchSettings.defaults() : haloSearch; + haloResourceDetail = haloResourceDetail == null + ? AgentHaloResourceDetailSettings.defaults() + : haloResourceDetail; + networkAccess = networkAccess == null + ? AgentNetworkAccessSettings.defaults() + : networkAccess; + } + + public static AgentSettings defaults() { + return new AgentSettings(null, List.of(), null, null, null, null); + } + + public List normalizedAiTools() { + if (aiTools instanceof List list) { + return list.stream() + .map(item -> OBJECT_MAPPER.convertValue(item, AgentToolConfig.class)) + .toList(); + } + if (aiTools instanceof String text && !text.isBlank()) { + try { + return OBJECT_MAPPER.readerForListOf(AgentToolConfig.class).readValue(text); + } catch (Exception e) { + return List.of(); + } + } + return List.of(); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public record AgentBuiltInCapabilities( + boolean pageContext, + boolean haloNavigation, + boolean haloContentSearch, + boolean networkAccess, + AgentCommentCapability commentCapability + ) { + public AgentBuiltInCapabilities { + commentCapability = commentCapability == null + ? AgentCommentCapability.ASSIST + : commentCapability; + } + + public static AgentBuiltInCapabilities defaults() { + return new AgentBuiltInCapabilities(true, true, true, false, + AgentCommentCapability.ASSIST); + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public record AgentToolSecurity( + Object allowedExternalOrigins, + boolean allowNewTab + ) { + public static AgentToolSecurity defaults() { + return new AgentToolSecurity(List.of(), false); + } + + public List normalizedAllowedExternalOrigins() { + return normalizeStringList(allowedExternalOrigins, "origin"); + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public record AgentHaloSearchSettings( + List allowedTypes, + int defaultLimit + ) { + public static AgentHaloSearchSettings defaults() { + return new AgentHaloSearchSettings(List.of(), 5); + } + + public List normalizedAllowedTypes() { + var normalized = normalizeStringList(allowedTypes); + return normalized.isEmpty() + ? List.of("post.content.halo.run", "singlepage.content.halo.run") + : normalized; + } + + public int normalizedDefaultLimit() { + return defaultLimit < 1 || defaultLimit > 20 ? 5 : defaultLimit; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public record AgentHaloResourceDetailSettings( + int maxContentChars + ) { + public AgentHaloResourceDetailSettings { + if (maxContentChars < 500) { + maxContentChars = 3000; + } + if (maxContentChars > 10000) { + maxContentChars = 10000; + } + } + + public static AgentHaloResourceDetailSettings defaults() { + return new AgentHaloResourceDetailSettings(3000); + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public record AgentNetworkAccessSettings( + Object allowedOrigins, + int maxResponseChars, + int timeoutSeconds + ) { + public static AgentNetworkAccessSettings defaults() { + return new AgentNetworkAccessSettings(List.of(), 4000, 5); + } + + public List normalizedAllowedOrigins() { + return normalizeStringList(allowedOrigins, "origin"); + } + + public int normalizedMaxResponseChars() { + return maxResponseChars < 1000 || maxResponseChars > 20000 + ? 4000 + : maxResponseChars; + } + + public int normalizedTimeoutSeconds() { + return timeoutSeconds < 1 || timeoutSeconds > 15 ? 5 : timeoutSeconds; + } + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public record AgentToolConfig( + String name, + boolean enabled, + String description, + JsonNode inputSchema, + AgentToolApproval approval, + AgentToolAuth requiredAuth, + JsonNode action, + JsonNode testInput + ) { + public AgentToolConfig { + approval = approval == null ? AgentToolApproval.DEFAULT : approval; + requiredAuth = requiredAuth == null ? AgentToolAuth.NONE : requiredAuth; + } + } + + @SuppressWarnings("unchecked") + private static List normalizeStringList(Object value, String objectField) { + if (!(value instanceof List list)) { + return List.of(); + } + List normalized = new ArrayList<>(); + for (var item : list) { + if (item instanceof String text && !text.isBlank()) { + normalized.add(text.trim()); + continue; + } + if (item instanceof Map map) { + var nested = map.get(objectField); + if (nested instanceof String text && !text.isBlank()) { + normalized.add(text.trim()); + } + } + } + return List.copyOf(normalized); + } + + private static List normalizeStringList(List list) { + return list.stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .map(String::trim) + .filter(text -> !text.isBlank()) + .toList(); + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentToolApproval.java b/src/main/java/run/halo/live2d/agent/AgentToolApproval.java new file mode 100644 index 0000000..1580e6e --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentToolApproval.java @@ -0,0 +1,33 @@ +package run.halo.live2d.agent; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum AgentToolApproval { + DEFAULT("default"), + NEVER("never"), + ALWAYS("always"); + + private final String value; + + AgentToolApproval(String value) { + this.value = value; + } + + @JsonValue + public String value() { + return value; + } + + @JsonCreator + public static AgentToolApproval from(String value) { + if (value != null) { + for (var approval : values()) { + if (approval.value.equals(value)) { + return approval; + } + } + } + return DEFAULT; + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentToolAuth.java b/src/main/java/run/halo/live2d/agent/AgentToolAuth.java new file mode 100644 index 0000000..18c4d5e --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentToolAuth.java @@ -0,0 +1,36 @@ +package run.halo.live2d.agent; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +public enum AgentToolAuth { + NONE("none"), + AUTHENTICATED("authenticated"); + + private final String value; + + AgentToolAuth(String value) { + this.value = value; + } + + @JsonValue + public String value() { + return value; + } + + @JsonCreator + public static AgentToolAuth from(String value) { + return from(value, NONE); + } + + public static AgentToolAuth from(String value, AgentToolAuth defaultValue) { + if (value != null) { + for (var auth : values()) { + if (auth.value.equals(value)) { + return auth; + } + } + } + return defaultValue; + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentToolNormalizer.java b/src/main/java/run/halo/live2d/agent/AgentToolNormalizer.java new file mode 100644 index 0000000..524fded --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentToolNormalizer.java @@ -0,0 +1,201 @@ +package run.halo.live2d.agent; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class AgentToolNormalizer { + private static final Pattern TOOL_NAME_PATTERN = Pattern.compile("^[a-z][a-z0-9_]{2,63}$"); + private static final Set SCALAR_TYPES = Set.of("string", "number", "integer", + "boolean"); + private static final Set PROPERTY_KEYWORDS = Set.of("type", "description", "enum", + "default", "minimum", "maximum", "minLength", "maxLength"); + private static final Set ROOT_KEYWORDS = Set.of("type", "properties", "required", + "description"); + private static final Set RESERVED_TOOL_NAMES = Set.of( + "perform_live2d_action", + "get_current_page_context", + "open_halo_resource", + "search_halo_resources", + "get_halo_resource_detail", + "get_latest_halo_resources", + "get_categories", + "get_tags", + "get_posts_by_category", + "get_posts_by_tag", + "get_pages", + "fetch_allowed_url", + "open_comment_area", + "get_recent_comments", + "draft_comment", + "submit_comment" + ); + + private final ObjectMapper objectMapper = new ObjectMapper(); + + public List normalizeCustomTools(AgentSettings settings) { + if (settings == null) { + return List.of(); + } + List normalized = new ArrayList<>(); + Set names = new HashSet<>(RESERVED_TOOL_NAMES); + for (var tool : settings.normalizedAiTools()) { + normalizeCustomTool(tool, names).ifPresent(normalized::add); + } + return List.copyOf(normalized); + } + + public java.util.Optional normalizeCustomTool( + AgentSettings.AgentToolConfig tool, Set usedNames) { + if (tool == null || !tool.enabled()) { + return java.util.Optional.empty(); + } + + var name = StringUtils.trimToEmpty(tool.name()).toLowerCase(Locale.ROOT); + if (!TOOL_NAME_PATTERN.matcher(name).matches()) { + log.warn("Ignore invalid Agent tool name: {}", tool.name()); + return java.util.Optional.empty(); + } + if (!usedNames.add(name)) { + log.warn("Ignore duplicated or reserved Agent tool name: {}", name); + return java.util.Optional.empty(); + } + + var description = StringUtils.trimToNull(tool.description()); + if (description == null) { + log.warn("Ignore Agent tool {} because description is blank", name); + return java.util.Optional.empty(); + } + + var inputSchema = normalizeSchema(tool.inputSchema()); + if (inputSchema == null) { + log.warn("Ignore Agent tool {} because input schema is invalid", name); + return java.util.Optional.empty(); + } + + var action = tool.action(); + var actionType = action == null || action.isNull() ? null : textValue(action, "type"); + if (!AgentBrowserActionType.SUPPORTED.contains(actionType)) { + log.warn("Ignore Agent tool {} because action type is invalid: {}", name, actionType); + return java.util.Optional.empty(); + } + + return java.util.Optional.of(new NormalizedAgentTool( + name, + description, + inputSchema, + tool.approval(), + tool.requiredAuth(), + actionType, + action, + tool.testInput() + )); + } + + @SuppressWarnings("unchecked") + public Map normalizeSchema(JsonNode schema) { + if (schema == null || schema.isNull()) { + return Map.of("type", "object", "properties", Map.of()); + } + if (!schema.isObject() || !"object".equals(textValue(schema, "type"))) { + return null; + } + if (!allowedKeys(schema, ROOT_KEYWORDS)) { + return null; + } + Map result = new LinkedHashMap<>(); + result.put("type", "object"); + putIfText(result, "description", textValue(schema, "description")); + + Map properties = new LinkedHashMap<>(); + var propertyNode = schema.get("properties"); + if (propertyNode != null && !propertyNode.isNull()) { + if (!propertyNode.isObject()) { + return null; + } + var fields = propertyNode.fields(); + while (fields.hasNext()) { + var entry = fields.next(); + if (!entry.getValue().isObject()) { + return null; + } + var propertySchema = normalizePropertySchema(entry.getValue()); + if (propertySchema == null) { + return null; + } + properties.put(entry.getKey(), propertySchema); + } + } + result.put("properties", properties); + + var required = schema.get("required"); + if (required != null && !required.isNull()) { + if (!required.isArray()) { + return null; + } + List requiredNames = new ArrayList<>(); + required.forEach(item -> { + if (item.isTextual() && properties.containsKey(item.asText())) { + requiredNames.add(item.asText()); + } + }); + if (!requiredNames.isEmpty()) { + result.put("required", requiredNames); + } + } + return result; + } + + private Map normalizePropertySchema(JsonNode schema) { + if (!allowedKeys(schema, PROPERTY_KEYWORDS)) { + return null; + } + var type = textValue(schema, "type"); + if (!SCALAR_TYPES.contains(type)) { + return null; + } + Map result = objectMapper.convertValue(schema, + new TypeReference>() { + }); + result.keySet().retainAll(PROPERTY_KEYWORDS); + result.put("type", type); + return result; + } + + private boolean allowedKeys(JsonNode node, Set allowed) { + var fields = node.fieldNames(); + while (fields.hasNext()) { + if (!allowed.contains(fields.next())) { + return false; + } + } + return true; + } + + private void putIfText(Map target, String key, String value) { + if (StringUtils.isNotBlank(value)) { + target.put(key, value); + } + } + + private String textValue(JsonNode source, String field) { + if (source == null || source.isNull()) { + return null; + } + var value = source.get(field); + return value == null || value.isNull() ? null : StringUtils.trimToNull(value.asText()); + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentToolService.java b/src/main/java/run/halo/live2d/agent/AgentToolService.java new file mode 100644 index 0000000..0a5182b --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentToolService.java @@ -0,0 +1,224 @@ +package run.halo.live2d.agent; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import run.halo.aifoundation.schema.JsonSchema; +import run.halo.aifoundation.tool.ToolDefinition; + +@Component +@RequiredArgsConstructor +public class AgentToolService { + private final AgentToolNormalizer normalizer; + private final HaloAgentPresetToolService haloPresetToolService; + + public AgentToolSet buildTools(AgentSettings settings, AgentAccessMode accessMode, + boolean authenticated) { + if (!accessMode.agentAllowed()) { + return AgentToolSet.disabled(); + } + + var resolvedSettings = settings == null ? AgentSettings.defaults() : settings; + List tools = new ArrayList<>(); + addBuiltInBrowserTools(tools, resolvedSettings); + addHaloPresetTools(tools, resolvedSettings); + normalizer.normalizeCustomTools(resolvedSettings).stream() + .filter(tool -> isAllowed(tool.requiredAuth(), authenticated)) + .map(this::toBrowserToolDefinition) + .forEach(tools::add); + return new AgentToolSet(true, List.copyOf(tools)); + } + + private void addHaloPresetTools(List tools, AgentSettings settings) { + var builtIn = settings.builtIn(); + if (builtIn.haloContentSearch()) { + tools.add(ToolDefinition.builder() + .name("search_halo_resources") + .description("使用 Halo 自身全文搜索引擎搜索公开内容资源，适合查找文章、页面或其他已接入搜索索引的公开内容。") + .inputSchema(JsonSchema.object() + .property("keyword", JsonSchema.string().description("搜索关键词")) + .property("limit", JsonSchema.integer().description("返回数量，1 到 20")) + .property("includeTypes", JsonSchema.array(JsonSchema.string().build()) + .description("可选内容类型列表，只会使用站点允许的类型")) + .required("keyword") + .build()) + .executor(context -> haloPresetToolService.searchHaloResources(context.getInput(), + settings)) + .build()); + tools.add(ToolDefinition.builder() + .name("get_halo_resource_detail") + .description("读取已查询到的公开 Halo 资源的有限详情，用于总结或介绍资源内容。") + .inputSchema(JsonSchema.object() + .property("resourceId", JsonSchema.string().description("可信资源 ID")) + .required("resourceId") + .build()) + .executor(context -> haloPresetToolService.getHaloResourceDetail(context.getInput(), + settings)) + .build()); + tools.add(ToolDefinition.builder() + .name("get_latest_halo_resources") + .description("查看站点最新公开内容资源。第一版稳定支持最新文章。") + .inputSchema(JsonSchema.object() + .property("limit", JsonSchema.integer().description("返回数量，1 到 20")) + .build()) + .executor(context -> haloPresetToolService.getLatestHaloResources( + context.getInput())) + .build()); + tools.add(ToolDefinition.builder() + .name("get_categories") + .description("查看站点公开分类列表。") + .inputSchema(JsonSchema.object() + .property("limit", JsonSchema.integer().description("返回数量，1 到 100")) + .build()) + .executor(context -> haloPresetToolService.getCategories(context.getInput())) + .build()); + tools.add(ToolDefinition.builder() + .name("get_tags") + .description("查看站点公开标签列表。") + .inputSchema(JsonSchema.object() + .property("limit", JsonSchema.integer().description("返回数量，1 到 100")) + .build()) + .executor(context -> haloPresetToolService.getTags(context.getInput())) + .build()); + tools.add(ToolDefinition.builder() + .name("get_posts_by_category") + .description("查看指定分类下的公开文章。") + .inputSchema(JsonSchema.object() + .property("categoryName", JsonSchema.string().description("分类元数据名称")) + .property("limit", JsonSchema.integer().description("返回数量，1 到 20")) + .required("categoryName") + .build()) + .executor(context -> haloPresetToolService.getPostsByCategory(context.getInput())) + .build()); + tools.add(ToolDefinition.builder() + .name("get_posts_by_tag") + .description("查看指定标签下的公开文章。") + .inputSchema(JsonSchema.object() + .property("tagName", JsonSchema.string().description("标签元数据名称")) + .property("limit", JsonSchema.integer().description("返回数量，1 到 20")) + .required("tagName") + .build()) + .executor(context -> haloPresetToolService.getPostsByTag(context.getInput())) + .build()); + tools.add(ToolDefinition.builder() + .name("get_pages") + .description("查看站点公开独立页面列表。") + .inputSchema(JsonSchema.object() + .property("limit", JsonSchema.integer().description("返回数量，1 到 100")) + .build()) + .executor(context -> haloPresetToolService.getPages(context.getInput())) + .build()); + } + if (builtIn.networkAccess()) { + tools.add(ToolDefinition.builder() + .name("fetch_allowed_url") + .description("通过后端读取站长白名单中的公网 URL。仅支持 GET，只能访问网络访问安全策略允许的 Origin，不能访问 localhost、内网或链路本地地址。适合读取公开 API、公开 JSON、文本页面或文档摘要。") + .inputSchema(JsonSchema.object() + .property("url", JsonSchema.string().description("要读取的完整公网 URL，必须属于站长配置的允许 Origin")) + .required("url") + .build()) + .executor(context -> haloPresetToolService.fetchAllowedUrl(context.getInput(), + settings)) + .build()); + } + if (builtIn.commentCapability() == AgentCommentCapability.ASSIST + || builtIn.commentCapability() == AgentCommentCapability.SUBMIT) { + tools.add(ToolDefinition.builder() + .name("draft_comment") + .description("滚动到评论区，并将评论草稿写入当前页面的评论输入框。该工具不会自动提交评论。") + .inputSchema(JsonSchema.object() + .property("content", JsonSchema.string().description("评论草稿内容")) + .required("content") + .build()) + .build()); + } + if (builtIn.commentCapability() == AgentCommentCapability.SUBMIT) { + tools.add(ToolDefinition.builder() + .name("submit_comment") + .description("滚动到评论区，写入评论内容，并在访客审批后尝试提交评论。只有站点评论流程允许且必要校验满足时才可成功。") + .inputSchema(JsonSchema.object() + .property("content", JsonSchema.string().description("评论内容")) + .required("content") + .build()) + .requiresApproval(true) + .build()); + } + } + + public String appendCapabilityPrompt(String systemMessage, AgentToolSet toolSet) { + if (toolSet == null || !toolSet.agentEnabled() || toolSet.tools().isEmpty()) { + return systemMessage + "\n\n【Agent 能力边界】\n" + + "当前站点未向访客开放 Agent 操作能力。你可以正常聊天，但不能承诺打开页面、提交内容或控制站点功能。"; + } + return systemMessage + "\n\n【Agent 能力】\n" + + "- 你可以在工具可用时协助访客执行已授权的站点操作。\n" + + "- 只能调用当前已声明的工具；不要承诺未声明或未授权的能力。\n" + + "- 执行评论、表单填写、页面定位等依赖当前页面结构的操作前，应先读取当前页面上下文；如果页面不具备对应能力，应如实说明。\n" + + "- 需要访客确认的操作必须等待确认，不能声称已经完成。\n" + + "- 导航、搜索和评论都应以工具结果为准。"; + } + + private void addBuiltInBrowserTools(List tools, AgentSettings settings) { + var builtIn = settings.builtIn(); + if (builtIn.pageContext()) { + tools.add(ToolDefinition.builder() + .name("get_current_page_context") + .description("读取当前访客页面上下文和可操作能力，例如页面标题、地址、主要标题、选中文本、评论区/评论输入框/提交按钮是否存在、页面表单摘要和站内链接摘要。不会读取表单当前值、Cookie 或本地存储。执行评论、表单填写或页面定位前应先使用该工具判断当前页面是否支持对应操作。") + .inputSchema(JsonSchema.object().build()) + .build()); + } + if (builtIn.haloNavigation()) { + tools.add(ToolDefinition.builder() + .name("open_halo_resource") + .description("打开刚由 Halo 查询工具返回的可信资源。只能打开工具结果中出现过的资源。") + .inputSchema(JsonSchema.object() + .property("resourceId", JsonSchema.string().description("可信资源 ID")) + .required("resourceId") + .build()) + .build()); + } + if (builtIn.commentCapability() != AgentCommentCapability.OFF) { + tools.add(ToolDefinition.builder() + .name("open_comment_area") + .description("打开或滚动到当前页面评论区。该工具不会自动提交评论。") + .inputSchema(JsonSchema.object().build()) + .build()); + } + } + + private ToolDefinition toBrowserToolDefinition(NormalizedAgentTool tool) { + return ToolDefinition.builder() + .name(tool.name()) + .description(descriptionWithPolicy(tool)) + .inputSchema(tool.inputSchema()) + .build(); + } + + private String descriptionWithPolicy(NormalizedAgentTool tool) { + String description = tool.description(); + if (tool.approval() == AgentToolApproval.ALWAYS) { + description += "。该工具需要访客确认后才会执行。"; + } + if ("dispatch-event".equals(tool.actionType())) { + description += "。该工具只会触发站点声明的事件，具体行为由站点实现。"; + } + if ("registered".equals(tool.actionType())) { + description += "。该工具由站点注册的前端执行器处理。"; + } + return description; + } + + private boolean isAllowed(AgentToolAuth requiredAuth, boolean authenticated) { + return requiredAuth != AgentToolAuth.AUTHENTICATED || authenticated; + } + + public static Map emptyObjectSchema() { + Map schema = new LinkedHashMap<>(); + schema.put("type", "object"); + schema.put("properties", Map.of()); + return schema; + } +} diff --git a/src/main/java/run/halo/live2d/agent/AgentToolSet.java b/src/main/java/run/halo/live2d/agent/AgentToolSet.java new file mode 100644 index 0000000..8fa33d7 --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/AgentToolSet.java @@ -0,0 +1,13 @@ +package run.halo.live2d.agent; + +import java.util.List; +import run.halo.aifoundation.tool.ToolDefinition; + +public record AgentToolSet( + boolean agentEnabled, + List tools +) { + public static AgentToolSet disabled() { + return new AgentToolSet(false, List.of()); + } +} diff --git a/src/main/java/run/halo/live2d/agent/HaloAgentPresetToolService.java b/src/main/java/run/halo/live2d/agent/HaloAgentPresetToolService.java new file mode 100644 index 0000000..546c274 --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/HaloAgentPresetToolService.java @@ -0,0 +1,464 @@ +package run.halo.live2d.agent; + +import java.io.IOException; +import java.io.InputStream; +import java.net.IDN; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.Duration; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Locale; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; +import run.halo.app.core.extension.content.Category; +import run.halo.app.core.extension.content.Post; +import run.halo.app.core.extension.content.SinglePage; +import run.halo.app.core.extension.content.Tag; +import run.halo.app.extension.ReactiveExtensionClient; +import run.halo.app.search.HaloDocument; +import run.halo.app.search.SearchOption; +import run.halo.app.search.SearchService; + +@Component +@RequiredArgsConstructor +public class HaloAgentPresetToolService { + public static final String TYPE_POST = "post.content.halo.run"; + public static final String TYPE_SINGLE_PAGE = "singlepage.content.halo.run"; + public static final String TYPE_CATEGORY = "category.content.halo.run"; + public static final String TYPE_TAG = "tag.content.halo.run"; + + private final SearchService searchService; + private final ReactiveExtensionClient extensionClient; + private final HttpClient httpClient = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)) + .build(); + + public Mono searchHaloResources(Map input, AgentSettings settings) { + var keyword = stringInput(input, "keyword"); + if (StringUtils.isBlank(keyword)) { + return Mono.just(failure("INVALID_INPUT", "keyword is required")); + } + var search = settings.haloSearch(); + var allowedTypes = new HashSet<>(search.normalizedAllowedTypes()); + var requestedTypes = stringList(input.get("includeTypes")); + var includeTypes = requestedTypes.isEmpty() + ? List.copyOf(allowedTypes) + : requestedTypes.stream().filter(allowedTypes::contains).toList(); + + var option = new SearchOption(); + option.setKeyword(keyword); + option.setLimit(limit(input, search.normalizedDefaultLimit(), 1, 20)); + option.setFilterExposed(true); + option.setFilterPublished(true); + option.setFilterRecycled(false); + option.setIncludeTypes(includeTypes); + option.setHighlightPreTag(""); + option.setHighlightPostTag(""); + return searchService.search(option) + .map(result -> Map.of( + "ok", true, + "keyword", keyword, + "total", result.getTotal() == null ? 0 : result.getTotal(), + "resources", result.getHits() == null ? List.of() + : result.getHits().stream().map(this::fromDocument).toList() + )); + } + + public Mono getHaloResourceDetail(Map input, AgentSettings settings) { + var resourceId = stringInput(input, "resourceId"); + if (StringUtils.isBlank(resourceId)) { + return Mono.just(failure("INVALID_INPUT", "resourceId is required")); + } + var parts = resourceId.split(":", 2); + if (parts.length != 2) { + return Mono.just(failure("INVALID_RESOURCE", "resourceId is invalid")); + } + var maxChars = settings.haloResourceDetail().maxContentChars(); + if (TYPE_POST.equals(parts[0])) { + return extensionClient.fetch(Post.class, parts[1]) + .filter(this::isPublicPost) + .map(post -> detailFromPost(post, maxChars)) + .cast(Object.class) + .defaultIfEmpty(failure("RESOURCE_NOT_FOUND", "post is not available")); + } + if (TYPE_SINGLE_PAGE.equals(parts[0])) { + return extensionClient.fetch(SinglePage.class, parts[1]) + .filter(this::isPublicPage) + .map(page -> detailFromPage(page, maxChars)) + .cast(Object.class) + .defaultIfEmpty(failure("RESOURCE_NOT_FOUND", "page is not available")); + } + return Mono.just(failure("UNSUPPORTED_RESOURCE", "resource type is not supported")); + } + + public Mono getLatestHaloResources(Map input) { + var limit = limit(input, 5, 1, 20); + return extensionClient.list(Post.class, this::isPublicPost, + Comparator.comparing(this::postTime, Comparator.nullsLast(Comparator.naturalOrder())) + .reversed()) + .take(limit) + .map(this::fromPost) + .collectList() + .map(resources -> Map.of("ok", true, "resources", resources)); + } + + public Mono getCategories(Map input) { + var limit = limit(input, 20, 1, 100); + return extensionClient.list(Category.class, category -> !category.isDeleted(), + Comparator.comparing(category -> category.getSpec().getDisplayName(), + Comparator.nullsLast(String::compareToIgnoreCase))) + .take(limit) + .map(this::fromCategory) + .collectList() + .map(resources -> Map.of("ok", true, "resources", resources)); + } + + public Mono getTags(Map input) { + var limit = limit(input, 20, 1, 100); + return extensionClient.list(Tag.class, tag -> true, + Comparator.comparing(tag -> tag.getSpec().getDisplayName(), + Comparator.nullsLast(String::compareToIgnoreCase))) + .take(limit) + .map(this::fromTag) + .collectList() + .map(resources -> Map.of("ok", true, "resources", resources)); + } + + public Mono getPostsByCategory(Map input) { + var category = stringInput(input, "categoryName"); + var limit = limit(input, 10, 1, 20); + return extensionClient.list(Post.class, + post -> isPublicPost(post) && contains(post.getSpec().getCategories(), category), + Comparator.comparing(this::postTime, Comparator.nullsLast(Comparator.naturalOrder())) + .reversed()) + .take(limit) + .map(this::fromPost) + .collectList() + .map(resources -> Map.of("ok", true, "resources", resources)); + } + + public Mono getPostsByTag(Map input) { + var tag = stringInput(input, "tagName"); + var limit = limit(input, 10, 1, 20); + return extensionClient.list(Post.class, + post -> isPublicPost(post) && contains(post.getSpec().getTags(), tag), + Comparator.comparing(this::postTime, Comparator.nullsLast(Comparator.naturalOrder())) + .reversed()) + .take(limit) + .map(this::fromPost) + .collectList() + .map(resources -> Map.of("ok", true, "resources", resources)); + } + + public Mono getPages(Map input) { + var limit = limit(input, 20, 1, 100); + return extensionClient.list(SinglePage.class, this::isPublicPage, + Comparator.comparing(page -> page.getSpec().getTitle(), + Comparator.nullsLast(String::compareToIgnoreCase))) + .take(limit) + .map(this::fromPage) + .collectList() + .map(resources -> Map.of("ok", true, "resources", resources)); + } + + public Mono fetchAllowedUrl(Map input, AgentSettings settings) { + return Mono.fromCallable(() -> validateNetworkTarget(stringInput(input, "url"), settings)) + .subscribeOn(Schedulers.boundedElastic()) + .flatMap(target -> { + var request = HttpRequest.newBuilder(target.uri()) + .GET() + .timeout(Duration.ofSeconds( + settings.networkAccess().normalizedTimeoutSeconds())) + .header("Accept", "text/plain, application/json, application/xml, text/*;q=0.9, */*;q=0.1") + .header("User-Agent", "Halo-Live2D-Agent/1.0") + .build(); + return Mono.fromFuture(httpClient.sendAsync(request, + HttpResponse.BodyHandlers.ofInputStream())) + .map(response -> readNetworkResponse(response, + settings.networkAccess().normalizedMaxResponseChars())) + .cast(Object.class); + }) + .onErrorResume(IllegalArgumentException.class, + throwable -> Mono.just(failure("NETWORK_ACCESS_DENIED", throwable.getMessage()))) + .onErrorResume(IOException.class, + throwable -> Mono.just(failure("NETWORK_READ_FAILED", "网络响应读取失败"))) + .onErrorResume(Exception.class, + throwable -> Mono.just(failure("NETWORK_REQUEST_FAILED", "后端网络请求失败"))); + } + + public Object commentUnavailable(String message) { + return failure("COMMENT_FLOW_REQUIRED", message); + } + + private Map fromDocument(HaloDocument document) { + return resource( + document.getType() + ":" + document.getMetadataName(), + document.getType(), + document.getMetadataName(), + document.getTitle(), + StringUtils.defaultIfBlank(document.getDescription(), excerpt(document.getContent(), 220)), + document.getPermalink() + ); + } + + private Map fromPost(Post post) { + return resource( + TYPE_POST + ":" + post.getMetadata().getName(), + TYPE_POST, + post.getMetadata().getName(), + post.getSpec().getTitle(), + excerpt(StringUtils.defaultIfBlank(post.getStatusOrDefault().getExcerpt(), + Optional.ofNullable(post.getSpec().getExcerpt()).map(Post.Excerpt::getRaw).orElse(null)), 220), + post.getStatusOrDefault().getPermalink() + ); + } + + private Map fromPage(SinglePage page) { + return resource( + TYPE_SINGLE_PAGE + ":" + page.getMetadata().getName(), + TYPE_SINGLE_PAGE, + page.getMetadata().getName(), + page.getSpec().getTitle(), + excerpt(StringUtils.defaultIfBlank(page.getStatusOrDefault().getExcerpt(), + Optional.ofNullable(page.getSpec().getExcerpt()).map(Post.Excerpt::getRaw).orElse(null)), 220), + page.getStatusOrDefault().getPermalink() + ); + } + + private Map fromCategory(Category category) { + return resource( + TYPE_CATEGORY + ":" + category.getMetadata().getName(), + TYPE_CATEGORY, + category.getMetadata().getName(), + category.getSpec().getDisplayName(), + category.getSpec().getDescription(), + category.getStatusOrDefault().getPermalink() + ); + } + + private Map fromTag(Tag tag) { + return resource( + TYPE_TAG + ":" + tag.getMetadata().getName(), + TYPE_TAG, + tag.getMetadata().getName(), + tag.getSpec().getDisplayName(), + tag.getSpec().getDescription(), + tag.getStatusOrDefault().getPermalink() + ); + } + + private Map detailFromPost(Post post, int maxChars) { + var content = excerpt(StringUtils.defaultIfBlank(post.getStatusOrDefault().getExcerpt(), + Optional.ofNullable(post.getSpec().getExcerpt()).map(Post.Excerpt::getRaw).orElse("")), maxChars); + return Map.of("ok", true, "resource", fromPost(post), "content", content, + "truncated", content.length() >= maxChars); + } + + private Map detailFromPage(SinglePage page, int maxChars) { + var content = excerpt(StringUtils.defaultIfBlank(page.getStatusOrDefault().getExcerpt(), + Optional.ofNullable(page.getSpec().getExcerpt()).map(Post.Excerpt::getRaw).orElse("")), maxChars); + return Map.of("ok", true, "resource", fromPage(page), "content", content, + "truncated", content.length() >= maxChars); + } + + private Map resource(String id, String type, String metadataName, String title, + String excerpt, String permalink) { + return Map.of( + "resourceId", id, + "resourceType", type, + "metadataName", metadataName, + "title", StringUtils.defaultString(title), + "excerpt", StringUtils.defaultString(excerpt), + "permalink", StringUtils.defaultString(permalink) + ); + } + + private Map failure(String code, String message) { + return Map.of("ok", false, "errorCode", code, "message", message); + } + + private NetworkTarget validateNetworkTarget(String rawUrl, AgentSettings settings) { + if (StringUtils.isBlank(rawUrl)) { + throw new IllegalArgumentException("url is required"); + } + URI uri; + try { + uri = new URI(rawUrl.trim()).normalize(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("url is invalid"); + } + if (!"https".equalsIgnoreCase(uri.getScheme()) + && !"http".equalsIgnoreCase(uri.getScheme())) { + throw new IllegalArgumentException("只允许访问 http 或 https URL"); + } + if (StringUtils.isNotBlank(uri.getUserInfo())) { + throw new IllegalArgumentException("URL 不能包含用户信息"); + } + var host = StringUtils.trimToNull(uri.getHost()); + if (host == null) { + throw new IllegalArgumentException("URL host is required"); + } + var normalizedOrigin = originOf(uri); + var allowedOrigins = settings.networkAccess().normalizedAllowedOrigins().stream() + .map(this::normalizeOrigin) + .flatMap(Optional::stream) + .toList(); + if (allowedOrigins.isEmpty() || !allowedOrigins.contains(normalizedOrigin)) { + throw new IllegalArgumentException("URL Origin 未被站长允许"); + } + if (isDangerousHost(host)) { + throw new IllegalArgumentException("禁止访问 localhost、内网或链路本地地址"); + } + try { + for (var address : InetAddress.getAllByName(host)) { + if (isDangerousAddress(address)) { + throw new IllegalArgumentException("禁止访问 localhost、内网或链路本地地址"); + } + } + } catch (IOException e) { + throw new IllegalArgumentException("无法解析目标地址"); + } + return new NetworkTarget(uri, normalizedOrigin); + } + + private Map readNetworkResponse(HttpResponse response, + int maxChars) { + try (var body = response.body()) { + var bytes = body.readNBytes(maxChars + 1); + var truncated = bytes.length > maxChars; + var length = Math.min(bytes.length, maxChars); + var text = new String(bytes, 0, length, StandardCharsets.UTF_8); + return Map.of( + "ok", true, + "url", response.uri().toString(), + "status", response.statusCode(), + "contentType", response.headers().firstValue("content-type").orElse(""), + "body", text, + "truncated", truncated + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Optional normalizeOrigin(String rawOrigin) { + if (StringUtils.isBlank(rawOrigin)) { + return Optional.empty(); + } + try { + return Optional.of(originOf(new URI(rawOrigin.trim()))); + } catch (URISyntaxException | IllegalArgumentException e) { + return Optional.empty(); + } + } + + private String originOf(URI uri) { + var scheme = StringUtils.lowerCase(uri.getScheme(), Locale.ROOT); + var host = IDN.toASCII(uri.getHost()).toLowerCase(Locale.ROOT); + var port = uri.getPort(); + var defaultPort = "https".equals(scheme) ? 443 : 80; + return port < 0 || port == defaultPort + ? scheme + "://" + host + : scheme + "://" + host + ":" + port; + } + + private boolean isDangerousHost(String host) { + var normalized = host.toLowerCase(Locale.ROOT); + return "localhost".equals(normalized) + || normalized.endsWith(".localhost") + || normalized.endsWith(".local"); + } + + private boolean isDangerousAddress(InetAddress address) { + return address.isAnyLocalAddress() + || address.isLoopbackAddress() + || address.isLinkLocalAddress() + || address.isSiteLocalAddress() + || address.isMulticastAddress() + || isUniqueLocalIpv6(address); + } + + private boolean isUniqueLocalIpv6(InetAddress address) { + if (!(address instanceof Inet6Address)) { + return false; + } + var firstByte = address.getAddress()[0] & 0xff; + return (firstByte & 0xfe) == 0xfc; + } + + private boolean isPublicPost(Post post) { + return post != null + && post.getSpec() != null + && post.isPublished() + && !post.isDeleted() + && Post.isPublic(post.getSpec()); + } + + private boolean isPublicPage(SinglePage page) { + return page != null + && page.getSpec() != null + && page.isPublished() + && !Boolean.TRUE.equals(page.getSpec().getDeleted()) + && (page.getSpec().getVisible() == null + || Post.VisibleEnum.PUBLIC.equals(page.getSpec().getVisible())); + } + + private Instant postTime(Post post) { + return Optional.ofNullable(post.getSpec().getPublishTime()) + .orElse(post.getMetadata().getCreationTimestamp()); + } + + private boolean contains(List values, String value) { + return StringUtils.isBlank(value) + || values != null && values.stream().anyMatch(item -> Objects.equals(item, value)); + } + + private int limit(Map input, int defaultValue, int min, int max) { + var value = input == null ? null : input.get("limit"); + int parsed = value instanceof Number number ? number.intValue() : defaultValue; + return Math.max(min, Math.min(max, parsed)); + } + + private String stringInput(Map input, String key) { + var value = input == null ? null : input.get(key); + return value == null ? null : StringUtils.trimToNull(String.valueOf(value)); + } + + private List stringList(Object value) { + if (!(value instanceof List list)) { + return List.of(); + } + return list.stream() + .filter(String.class::isInstance) + .map(String.class::cast) + .filter(StringUtils::isNotBlank) + .toList(); + } + + private String excerpt(String content, int maxChars) { + var text = StringUtils.defaultString(content).replaceAll("\\s+", " ").trim(); + if (text.length() <= maxChars) { + return text; + } + return text.substring(0, maxChars); + } + + private record NetworkTarget(URI uri, String origin) { + } +} diff --git a/src/main/java/run/halo/live2d/agent/NormalizedAgentTool.java b/src/main/java/run/halo/live2d/agent/NormalizedAgentTool.java new file mode 100644 index 0000000..7178dff --- /dev/null +++ b/src/main/java/run/halo/live2d/agent/NormalizedAgentTool.java @@ -0,0 +1,16 @@ +package run.halo.live2d.agent; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Map; + +public record NormalizedAgentTool( + String name, + String description, + Map inputSchema, + AgentToolApproval approval, + AgentToolAuth requiredAuth, + String actionType, + JsonNode action, + JsonNode testInput +) { +} diff --git a/src/main/java/run/halo/live2d/chat/AIChatServiceImpl.java b/src/main/java/run/halo/live2d/chat/AIChatServiceImpl.java index 3c3374c..ca7f1db 100644 --- a/src/main/java/run/halo/live2d/chat/AIChatServiceImpl.java +++ b/src/main/java/run/halo/live2d/chat/AIChatServiceImpl.java @@ -6,17 +6,20 @@ import org.springframework.stereotype.Component; import reactor.core.publisher.Mono; import run.halo.aifoundation.AiModelService; -import run.halo.aifoundation.chat.ReasoningOptions; +import run.halo.aifoundation.chat.StopCondition; import run.halo.aifoundation.exception.ModelDisabledException; import run.halo.aifoundation.exception.ModelNotFoundException; import run.halo.aifoundation.exception.ProviderApiException; import run.halo.aifoundation.exception.ProviderDisabledException; +import run.halo.aifoundation.ui.InvalidUIMessageException; import run.halo.aifoundation.ui.UIMessageChatHandlers; import run.halo.aifoundation.ui.UIMessageChatRequest; import run.halo.aifoundation.ui.UIMessageChunks; import run.halo.aifoundation.ui.UIMessageStreamResponse; import run.halo.aifoundation.ui.UIMessageStreams; +import run.halo.aifoundation.tool.ToolChoice; import run.halo.app.plugin.extensionpoint.ExtensionGetter; +import run.halo.live2d.agent.AgentToolSet; @Slf4j @Component @@ -31,14 +34,11 @@ private Mono aiModelService() { @Override public Mono streamChatCompletion(String modelName, - String systemMessage, UIMessageChatRequest chatRequest) { + String systemMessage, UIMessageChatRequest chatRequest, AgentToolSet agentToolSet) { if (StringUtils.isBlank(modelName)) { return Mono.just(errorResponse("请先在插件设置中配置 Halo AI 模型")); } - log.debug("Stream Halo AI text generation with model: {}, messages: {}", modelName, - chatRequest.messages()); - return aiModelService() .flatMap(service -> service.languageModel(modelName)) .map(model -> { @@ -47,14 +47,18 @@ public Mono streamChatCompletion(String modelName, .chatRequest(chatRequest) .request(builder -> builder .system(systemMessage) - .reasoning(ReasoningOptions.disabled())) + .tools(agentToolSet == null ? null : agentToolSet.tools()) + .toolChoice(agentToolSet == null || agentToolSet.tools().isEmpty() + ? ToolChoice.none() + : ToolChoice.auto()) + .stopWhen(agentToolSet == null || agentToolSet.tools().isEmpty() + ? null + : StopCondition.stepCountIs(3))) .onError(this::resolveErrorMessage)); return chat.response(); }) .onErrorResume(throwable -> { - log.error("Error occurred while generating Halo AI chat result, model: {}", - modelName, - throwable); + logChatError(modelName, throwable); return Mono.just(errorResponse(resolveErrorMessage(throwable))); }); } @@ -71,6 +75,9 @@ private String resolveErrorMessageText(String message) { } private String resolveErrorMessage(Throwable throwable) { + if (throwable instanceof InvalidUIMessageException) { + return "对话上下文异常了，请刷新后重试"; + } if (throwable instanceof ModelNotFoundException || throwable instanceof ModelDisabledException) { return "当前配置的 Halo AI 模型不可用，请联系站长检查配置"; @@ -87,4 +94,14 @@ private String resolveErrorMessage(Throwable throwable) { return StringUtils.defaultIfBlank(throwable.getMessage(), "对话接口异常了哦～快去联系我的主人吧！"); } + + private void logChatError(String modelName, Throwable throwable) { + if (throwable instanceof InvalidUIMessageException invalid) { + log.error("Invalid Halo UI messages for model: {}, issues: {}", modelName, + invalid.issues(), throwable); + return; + } + log.error("Error occurred while generating Halo AI chat result, model: {}", + modelName, throwable); + } } diff --git a/src/main/java/run/halo/live2d/chat/AiChatEndpoint.java b/src/main/java/run/halo/live2d/chat/AiChatEndpoint.java index 77791a6..5d3c8c8 100644 --- a/src/main/java/run/halo/live2d/chat/AiChatEndpoint.java +++ b/src/main/java/run/halo/live2d/chat/AiChatEndpoint.java @@ -32,6 +32,9 @@ import run.halo.aifoundation.ui.UIMessageChunk; import run.halo.aifoundation.ui.UIMessageStreamResponse; import run.halo.aifoundation.ui.UIMessageTransportCodec; +import run.halo.live2d.agent.AgentAccessMode; +import run.halo.live2d.agent.AgentSettings; +import run.halo.live2d.agent.AgentToolService; @Slf4j @Component @@ -42,6 +45,8 @@ public class AiChatEndpoint implements CustomEndpoint { private final AiChatService aiChatService; + private final AgentToolService agentToolService; + private final ObjectMapper objectMapper = new ObjectMapper(); @Override @@ -95,10 +100,17 @@ private Mono chatCompletion(UIMessageChatRequest } var baseSetting = aiChatConfig.aiChatBaseSetting(); - - if (baseSetting.isAnonymous()) { - return aiChatService.streamChatCompletion( - baseSetting.modelName(), baseSetting.systemMessage(), chatRequest); + var accessMode = baseSetting.resolvedAccessMode(); + + if (!accessMode.authenticationRequired()) { + return loadAgentSettings() + .map(settings -> agentToolService.buildTools(settings, accessMode, false)) + .flatMap(toolSet -> aiChatService.streamChatCompletion( + baseSetting.modelName(), + agentToolService.appendCapabilityPrompt( + baseSetting.systemMessage(), toolSet), + chatRequest, + toolSet)); } return ReactiveSecurityContextHolder.getContext() @@ -106,11 +118,24 @@ private Mono chatCompletion(UIMessageChatRequest .filter(this::isAuthenticated) .switchIfEmpty(Mono.error( new ResponseStatusException(HttpStatus.UNAUTHORIZED, "请先登录"))) - .flatMap(authentication -> aiChatService.streamChatCompletion( - baseSetting.modelName(), baseSetting.systemMessage(), chatRequest)); + .flatMap(authentication -> loadAgentSettings() + .map(settings -> agentToolService.buildTools(settings, accessMode, true)) + .flatMap(toolSet -> aiChatService.streamChatCompletion( + baseSetting.modelName(), + agentToolService.appendCapabilityPrompt( + baseSetting.systemMessage(), toolSet), + chatRequest, + toolSet))); }); } + private Mono loadAgentSettings() { + return reactiveSettingFetcher.get("agent") + .map(node -> objectMapper.convertValue(node, AgentSettings.class)) + .defaultIfEmpty(AgentSettings.defaults()) + .onErrorReturn(AgentSettings.defaults()); + } + private boolean isAuthenticated(Authentication authentication) { return !isAnonymousUser(authentication.getName()) && authentication.isAuthenticated(); @@ -142,7 +167,8 @@ record AiChatConfig(boolean isAiChat, AiChatBaseSetting aiChatBaseSetting) { } } - record AiChatBaseSetting(boolean isAnonymous, String systemMessage, String modelName) { + record AiChatBaseSetting(boolean isAnonymous, String accessMode, String systemMessage, + String modelName) { AiChatBaseSetting { if (StringUtils.isBlank(systemMessage)) { throw new IllegalArgumentException("system message must not be null"); @@ -151,6 +177,10 @@ record AiChatBaseSetting(boolean isAnonymous, String systemMessage, String model throw new IllegalArgumentException("model name must not be null"); } } + + AgentAccessMode resolvedAccessMode() { + return AgentAccessMode.from(accessMode, isAnonymous); + } } @Override diff --git a/src/main/java/run/halo/live2d/chat/AiChatService.java b/src/main/java/run/halo/live2d/chat/AiChatService.java index 61e5db7..cb4e01a 100644 --- a/src/main/java/run/halo/live2d/chat/AiChatService.java +++ b/src/main/java/run/halo/live2d/chat/AiChatService.java @@ -3,11 +3,12 @@ import reactor.core.publisher.Mono; import run.halo.aifoundation.ui.UIMessageChatRequest; import run.halo.aifoundation.ui.UIMessageStreamResponse; +import run.halo.live2d.agent.AgentToolSet; public interface AiChatService { Mono streamChatCompletion(String modelName, String systemMessage, - UIMessageChatRequest chatRequest); + UIMessageChatRequest chatRequest, AgentToolSet agentToolSet); UIMessageStreamResponse errorResponse(String message); } diff --git a/src/main/resources/extensions/settings.yaml b/src/main/resources/extensions/settings.yaml index 3731eb4..1a19018 100644 --- a/src/main/resources/extensions/settings.yaml +++ b/src/main/resources/extensions/settings.yaml @@ -165,7 +165,7 @@ spec: if: "$get(isAiChat).value === true" label: 聊天基本设置 value: - isAnonymous: true + accessMode: anonymous_chat modelName: "" chatContextRounds: 20 requestAcceptedMessage: "收到啦，马上就来陪你啦～" @@ -200,11 +200,21 @@ spec: - 在合适场景下可以表现出贴心、活泼、害羞、关心等情绪，但不要夸张失控。 chunkTimeout: 10 showChatMessageTimeout: 10 + autoContinuationMessageMinVisibleMs: 1500 children: - - $formkit: switch - label: 是否开启公共聊天 - help: 关闭后，用户需要登录后才能与 Live2d 进行对话 - name: isAnonymous + - $formkit: select + label: 访问模式 + help: 控制访客能否对话，以及是否允许 Live2D Agent 操作站点能力。旧版公共聊天开启会按“匿名用户可对话”兼容。 + name: accessMode + options: + - value: anonymous_chat + label: 匿名用户可对话 + - value: anonymous_chat_agent + label: 匿名用户可对话和使用 Agent + - value: authenticated_chat + label: 登录用户可对话 + - value: authenticated_chat_agent + label: 登录用户可对话和使用 Agent - $formkit: aiModelSelector name: modelName label: 对话模型 @@ -254,6 +264,142 @@ spec: help: 获取到完整消息后，看板娘展示的时间 name: showChatMessageTimeout validation: Number + - $formkit: text + label: Agent 自动续写前消息最短展示时间（毫秒） + help: Agent 自动调用工具并继续回复时，上一段看板娘回复至少展示这段时间，避免被后续回复过快覆盖；设置为 0 表示不延迟 + name: autoContinuationMessageMinVisibleMs + validation: Number|between:0,10000 + - group: agent + label: Agent 能力 + formSchema: + - $formkit: group + name: builtIn + label: 预设能力 + value: + pageContext: true + haloNavigation: true + haloContentSearch: true + networkAccess: false + commentCapability: assist + children: + - $formkit: switch + name: pageContext + label: 当前页面上下文 + help: 允许模型读取当前页面标题、地址和选中文本等基础上下文 + - $formkit: switch + name: haloNavigation + label: Halo 导航 + help: 允许模型打开可信的 Halo 资源链接 + - $formkit: switch + name: haloContentSearch + label: Halo 内容搜索 + help: 允许模型通过 Halo 搜索引擎查询公开内容 + - $formkit: switch + name: networkAccess + label: 网络访问 + help: 允许模型通过后端工具读取站长白名单中的公网 URL。默认关闭，且会阻止 localhost、内网、链路本地地址和未授权 Origin。 + - $formkit: select + name: commentCapability + label: 评论能力 + help: 关闭：不向模型开放评论相关工具。辅助：允许模型打开评论区、读取当前评论状态并生成评论草稿，但不提交。提交：在站点允许且访客确认后，允许模型尝试提交评论。 + value: assist + options: + - value: off + label: 关闭 + - value: assist + label: 辅助 + - value: submit + label: 提交 + - $formkit: group + name: toolSecurity + label: 安全策略 + value: + allowedExternalOrigins: [] + allowNewTab: false + children: + - $formkit: repeater + name: allowedExternalOrigins + label: 允许打开的外部站点 + help: 仅填写 origin，例如 https://github.com。外链仍会默认请求访客确认。 + value: [] + children: + - $formkit: text + name: origin + label: Origin + - $formkit: switch + name: allowNewTab + label: 允许新窗口打开 + value: false + - $formkit: group + name: haloSearch + label: Halo 搜索 + value: + allowedTypes: + - post.content.halo.run + - singlepage.content.halo.run + defaultLimit: 5 + children: + - $formkit: checkbox + name: allowedTypes + label: 允许搜索的内容类型 + help: 只有选中的公开内容类型会暴露给 Agent 搜索。Moment 等类型需要对应插件已接入 Halo 搜索索引。 + value: + - post.content.halo.run + - singlepage.content.halo.run + options: + - value: post.content.halo.run + label: 文章 + - value: singlepage.content.halo.run + label: 独立页面 + - value: moment.moment.halo.run + label: 瞬间 + - $formkit: text + name: defaultLimit + label: 默认返回数量 + validation: Number|between:1,20 + - $formkit: group + name: networkAccess + label: 网络访问安全策略 + value: + allowedOrigins: [] + maxResponseChars: 4000 + timeoutSeconds: 5 + children: + - $formkit: repeater + name: allowedOrigins + label: 允许访问的 Origin + help: 仅填写可信公网 Origin，例如 https://api.example.com。不会跟随重定向，也不会访问 localhost、内网或链路本地地址。 + value: [] + children: + - $formkit: text + name: origin + label: Origin + - $formkit: text + name: maxResponseChars + label: 最大响应字符数 + help: 超出部分会被截断，范围 1000 到 20000，默认 4000 + validation: Number|between:1000,20000 + - $formkit: text + name: timeoutSeconds + label: 请求超时（秒） + help: 后端网络请求超时时间，范围 1 到 15，默认 5 + validation: Number|between:1,15 + - $formkit: group + name: haloResourceDetail + label: 资源详情 + value: + maxContentChars: 3000 + children: + - $formkit: text + name: maxContentChars + label: 最大正文字符数 + validation: Number|between:500,10000 + - $formkit: textarea + name: aiTools + label: 自定义 Agent 工具 JSON + help: 高级配置项。填写工具数组 JSON，支持 navigate、scroll-to、highlight、dispatch-event、registered。 + rows: 12 + value: "[]" - group: advanced label: 高级设置 formSchema: diff --git a/src/test/java/run/halo/live2d/agent/AgentToolNormalizerTest.java b/src/test/java/run/halo/live2d/agent/AgentToolNormalizerTest.java new file mode 100644 index 0000000..331dece --- /dev/null +++ b/src/test/java/run/halo/live2d/agent/AgentToolNormalizerTest.java @@ -0,0 +1,130 @@ +package run.halo.live2d.agent; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import org.junit.jupiter.api.Test; + +class AgentToolNormalizerTest { + private final ObjectMapper objectMapper = new ObjectMapper(); + private final AgentToolNormalizer normalizer = new AgentToolNormalizer(); + + @Test + void mapsLegacyAccessModeToChatOnlyModes() { + assertThat(AgentAccessMode.from(null, true)).isEqualTo(AgentAccessMode.ANONYMOUS_CHAT); + assertThat(AgentAccessMode.from(null, false)).isEqualTo( + AgentAccessMode.AUTHENTICATED_CHAT); + } + + @Test + void mapsFormValuesToAgentEnums() throws Exception { + var settings = objectMapper.readValue( + """ + { + "builtIn": { + "commentCapability": "assist" + }, + "aiTools": [{ + "name": "open_contact_form", + "enabled": true, + "description": "打开留言面板", + "approval": "always", + "requiredAuth": "authenticated", + "action": { + "type": "registered" + } + }] + } + """, + AgentSettings.class); + + assertThat(settings.builtIn().commentCapability()).isEqualTo(AgentCommentCapability.ASSIST); + assertThat(settings.normalizedAiTools()).hasSize(1); + assertThat(settings.normalizedAiTools().getFirst().approval()) + .isEqualTo(AgentToolApproval.ALWAYS); + assertThat(settings.normalizedAiTools().getFirst().requiredAuth()) + .isEqualTo(AgentToolAuth.AUTHENTICATED); + assertThat(objectMapper.writeValueAsString(settings.builtIn())) + .contains("\"commentCapability\":\"assist\""); + } + + @Test + void normalizesValidCustomTool() throws Exception { + var settings = new AgentSettings( + null, + """ + [{ + "name": "open_contact_form", + "enabled": true, + "description": "打开留言面板", + "inputSchema": { + "type": "object", + "properties": { + "message": { + "type": "string", + "description": "留言草稿" + } + }, + "required": ["message"] + }, + "action": { + "type": "dispatch-event", + "event": "site:open-contact-form" + } + }] + """, + null, + null, + null, + null + ); + + var tools = normalizer.normalizeCustomTools(settings); + + assertThat(tools).hasSize(1); + assertThat(tools.getFirst().name()).isEqualTo("open_contact_form"); + assertThat(tools.getFirst().actionType()).isEqualTo("dispatch-event"); + assertThat(tools.getFirst().inputSchema()).containsEntry("type", "object"); + } + + @Test + void rejectsUnsupportedSchemaAndReservedNames() throws Exception { + var tools = List.of( + new AgentSettings.AgentToolConfig( + "perform_live2d_action", + true, + "reserved", + objectMapper.readTree("{\"type\":\"object\",\"properties\":{}}"), + AgentToolApproval.DEFAULT, + AgentToolAuth.NONE, + objectMapper.readTree("{\"type\":\"registered\"}"), + null + ), + new AgentSettings.AgentToolConfig( + "open_bad_tool", + true, + "bad schema", + objectMapper.readTree( + "{\"type\":\"object\",\"properties\":{\"items\":{\"type\":\"array\"}}}"), + AgentToolApproval.DEFAULT, + AgentToolAuth.NONE, + objectMapper.readTree("{\"type\":\"registered\"}"), + null + ), + new AgentSettings.AgentToolConfig( + "old_live2d_action", + true, + "unsupported action", + objectMapper.readTree("{\"type\":\"object\",\"properties\":{}}"), + AgentToolApproval.DEFAULT, + AgentToolAuth.NONE, + objectMapper.readTree("{\"type\":\"perform-live2d-action\"}"), + null + ) + ); + var settings = new AgentSettings(null, tools, null, null, null, null); + + assertThat(normalizer.normalizeCustomTools(settings)).isEmpty(); + } +} diff --git a/src/test/java/run/halo/live2d/agent/AgentToolServiceTest.java b/src/test/java/run/halo/live2d/agent/AgentToolServiceTest.java new file mode 100644 index 0000000..3880b23 --- /dev/null +++ b/src/test/java/run/halo/live2d/agent/AgentToolServiceTest.java @@ -0,0 +1,119 @@ +package run.halo.live2d.agent; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import org.junit.jupiter.api.Test; + +class AgentToolServiceTest { + private final ObjectMapper objectMapper = new ObjectMapper(); + private final AgentToolNormalizer normalizer = new AgentToolNormalizer(); + + @Test + void doesNotExposeToolsWhenAgentDisabled() { + var service = new AgentToolService(normalizer, null); + + var tools = service.buildTools(AgentSettings.defaults(), AgentAccessMode.ANONYMOUS_CHAT, + false); + + assertThat(tools.agentEnabled()).isFalse(); + assertThat(tools.tools()).isEmpty(); + } + + @Test + void filtersAuthenticatedCustomToolsForAnonymousVisitors() throws Exception { + var service = new AgentToolService(normalizer, null); + var settings = new AgentSettings( + new AgentSettings.AgentBuiltInCapabilities(false, false, false, false, + AgentCommentCapability.OFF), + List.of(new AgentSettings.AgentToolConfig( + "secure_action", + true, + "登录后可用能力", + objectMapper.readTree("{\"type\":\"object\",\"properties\":{}}"), + AgentToolApproval.DEFAULT, + AgentToolAuth.AUTHENTICATED, + objectMapper.readTree("{\"type\":\"registered\"}"), + null + )), + null, + null, + null, + null + ); + + assertThat(service.buildTools(settings, AgentAccessMode.ANONYMOUS_CHAT_AGENT, false) + .tools()).isEmpty(); + assertThat(service.buildTools(settings, AgentAccessMode.ANONYMOUS_CHAT_AGENT, true) + .tools()).extracting("name").containsExactly("secure_action"); + } + + @Test + void assistCommentCapabilityExposesDraftButNotSubmitTool() { + var service = new AgentToolService(normalizer, null); + var settings = new AgentSettings( + new AgentSettings.AgentBuiltInCapabilities(false, false, false, false, + AgentCommentCapability.ASSIST), + List.of(), + null, + null, + null, + null + ); + + var tools = service.buildTools(settings, AgentAccessMode.ANONYMOUS_CHAT_AGENT, false) + .tools(); + + assertThat(tools).extracting("name") + .contains("open_comment_area", "draft_comment") + .doesNotContain("submit_comment"); + assertThat(tools.stream().filter(tool -> "draft_comment".equals(tool.getName())) + .findFirst().orElseThrow().getExecutor()).isNull(); + } + + @Test + void submitCommentCapabilityExposesApprovedBrowserSubmitTool() { + var service = new AgentToolService(normalizer, null); + var settings = new AgentSettings( + new AgentSettings.AgentBuiltInCapabilities(false, false, false, false, + AgentCommentCapability.SUBMIT), + List.of(), + null, + null, + null, + null + ); + + var submitTool = service.buildTools(settings, AgentAccessMode.ANONYMOUS_CHAT_AGENT, false) + .tools() + .stream() + .filter(tool -> "submit_comment".equals(tool.getName())) + .findFirst() + .orElseThrow(); + + assertThat(submitTool.getExecutor()).isNull(); + assertThat(submitTool.getApprovalPolicy()).isNotNull(); + } + + @Test + void networkAccessToolIsExposedOnlyWhenEnabled() { + var service = new AgentToolService(normalizer, null); + var disabled = AgentSettings.defaults(); + var enabled = new AgentSettings( + new AgentSettings.AgentBuiltInCapabilities(false, false, false, true, + AgentCommentCapability.OFF), + List.of(), + null, + null, + null, + new AgentSettings.AgentNetworkAccessSettings( + List.of("https://api.example.com"), 4000, 5) + ); + + assertThat(service.buildTools(disabled, AgentAccessMode.ANONYMOUS_CHAT_AGENT, false) + .tools()).extracting("name").doesNotContain("fetch_allowed_url"); + assertThat(service.buildTools(enabled, AgentAccessMode.ANONYMOUS_CHAT_AGENT, false) + .tools()).extracting("name").contains("fetch_allowed_url"); + } +} diff --git a/src/test/java/run/halo/live2d/agent/HaloAgentPresetToolServiceTest.java b/src/test/java/run/halo/live2d/agent/HaloAgentPresetToolServiceTest.java new file mode 100644 index 0000000..516f6ef --- /dev/null +++ b/src/test/java/run/halo/live2d/agent/HaloAgentPresetToolServiceTest.java @@ -0,0 +1,107 @@ +package run.halo.live2d.agent; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; +import run.halo.app.extension.ReactiveExtensionClient; +import run.halo.app.search.HaloDocument; +import run.halo.app.search.SearchOption; +import run.halo.app.search.SearchResult; +import run.halo.app.search.SearchService; + +class HaloAgentPresetToolServiceTest { + + @Test + @SuppressWarnings("unchecked") + void searchHaloResourcesUsesSearchServiceAndMapsBoundedResources() { + var searchService = mock(SearchService.class); + var extensionClient = mock(ReactiveExtensionClient.class); + var service = new HaloAgentPresetToolService(searchService, extensionClient); + var result = new SearchResult(); + result.setKeyword("Halo"); + result.setTotal(1L); + result.setHits(List.of(document())); + when(searchService.search(any(SearchOption.class))).thenReturn(Mono.just(result)); + + var output = (Map) service.searchHaloResources( + Map.of("keyword", "Halo", "includeTypes", List.of("post.content.halo.run")), + AgentSettings.defaults()).block(); + + assertThat(output).containsEntry("ok", true); + var resources = (List>) output.get("resources"); + assertThat(resources).hasSize(1); + assertThat(resources.getFirst()) + .containsEntry("resourceId", "post.content.halo.run:demo-post") + .containsEntry("title", "Demo Post") + .containsEntry("permalink", "https://example.com/archives/demo"); + verify(searchService).search(any(SearchOption.class)); + } + + @Test + @SuppressWarnings("unchecked") + void fetchAllowedUrlRequiresAllowedOrigin() { + var service = new HaloAgentPresetToolService(mock(SearchService.class), + mock(ReactiveExtensionClient.class)); + + var output = (Map) service.fetchAllowedUrl( + Map.of("url", "https://api.example.com/data"), + AgentSettings.defaults()).block(); + + assertThat(output) + .containsEntry("ok", false) + .containsEntry("errorCode", "NETWORK_ACCESS_DENIED"); + } + + @Test + @SuppressWarnings("unchecked") + void fetchAllowedUrlRejectsPrivateTargetsEvenWhenAllowed() { + var service = new HaloAgentPresetToolService(mock(SearchService.class), + mock(ReactiveExtensionClient.class)); + var settings = new AgentSettings( + new AgentSettings.AgentBuiltInCapabilities(false, false, false, true, + AgentCommentCapability.OFF), + List.of(), + null, + null, + null, + new AgentSettings.AgentNetworkAccessSettings( + List.of("http://127.0.0.1"), 4000, 5) + ); + + var output = (Map) service.fetchAllowedUrl( + Map.of("url", "http://127.0.0.1/secret"), settings).block(); + + assertThat(output) + .containsEntry("ok", false) + .containsEntry("errorCode", "NETWORK_ACCESS_DENIED") + .extracting("message") + .asString() + .contains("禁止访问"); + } + + private HaloDocument document() { + var document = new HaloDocument(); + document.setId("doc-1"); + document.setMetadataName("demo-post"); + document.setTitle("Demo Post"); + document.setDescription("Demo Description"); + document.setContent("Demo Content"); + document.setPublished(true); + document.setExposed(true); + document.setRecycled(false); + document.setOwnerName("admin"); + document.setCreationTimestamp(Instant.now()); + document.setUpdateTimestamp(Instant.now()); + document.setPermalink("https://example.com/archives/demo"); + document.setType("post.content.halo.run"); + return document; + } +} From 96e8768ea2accb7e8cd9080d63aefaf6e8674f9b Mon Sep 17 00:00:00 2001 From: LIlGG <1103069291@qq.com> Date: Tue, 16 Jun 2026 18:52:44 +0800 Subject: [PATCH 8/9] chore: add codex openspec skills --- .codex/skills/openspec-apply-change/SKILL.md | 159 ++++++++++ .../skills/openspec-archive-change/SKILL.md | 117 +++++++ .codex/skills/openspec-explore/SKILL.md | 287 ++++++++++++++++++ .codex/skills/openspec-propose/SKILL.md | 111 +++++++ 4 files changed, 674 insertions(+) create mode 100644 .codex/skills/openspec-apply-change/SKILL.md create mode 100644 .codex/skills/openspec-archive-change/SKILL.md create mode 100644 .codex/skills/openspec-explore/SKILL.md create mode 100644 .codex/skills/openspec-propose/SKILL.md diff --git a/.codex/skills/openspec-apply-change/SKILL.md b/.codex/skills/openspec-apply-change/SKILL.md new file mode 100644 index 0000000..db4d8ce --- /dev/null +++ b/.codex/skills/openspec-apply-change/SKILL.md @@ -0,0 +1,159 @@ +--- +name: openspec-apply-change +description: Implement tasks from an OpenSpec change. Use when the user wants to start implementing, continue implementation, or work through tasks. +license: MIT +compatibility: Requires openspec CLI. +metadata: + author: openspec + version: "1.0" + generatedBy: "1.4.1" +--- + +Implement tasks from an OpenSpec change. + +**Input**: Optionally specify a change name. If omitted, check if it can be inferred from conversation context. If vague or ambiguous you MUST prompt for available changes. + +**Steps** + +1. **Select the change** + + If a name is provided, use it. Otherwise: + - Infer from conversation context if the user mentioned a change + - Auto-select if only one active change exists + - If ambiguous, run `openspec list --json` to get available changes and use the **AskUserQuestion tool** to let the user select + + Always announce: "Using change: " and how to override (e.g., `/opsx:apply `). + +2. **Check status to understand the schema** + ```bash + openspec status --change "" --json + ``` + Parse the JSON to understand: + - `schemaName`: The workflow being used (e.g., "spec-driven") + - `planningHome`, `changeRoot`, and `actionContext`: planning scope and edit constraints + - Which artifact contains the tasks (typically "tasks" for spec-driven, check status for others) + +3. **Get apply instructions** + + ```bash + openspec instructions apply --change "" --json + ``` + + This returns: + - `contextFiles`: artifact ID -> array of concrete file paths (varies by schema - could be proposal/specs/design/tasks or spec/tests/implementation/docs) + - Progress (total, complete, remaining) + - Task list with status + - Dynamic instruction based on current state + + **Handle states:** + - If `state: "blocked"` (missing artifacts): show message, suggest using openspec-continue-change + - If `state: "all_done"`: congratulate, suggest archive + - Otherwise: proceed to implementation + + **Workspace guard:** If status JSON reports `actionContext.mode: "workspace-planning"` and `allowedEditRoots` is empty, explain that full workspace apply is not supported in this slice. Treat linked repos and folders as read-only context, ask the user to select an affected area through an explicit implementation workflow, and STOP before editing files. + +4. **Read context files** + + Read every file path listed under `contextFiles` from the apply instructions output. + The files depend on the schema being used: + - **spec-driven**: proposal, specs, design, tasks + - Other schemas: follow the contextFiles from CLI output + +5. **Show current progress** + + Display: + - Schema being used + - Progress: "N/M tasks complete" + - Remaining tasks overview + - Dynamic instruction from CLI + +6. **Implement tasks (loop until done or blocked)** + + For each pending task: + - Show which task is being worked on + - Make the code changes required + - Keep changes minimal and focused + - Mark task complete in the tasks file: `- [ ]` → `- [x]` + - Continue to next task + + **Pause if:** + - Task is unclear → ask for clarification + - Implementation reveals a design issue → suggest updating artifacts + - Error or blocker encountered → report and wait for guidance + - User interrupts + +7. **On completion or pause, show status** + + Display: + - Tasks completed this session + - Overall progress: "N/M tasks complete" + - If all done: suggest archive + - If paused: explain why and wait for guidance + +**Output During Implementation** + +``` +## Implementing: (schema: ) + +Working on task 3/7: +[...implementation happening...] +✓ Task complete + +Working on task 4/7: +[...implementation happening...] +✓ Task complete +``` + +**Output On Completion** + +``` +## Implementation Complete + +**Change:** +**Schema:** +**Progress:** 7/7 tasks complete ✓ + +### Completed This Session +- [x] Task 1 +- [x] Task 2 +... + +All tasks complete! Ready to archive this change. +``` + +**Output On Pause (Issue Encountered)** + +``` +## Implementation Paused + +**Change:** +**Schema:** +**Progress:** 4/7 tasks complete + +### Issue Encountered + + +**Options:** +1.