From cd1b84ae4d0e749d2045b596c25528518064ab3e Mon Sep 17 00:00:00 2001 From: Copilot App <223556219+Copilot@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:42:09 +0800 Subject: [PATCH 1/4] Add chat file attachments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 1fe75171-fea2-45ef-9401-bf8a4ea0f2ba --- desktop/renderer/env.d.ts | 31 ++- .../src/components/chat/ChatAttachments.vue | 129 +++++++++++++ .../src/components/chat/ChatMessageList.vue | 16 +- desktop/renderer/src/i18n/en-US.ts | 7 + desktop/renderer/src/i18n/zh-CN.ts | 6 + desktop/renderer/src/stores/chat.test.ts | 117 +++++++++++ desktop/renderer/src/stores/chat.ts | 164 ++++++++++++---- desktop/renderer/src/views/ChatView.vue | 153 +++++++++++++-- desktop/src/chat-attachments.test.ts | 179 +++++++++++++++++ desktop/src/chat-attachments.ts | 182 ++++++++++++++++++ desktop/src/constants.ts | 8 + desktop/src/gateway-client.test.ts | 48 +++++ desktop/src/gateway-client.ts | 17 +- desktop/src/main.ts | 31 ++- desktop/src/preload.ts | 13 +- 15 files changed, 1039 insertions(+), 62 deletions(-) create mode 100644 desktop/renderer/src/components/chat/ChatAttachments.vue create mode 100644 desktop/src/chat-attachments.test.ts create mode 100644 desktop/src/chat-attachments.ts diff --git a/desktop/renderer/env.d.ts b/desktop/renderer/env.d.ts index cc90927..b097ffa 100644 --- a/desktop/renderer/env.d.ts +++ b/desktop/renderer/env.d.ts @@ -25,6 +25,28 @@ interface AppSettings { privacyLevel: string; } +interface ChatAttachment { + type: "image" | "file"; + mimeType: string; + fileName: string; + size: number; + content: string; +} + +type AttachmentRejectionReason = "file_too_large" | "total_too_large" | "read_failed"; + +interface AttachmentRejection { + fileName: string; + reason: AttachmentRejectionReason; + size?: number; + limit?: number; +} + +interface OpenFilesResult { + attachments: ChatAttachment[]; + rejections: AttachmentRejection[]; +} + type GitHubCopilotLoginEvent = | { sessionId: string; @@ -182,7 +204,11 @@ interface OpenClawAPI { }; chat: { isConnected(): Promise; - sendMessage(sessionKey: string, message: string): Promise; + sendMessage( + sessionKey: string, + message: string, + attachments?: ChatAttachment[], + ): Promise; loadHistory(sessionKey: string): Promise<{ messages?: unknown[]; thinkingLevel?: string }>; abort(sessionKey: string): Promise; deleteSession(sessionKey: string): Promise; @@ -287,6 +313,9 @@ interface OpenClawAPI { shell: { openExternal(url: string): Promise; }; + dialog: { + openFiles(currentTotalBytes?: number): Promise; + }; sandbox: { getStatus(): Promise<{ available: boolean; diff --git a/desktop/renderer/src/components/chat/ChatAttachments.vue b/desktop/renderer/src/components/chat/ChatAttachments.vue new file mode 100644 index 0000000..6f4a2ac --- /dev/null +++ b/desktop/renderer/src/components/chat/ChatAttachments.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/desktop/renderer/src/components/chat/ChatMessageList.vue b/desktop/renderer/src/components/chat/ChatMessageList.vue index e0d1041..b6d2c2c 100644 --- a/desktop/renderer/src/components/chat/ChatMessageList.vue +++ b/desktop/renderer/src/components/chat/ChatMessageList.vue @@ -286,9 +286,16 @@ -
- {{ getMessageText(msg) }} -
+ @@ -482,6 +489,7 @@ import { stripPlanMarkers, } from "@/composables/usePlanProgress"; import PlanProgressPanel from "./PlanProgress.vue"; +import ChatAttachments from "./ChatAttachments.vue"; import searchGif from "@/assets/openclaw_search_preview_transparent.gif"; import bookGif from "@/assets/book.gif"; import binocularsGif from "@/assets/binoculars.gif"; @@ -911,7 +919,7 @@ function cancelEdit() { async function confirmEdit() { const text = editText.value.trim(); - if (!text || !chatStore.wsConnected) return; + if (!text || !chatStore.wsConnected || chatStore.sending || chatStore.streaming) return; cancelEdit(); await chatStore.sendMessage(text); } diff --git a/desktop/renderer/src/i18n/en-US.ts b/desktop/renderer/src/i18n/en-US.ts index a7d4a2a..14f31e4 100644 --- a/desktop/renderer/src/i18n/en-US.ts +++ b/desktop/renderer/src/i18n/en-US.ts @@ -120,6 +120,13 @@ export default { "chat.stopGeneration": "Stop Generating", "chat.stopTask": "Stop task", "chat.addAttachment": "Add attachment", + "chat.attachment": "Attachment", + "chat.removeAttachment": "Remove {file}", + "chat.attachment.fileTooLarge": "{file} exceeds the {limit} per-file attachment limit.", + "chat.attachment.totalTooLarge": + "{file} would exceed the {limit} total attachment limit for this message.", + "chat.attachment.readFailed": "MicroClaw could not read {file}.", + "chat.attachment.pickerFailed": "Could not add attachments: {error}", "chat.send": "Send", "chat.editHint": "Double-click to edit and resend", "chat.editConfirm": "Send", diff --git a/desktop/renderer/src/i18n/zh-CN.ts b/desktop/renderer/src/i18n/zh-CN.ts index cfd1d03..a9b6d2e 100644 --- a/desktop/renderer/src/i18n/zh-CN.ts +++ b/desktop/renderer/src/i18n/zh-CN.ts @@ -115,6 +115,12 @@ export default { "chat.stopGeneration": "停止生成", "chat.stopTask": "停止任务", "chat.addAttachment": "添加附件", + "chat.attachment": "附件", + "chat.removeAttachment": "移除 {file}", + "chat.attachment.fileTooLarge": "{file} 超过了单个附件 {limit} 的大小限制。", + "chat.attachment.totalTooLarge": "{file} 会导致本条消息的附件总大小超过 {limit}。", + "chat.attachment.readFailed": "MicroClaw 无法读取 {file}。", + "chat.attachment.pickerFailed": "无法添加附件：{error}", "chat.send": "发送", "chat.editHint": "双击编辑并重新发送", "chat.editConfirm": "发送", diff --git a/desktop/renderer/src/stores/chat.test.ts b/desktop/renderer/src/stores/chat.test.ts index 0b0410a..4859561 100644 --- a/desktop/renderer/src/stores/chat.test.ts +++ b/desktop/renderer/src/stores/chat.test.ts @@ -598,6 +598,123 @@ describe("useChatStore — draft sessions", () => { }); }); +describe("useChatStore — attachments", () => { + const attachment: ChatAttachment = { + type: "image", + mimeType: "image/png", + fileName: "pixel.png", + size: 3, + content: "AQID", + }; + + beforeEach(() => { + Object.keys(storage).forEach((k) => delete storage[k]); + setActivePinia(createPinia()); + mockSendMessage.mockReset().mockResolvedValue(undefined); + mockLoadHistory.mockResolvedValue({ messages: [] }); + }); + + it("forwards attachments and adds them to the optimistic user message", async () => { + const store = useChatStore(); + store.sessionKey = "session-attachments"; + + await expect(store.sendMessage("describe this", [attachment])).resolves.toBe(true); + + expect(mockSendMessage).toHaveBeenCalledWith( + "session-attachments", + "describe this", + [attachment], + ); + expect(store.messages.at(-1)).toMatchObject({ + role: "user", + attachments: [attachment], + }); + }); + + it("supports an attachment-only message", async () => { + const store = useChatStore(); + + await expect(store.sendMessage("", [attachment])).resolves.toBe(true); + + expect(mockSendMessage).toHaveBeenCalledWith("main", "", [attachment]); + expect(store.messages.at(-1)?.content).toEqual([]); + }); + + it("removes the optimistic message and reports failure when IPC rejects", async () => { + const store = useChatStore(); + mockSendMessage.mockRejectedValueOnce(new Error("Gateway not connected")); + + await expect(store.sendMessage("retry me", [attachment])).resolves.toBe(false); + + expect(store.messages).toEqual([]); + expect(store.lastError?.raw).toContain("Gateway not connected"); + }); + + it("rejects a concurrent send while the first message is in flight", async () => { + const store = useChatStore(); + let resolveSend: (() => void) | undefined; + mockSendMessage.mockReturnValueOnce( + new Promise((resolve) => { + resolveSend = resolve; + }), + ); + + const firstSend = store.sendMessage("first", [attachment]); + await vi.waitFor(() => expect(mockSendMessage).toHaveBeenCalledTimes(1)); + + await expect(store.sendMessage("second", [attachment])).resolves.toBe(false); + expect(mockSendMessage).toHaveBeenCalledTimes(1); + + resolveSend?.(); + await expect(firstSend).resolves.toBe(true); + }); + + it("normalizes gateway-style base64 attachment blocks", () => { + const store = useChatStore(); + + expect( + store.getMessageAttachments({ + role: "user", + content: [ + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "AQID" }, + }, + ], + }), + ).toEqual([ + { + type: "image", + mimeType: "image/png", + fileName: "", + size: 0, + content: "AQID", + }, + ]); + }); + + it("normalizes persisted gateway media metadata", () => { + const store = useChatStore(); + + expect( + store.getMessageAttachments({ + role: "user", + content: "see attached", + MediaPaths: ["C:\\openclaw\\media\\report.pdf"], + MediaTypes: ["application/pdf"], + }), + ).toEqual([ + { + type: "file", + mimeType: "application/pdf", + fileName: "report.pdf", + size: 0, + content: "", + }, + ]); + }); +}); + describe("useChatStore — session deletion", () => { beforeEach(() => { Object.keys(storage).forEach((k) => delete storage[k]); diff --git a/desktop/renderer/src/stores/chat.ts b/desktop/renderer/src/stores/chat.ts index 85a1636..4b54d80 100644 --- a/desktop/renderer/src/stores/chat.ts +++ b/desktop/renderer/src/stores/chat.ts @@ -18,6 +18,7 @@ export interface ChatMessage { content: unknown; // string or content-block array timestamp?: number; text?: string; + attachments?: ChatAttachment[]; } /** @@ -417,6 +418,79 @@ export const useChatStore = defineStore("chat", () => { return null; } + function getMessageAttachments(message: unknown): ChatAttachment[] { + const m = message as Record; + const mediaPaths = [ + ...(Array.isArray(m.MediaPaths) ? m.MediaPaths : []), + ...(typeof m.MediaPath === "string" ? [m.MediaPath] : []), + ]; + const mediaTypes = [ + ...(Array.isArray(m.MediaTypes) ? m.MediaTypes : []), + ...(typeof m.MediaType === "string" ? [m.MediaType] : []), + ]; + const candidates = [ + ...(Array.isArray(m.attachments) ? m.attachments : []), + ...(Array.isArray(m.content) ? m.content : []), + ...mediaPaths.map((mediaPath, index) => ({ + type: + typeof mediaTypes[index] === "string" && + (mediaTypes[index] as string).startsWith("image/") + ? "image" + : "file", + mimeType: + typeof mediaTypes[index] === "string" + ? (mediaTypes[index] as string) + : "application/octet-stream", + fileName: + typeof mediaPath === "string" ? mediaPath.split(/[\\/]/).at(-1) || "" : "", + size: 0, + content: "", + })), + ]; + const attachments: ChatAttachment[] = []; + + for (const candidate of candidates) { + if (!candidate || typeof candidate !== "object") continue; + const block = candidate as Record; + const source = + block.source && typeof block.source === "object" + ? (block.source as Record) + : undefined; + const mimeType = + (typeof block.mimeType === "string" && block.mimeType) || + (typeof block.media_type === "string" && block.media_type) || + (typeof source?.media_type === "string" && source.media_type) || + ""; + const blockType = typeof block.type === "string" ? block.type : ""; + if (!mimeType && blockType !== "image" && blockType !== "file") continue; + const content = + (typeof block.content === "string" && block.content) || + (typeof block.data === "string" && block.data) || + (typeof source?.data === "string" && source.data) || + ""; + attachments.push({ + type: mimeType.startsWith("image/") || blockType === "image" ? "image" : "file", + mimeType: mimeType || "application/octet-stream", + fileName: + (typeof block.fileName === "string" && block.fileName) || + (typeof block.name === "string" && block.name) || + "", + size: typeof block.size === "number" ? block.size : 0, + content, + }); + } + + return attachments.filter( + (attachment, index) => + attachments.findIndex( + (item) => + item.fileName === attachment.fileName && + item.mimeType === attachment.mimeType && + item.content === attachment.content, + ) === index, + ); + } + /** * Extract only the text content from a message, excluding tool_use, * tool_result, and thinking blocks. Used for rendering the main chat bubble. @@ -524,7 +598,7 @@ export const useChatStore = defineStore("chat", () => { // ── Actions ── - /** Fast shallow comparison: same length, same role, same text content. */ + /** Fast shallow comparison: same length, role, text, and attachment identity. */ function _messagesEqual(a: ChatMessage[], b: ChatMessage[]): boolean { if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) { @@ -532,6 +606,13 @@ export const useChatStore = defineStore("chat", () => { const ta = extractText(a[i]); const tb = extractText(b[i]); if (ta !== tb) return false; + const aa = getMessageAttachments(a[i]) + .map((attachment) => `${attachment.fileName}:${attachment.mimeType}:${attachment.size}`) + .join("|"); + const ab = getMessageAttachments(b[i]) + .map((attachment) => `${attachment.fileName}:${attachment.mimeType}:${attachment.size}`) + .join("|"); + if (aa !== ab) return false; } return true; } @@ -622,54 +703,68 @@ export const useChatStore = defineStore("chat", () => { } /** Send a message to the current session. */ - async function sendMessage(text: string) { + async function sendMessage(text: string, attachments: ChatAttachment[] = []): Promise { const msg = text.trim(); - if (!msg) return; - - // Privacy protection: scan for PII based on privacy level - let finalMsg = msg; - const privacySettings = await window.openclaw.settings.get(); - const privacyLevel = privacySettings?.privacyLevel ?? "balanced"; - if (privacyLevel !== "basic") { - const piiMatches = scanPii(msg); - if (privacyLevel === "strict" && piiMatches.length > 0) { - // Auto-redact in strict mode - finalMsg = redactPii(msg); - } - // In balanced mode, piiMatches are available for UI warning (future) - } - - // Optimistic: add user message locally - messages.value = [ - ...messages.value, - { role: "user", content: [{ type: "text", text: finalMsg }], timestamp: Date.now() }, - ]; - _updateLastPreview(); + if ((!msg && attachments.length === 0) || sending.value || streaming.value) return false; sending.value = true; lastError.value = null; - streamText.value = ""; - streamTextOffset.value = 0; - _toolPhaseActive.value = false; - streamToolCalls.value = []; - streamStartedAt.value = Date.now(); - lastStreamEventAt.value = Date.now(); - streaming.value = true; - + let optimisticTimestamp: number | undefined; try { + // Privacy protection: scan for PII based on privacy level + let finalMsg = msg; + const privacySettings = await window.openclaw.settings.get(); + const privacyLevel = privacySettings?.privacyLevel ?? "balanced"; + if (privacyLevel !== "basic") { + const piiMatches = scanPii(msg); + if (privacyLevel === "strict" && piiMatches.length > 0) { + // Auto-redact in strict mode + finalMsg = redactPii(msg); + } + // In balanced mode, piiMatches are available for UI warning (future) + } + + // Optimistic: add user message locally + optimisticTimestamp = Date.now(); + const optimisticMessage: ChatMessage = { + role: "user", + content: finalMsg ? [{ type: "text", text: finalMsg }] : [], + timestamp: optimisticTimestamp, + attachments: attachments.map((attachment) => ({ ...attachment })), + }; + messages.value = [...messages.value, optimisticMessage]; + _updateLastPreview(); + + streamText.value = ""; + streamTextOffset.value = 0; + _toolPhaseActive.value = false; + streamToolCalls.value = []; + streamStartedAt.value = Date.now(); + lastStreamEventAt.value = Date.now(); + streaming.value = true; + await window.openclaw.chat.sendMessage( resolvedSessionKey.value || sessionKey.value, finalMsg, + attachments.length ? attachments : undefined, ); + return true; } catch (err) { const error = String(err); lastError.value = classifyChatError(error); streaming.value = false; - sending.value = false; + streamStartedAt.value = null; lastStreamEventAt.value = null; - return; + if (optimisticTimestamp !== undefined) { + messages.value = messages.value.filter( + (message) => message.role !== "user" || message.timestamp !== optimisticTimestamp, + ); + _updateLastPreview(); + } + return false; + } finally { + sending.value = false; } - sending.value = false; } /** Handle an incoming chat event from the gateway. */ @@ -1367,6 +1462,7 @@ export const useChatStore = defineStore("chat", () => { pendingPrompt, extractText, extractTextOnly, + getMessageAttachments, extractThinking, extractToolUseBlocks, switchSession, diff --git a/desktop/renderer/src/views/ChatView.vue b/desktop/renderer/src/views/ChatView.vue index 0c83025..a528c2a 100644 --- a/desktop/renderer/src/views/ChatView.vue +++ b/desktop/renderer/src/views/ChatView.vue @@ -59,6 +59,12 @@
+
- @@ -38,6 +46,7 @@