diff --git a/desktop/renderer/env.d.ts b/desktop/renderer/env.d.ts index cc90927..529fb47 100644 --- a/desktop/renderer/env.d.ts +++ b/desktop/renderer/env.d.ts @@ -25,6 +25,29 @@ interface AppSettings { privacyLevel: string; } +interface ChatAttachment { + type: "image" | "file"; + mimeType: string; + fileName: string; + size: number; + content: string; + mediaPath?: string; +} + +type AttachmentRejectionReason = "file_too_large" | "total_too_large" | "read_failed"; + +interface AttachmentRejection { + fileName: string; + reason: AttachmentRejectionReason; + size?: number; + limit?: number; +} + +interface OpenFilesResult { + attachments: ChatAttachment[]; + rejections: AttachmentRejection[]; +} + type GitHubCopilotLoginEvent = | { sessionId: string; @@ -182,7 +205,11 @@ interface OpenClawAPI { }; chat: { isConnected(): Promise; - sendMessage(sessionKey: string, message: string): Promise; + sendMessage( + sessionKey: string, + message: string, + attachments?: ChatAttachment[], + ): Promise; loadHistory(sessionKey: string): Promise<{ messages?: unknown[]; thinkingLevel?: string }>; abort(sessionKey: string): Promise; deleteSession(sessionKey: string): Promise; @@ -287,6 +314,12 @@ interface OpenClawAPI { shell: { openExternal(url: string): Promise; }; + attachment: { + open(attachment: ChatAttachment): Promise<{ ok: boolean; error?: string }>; + }; + dialog: { + openFiles(currentTotalBytes?: number): Promise; + }; sandbox: { getStatus(): Promise<{ available: boolean; diff --git a/desktop/renderer/src/components/chat/ChatAttachments.test.ts b/desktop/renderer/src/components/chat/ChatAttachments.test.ts new file mode 100644 index 0000000..a927683 --- /dev/null +++ b/desktop/renderer/src/components/chat/ChatAttachments.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { mount } from "@vue/test-utils"; +import ChatAttachments from "./ChatAttachments.vue"; + +const openAttachment = vi.fn().mockResolvedValue({ ok: true }); + +beforeEach(() => { + openAttachment.mockClear(); + Object.defineProperty(window, "openclaw", { + configurable: true, + value: { attachment: { open: openAttachment } }, + }); +}); + +describe("ChatAttachments", () => { + const attachment: ChatAttachment = { + type: "file", + mimeType: "text/plain", + fileName: "notes.txt", + size: 5, + content: "aGVsbG8=", + }; + + it("opens a content-bearing attachment when its card is clicked", async () => { + const wrapper = mount(ChatAttachments, { + props: { attachments: [attachment] }, + }); + + await wrapper.get(".chat-attachment").trigger("click"); + + expect(openAttachment).toHaveBeenCalledWith(attachment); + expect(wrapper.get(".chat-attachment").attributes("role")).toBe("button"); + }); + + it("removes without opening when the remove button is clicked", async () => { + const wrapper = mount(ChatAttachments, { + props: { attachments: [attachment], removable: true }, + }); + + await wrapper.get(".chat-attachment__remove").trigger("click"); + + expect(openAttachment).not.toHaveBeenCalled(); + expect(wrapper.emitted("remove")).toEqual([[0]]); + }); + + it("does not open when keyboard events originate from the remove button", async () => { + const wrapper = mount(ChatAttachments, { + props: { attachments: [attachment], removable: true }, + }); + const removeButton = wrapper.get(".chat-attachment__remove"); + + await removeButton.trigger("keydown", { key: "Enter" }); + await removeButton.trigger("click"); + + expect(openAttachment).not.toHaveBeenCalled(); + expect(wrapper.emitted("remove")).toEqual([[0]]); + }); + + it("does not show clickable affordance for an unavailable history attachment", () => { + const wrapper = mount(ChatAttachments, { + props: { + attachments: [{ ...attachment, content: "", mediaPath: undefined }], + }, + }); + + expect(wrapper.get(".chat-attachment").attributes("role")).toBeUndefined(); + expect(wrapper.get(".chat-attachment").classes()).not.toContain("chat-attachment--openable"); + }); + + it("opens a history attachment with an absolute inbound media path", async () => { + const historyAttachment = { + ...attachment, + content: "", + mediaPath: "C:\\Users\\sunt\\.openclaw\\media\\inbound\\notes.txt", + }; + const wrapper = mount(ChatAttachments, { + props: { attachments: [historyAttachment] }, + }); + + await wrapper.get(".chat-attachment").trigger("click"); + + expect(openAttachment).toHaveBeenCalledWith(historyAttachment); + }); +}); diff --git a/desktop/renderer/src/components/chat/ChatAttachments.vue b/desktop/renderer/src/components/chat/ChatAttachments.vue new file mode 100644 index 0000000..5458772 --- /dev/null +++ b/desktop/renderer/src/components/chat/ChatAttachments.vue @@ -0,0 +1,177 @@ + + + + + diff --git a/desktop/renderer/src/components/chat/ChatMessageList.vue b/desktop/renderer/src/components/chat/ChatMessageList.vue index e0d1041..449fdfb 100644 --- a/desktop/renderer/src/components/chat/ChatMessageList.vue +++ b/desktop/renderer/src/components/chat/ChatMessageList.vue @@ -286,9 +286,16 @@ -
- {{ getMessageText(msg) }} -
+ @@ -482,6 +489,7 @@ import { stripPlanMarkers, } from "@/composables/usePlanProgress"; import PlanProgressPanel from "./PlanProgress.vue"; +import ChatAttachments from "./ChatAttachments.vue"; import searchGif from "@/assets/openclaw_search_preview_transparent.gif"; import bookGif from "@/assets/book.gif"; import binocularsGif from "@/assets/binoculars.gif"; @@ -911,7 +919,7 @@ function cancelEdit() { async function confirmEdit() { const text = editText.value.trim(); - if (!text || !chatStore.wsConnected) return; + if (!text || !chatStore.wsConnected || chatStore.sending || chatStore.streaming) return; cancelEdit(); await chatStore.sendMessage(text); } @@ -982,6 +990,10 @@ function handleEditKeydown(e: KeyboardEvent) { cursor: default; } +.chat-bubble.user .chat-message-attachments { + margin-bottom: 8px; +} + .chat-bubble.user.editable { cursor: pointer; } diff --git a/desktop/renderer/src/i18n/en-US.ts b/desktop/renderer/src/i18n/en-US.ts index a7d4a2a..8b34cd3 100644 --- a/desktop/renderer/src/i18n/en-US.ts +++ b/desktop/renderer/src/i18n/en-US.ts @@ -120,6 +120,15 @@ export default { "chat.stopGeneration": "Stop Generating", "chat.stopTask": "Stop task", "chat.addAttachment": "Add attachment", + "chat.attachment": "Attachment", + "chat.image": "Image", + "chat.removeAttachment": "Remove {file}", + "chat.attachment.fileTooLarge": "{file} exceeds the {limit} per-file attachment limit.", + "chat.attachment.totalTooLarge": + "{file} would exceed the {limit} total attachment limit for this message.", + "chat.attachment.readFailed": "MicroClaw could not read {file}.", + "chat.attachment.pickerFailed": "Could not add attachments: {error}", + "chat.attachment.openFailed": "Couldn't open attachment: {error}", "chat.send": "Send", "chat.editHint": "Double-click to edit and resend", "chat.editConfirm": "Send", @@ -243,7 +252,8 @@ export default { "settings.toolCalls": "Tool Calls", "settings.modelBreakdown": "Model Usage Breakdown", "settings.refresh": "Refresh", - "settings.usageFooter": "Data comes from the MicroClaw gateway and is available only while the gateway is running.", + "settings.usageFooter": + "Data comes from the MicroClaw gateway and is available only while the gateway is running.", "settings.callsSuffix": "calls", "settings.inputSuffix": "input", "settings.outputSuffix": "output", @@ -687,7 +697,8 @@ export default { // ── Skills (dev-only panel) ── "skills.dev.tab": "Skills", "skills.dev.title": "Agent Skills (Dev)", - "skills.dev.subtitle": "Toggle which skills each agent can use, then apply and reload the gateway. Development builds only.", + "skills.dev.subtitle": + "Toggle which skills each agent can use, then apply and reload the gateway. Development builds only.", "skills.dev.agent": "Agent", "skills.dev.selectAll": "Select all", "skills.dev.clearAll": "Clear all", @@ -705,8 +716,10 @@ export default { "skills.dev.modelVisibleSummary": "Model-visible: {visible} / {total}", "skills.dev.statusFailed": "Status unavailable: {error}", "skills.dev.statusChecking": "Checking skill status… this can take up to a minute.", - "skills.dev.statusCheckingAll": "Checking all agents ({current}/{total})… this can take a few minutes.", - "skills.dev.statusNotChecked": "Status not checked for this agent — click “Refresh all agents” (takes a few minutes).", + "skills.dev.statusCheckingAll": + "Checking all agents ({current}/{total})… this can take a few minutes.", + "skills.dev.statusNotChecked": + "Status not checked for this agent — click “Refresh all agents” (takes a few minutes).", "skills.dev.badge.visible": "Model-visible", "skills.dev.badge.hidden": "Not injected", "skills.dev.badge.unknown": "No status", diff --git a/desktop/renderer/src/i18n/zh-CN.ts b/desktop/renderer/src/i18n/zh-CN.ts index cfd1d03..9447aaa 100644 --- a/desktop/renderer/src/i18n/zh-CN.ts +++ b/desktop/renderer/src/i18n/zh-CN.ts @@ -115,6 +115,14 @@ export default { "chat.stopGeneration": "停止生成", "chat.stopTask": "停止任务", "chat.addAttachment": "添加附件", + "chat.attachment": "附件", + "chat.image": "图片", + "chat.removeAttachment": "移除 {file}", + "chat.attachment.fileTooLarge": "{file} 超过了单个附件 {limit} 的大小限制。", + "chat.attachment.totalTooLarge": "{file} 会导致本条消息的附件总大小超过 {limit}。", + "chat.attachment.readFailed": "MicroClaw 无法读取 {file}。", + "chat.attachment.pickerFailed": "无法添加附件：{error}", + "chat.attachment.openFailed": "无法打开附件：{error}", "chat.send": "发送", "chat.editHint": "双击编辑并重新发送", "chat.editConfirm": "发送", @@ -396,7 +404,8 @@ export default { "settings.editCustomModel": "编辑自定义模型", "settings.connected": "已连接", "settings.port": "端口", - "settings.portDesc": "更改端口后网关将自动重启。如果所选端口被占用，MicroClaw 将尝试使用相邻端口。", + "settings.portDesc": + "更改端口后网关将自动重启。如果所选端口被占用，MicroClaw 将尝试使用相邻端口。", "settings.webSearch": "联网搜索", "settings.reasoningOffDesc": "推理关闭", "settings.reasoningDesc": "推理 {level}", @@ -679,7 +688,8 @@ export default { "skills.dev.statusFailed": "无法获取状态：{error}", "skills.dev.statusChecking": "正在检查技能状态…最长可能需要一分钟。", "skills.dev.statusCheckingAll": "正在检查所有 agent（{current}/{total}）……可能需要几分钟。", - "skills.dev.statusNotChecked": "尚未检查该智能体的技能状态——请点击“刷新所有 agent 状态”（可能需要几分钟）。", + "skills.dev.statusNotChecked": + "尚未检查该智能体的技能状态——请点击“刷新所有 agent 状态”（可能需要几分钟）。", "skills.dev.badge.visible": "模型可见", "skills.dev.badge.hidden": "未注入", "skills.dev.badge.unknown": "无状态", diff --git a/desktop/renderer/src/stores/chat.test.ts b/desktop/renderer/src/stores/chat.test.ts index 0b0410a..deebd02 100644 --- a/desktop/renderer/src/stores/chat.test.ts +++ b/desktop/renderer/src/stores/chat.test.ts @@ -504,9 +504,7 @@ describe("useChatStore — session key learning", () => { expect(store.sessionKey).toBe("agent:main:session-abc"); expect(store.resolvedSessionKey).toBe("agent:main:session-abc"); - expect(sessionStore.sessions.map((session) => session.key)).toEqual([ - "agent:main:session-abc", - ]); + expect(sessionStore.sessions.map((session) => session.key)).toEqual(["agent:main:session-abc"]); }); it("drops events from unrelated sessions", () => { @@ -598,6 +596,328 @@ describe("useChatStore — draft sessions", () => { }); }); +describe("useChatStore — attachments", () => { + const attachment: ChatAttachment = { + type: "image", + mimeType: "image/png", + fileName: "pixel.png", + size: 3, + content: "AQID", + }; + + beforeEach(() => { + Object.keys(storage).forEach((k) => delete storage[k]); + setActivePinia(createPinia()); + mockSendMessage.mockReset().mockResolvedValue(undefined); + mockLoadHistory.mockResolvedValue({ messages: [] }); + }); + + it("forwards attachments and adds them to the optimistic user message", async () => { + const store = useChatStore(); + store.sessionKey = "session-attachments"; + + await expect(store.sendMessage("describe this", [attachment])).resolves.toBe(true); + + expect(mockSendMessage).toHaveBeenCalledWith("session-attachments", "describe this", [ + attachment, + ]); + expect(store.messages.at(-1)).toMatchObject({ + role: "user", + attachments: [attachment], + }); + }); + + it("supports an attachment-only message", async () => { + const store = useChatStore(); + + await expect(store.sendMessage("", [attachment])).resolves.toBe(true); + + expect(mockSendMessage).toHaveBeenCalledWith("main", "", [attachment]); + expect(store.messages.at(-1)?.content).toEqual([]); + }); + + it("removes the optimistic message and reports failure when IPC rejects", async () => { + const store = useChatStore(); + mockSendMessage.mockRejectedValueOnce(new Error("Gateway not connected")); + + await expect(store.sendMessage("retry me", [attachment])).resolves.toBe(false); + + expect(store.messages).toEqual([]); + expect(store.lastError?.raw).toContain("Gateway not connected"); + }); + + it("rejects a concurrent send while the first message is in flight", async () => { + const store = useChatStore(); + let resolveSend: (() => void) | undefined; + mockSendMessage.mockReturnValueOnce( + new Promise((resolve) => { + resolveSend = resolve; + }), + ); + + const firstSend = store.sendMessage("first", [attachment]); + await vi.waitFor(() => expect(mockSendMessage).toHaveBeenCalledTimes(1)); + + await expect(store.sendMessage("second", [attachment])).resolves.toBe(false); + expect(mockSendMessage).toHaveBeenCalledTimes(1); + + resolveSend?.(); + await expect(firstSend).resolves.toBe(true); + }); + + it("normalizes gateway-style base64 attachment blocks", () => { + const store = useChatStore(); + + expect( + store.getMessageAttachments({ + role: "user", + content: [ + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "AQID" }, + }, + ], + }), + ).toEqual([ + { + type: "image", + mimeType: "image/png", + fileName: "", + size: 0, + content: "AQID", + }, + ]); + }); + + it("normalizes persisted gateway media metadata", () => { + const store = useChatStore(); + + expect( + store.getMessageAttachments({ + role: "user", + content: "see attached", + MediaPaths: ["C:\\openclaw\\media\\report.pdf"], + MediaTypes: ["application/pdf"], + }), + ).toEqual([ + { + type: "file", + mimeType: "application/pdf", + fileName: "report.pdf", + size: 0, + content: "", + mediaPath: "C:\\openclaw\\media\\report.pdf", + }, + ]); + }); + + it("keeps distinct extensionless media paths as separate attachments", () => { + const store = useChatStore(); + const attachments = store.getMessageAttachments({ + role: "user", + content: "two images", + MediaPaths: [ + "media://inbound/8f6cbb00-1234-4abc-8def-1234567890ab", + "media://inbound/9f6cbb00-1234-4abc-8def-1234567890ab", + ], + MediaTypes: ["image/jpeg", "image/jpeg"], + }); + + expect(attachments).toHaveLength(2); + expect(attachments.map((attachment) => attachment.mediaPath)).toEqual([ + "media://inbound/8f6cbb00-1234-4abc-8def-1234567890ab", + "media://inbound/9f6cbb00-1234-4abc-8def-1234567890ab", + ]); + }); + + it("uses a generic image name for an absolute UUID image path", () => { + const store = useChatStore(); + const [attachment] = store.getMessageAttachments({ + role: "user", + content: "image", + MediaPaths: [ + "C:\\Users\\sunt\\.openclaw\\media\\inbound\\1be4ae56-6c48-4787-8fef-04c0cb29e130.png", + ], + MediaTypes: ["image/png"], + }); + + expect(attachment.fileName).toBe(""); + expect(attachment.mediaPath).toContain("\\media\\inbound\\"); + }); + + it("removes the gateway UUID suffix from a non-image inbound filename", () => { + const store = useChatStore(); + const [attachment] = store.getMessageAttachments({ + role: "user", + content: "text", + MediaPaths: [ + "C:\\Users\\sunt\\.openclaw\\media\\inbound\\permission_test---499c16ac-1817-41d7-87e3-045e7aed4fa2.txt", + ], + MediaTypes: ["text/plain"], + }); + + expect(attachment.fileName).toBe("permission_test.txt"); + }); + + it("sanitizes an attachment gateway echo and uses only MediaPaths for its chips", async () => { + const store = useChatStore(); + const question = "What is shown in the image and text file?"; + mockLoadHistory.mockResolvedValueOnce({ + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: + "[media attached: media://inbound/permission_test---abc123.txt (text/plain)]\n" + + `${question}\n\n` + + '\n\n' + + '<<>>\n' + + "Source: External\n---\nsecret file contents\n" + + '<<>>\n' + + "", + }, + { type: "image", data: "/9j/..." }, + ], + MediaPaths: [ + "media://inbound/permission_test---abc123.txt", + "media://inbound/8f6cbb00-1234-4abc-8def-1234567890ab", + ], + MediaTypes: ["text/plain", "image/jpeg"], + }, + ], + }); + + await store.loadHistory(); + + expect(store.extractText(store.messages[0])).toBe(question); + expect(store.getMessageAttachments(store.messages[0])).toEqual([ + { + type: "file", + mimeType: "text/plain", + fileName: "permission_test---abc123.txt", + size: 0, + content: "", + mediaPath: "media://inbound/permission_test---abc123.txt", + }, + { + type: "image", + mimeType: "image/jpeg", + fileName: "", + size: 0, + content: "", + mediaPath: "media://inbound/8f6cbb00-1234-4abc-8def-1234567890ab", + }, + ]); + }); + + it("preserves optimistic text and original filenames across the gateway echo", async () => { + const store = useChatStore(); + const question = "What is shown in both files?"; + const attachments: ChatAttachment[] = [ + attachment, + { + type: "file", + mimeType: "text/plain", + fileName: "permission_test.txt", + size: 5, + content: "aGVsbG8=", + }, + ]; + + await store.sendMessage(question, attachments); + mockLoadHistory.mockResolvedValueOnce({ + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: + "[media attached: media://inbound/permission_test---abc123.txt (text/plain)]\n" + + `${question}\n\n` + + '<<>>file contents' + + '<<>>', + }, + { type: "image", data: "/9j/..." }, + ], + MediaPaths: [ + "media://inbound/permission_test---abc123.txt", + "media://inbound/8f6cbb00-1234-4abc-8def-1234567890ab", + ], + MediaTypes: ["text/plain", "image/jpeg"], + }, + ], + }); + + await store.loadHistory(); + + expect(store.extractText(store.messages[0])).toBe(question); + expect(store.getMessageAttachments(store.messages[0])).toEqual(attachments); + }); + + it("does not sanitize user text without attachment evidence", () => { + const store = useChatStore(); + const text = " Please explain this literal example. "; + + expect(store.extractText({ role: "user", content: text })).toBe(text); + }); + + it("does not apply optimistic metadata to a different attachment message", async () => { + const store = useChatStore(); + await store.sendMessage("My local question", [attachment]); + mockLoadHistory.mockResolvedValueOnce({ + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "[media attached: media://inbound/other.txt (text/plain)]\nA remote question", + }, + ], + MediaPaths: ["media://inbound/other.txt"], + MediaTypes: ["text/plain"], + }, + ], + }); + + await store.loadHistory(); + + expect(store.extractText(store.messages[0])).toBe("A remote question"); + expect(store.getMessageAttachments(store.messages[0])[0].fileName).toBe("other.txt"); + expect(store.extractText(store.messages[1])).toBe("My local question"); + expect(store.getMessageAttachments(store.messages[1])[0].fileName).toBe("pixel.png"); + }); + + it("does not reconcile a repeated prompt to an older history turn", async () => { + const store = useChatStore(); + const repeatedQuestion = "Describe this attachment"; + const oldGatewayMessage = { + role: "user", + content: [ + { + type: "text", + text: `[media attached: media://inbound/old.txt (text/plain)]\n${repeatedQuestion}`, + }, + ], + MediaPaths: ["media://inbound/old.txt"], + MediaTypes: ["text/plain"], + }; + mockLoadHistory.mockResolvedValueOnce({ messages: [oldGatewayMessage] }); + await store.loadHistory(); + + await store.sendMessage(repeatedQuestion, [attachment]); + mockLoadHistory.mockResolvedValueOnce({ messages: [oldGatewayMessage] }); + await store.loadHistory(); + + expect(store.messages).toHaveLength(2); + expect(store.getMessageAttachments(store.messages[0])[0].fileName).toBe("old.txt"); + expect(store.getMessageAttachments(store.messages[1])[0].fileName).toBe("pixel.png"); + }); +}); + describe("useChatStore — session deletion", () => { beforeEach(() => { Object.keys(storage).forEach((k) => delete storage[k]); diff --git a/desktop/renderer/src/stores/chat.ts b/desktop/renderer/src/stores/chat.ts index 85a1636..341b63c 100644 --- a/desktop/renderer/src/stores/chat.ts +++ b/desktop/renderer/src/stores/chat.ts @@ -18,6 +18,8 @@ export interface ChatMessage { content: unknown; // string or content-block array timestamp?: number; text?: string; + attachments?: ChatAttachment[]; + originalText?: string; } /** @@ -379,10 +381,61 @@ export const useChatStore = defineStore("chat", () => { return text; } + function sanitizeUserAttachmentText(text: string): string { + return text + .replace(/^(?:\[media attached:[^\r\n]*\]\r?\n?)+/i, "") + .replace(/]*>[\s\S]*?<\/file>/gi, "") + .trim(); + } + + function hasAttachmentEvidence(message: unknown): boolean { + const m = message as Record; + if ( + (Array.isArray(m.attachments) && m.attachments.length > 0) || + (Array.isArray(m.MediaPaths) && m.MediaPaths.length > 0) || + typeof m.MediaPath === "string" + ) { + return true; + } + if ( + Array.isArray(m.content) && + m.content.some((candidate) => { + if (!candidate || typeof candidate !== "object") return false; + const block = candidate as Record; + return block.type === "image" || block.type === "file"; + }) + ) { + return true; + } + const rawText = + typeof m.content === "string" + ? m.content + : typeof m.text === "string" + ? m.text + : Array.isArray(m.content) + ? m.content + .filter( + (block): block is Record => !!block && typeof block === "object", + ) + .map((block) => + block.type === "text" && typeof block.text === "string" ? block.text : "", + ) + .join("\n") + : ""; + return /^\[media attached:[^\r\n]*\]/i.test(rawText); + } + function extractText(message: unknown): string | null { const m = message as Record; - if (typeof m.content === "string") return m.content; - if (typeof m.text === "string") return m.text; + const isUserMessage = typeof m.role === "string" && m.role.toLowerCase() === "user"; + const shouldSanitize = isUserMessage && hasAttachmentEvidence(message); + if (isUserMessage && typeof m.originalText === "string") return m.originalText; + if (typeof m.content === "string") { + return shouldSanitize ? sanitizeUserAttachmentText(m.content) : m.content; + } + if (typeof m.text === "string") { + return shouldSanitize ? sanitizeUserAttachmentText(m.text) : m.text; + } if (Array.isArray(m.content)) { const parts: string[] = []; for (const block of m.content as Array>) { @@ -412,11 +465,158 @@ export const useChatStore = defineStore("chat", () => { } } } - return parts.length > 0 ? parts.join("\n\n") : null; + if (parts.length === 0) return null; + const text = parts.join("\n\n"); + return shouldSanitize ? sanitizeUserAttachmentText(text) : text; } return null; } + function getMessageAttachments(message: unknown): ChatAttachment[] { + const m = message as Record; + const mediaPaths = [ + ...(Array.isArray(m.MediaPaths) ? m.MediaPaths : []), + ...(typeof m.MediaPath === "string" ? [m.MediaPath] : []), + ]; + const mediaTypes = [ + ...(Array.isArray(m.MediaTypes) ? m.MediaTypes : []), + ...(typeof m.MediaType === "string" ? [m.MediaType] : []), + ]; + const optimisticAttachments = Array.isArray(m.attachments) ? m.attachments : []; + const candidates = + optimisticAttachments.length > 0 + ? optimisticAttachments + : mediaPaths.length > 0 + ? mediaPaths.map((mediaPath, index) => { + const mimeType = + typeof mediaTypes[index] === "string" + ? (mediaTypes[index] as string) + : "application/octet-stream"; + const rawName = + typeof mediaPath === "string" + ? mediaPath + .replace(/^media:\/\/inbound\//i, "") + .split(/[\\/]/) + .at(-1) || "" + : ""; + const extensionIndex = rawName.lastIndexOf("."); + const nameWithoutExtension = + extensionIndex > 0 ? rawName.slice(0, extensionIndex) : rawName; + const isBareUuid = + mimeType.startsWith("image/") && + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + nameWithoutExtension, + ); + const displayName = mimeType.startsWith("image/") + ? rawName + : rawName.replace( + /---[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}(?=\.[^.]+$)/i, + "", + ); + return { + type: mimeType.startsWith("image/") ? "image" : "file", + mimeType, + fileName: isBareUuid ? "" : displayName, + size: 0, + content: "", + mediaPath: typeof mediaPath === "string" ? mediaPath : undefined, + }; + }) + : Array.isArray(m.content) + ? m.content + : []; + const attachments: ChatAttachment[] = []; + + for (const candidate of candidates) { + if (!candidate || typeof candidate !== "object") continue; + const block = candidate as Record; + const source = + block.source && typeof block.source === "object" + ? (block.source as Record) + : undefined; + const mimeType = + (typeof block.mimeType === "string" && block.mimeType) || + (typeof block.media_type === "string" && block.media_type) || + (typeof source?.media_type === "string" && source.media_type) || + ""; + const blockType = typeof block.type === "string" ? block.type : ""; + if (!mimeType && blockType !== "image" && blockType !== "file") continue; + const content = + (typeof block.content === "string" && block.content) || + (typeof block.data === "string" && block.data) || + (typeof source?.data === "string" && source.data) || + ""; + attachments.push({ + type: mimeType.startsWith("image/") || blockType === "image" ? "image" : "file", + mimeType: mimeType || "application/octet-stream", + fileName: + (typeof block.fileName === "string" && block.fileName) || + (typeof block.name === "string" && block.name) || + "", + size: typeof block.size === "number" ? block.size : 0, + content, + mediaPath: typeof block.mediaPath === "string" ? block.mediaPath : undefined, + }); + } + + return attachments.filter( + (attachment, index) => + attachments.findIndex( + (item) => + item.fileName === attachment.fileName && + item.mimeType === attachment.mimeType && + item.content === attachment.content && + item.mediaPath === attachment.mediaPath, + ) === index, + ); + } + + function preserveOptimisticAttachmentMetadata( + currentMessages: ChatMessage[], + historyMessages: ChatMessage[], + ): ChatMessage[] { + const currentAttachmentMessages = currentMessages.filter( + (message) => + message.role.toLowerCase() === "user" && getMessageAttachments(message).length > 0, + ); + const optimisticMessages = currentAttachmentMessages + .map((message, attachmentOrdinal) => ({ message, attachmentOrdinal })) + .filter( + ( + item, + ): item is { + message: ChatMessage & { originalText: string; attachments: ChatAttachment[] }; + attachmentOrdinal: number; + } => typeof item.message.originalText === "string" && !!item.message.attachments?.length, + ); + if (optimisticMessages.length === 0) return historyMessages; + + const reconciled = [...historyMessages]; + const historyAttachmentIndexes = reconciled + .map((message, index) => ({ message, index })) + .filter( + ({ message }) => + message.role.toLowerCase() === "user" && getMessageAttachments(message).length > 0, + ) + .map(({ index }) => index); + const unmatchedOptimistic: ChatMessage[] = []; + for (const { message: optimisticMessage, attachmentOrdinal } of optimisticMessages) { + const historyIndex = historyAttachmentIndexes[attachmentOrdinal]; + const historyMessage = historyIndex === undefined ? undefined : reconciled[historyIndex]; + if (historyMessage && extractText(historyMessage) === optimisticMessage.originalText) { + reconciled[historyIndex] = { + ...historyMessage, + originalText: optimisticMessage.originalText, + attachments: optimisticMessage.attachments, + }; + } else { + unmatchedOptimistic.push(optimisticMessage); + } + } + reconciled.push(...unmatchedOptimistic); + return reconciled; + } + /** * Extract only the text content from a message, excluding tool_use, * tool_result, and thinking blocks. Used for rendering the main chat bubble. @@ -524,7 +724,7 @@ export const useChatStore = defineStore("chat", () => { // ── Actions ── - /** Fast shallow comparison: same length, same role, same text content. */ + /** Fast shallow comparison: same length, role, text, and attachment identity. */ function _messagesEqual(a: ChatMessage[], b: ChatMessage[]): boolean { if (a.length !== b.length) return false; for (let i = 0; i < a.length; i++) { @@ -532,6 +732,13 @@ export const useChatStore = defineStore("chat", () => { const ta = extractText(a[i]); const tb = extractText(b[i]); if (ta !== tb) return false; + const aa = getMessageAttachments(a[i]) + .map((attachment) => `${attachment.fileName}:${attachment.mimeType}:${attachment.size}`) + .join("|"); + const ab = getMessageAttachments(b[i]) + .map((attachment) => `${attachment.fileName}:${attachment.mimeType}:${attachment.size}`) + .join("|"); + if (aa !== ab) return false; } return true; } @@ -595,11 +802,12 @@ export const useChatStore = defineStore("chat", () => { const filtered = raw .map((m) => stripSystemLines(m)) .filter((m): m is ChatMessage => m != null && !isGatewaySystemMessage(m)); + const reconciled = preserveOptimisticAttachmentMetadata(messages.value, filtered); // Only replace messages if content actually changed — avoids // unnecessary Vue reactivity triggers and DOM re-renders. - const changed = !_messagesEqual(messages.value, filtered); + const changed = !_messagesEqual(messages.value, reconciled); if (changed) { - messages.value = filtered; + messages.value = reconciled; // Update last message preview only when content changed _updateLastPreview(); } @@ -622,54 +830,69 @@ export const useChatStore = defineStore("chat", () => { } /** Send a message to the current session. */ - async function sendMessage(text: string) { + async function sendMessage(text: string, attachments: ChatAttachment[] = []): Promise { const msg = text.trim(); - if (!msg) return; - - // Privacy protection: scan for PII based on privacy level - let finalMsg = msg; - const privacySettings = await window.openclaw.settings.get(); - const privacyLevel = privacySettings?.privacyLevel ?? "balanced"; - if (privacyLevel !== "basic") { - const piiMatches = scanPii(msg); - if (privacyLevel === "strict" && piiMatches.length > 0) { - // Auto-redact in strict mode - finalMsg = redactPii(msg); - } - // In balanced mode, piiMatches are available for UI warning (future) - } - - // Optimistic: add user message locally - messages.value = [ - ...messages.value, - { role: "user", content: [{ type: "text", text: finalMsg }], timestamp: Date.now() }, - ]; - _updateLastPreview(); + if ((!msg && attachments.length === 0) || sending.value || streaming.value) return false; sending.value = true; lastError.value = null; - streamText.value = ""; - streamTextOffset.value = 0; - _toolPhaseActive.value = false; - streamToolCalls.value = []; - streamStartedAt.value = Date.now(); - lastStreamEventAt.value = Date.now(); - streaming.value = true; - + let optimisticTimestamp: number | undefined; try { + // Privacy protection: scan for PII based on privacy level + let finalMsg = msg; + const privacySettings = await window.openclaw.settings.get(); + const privacyLevel = privacySettings?.privacyLevel ?? "balanced"; + if (privacyLevel !== "basic") { + const piiMatches = scanPii(msg); + if (privacyLevel === "strict" && piiMatches.length > 0) { + // Auto-redact in strict mode + finalMsg = redactPii(msg); + } + // In balanced mode, piiMatches are available for UI warning (future) + } + + // Optimistic: add user message locally + optimisticTimestamp = Date.now(); + const optimisticMessage: ChatMessage = { + role: "user", + content: finalMsg ? [{ type: "text", text: finalMsg }] : [], + timestamp: optimisticTimestamp, + attachments: attachments.map((attachment) => ({ ...attachment })), + originalText: finalMsg, + }; + messages.value = [...messages.value, optimisticMessage]; + _updateLastPreview(); + + streamText.value = ""; + streamTextOffset.value = 0; + _toolPhaseActive.value = false; + streamToolCalls.value = []; + streamStartedAt.value = Date.now(); + lastStreamEventAt.value = Date.now(); + streaming.value = true; + await window.openclaw.chat.sendMessage( resolvedSessionKey.value || sessionKey.value, finalMsg, + attachments.length ? attachments : undefined, ); + return true; } catch (err) { const error = String(err); lastError.value = classifyChatError(error); streaming.value = false; - sending.value = false; + streamStartedAt.value = null; lastStreamEventAt.value = null; - return; + if (optimisticTimestamp !== undefined) { + messages.value = messages.value.filter( + (message) => message.role !== "user" || message.timestamp !== optimisticTimestamp, + ); + _updateLastPreview(); + } + return false; + } finally { + sending.value = false; } - sending.value = false; } /** Handle an incoming chat event from the gateway. */ @@ -865,8 +1088,7 @@ export const useChatStore = defineStore("chat", () => { const { phase, name, toolCallId, meta } = payload.data; const args = (payload.data as Record).args as - | Record - | undefined; + Record | undefined; if (phase === "start") { // Build a descriptive display name combining tool name + primary argument let displayName = name; @@ -1290,8 +1512,7 @@ export const useChatStore = defineStore("chat", () => { // Without a canonical key, keep the generated draft in memory until // its first message instead of persisting an empty sidebar entry. const newKey = - mainSessionKey.value ?? - `session-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + mainSessionKey.value ?? `session-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; pendingSessionAgentId.value = mainSessionKey.value ? "main" : undefined; sessionKey.value = newKey; resolvedSessionKey.value = mainSessionKey.value ? newKey : null; @@ -1367,6 +1588,7 @@ export const useChatStore = defineStore("chat", () => { pendingPrompt, extractText, extractTextOnly, + getMessageAttachments, extractThinking, extractToolUseBlocks, switchSession, diff --git a/desktop/renderer/src/views/ChatView.vue b/desktop/renderer/src/views/ChatView.vue index 0c83025..a528c2a 100644 --- a/desktop/renderer/src/views/ChatView.vue +++ b/desktop/renderer/src/views/ChatView.vue @@ -59,6 +59,12 @@
+
-