diff --git a/canvas/src/ChatPanel.tsx b/canvas/src/ChatPanel.tsx index 5b3a9d17..a8110e94 100644 --- a/canvas/src/ChatPanel.tsx +++ b/canvas/src/ChatPanel.tsx @@ -47,9 +47,10 @@ import { Fragment, useContext, useEffect, useRef, useState } from "react"; import { useValue } from "tldraw"; import { - IMAGE_TYPES, + ATTACH_TYPES, MAX_IMAGE_BYTES, MAX_IMAGES, + SVG_TYPE, type AgentId, type AgentModel, } from "./agents"; @@ -63,6 +64,7 @@ import { ClaudeMark } from "./ClaudeMark"; import { CodexMark } from "./CodexMark"; import { Check, ClockRewind, Image, Plus } from "./geistIcons"; import { renderMarkdown } from "./markdown"; +import { rasterizeSvg } from "./svgRaster"; const RUNS_KEY = "sp-chat-runs"; const AGENT_KEY = "sp-chat-agent"; @@ -438,20 +440,28 @@ export function ChatPanel() { (adds.current = adds.current .then(async () => { const given = [...(list ?? [])]; - const picked = given.filter((f) => IMAGE_TYPES.includes(f.type)); + const taken = given.filter((f) => ATTACH_TYPES.includes(f.type)); // A file the agent could not look at is refused here rather than by the server, and said: // the file dialog offers only these, but a drop, a paste and the canvas hand over anything. setSendError( - picked.length < given.length - ? "only png, jpeg, gif and webp pictures can be attached" + taken.length < given.length + ? "only png, jpeg, gif, webp and svg pictures can be attached" : null, ); - if (picked.length === 0) return; + if (taken.length === 0) return; // What cannot fit whatever the tray holds is refused before it is read: a read is the - // whole file in memory, a third larger. + // whole file in memory, a third larger. Measured on the files as they arrived, since + // that is what a drawing has to hold in memory too. const tooBig = `those are too large to attach; keep them under about ${MAX_IMAGE_BYTES / 1_000_000} MB together`; - if (picked.reduce((n, f) => n + f.size, 0) > MAX_IMAGE_BYTES) + if (taken.reduce((n, f) => n + f.size, 0) > MAX_IMAGE_BYTES) return setSendError(tooBig); + // A vector is drawn into a PNG here, at the one door every attachment comes through — + // the picker, a paste, a drop, and the + on a canvas shape, which hands its asset over + // with whatever type the asset has. Past this line everything is an IMAGE_TYPE, so the + // tray, the limits, the preview and the agent all go on seeing what they always saw. + const picked = await Promise.all( + taken.map((f) => (f.type === SVG_TYPE ? rasterizeSvg(f) : f)), + ); // Read first and numbered after, which is what lets the same picture keep the number it // already has: pressing + on one twice is one picture said twice, not two — and not a // twenty-first, so the limits are over what is new. Still numbered in the order they @@ -492,10 +502,13 @@ export function ChatPanel() { // rather than the one React has not rendered yet. tray.current = [...tray.current, ...fresh].sort((x, y) => x.n - y.n); setAttached(tray.current); - // One picture, pointed at on the canvas: the sentence says which one without a second - // click on the tile that has just appeared, and says it once however many times it is - // pointed at. A pick, a paste or a drop is a handful at once, and which of them the - // message is about is still to be said. + // Numbered into the sentence where the caret already is, so the picture the writer has + // just put there is named without a second click on the tile that has appeared above — + // and named once however many times it is added. Both ways in that land on one picture + // do this: the + on a canvas shape, and a paste, which is the screenshot in the + // clipboard going into the sentence being typed. A pick and a drop are a handful chosen + // at a distance from the caret, and which of them the message is about is still to be + // said. const box = composer.current; if (cite && box) for (const image of said) @@ -1042,7 +1055,7 @@ export function ChatPanel() { onPaste={(e) => { e.preventDefault(); if (e.clipboardData.files.length) - return void addImages(e.clipboardData.files); + return void addImages(e.clipboardData.files, true); // The text and not the markup that came with it: the box holds the chips it made // itself and nothing else. execCommand because it is the only insert that native // undo still knows about. @@ -1177,7 +1190,7 @@ export function ChatPanel() { { diff --git a/canvas/src/agents.ts b/canvas/src/agents.ts index 20b06729..a0279fd2 100644 --- a/canvas/src/agents.ts +++ b/canvas/src/agents.ts @@ -40,6 +40,14 @@ export const IMAGE_TYPES = [ "image/gif", "image/webp", ]; +export const SVG_TYPE = "image/svg+xml"; +/** + * What the composer takes at the door, which is one more than what travels: an SVG is accepted + * and drawn into a PNG on the way in (svgRaster.ts), so the vector itself never reaches the tray, + * the server or the agent. The two lists differ by exactly that conversion — anything accepted + * here is an IMAGE_TYPE by the time it is numbered. + */ +export const ATTACH_TYPES = [...IMAGE_TYPES, SVG_TYPE]; export const MAX_IMAGES = 20; export const MAX_IMAGE_BYTES = 24_000_000; diff --git a/canvas/src/svgRaster.test.ts b/canvas/src/svgRaster.test.ts new file mode 100644 index 00000000..2e20366d --- /dev/null +++ b/canvas/src/svgRaster.test.ts @@ -0,0 +1,45 @@ +// @vitest-environment jsdom +import { describe, expect, it } from "vitest"; +import { rasterSize } from "./svgRaster"; + +/** What the browser reports for a vector it could not measure, which is the interesting case. */ +const unmeasured = { w: 0, h: 0 }; + +describe("rasterSize", () => { + it("scales a stated size so its long edge is the raster edge", () => { + const svg = + ''; + expect(rasterSize(svg, unmeasured)).toEqual({ w: 1024, h: 512 }); + }); + + it("keeps a viewBox-only icon square rather than taking the browser's 300x150", () => { + const svg = ''; + expect(rasterSize(svg, { w: 300, h: 150 })).toEqual({ w: 1024, h: 1024 }); + }); + + it("ignores a relative width and falls through to the viewBox", () => { + const svg = ''; + expect(rasterSize(svg, unmeasured)).toEqual({ w: 1024, h: 256 }); + }); + + it("reads the root's own size, not a child's", () => { + const svg = ''; + expect(rasterSize(svg, unmeasured)).toEqual({ w: 512, h: 1024 }); + }); + + it("falls back to what the browser measured when the markup states nothing", () => { + expect(rasterSize("", { w: 64, h: 32 })).toEqual({ w: 1024, h: 512 }); + }); + + it("gives an unparseable file a square rather than a zero-sized canvas", () => { + expect(rasterSize(" { + // 5000:1 scales to 0.2 of a pixel. Rounded it is nothing, and a canvas with a zero side + // hands toBlob back no blob at all, which rasterizeSvg then reports as a failed drawing. + const svg = + ''; + expect(rasterSize(svg, unmeasured)).toEqual({ w: 1024, h: 1 }); + }); +}); diff --git a/canvas/src/svgRaster.ts b/canvas/src/svgRaster.ts new file mode 100644 index 00000000..05d8b972 --- /dev/null +++ b/canvas/src/svgRaster.ts @@ -0,0 +1,104 @@ +/** + * An SVG on its way into the composer, drawn into a PNG. + * + * Two reasons, and either alone would be enough. The CLIs read png, jpeg, gif and webp and + * nothing else, so a vector handed over whole is a picture the agent cannot look at — it would + * travel the whole pipe to be refused at the far end. And an SVG is a document rather than an + * image, which is why IMAGE_TYPES (agents.ts) does not carry one: the server serves an + * attachment back on its own origin. + * + * The drawing goes through an ``, and that is what makes this safe rather than merely + * convenient. An SVG loaded that way renders in the browser's secure static mode, where a script + * does not run, an external reference is not fetched and no other document is reachable. Pixels + * come out; nothing SVG-shaped is stored, sent or served back, so the allowlist stays as it is. + */ + +/** + * The long edge of the PNG. A vector has no pixels of its own to be scaled up past — unlike the + * photographs in an image row, which App.tsx is careful never to draw larger than they are — so + * every one is drawn at this size rather than at whatever it claims: a 24px icon attached at 24px + * is a picture the agent can see nothing in. 1024² of PNG is a few hundred KB against the 24 MB + * the tray holds. + */ +const RASTER_EDGE = 1024; + +/** + * What to draw the vector at, from its own markup: the `width`/`height` pair if it states one in + * absolute units, else the `viewBox`, else whatever the browser made of it. + * + * Parsed from the text rather than read off the loaded `` because a viewBox-only SVG — which + * is most icons — has a ratio but no intrinsic size, and browsers disagree on what naturalWidth + * then reports: the CSS default 300×150 in some, the viewBox in others. Taking the wrong one is a + * squashed logo, and a logo is the thing people attach these for. + */ +export function rasterSize( + markup: string, + natural: { w: number; h: number }, +): { w: number; h: number } { + // Malformed markup parses to a document, whose attributes are all absent: that + // falls through to `natural` on its own, so there is nothing here to catch. + const root = new DOMParser().parseFromString( + markup, + "image/svg+xml", + ).documentElement; + const px = (raw: string | null) => { + const n = Number.parseFloat(raw ?? ""); + // A percentage or an em is a size relative to a box this has no business inventing. + return Number.isFinite(n) && n > 0 && !/%|e[mx]\s*$/i.test(raw ?? "") + ? n + : 0; + }; + const box = (root.getAttribute("viewBox") ?? "").split(/[\s,]+/).map(Number); + const side = (attr: string, i: number, fallback: number) => + px(root.getAttribute(attr)) || + (Number.isFinite(box[i]) && box[i]! > 0 ? box[i]! : 0) || + fallback; + const w = side("width", 2, natural.w); + const h = side("height", 3, natural.h); + // A vector that states no size anywhere and that the browser measured at nothing. Square is the + // only guess left, and it is better than a zero-sized canvas, which throws. + if (!(w > 0) || !(h > 0)) return { w: RASTER_EDGE, h: RASTER_EDGE }; + const k = RASTER_EDGE / Math.max(w, h); + // Rounded up off zero, not just rounded. Past about 2048:1 — a hairline rule, a wide divider — + // the short edge lands under half a pixel and rounds away, and a canvas with a zero side draws + // nothing for toBlob to hand back. One pixel of a sliver is the sliver; none of it throws. + const edge = (side: number) => Math.max(1, Math.round(side * k)); + return { w: edge(w), h: edge(h) }; +} + +/** The same picture as a PNG, named for the file it came from. Rejects if it will not draw. */ +export async function rasterizeSvg(file: File): Promise { + const markup = await file.text(); + // An object URL rather than the text as a data URL: same origin either way, so the canvas is + // not tainted and toBlob is allowed, and a large icon is not carried twice through base64. + const url = URL.createObjectURL(file); + try { + const img = new Image(); + await new Promise((drawn, fail) => { + img.onload = () => drawn(); + img.onerror = () => fail(new Error(`${file.name} is not a picture`)); + img.src = url; + }); + const { w, h } = rasterSize(markup, { + w: img.naturalWidth, + h: img.naturalHeight, + }); + const surface = document.createElement("canvas"); + surface.width = w; + surface.height = h; + const pen = surface.getContext("2d"); + if (!pen) throw new Error("this browser draws no 2d canvas"); + // Left transparent where the vector is transparent. Painting a background in would be a + // guess at the page it is meant to sit on, and would bury a logo drawn in white. + pen.drawImage(img, 0, 0, w, h); + const png = await new Promise((done) => + surface.toBlob(done, "image/png"), + ); + if (!png) throw new Error(`${file.name} could not be drawn`); + return new File([png], `${file.name.replace(/\.svg$/i, "")}.png`, { + type: "image/png", + }); + } finally { + URL.revokeObjectURL(url); + } +}