canvas: an SVG is attached by drawing it, not by refusing it

canvas/src/ChatPanel.tsx

@@ -47,9 +47,10 @@
 import { Fragment, useContext, useEffect, useRef, useState } from "react";
 import { useValue } from "tldraw";
 import {
-  IMAGE_TYPES,
+  ATTACH_TYPES,
   MAX_IMAGE_BYTES,
   MAX_IMAGES,
+  SVG_TYPE,
   type AgentId,
   type AgentModel,
 } from "./agents";
@@ -63,6 +64,7 @@ import { ClaudeMark } from "./ClaudeMark";
 import { CodexMark } from "./CodexMark";
 import { Check, ClockRewind, Image, Plus } from "./geistIcons";
 import { renderMarkdown } from "./markdown";
+import { rasterizeSvg } from "./svgRaster";
 
 const RUNS_KEY = "sp-chat-runs";
 const AGENT_KEY = "sp-chat-agent";
@@ -438,20 +440,28 @@ export function ChatPanel() {
     (adds.current = adds.current
       .then(async () => {
         const given = [...(list ?? [])];
-        const picked = given.filter((f) => IMAGE_TYPES.includes(f.type));
+        const taken = given.filter((f) => ATTACH_TYPES.includes(f.type));
         // A file the agent could not look at is refused here rather than by the server, and said:
         // the file dialog offers only these, but a drop, a paste and the canvas hand over anything.
         setSendError(
-          picked.length < given.length
-            ? "only png, jpeg, gif and webp pictures can be attached"
+          taken.length < given.length
+            ? "only png, jpeg, gif, webp and svg pictures can be attached"
             : null,
         );
-        if (picked.length === 0) return;
+        if (taken.length === 0) return;
         // What cannot fit whatever the tray holds is refused before it is read: a read is the
-        // whole file in memory, a third larger.
+        // whole file in memory, a third larger. Measured on the files as they arrived, since
+        // that is what a drawing has to hold in memory too.
         const tooBig = `those are too large to attach; keep them under about ${MAX_IMAGE_BYTES / 1_000_000} MB together`;
-        if (picked.reduce((n, f) => n + f.size, 0) > MAX_IMAGE_BYTES)
+        if (taken.reduce((n, f) => n + f.size, 0) > MAX_IMAGE_BYTES)
           return setSendError(tooBig);
+        // A vector is drawn into a PNG here, at the one door every attachment comes through —
+        // the picker, a paste, a drop, and the + on a canvas shape, which hands its asset over
+        // with whatever type the asset has. Past this line everything is an IMAGE_TYPE, so the
+        // tray, the limits, the preview and the agent all go on seeing what they always saw.
+        const picked = await Promise.all(
+          taken.map((f) => (f.type === SVG_TYPE ? rasterizeSvg(f) : f)),
+        );
         // Read first and numbered after, which is what lets the same picture keep the number it
         // already has: pressing + on one twice is one picture said twice, not two — and not a
         // twenty-first, so the limits are over what is new. Still numbered in the order they
@@ -492,10 +502,13 @@ export function ChatPanel() {
         // rather than the one React has not rendered yet.
         tray.current = [...tray.current, ...fresh].sort((x, y) => x.n - y.n);
         setAttached(tray.current);
-        // One picture, pointed at on the canvas: the sentence says which one without a second
-        // click on the tile that has just appeared, and says it once however many times it is
-        // pointed at. A pick, a paste or a drop is a handful at once, and which of them the
-        // message is about is still to be said.
+        // Numbered into the sentence where the caret already is, so the picture the writer has
+        // just put there is named without a second click on the tile that has appeared above —
+        // and named once however many times it is added. Both ways in that land on one picture
+        // do this: the + on a canvas shape, and a paste, which is the screenshot in the
+        // clipboard going into the sentence being typed. A pick and a drop are a handful chosen
+        // at a distance from the caret, and which of them the message is about is still to be
+        // said.
         const box = composer.current;
         if (cite && box)
           for (const image of said)
@@ -1042,7 +1055,7 @@ export function ChatPanel() {
           onPaste={(e) => {
             e.preventDefault();
             if (e.clipboardData.files.length)
-              return void addImages(e.clipboardData.files);
+              return void addImages(e.clipboardData.files, true);
             // The text and not the markup that came with it: the box holds the chips it made
             // itself and nothing else. execCommand because it is the only insert that native
             // undo still knows about.
@@ -1177,7 +1190,7 @@ export function ChatPanel() {
         <input
           ref={files}
           type="file"
-          accept={IMAGE_TYPES.join(",")}
+          accept={ATTACH_TYPES.join(",")}
           multiple
           hidden
           onChange={(e) => {

canvas/src/agents.ts

@@ -40,6 +40,14 @@ export const IMAGE_TYPES = [
   "image/gif",
   "image/webp",
 ];
+export const SVG_TYPE = "image/svg+xml";
+/**
+ * What the composer takes at the door, which is one more than what travels: an SVG is accepted
+ * and drawn into a PNG on the way in (svgRaster.ts), so the vector itself never reaches the tray,
+ * the server or the agent. The two lists differ by exactly that conversion — anything accepted
+ * here is an IMAGE_TYPE by the time it is numbered.
+ */
+export const ATTACH_TYPES = [...IMAGE_TYPES, SVG_TYPE];
 export const MAX_IMAGES = 20;
 export const MAX_IMAGE_BYTES = 24_000_000;
 

canvas/src/svgRaster.test.ts

@@ -0,0 +1,45 @@
+// @vitest-environment jsdom
+import { describe, expect, it } from "vitest";
+import { rasterSize } from "./svgRaster";
+
+/** What the browser reports for a vector it could not measure, which is the interesting case. */
+const unmeasured = { w: 0, h: 0 };
+
+describe("rasterSize", () => {
+  it("scales a stated size so its long edge is the raster edge", () => {
+    const svg =
+      '<svg width="200" height="100" xmlns="http://www.w3.org/2000/svg"/>';
+    expect(rasterSize(svg, unmeasured)).toEqual({ w: 1024, h: 512 });
+  });
+
+  it("keeps a viewBox-only icon square rather than taking the browser's 300x150", () => {
+    const svg = '<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"/>';
+    expect(rasterSize(svg, { w: 300, h: 150 })).toEqual({ w: 1024, h: 1024 });
+  });
+
+  it("ignores a relative width and falls through to the viewBox", () => {
+    const svg = '<svg width="100%" height="100%" viewBox="0 0 40 10"/>';
+    expect(rasterSize(svg, unmeasured)).toEqual({ w: 1024, h: 256 });
+  });
+
+  it("reads the root's own size, not a child's", () => {
+    const svg = '<svg viewBox="0 0 10 20"><rect width="999" height="1"/></svg>';
+    expect(rasterSize(svg, unmeasured)).toEqual({ w: 512, h: 1024 });
+  });
+
+  it("falls back to what the browser measured when the markup states nothing", () => {
+    expect(rasterSize("<svg/>", { w: 64, h: 32 })).toEqual({ w: 1024, h: 512 });
+  });
+
+  it("gives an unparseable file a square rather than a zero-sized canvas", () => {
+    expect(rasterSize("<svg", unmeasured)).toEqual({ w: 1024, h: 1024 });
+  });
+
+  it("keeps a pixel of a sliver too thin to round to one", () => {
+    // 5000:1 scales to 0.2 of a pixel. Rounded it is nothing, and a canvas with a zero side
+    // hands toBlob back no blob at all, which rasterizeSvg then reports as a failed drawing.
+    const svg =
+      '<svg viewBox="0 0 5000 1" xmlns="http://www.w3.org/2000/svg"/>';
+    expect(rasterSize(svg, unmeasured)).toEqual({ w: 1024, h: 1 });
+  });
+});

canvas/src/svgRaster.ts

@@ -0,0 +1,104 @@
+/**
+ * An SVG on its way into the composer, drawn into a PNG.
+ *
+ * Two reasons, and either alone would be enough. The CLIs read png, jpeg, gif and webp and
+ * nothing else, so a vector handed over whole is a picture the agent cannot look at — it would
+ * travel the whole pipe to be refused at the far end. And an SVG is a document rather than an
+ * image, which is why IMAGE_TYPES (agents.ts) does not carry one: the server serves an
+ * attachment back on its own origin.
+ *
+ * The drawing goes through an `<img>`, and that is what makes this safe rather than merely
+ * convenient. An SVG loaded that way renders in the browser's secure static mode, where a script
+ * does not run, an external reference is not fetched and no other document is reachable. Pixels
+ * come out; nothing SVG-shaped is stored, sent or served back, so the allowlist stays as it is.
+ */
+
+/**
+ * The long edge of the PNG. A vector has no pixels of its own to be scaled up past — unlike the
+ * photographs in an image row, which App.tsx is careful never to draw larger than they are — so
+ * every one is drawn at this size rather than at whatever it claims: a 24px icon attached at 24px
+ * is a picture the agent can see nothing in. 1024² of PNG is a few hundred KB against the 24 MB
+ * the tray holds.
+ */
+const RASTER_EDGE = 1024;
+
+/**
+ * What to draw the vector at, from its own markup: the `width`/`height` pair if it states one in
+ * absolute units, else the `viewBox`, else whatever the browser made of it.
+ *
+ * Parsed from the text rather than read off the loaded `<img>` because a viewBox-only SVG — which
+ * is most icons — has a ratio but no intrinsic size, and browsers disagree on what naturalWidth
+ * then reports: the CSS default 300×150 in some, the viewBox in others. Taking the wrong one is a
+ * squashed logo, and a logo is the thing people attach these for.
+ */
+export function rasterSize(
+  markup: string,
+  natural: { w: number; h: number },
+): { w: number; h: number } {
+  // Malformed markup parses to a <parsererror> document, whose attributes are all absent: that
+  // falls through to `natural` on its own, so there is nothing here to catch.
+  const root = new DOMParser().parseFromString(
+    markup,
+    "image/svg+xml",
+  ).documentElement;
+  const px = (raw: string | null) => {
+    const n = Number.parseFloat(raw ?? "");
+    // A percentage or an em is a size relative to a box this has no business inventing.
+    return Number.isFinite(n) && n > 0 && !/%|e[mx]\s*$/i.test(raw ?? "")
+      ? n
+      : 0;
+  };
+  const box = (root.getAttribute("viewBox") ?? "").split(/[\s,]+/).map(Number);
+  const side = (attr: string, i: number, fallback: number) =>
+    px(root.getAttribute(attr)) ||
+    (Number.isFinite(box[i]) && box[i]! > 0 ? box[i]! : 0) ||
+    fallback;
+  const w = side("width", 2, natural.w);
+  const h = side("height", 3, natural.h);
+  // A vector that states no size anywhere and that the browser measured at nothing. Square is the
+  // only guess left, and it is better than a zero-sized canvas, which throws.
+  if (!(w > 0) || !(h > 0)) return { w: RASTER_EDGE, h: RASTER_EDGE };
+  const k = RASTER_EDGE / Math.max(w, h);
+  // Rounded up off zero, not just rounded. Past about 2048:1 — a hairline rule, a wide divider —
+  // the short edge lands under half a pixel and rounds away, and a canvas with a zero side draws
+  // nothing for toBlob to hand back. One pixel of a sliver is the sliver; none of it throws.
+  const edge = (side: number) => Math.max(1, Math.round(side * k));
+  return { w: edge(w), h: edge(h) };
+}
+
+/** The same picture as a PNG, named for the file it came from. Rejects if it will not draw. */
+export async function rasterizeSvg(file: File): Promise<File> {
+  const markup = await file.text();
+  // An object URL rather than the text as a data URL: same origin either way, so the canvas is
+  // not tainted and toBlob is allowed, and a large icon is not carried twice through base64.
+  const url = URL.createObjectURL(file);
+  try {
+    const img = new Image();
+    await new Promise<void>((drawn, fail) => {
+      img.onload = () => drawn();
+      img.onerror = () => fail(new Error(`${file.name} is not a picture`));
+      img.src = url;
+    });
+    const { w, h } = rasterSize(markup, {
+      w: img.naturalWidth,
+      h: img.naturalHeight,
+    });
+    const surface = document.createElement("canvas");
+    surface.width = w;
+    surface.height = h;
+    const pen = surface.getContext("2d");
+    if (!pen) throw new Error("this browser draws no 2d canvas");
+    // Left transparent where the vector is transparent. Painting a background in would be a
+    // guess at the page it is meant to sit on, and would bury a logo drawn in white.
+    pen.drawImage(img, 0, 0, w, h);
+    const png = await new Promise<Blob | null>((done) =>
+      surface.toBlob(done, "image/png"),
+    );
+    if (!png) throw new Error(`${file.name} could not be drawn`);
+    return new File([png], `${file.name.replace(/\.svg$/i, "")}.png`, {
+      type: "image/png",
+    });
+  } finally {
+    URL.revokeObjectURL(url);
+  }
+}