From 96f9f33fe90c40fbb66ed6f8f42c73418d7572da Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 17:30:59 -0400 Subject: [PATCH 01/17] feat(images) I1: carry images to OpenAI-compatible providers, and stop dropping blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First slice of docs/IMAGES.md. No UI yet — this is the wire seam, and half of it is a bug that predates images and is worth fixing on its own. THE SILENT DROP. toOpenAIMessages' user-message loop handled tool_result and text and fell through on everything else. A block type it had never seen vanished between the composer and the wire with no error, no warning and no log line — and the model then answered confidently about content it was never sent. A user would reasonably conclude it hallucinates. That fall-through is now a throw naming the type. Checked before making it throw: the only block types this codebase produces are text, tool_use and tool_result, so nothing in production reaches the new error path. IMAGES. Anthropic carries bytes in `source` (base64 + media_type, or url); OpenAI takes one `url` that is either a real URL or a data: URI. toOpenAIImagePart maps between them and refuses a `file` source outright — Files API references are Anthropic-only, and there is nothing to translate them to, so sending SOMETHING would mean sending a request whose subject is missing. Two ordering decisions: - Images lead the turn. The model reads them best before the text that asks about them, and it keeps markLastOpenAICacheable's breakpoint (which lands on the LAST block) on a text block rather than on an image. - A text-only turn still emits a plain string, not a one-element block array. Widening every turn would change the bytes of every cached prefix for no gain. Wrote the tests against the wrong signature first — toOpenAIMessages is (system, messages, opts), not (messages) — and got 'system' !== 'user' rather than a passing test, which is the failure mode I would rather have. Bypass-verified, each by reverting the fix: - the silent drop restored (the bug this slice exists for) - image blocks no longer recognised - text ordered before the image - malformed base64 no longer refused; a Files-API source sent as an empty url - text-only turns widened to block arrays 32 tests in translate (was 24), all suites green. --- .../levelcode-ai/providers/translate.js | 45 ++++++++- .../levelcode-ai/test/translate.test.js | 92 +++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) diff --git a/extensions/levelcode-ai/providers/translate.js b/extensions/levelcode-ai/providers/translate.js index 41df2f0..f046d83 100644 --- a/extensions/levelcode-ai/providers/translate.js +++ b/extensions/levelcode-ai/providers/translate.js @@ -33,6 +33,30 @@ function toOpenAITools(tools) { })); } +/** + * Anthropic image block → OpenAI `image_url` part. + * + * Anthropic carries the bytes in `source` (base64 + media_type, or a url); OpenAI-compatible APIs take + * one `url` field that is either a real URL or a `data:` URI. Everything else about the block is ours. + */ +function toOpenAIImagePart(b) { + const src = b && b.source; + if (!src) { throw new Error('translate: image block has no source'); } + if (src.type === 'url') { + if (!src.url) { throw new Error('translate: image block has a url source with no url'); } + return { type: 'image_url', image_url: { url: src.url } }; + } + if (src.type === 'base64') { + if (!src.media_type || !src.data) { + throw new Error('translate: image block is missing media_type or data'); + } + return { type: 'image_url', image_url: { url: 'data:' + src.media_type + ';base64,' + src.data } }; + } + // `file` (Files API) is Anthropic-only — there is no OpenAI equivalent to translate it to, so + // refuse rather than send a request whose subject is missing. + throw new Error('translate: image source type not supported on this provider: ' + String(src.type)); +} + /** Coerce a tool_result's content (string | array of blocks | other) to a plain string for OpenAI. */ function toolResultText(content) { if (typeof content === 'string') { return content; } @@ -92,15 +116,34 @@ function toOpenAIMessages(system, messages, opts) { if (typeof m.content === 'string') { out.push({ role: 'user', content: m.content }); continue; } const blocks = Array.isArray(m.content) ? m.content : []; let trailingText = ''; + const images = []; for (const b of blocks) { if (!b) { continue; } if (b.type === 'tool_result') { out.push({ role: 'tool', tool_call_id: b.tool_use_id, content: toolResultText(b.content) }); } else if (b.type === 'text') { trailingText += (trailingText ? '\n' : '') + (b.text || ''); + } else if (b.type === 'image') { + images.push(toOpenAIImagePart(b)); + } else { + // Loudly, not silently. This loop used to fall through on anything it did not + // recognise, so a block type it had never seen vanished between the composer and + // the wire with no error and no log line — the model would then answer confidently + // about content it was never sent. A type we do not understand is a bug in us. + throw new Error('translate: unsupported content block in a user message: ' + String(b.type)); } } - if (trailingText) { out.push({ role: 'user', content: trailingText }); } + // Images first: the model reads them best before the text that asks about them, and it keeps + // markLastOpenAICacheable's breakpoint on a text block rather than on an image. + if (images.length) { + const content = images.slice(); + if (trailingText) { content.push({ type: 'text', text: trailingText }); } + out.push({ role: 'user', content }); + } else if (trailingText) { + // No image — keep the plain string. Widening every text-only turn to a block array would + // change the bytes of every cached prefix for no gain. + out.push({ role: 'user', content: trailingText }); + } } if (cache) { markLastOpenAICacheable(out); } return out; diff --git a/extensions/levelcode-ai/test/translate.test.js b/extensions/levelcode-ai/test/translate.test.js index 28c2359..566bc15 100644 --- a/extensions/levelcode-ai/test/translate.test.js +++ b/extensions/levelcode-ai/test/translate.test.js @@ -261,4 +261,96 @@ test('isAnthropicFamily: gates cache_control writes to Claude upstreams only', ( assert.strictEqual(O.isAnthropicFamily(''), false); }); +// ── images (I1) ───────────────────────────────────────────────────────────────────────────────── + +test('IMAGE: a base64 block becomes an OpenAI image_url data URI, ahead of the text', () => { + const out = T.toOpenAIMessages('', [{ role: 'user', content: [ + { type: 'text', text: 'why does this look wrong?' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AAAB' } } + ] }]); + assert.strictEqual(out.length, 1); + assert.strictEqual(out[0].role, 'user'); + assert.ok(Array.isArray(out[0].content), 'a turn carrying an image must use block content'); + // Images lead: the model reads them best before the text, and it keeps the cache breakpoint + // (which lands on the LAST block) on text rather than on an image. + assert.strictEqual(out[0].content[0].type, 'image_url', 'the image must come first'); + assert.strictEqual(out[0].content[0].image_url.url, 'data:image/png;base64,AAAB'); + assert.strictEqual(out[0].content[1].type, 'text'); + assert.strictEqual(out[0].content[1].text, 'why does this look wrong?'); +}); + +test('IMAGE: a url source passes through as a url, not re-encoded', () => { + const out = T.toOpenAIMessages('', [{ role: 'user', content: [ + { type: 'image', source: { type: 'url', url: 'https://example.test/a.png' } } + ] }]); + assert.strictEqual(out[0].content[0].image_url.url, 'https://example.test/a.png'); +}); + +test('IMAGE: several images in one turn all survive', () => { + const out = T.toOpenAIMessages('', [{ role: 'user', content: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'A' } }, + { type: 'image', source: { type: 'base64', media_type: 'image/webp', data: 'B' } }, + { type: 'text', text: 'compare these' } + ] }]); + assert.strictEqual(out[0].content.filter((c) => c.type === 'image_url').length, 2); + assert.strictEqual(out[0].content[2].text, 'compare these'); +}); + +test('IMAGE: a text-only turn still emits a plain string, not a block array', () => { + // Widening every text-only turn would change the bytes of every cached prefix for no gain. + const out = T.toOpenAIMessages('', [{ role: 'user', content: [{ type: 'text', text: 'hello' }] }]); + assert.strictEqual(typeof out[0].content, 'string', 'text-only turns must not become block arrays'); + assert.strictEqual(out[0].content, 'hello'); +}); + +test('IMAGE: an image with no text emits the image alone', () => { + const out = T.toOpenAIMessages('', [{ role: 'user', content: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'A' } } + ] }]); + assert.strictEqual(out.length, 1); + assert.strictEqual(out[0].content.length, 1); + assert.strictEqual(out[0].content[0].type, 'image_url'); +}); + +test('LOUD: an unrecognised block throws instead of vanishing', () => { + // THE bug this slice exists for. The loop used to fall through on anything it did not know, so + // the block disappeared between composer and wire with no error and no log line — and the model + // answered confidently about content it was never sent. + assert.throws( + () => T.toOpenAIMessages('', [{ role: 'user', content: [ + { type: 'text', text: 'look at this' }, + { type: 'video', source: { type: 'base64', media_type: 'video/mp4', data: 'A' } } + ] }]), + /unsupported content block/, + 'an unknown block type must fail loudly, naming the type' + ); +}); + +test('LOUD: a malformed image throws rather than sending a request missing its subject', () => { + const bad = [ + [{ type: 'image' }, /no source/], + [{ type: 'image', source: { type: 'base64', media_type: 'image/png' } }, /media_type or data/], + [{ type: 'image', source: { type: 'base64', data: 'A' } }, /media_type or data/], + [{ type: 'image', source: { type: 'url' } }, /no url/], + // Files API references are Anthropic-only; there is nothing to translate them to. + [{ type: 'image', source: { type: 'file', file_id: 'file_1' } }, /source type not supported/] + ]; + for (const [block, re] of bad) { + assert.throws(() => T.toOpenAIMessages('', [{ role: 'user', content: [block] }]), re, + 'malformed image should throw: ' + JSON.stringify(block)); + } +}); + +test('IMAGE: tool_result still splits out to its own tool message alongside an image', () => { + const out = T.toOpenAIMessages('', [{ role: 'user', content: [ + { type: 'tool_result', tool_use_id: 'tu_1', content: 'ok' }, + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'A' } }, + { type: 'text', text: 'and this' } + ] }]); + assert.strictEqual(out[0].role, 'tool'); + assert.strictEqual(out[0].tool_call_id, 'tu_1'); + assert.strictEqual(out[1].role, 'user'); + assert.strictEqual(out[1].content[0].type, 'image_url'); +}); + console.log('\ntranslate: ' + n + ' tests passed.'); From 5ea450e25ae725d704786871d0547413604b8ab1 Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 17:34:22 -0400 Subject: [PATCH 02/17] feat(images) I2+I3: the vision gate, and the arithmetic behind a pasted screenshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I2 — supportsVisionForModel, beside supportsToolsForModel, reading the `vision` flag that has been sitting in the catalog unread since the multi-provider work. Deliberately STRICTER than the tools gate. That one defaults unknown models to tools:true, because most modern chat models have tools and refusing the agent is the bigger loss. Vision inverts the trade: attaching to a blind model costs the user a composed message and returns a provider error — or worse, a confident answer about the text alone — while a disabled attach button that names a model which CAN see costs one click. The catalog marks every vision model we know and the family heuristic covers the big three, so the strict default bites only on genuinely unrecognised ids. I3 — imageCost.js. Pure geometry and cost; no canvas, no fs, no vscode. The webview does the pixel work, this decides what the pixel work should aim for and tells the context meter what the result costs. THE NUMBERS ARE NOT ESTIMATES. The formula I had in mind (w*h/750) is stale and wrong. Claude sees 28x28 patches, so cost is ceil(w/28)*ceil(h/28), and each tier caps both the long edge and the token count. The doc-parity test reproduces every worked example in the vision documentation on both tiers — 1092^2 -> 1521, 1000^2 -> 1296, 1920x1080 -> 2691 high-res and 1456x819/1560 standard, 3840x2160 -> 2576x1449/4784. Three decisions worth naming: - Binary search on the scale, not a step-down loop. Stepping lands a few pixels short and misreports the size — which matters here because it is also what the UI shows. - clientScale returns exactly 1 when nothing should happen, so the caller can skip re-encoding and forward the ORIGINAL bytes. Re-encoding an untouched image only stacks artifacts, worst on the screenshots of text that are most of what gets pasted. - Unknown model -> standard tier, and unknown image size -> charged the tier cap. Both fail toward OVER-counting. Under-counting is the direction that lets a conversation full of images look empty to the compaction cut. Bypass-verified, each by reverting the fix: - the stale w*h/750 formula - upscaling allowed (the `sips -Z` trap that grew a capture 40KB -> 89KB) - step-down instead of binary search - unknown model treated as high-res; unknown-size image costed at zero - cap 0 collapsing an image to nothing - aspect ratio skewed, in both fitToTier and clientTarget 9 tests in imageCost, all suites green. --- extensions/levelcode-ai/imageCost.js | 114 ++++++++++++++++++ extensions/levelcode-ai/providers/catalog.js | 19 ++- .../levelcode-ai/test/imageCost.test.js | 112 +++++++++++++++++ 3 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 extensions/levelcode-ai/imageCost.js create mode 100644 extensions/levelcode-ai/test/imageCost.test.js diff --git a/extensions/levelcode-ai/imageCost.js b/extensions/levelcode-ai/imageCost.js new file mode 100644 index 0000000..80a3397 --- /dev/null +++ b/extensions/levelcode-ai/imageCost.js @@ -0,0 +1,114 @@ +/*--------------------------------------------------------------------------------------------- + * Image geometry and cost — the arithmetic behind attaching a screenshot. + * + * Pure: no canvas, no fs, no vscode. The webview does the actual pixel work; this decides what + * the pixel work should aim for, and tells the context meter what the result costs. + * + * THE NUMBERS ARE NOT ESTIMATES. Claude sees images as 28x28 patches, so an image costs + * ceil(w/28) * ceil(h/28) visual tokens, and each model tier caps both the long edge and the + * token count, downscaling past either. This module reproduces every worked example in the + * vision documentation: 1092^2 -> 1521, 1000^2 -> 1296, 1920x1080 -> 2691 (high-res) / 1456x819 + * at 1560 (standard), 3840x2160 -> 2576x1449 at 4784. test/imageCost.test.js pins all of them. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +'use strict'; + +/** One visual token per 28x28 patch, ceiling on each axis independently. */ +const PATCH = 28; + +/** + * Per-tier limits. High-res is Claude 4.7 and later; everything else is standard. + * Both are enforced server-side — an image over either limit is downscaled before the model sees + * it, which is why sending more pixels than this buys latency rather than fidelity. + */ +const TIERS = { + high: { edge: 2576, tokens: 4784 }, + standard: { edge: 1568, tokens: 1568 } +}; + +/** Visual tokens for an image at these dimensions. */ +function visualTokens(w, h) { + if (!(w > 0) || !(h > 0)) { return 0; } + return Math.ceil(w / PATCH) * Math.ceil(h / PATCH); +} + +/** Which tier a model id lands in. Unknown -> standard, so we never UNDER-count a cost. */ +function tierFor(modelId) { + const id = String(modelId || '').toLowerCase(); + // Claude 4.7+ (including the 5 line) is the high-resolution tier. + if (/claude-(opus|sonnet|fable|mythos)-5/.test(id)) { return 'high'; } + if (/claude-opus-4-(7|8|9)/.test(id)) { return 'high'; } + return 'standard'; +} + +/** + * What the server will actually process, given a source size and a tier. + * + * The rule is the largest scale (never above 1) whose patch grid fits the tier's token cap, with + * the long edge bounded too. Binary search rather than stepping the scale down: a step-down loop + * lands a few pixels short and misreports the size, which matters because this is also what the + * UI shows the user. + * + * NEVER SCALES UP. A source already inside both limits comes back untouched — upscaling costs + * bytes and tokens and adds no information. + */ +function fitToTier(w, h, tier) { + const t = TIERS[tier] || TIERS.standard; + if (!(w > 0) || !(h > 0)) { return { w: 0, h: 0, tokens: 0, scaled: false }; } + + let hi = Math.min(1, t.edge / Math.max(w, h)); + const at = (s) => [Math.round(w * s), Math.round(h * s)]; + + if (visualTokens(...at(hi)) <= t.tokens) { + const [ow, oh] = at(hi); + return { w: ow, h: oh, tokens: visualTokens(ow, oh), scaled: hi < 1 }; + } + let lo = 0; + for (let i = 0; i < 60; i++) { + const mid = (lo + hi) / 2; + if (visualTokens(...at(mid)) <= t.tokens) { lo = mid; } else { hi = mid; } + } + const [ow, oh] = at(lo); + return { w: ow, h: oh, tokens: visualTokens(ow, oh), scaled: true }; +} + +/** + * The scale the CLIENT should apply before sending, for a configured long-edge cap. + * + * Separate from fitToTier on purpose. The server caps cost whatever we do, so this is not a + * safety measure — it is a deliberate fidelity-for-cost trade the user can configure, and a + * defence against the wire (bytes, latency, the request size limit). + * + * Returns exactly 1 when nothing should happen, so the caller can skip re-encoding entirely and + * forward the original bytes. Re-encoding an untouched image only stacks compression artifacts, + * which is worst on the screenshots of text that are most of what gets pasted. + */ +function clientScale(w, h, cap) { + if (!(cap > 0) || !(w > 0) || !(h > 0)) { return 1; } + return Math.min(1, cap / Math.max(w, h)); +} + +/** Apply clientScale, rounded to whole pixels. Never larger than the source. */ +function clientTarget(w, h, cap) { + const s = clientScale(w, h, cap); + return s === 1 ? { w, h, scaled: false } : { w: Math.round(w * s), h: Math.round(h * s), scaled: true }; +} + +/** + * What one image block costs the context meter. + * + * This exists because estimateMsgTokens measures JSON.stringify().length / 4, which is sound for + * text and catastrophic for an image: base64 books about a third of its byte count as tokens, so + * a 1MB screenshot reads as ~333,000 — larger than most context windows — and the compaction cut + * fires on the first paste and evicts real conversation history. With bytes on disk and only a + * ref in the message the same estimator swings the other way and under-counts a ~1800-token image + * as ~18. Both are wrong; this is the number. + */ +function imageBlockTokens(block, modelId) { + if (!block || block.type !== 'image') { return 0; } + const w = Number(block.w) || 0, h = Number(block.h) || 0; + if (!w || !h) { return TIERS[tierFor(modelId)].tokens; } // unknown size: assume the cap, never zero + return fitToTier(w, h, tierFor(modelId)).tokens; +} + +module.exports = { PATCH, TIERS, visualTokens, tierFor, fitToTier, clientScale, clientTarget, imageBlockTokens }; diff --git a/extensions/levelcode-ai/providers/catalog.js b/extensions/levelcode-ai/providers/catalog.js index 3ee6c76..f7080fc 100644 --- a/extensions/levelcode-ai/providers/catalog.js +++ b/extensions/levelcode-ai/providers/catalog.js @@ -116,6 +116,23 @@ function supportsToolsForModel(providerId, modelId) { return modelCaps(modelId).tools !== false; } +/** + * Whether an image may be attached for this provider+model. + * + * Deliberately STRICTER than supportsToolsForModel. That one defaults unknown models to tools:true, + * because most modern chat models have tools and refusing the agent is the bigger loss. Vision + * inverts that trade: an unknown model is assumed NOT to see. Attaching to a blind model costs the + * user a composed message and returns a provider error (or, worse, a confident answer about text + * alone); a disabled attach button that names a model which can see costs them one click. The + * catalog already marks every vision model we know, and heuristicCaps covers the big families, so + * the strict default bites only on genuinely unrecognised ids. + */ +function supportsVisionForModel(providerId, modelId) { + const p = getProvider(providerId); + if (!p) { return false; } + return modelCaps(modelId).vision === true; +} + /** The model's context window (tokens), or `fallback` (then 200000) when unknown. */ function contextWindowFor(providerId, modelId, fallback) { return modelCaps(modelId).context || fallback || 200000; @@ -238,7 +255,7 @@ async function getModelChoices(providerId, opts) { module.exports = { CAPS, modelCaps, baseName, heuristicCaps, - supportsToolsForModel, contextWindowFor, fastCompletionModel, + supportsToolsForModel, supportsVisionForModel, contextWindowFor, fastCompletionModel, describeCaps, describeModel, mapOpenRouterModels, mapModelIds, fetchModels, getModelChoices }; diff --git a/extensions/levelcode-ai/test/imageCost.test.js b/extensions/levelcode-ai/test/imageCost.test.js new file mode 100644 index 0000000..4581417 --- /dev/null +++ b/extensions/levelcode-ai/test/imageCost.test.js @@ -0,0 +1,112 @@ +/*--------------------------------------------------------------------------------------------- + * Image geometry + cost — run: node test/imageCost.test.js + * + * The load-bearing test here is DOC PARITY: every worked example in the vision documentation, + * reproduced. If these drift, the context meter is lying and the UI is showing the user a size + * the server will not actually use. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +'use strict'; + +const assert = require('assert'); +const C = require('../imageCost'); + +let n = 0; +function test(name, fn) { fn(); n++; console.log(' ok - ' + name); } + +test('PATCHES: cost is a ceiling division on each axis, independently', () => { + assert.strictEqual(C.visualTokens(28, 28), 1); + assert.strictEqual(C.visualTokens(29, 28), 2, 'one pixel over buys a whole column of patches'); + assert.strictEqual(C.visualTokens(28, 29), 2, '…and a whole row'); + assert.strictEqual(C.visualTokens(1092, 1092), 1521); + assert.strictEqual(C.visualTokens(0, 100), 0); +}); + +test('DOC PARITY: every worked example in the vision docs, both tiers', () => { + // [w, h, standard "WxH/tokens", high-res "WxH/tokens"] + const rows = [ + [1092, 1092, '1092x1092/1521', '1092x1092/1521'], + [1000, 1000, '1000x1000/1296', '1000x1000/1296'], + [1920, 1080, '1456x819/1560', '1920x1080/2691'], + [3840, 2160, '1456x819/1560', '2576x1449/4784'], + [200, 200, '200x200/64', '200x200/64'] + ]; + for (const [w, h, std, hi] of rows) { + const a = C.fitToTier(w, h, 'standard'), b = C.fitToTier(w, h, 'high'); + assert.strictEqual(`${a.w}x${a.h}/${a.tokens}`, std, `standard tier, ${w}x${h}`); + assert.strictEqual(`${b.w}x${b.h}/${b.tokens}`, hi, `high-res tier, ${w}x${h}`); + } +}); + +test('CAPS: nothing escapes its tier, at any source size', () => { + for (const [w, h] of [[8000, 8000], [8000, 200], [200, 8000], [4032, 3024], [3024, 1964]]) { + for (const tier of ['standard', 'high']) { + const r = C.fitToTier(w, h, tier); + assert.ok(r.tokens <= C.TIERS[tier].tokens, `${w}x${h} ${tier}: ${r.tokens} over the token cap`); + assert.ok(Math.max(r.w, r.h) <= C.TIERS[tier].edge, `${w}x${h} ${tier}: over the long-edge cap`); + } + } +}); + +test('NEVER UPSCALE: a small source comes back untouched', () => { + // Writing the plan, `sips -Z 1568` GREW a 1160x480 capture from 40KB to 89KB by scaling it up + // to meet the cap. Upscaling costs bytes and tokens and adds no information. + for (const [w, h] of [[100, 50], [1160, 480], [1568, 1018], [2576, 1449]]) { + const hi = C.fitToTier(w, h, 'high'); + assert.ok(hi.w <= w && hi.h <= h, `${w}x${h} was scaled UP to ${hi.w}x${hi.h}`); + assert.strictEqual(C.clientScale(w, h, 4000), 1, 'a cap above the source must be a no-op'); + const t = C.clientTarget(w, h, 4000); + assert.deepStrictEqual([t.w, t.h, t.scaled], [w, h, false]); + } +}); + +test('ASPECT: downscaling preserves the ratio to within a pixel', () => { + for (const [w, h] of [[3840, 2160], [3024, 1964], [4032, 3024], [1920, 1200]]) { + const r = C.fitToTier(w, h, 'high'); + assert.ok(Math.abs((r.w / r.h) - (w / h)) < 0.01, `${w}x${h} -> ${r.w}x${r.h} skewed the aspect`); + } +}); + +test('CLIENT CAP: scale is a no-op at 1, so the caller can skip re-encoding', () => { + // A factor of exactly 1 is the signal to forward the ORIGINAL bytes. Re-encoding an untouched + // image only stacks compression artifacts, worst on the screenshots of text people paste. + assert.strictEqual(C.clientScale(1000, 800, 1568), 1); + assert.strictEqual(C.clientTarget(1000, 800, 1568).scaled, false); + const t = C.clientTarget(3840, 2160, 1568); + assert.deepStrictEqual([t.w, t.h, t.scaled], [1568, 882, true]); + assert.strictEqual(C.clientScale(3840, 2160, 0), 1, 'cap 0 means no cap, not a zero-size image'); +}); + +test('TIER: 4.7-and-later is high-res; anything unrecognised is standard, never the reverse', () => { + for (const id of ['claude-opus-5', 'claude-sonnet-5', 'anthropic/claude-opus-4-8', 'claude-fable-5']) { + assert.strictEqual(C.tierFor(id), 'high', id); + } + for (const id of ['claude-opus-4-6', 'gpt-4o', 'some-unknown-model', '', null]) { + assert.strictEqual(C.tierFor(id), 'standard', String(id)); + } +}); + +test('METER: an image block costs its real visual tokens, not its JSON length', () => { + // The bug this replaces: estimateMsgTokens is JSON.stringify(m).length / 4, which charges a + // base64 image about a third of its BYTE count — a 1MB screenshot books ~333,000 phantom + // tokens, more than most context windows, and the compaction cut evicts real history. + const block = { type: 'image', ref: 'a'.repeat(64), w: 3840, h: 2160, media_type: 'image/png' }; + assert.strictEqual(C.imageBlockTokens(block, 'claude-opus-5'), 4784); + assert.strictEqual(C.imageBlockTokens(block, 'gpt-4o'), 1560, 'standard tier costs less'); + + const naive = Math.round(JSON.stringify(block).length / 4); + assert.ok(C.imageBlockTokens(block, 'claude-opus-5') > naive * 10, + 'a ref is ~18 tokens by JSON length and ~4784 in reality — the meter must not under-count either'); + + assert.strictEqual(C.imageBlockTokens({ type: 'text', text: 'hi' }, 'claude-opus-5'), 0); + assert.strictEqual(C.imageBlockTokens(null, 'claude-opus-5'), 0); +}); + +test('METER: an image of unknown size is charged the cap, never zero', () => { + // A missing dimension must fail toward over-counting. Charging zero would let a conversation + // full of images look empty to the compaction cut. + assert.strictEqual(C.imageBlockTokens({ type: 'image', ref: 'x' }, 'claude-opus-5'), 4784); + assert.strictEqual(C.imageBlockTokens({ type: 'image', ref: 'x', w: 100 }, 'gpt-4o'), 1568); +}); + +console.log('\nimageCost: ' + n + ' tests passed.'); From c7a46aea33eb1dea2d4670afaaa33eac4c8843ee Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 17:58:04 -0400 Subject: [PATCH 03/17] fix(images): correct the compaction claim repeated in the shipped code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The I3 comment (and its test) carried the same overstatement a reviewer caught on #89: that a bad token estimate makes findCompactionCut evict real conversation history. It does not. findCompactionCut cuts on message count and goal boundaries and never reads a token number; compactAgentMemory uses estimateMsgTokens only for its before/after report. Checked by opening both, which is what I should have done before writing it. Today the consequence is a UI meter that reads wildly high the moment an image is attached — telling someone to start a new chat when they are nowhere near full. It becomes a correctness bug the day anything automatic keys off that number. Both files now say that instead. Fixing the comment rather than only the doc: a false causal claim in the source outlives a false claim in a design note, because the next person to touch imageCost.js reads the comment and not the PR thread. --- extensions/levelcode-ai/imageCost.js | 13 ++++++++----- extensions/levelcode-ai/test/imageCost.test.js | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/extensions/levelcode-ai/imageCost.js b/extensions/levelcode-ai/imageCost.js index 80a3397..e8faf70 100644 --- a/extensions/levelcode-ai/imageCost.js +++ b/extensions/levelcode-ai/imageCost.js @@ -98,11 +98,14 @@ function clientTarget(w, h, cap) { * What one image block costs the context meter. * * This exists because estimateMsgTokens measures JSON.stringify().length / 4, which is sound for - * text and catastrophic for an image: base64 books about a third of its byte count as tokens, so - * a 1MB screenshot reads as ~333,000 — larger than most context windows — and the compaction cut - * fires on the first paste and evicts real conversation history. With bytes on disk and only a - * ref in the message the same estimator swings the other way and under-counts a ~1800-token image - * as ~18. Both are wrong; this is the number. + * text and catastrophic for an image: base64 books about a third of its byte count as tokens, so a + * 1MB screenshot reads as ~333,000 — larger than most context windows — for something that really + * costs ~4,800. With bytes on disk and only a ref in the message the same estimator swings the + * other way and under-counts a ~1800-token image as ~18. Both are wrong; this is the number. + * + * Scope, checked rather than assumed (a reviewer caught an earlier overstatement): today this only + * misreports the UI meter — findCompactionCut cuts on message count and goal boundaries and never + * reads a token number. It becomes a correctness bug the day anything automatic keys off it. */ function imageBlockTokens(block, modelId) { if (!block || block.type !== 'image') { return 0; } diff --git a/extensions/levelcode-ai/test/imageCost.test.js b/extensions/levelcode-ai/test/imageCost.test.js index 4581417..d2d96db 100644 --- a/extensions/levelcode-ai/test/imageCost.test.js +++ b/extensions/levelcode-ai/test/imageCost.test.js @@ -89,7 +89,8 @@ test('TIER: 4.7-and-later is high-res; anything unrecognised is standard, never test('METER: an image block costs its real visual tokens, not its JSON length', () => { // The bug this replaces: estimateMsgTokens is JSON.stringify(m).length / 4, which charges a // base64 image about a third of its BYTE count — a 1MB screenshot books ~333,000 phantom - // tokens, more than most context windows, and the compaction cut evicts real history. + // tokens, more than most context windows, for something that really costs ~4,800. That + // misreports the UI meter today; it would be a correctness bug under any auto-compaction. const block = { type: 'image', ref: 'a'.repeat(64), w: 3840, h: 2160, media_type: 'image/png' }; assert.strictEqual(C.imageBlockTokens(block, 'claude-opus-5'), 4784); assert.strictEqual(C.imageBlockTokens(block, 'gpt-4o'), 1560, 'standard tier costs less'); From e66d52402635cfab5131a6f314bc09aa7d02903b Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 18:22:51 -0400 Subject: [PATCH 04/17] feat(images) I4: store pasted images locally, and make the meter tell the truth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local, session-attached. Nothing is uploaded. A screenshot of someone's proprietary code never leaves their machine — which is also the only shape that works for BYOK, where the editor talks to the provider directly and a detour through our infrastructure would add a failure mode and contradict the promise that we are not in the middle. WHY A SIBLING media/ DIRECTORY RATHER THAN INLINE BASE64. Claude Code inlines image bytes in its own JSONL and that works fine there — I measured it: 24 images, all base64, in a 34MB transcript. It does not work here, for a reason specific to this codebase. sessionStore.scanProject does readFileSync + JSON.parse on EVERY session file in a project whenever index.json is missing, malformed, or on an older schema — first run, and after any schema bump. Inlined bytes would make drawing a list of session titles parse every screenshot in every session. Refs keep that scan cheap, keep the transcript greppable, and dedupe the re-paste that follows a failed send. Content-addressed by sha256, so the same screenshot twice is one file. Written tmp+rename, because a crash mid-write must never leave a truncated file under a hash claiming to describe its full contents. Capped at 5MB of bytes — the Claude API allows 10MB of base64, Bedrock and Vertex 5MB, and base64 inflates by 4/3, so 5MB of bytes is what is safe everywhere. REFS ARE NOT PATHS. A ref is read straight out of a session file, which is data on disk a user can edit. isRef pins the shape to 64 hex characters plus a known extension, and read() returns null rather than touching anything else — `../../etc/passwd` is not a ref. THE METER. estimateMsgTokens now counts an image by its real visual cost and excludes its JSON entirely. chars/4 is sound for text and wrong for an image in EITHER shape: inline base64 books about a third of its byte count (a 1MB screenshot read as ~350,000 tokens), while a stored ref is 64 hex characters and reads as ~18 — for the same ~4,800. A test pins that the storage shape does not move the number, which is the point of the design. Bypass-verified, each by reverting the fix: - path traversal accepted as a ref - not content-addressed, so duplicate pastes duplicate files - oversize and unsupported media types accepted - a missing file silently becoming empty text instead of throwing - tmp file left behind by a non-atomic write - the meter counting images by JSON length again; and costing them nothing at all 11 tests in imageStore, 36 suites green. --- extensions/levelcode-ai/agentMemory.js | 30 ++++- extensions/levelcode-ai/imageStore.js | 116 +++++++++++++++++ .../levelcode-ai/test/imageStore.test.js | 122 ++++++++++++++++++ 3 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 extensions/levelcode-ai/imageStore.js create mode 100644 extensions/levelcode-ai/test/imageStore.test.js diff --git a/extensions/levelcode-ai/agentMemory.js b/extensions/levelcode-ai/agentMemory.js index 5f9f2cf..a4c9a3e 100644 --- a/extensions/levelcode-ai/agentMemory.js +++ b/extensions/levelcode-ai/agentMemory.js @@ -6,6 +6,8 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; + +const { imageBlockTokens } = require('./imageCost'); /** A "goal boundary": a user message with plain STRING content (a fresh user turn, never a tool_result). * It is the only splice point that cannot orphan a tool_use/tool_result pair — tool results always sit * in the message immediately after their tool_use, so any pair is wholly on one side of such a cut. */ @@ -34,10 +36,32 @@ function findCompactionCut(msgs, keepRecent) { return cut; } -/** Rough token estimate for a message list — the house chars/4 heuristic, used only for the UI meter. */ -function estimateMsgTokens(msgs) { +/** + * Rough token estimate for a message list — the house chars/4 heuristic, used only for the UI meter. + * + * Images are counted by their real visual cost, not by their JSON. chars/4 is sound for text and + * wrong for an image in whichever shape it takes: inline base64 books about a third of its byte + * count (a 1MB screenshot reads as ~333,000 tokens, more than most context windows, for something + * that really costs ~4,800), and a stored ref swings the other way — 64 hex characters read as ~18 + * tokens for the same ~4,800. Both would make the meter lie about how much room is left. + * + * `modelId` picks the resolution tier; omitting it costs the standard tier, which over-counts + * rather than under-counts. See imageCost.js. + */ +function estimateMsgTokens(msgs, modelId) { if (!Array.isArray(msgs)) { return 0; } - return Math.round(msgs.reduce((n, m) => n + JSON.stringify(m).length, 0) / 4); + let chars = 0; + let imageTokens = 0; + for (const m of msgs) { + if (!m) { continue; } + if (!Array.isArray(m.content)) { chars += JSON.stringify(m).length; continue; } + chars += 24; // role + envelope, roughly what the object costs around its blocks + for (const b of m.content) { + if (b && b.type === 'image') { imageTokens += imageBlockTokens(b, modelId); } + else { chars += JSON.stringify(b).length; } + } + } + return Math.round(chars / 4) + imageTokens; } module.exports = { isGoalBoundary, findCompactionCut, estimateMsgTokens }; diff --git a/extensions/levelcode-ai/imageStore.js b/extensions/levelcode-ai/imageStore.js new file mode 100644 index 0000000..640c10a --- /dev/null +++ b/extensions/levelcode-ai/imageStore.js @@ -0,0 +1,116 @@ +/*--------------------------------------------------------------------------------------------- + * Pasted images on disk — content-addressed, beside the session that used them. + * + * LOCAL, SESSION-ATTACHED. Nothing is uploaded. A screenshot of someone's proprietary code + * never leaves their machine, which is also the only shape that works for BYOK, where the + * editor talks to the provider directly and a detour through our infrastructure would both add + * a failure mode and contradict the promise that we are not in the middle. + * + * WHY A SIBLING DIRECTORY RATHER THAN INLINE BASE64. Claude Code inlines image bytes in its + * own JSONL transcript and that works fine there. It does not work here, and the reason is + * specific to this codebase: sessionStore.scanProject readFileSync + JSON.parses EVERY session + * file in a project whenever index.json is missing, malformed, or on an older schema — which + * happens on first run and after any schema bump. Inlined bytes would make drawing a list of + * session titles parse every screenshot in every session. Refs keep that scan cheap, keep the + * transcript greppable, and dedupe the re-paste that follows a failed send. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + +/** Claude accepts exactly these. Anything else is refused before it reaches a provider. */ +const MEDIA_EXT = { + 'image/png': 'png', + 'image/jpeg': 'jpg', + 'image/gif': 'gif', + 'image/webp': 'webp' +}; + +/** + * Per-image ceiling. The Claude API's own limit is 10MB of base64 (5MB on Bedrock and Vertex), + * and base64 inflates by 4/3 — so 5MB of BYTES is the largest thing that is safe everywhere. + * Normalization should keep real pastes far under this; the cap is for the pathological file. + */ +const MAX_BYTES = 5 * 1024 * 1024; + +function mediaDir(root, slug) { return path.join(root, slug, 'media'); } +function refPath(root, slug, ref) { return path.join(mediaDir(root, slug), ref); } + +/** true for a ref this module could have produced — 64 hex chars, a known extension, no path parts. */ +function isRef(ref) { + return typeof ref === 'string' && /^[0-9a-f]{64}\.(png|jpg|gif|webp)$/.test(ref); +} + +/** + * Store bytes and return the ref that identifies them. + * + * Content-addressed: the same screenshot pasted twice is one file, which is exactly what happens + * when someone re-pastes after a send fails. Writing is skipped when the file already exists, so + * a duplicate paste costs a hash and a stat. + */ +function put(root, slug, base64, mediaType) { + const ext = MEDIA_EXT[mediaType]; + if (!ext) { throw new Error('imageStore: unsupported media type: ' + String(mediaType)); } + const buf = Buffer.from(String(base64 || ''), 'base64'); + if (!buf.length) { throw new Error('imageStore: empty image'); } + if (buf.length > MAX_BYTES) { + throw new Error('imageStore: image is ' + Math.round(buf.length / 1024) + 'KB, over the ' + + Math.round(MAX_BYTES / 1024) + 'KB limit'); + } + const ref = crypto.createHash('sha256').update(buf).digest('hex') + '.' + ext; + const dest = refPath(root, slug, ref); + if (!fs.existsSync(dest)) { + fs.mkdirSync(mediaDir(root, slug), { recursive: true }); + // tmp + rename: a crash mid-write must never leave a truncated file under a hash that + // claims to describe its full contents. + const tmp = dest + '.' + process.pid + '.tmp'; + fs.writeFileSync(tmp, buf); + fs.renameSync(tmp, dest); + } + return { ref, bytes: buf.length }; +} + +/** Read bytes back as base64 for a provider request. Returns null when the file is gone. */ +function read(root, slug, ref) { + if (!isRef(ref)) { return null; } + try { return fs.readFileSync(refPath(root, slug, ref)).toString('base64'); } + catch { return null; } +} + +/** The media type a ref implies, from its extension. */ +function mediaTypeOf(ref) { + if (!isRef(ref)) { return null; } + const ext = ref.slice(ref.lastIndexOf('.') + 1); + return Object.keys(MEDIA_EXT).find((k) => MEDIA_EXT[k] === ext) || null; +} + +/** + * A stored `{type:'image', ref, …}` block → the Anthropic wire block, bytes and all. + * + * Called only when a request is being built, and the result is never retained: the conversation, + * the session log and the token meter all keep the ref. Throws when the file is missing, because + * a request that silently drops its subject is the failure this whole feature exists to avoid. + */ +function materialize(root, slug, block) { + if (!block || block.type !== 'image') { return block; } + if (block.source) { return block; } // already materialized (or an inline block from elsewhere) + const data = read(root, slug, block.ref); + if (!data) { throw new Error('imageStore: attached image is missing from disk: ' + String(block.ref)); } + return { type: 'image', source: { type: 'base64', media_type: mediaTypeOf(block.ref), data } }; +} + +/** Refs still referenced by these messages — the keep-set for a sweep. */ +function refsIn(msgs) { + const out = new Set(); + for (const m of (Array.isArray(msgs) ? msgs : [])) { + for (const b of (Array.isArray(m && m.content) ? m.content : [])) { + if (b && b.type === 'image' && isRef(b.ref)) { out.add(b.ref); } + } + } + return out; +} + +module.exports = { MEDIA_EXT, MAX_BYTES, mediaDir, refPath, isRef, put, read, mediaTypeOf, materialize, refsIn }; diff --git a/extensions/levelcode-ai/test/imageStore.test.js b/extensions/levelcode-ai/test/imageStore.test.js new file mode 100644 index 0000000..cb7b0c6 --- /dev/null +++ b/extensions/levelcode-ai/test/imageStore.test.js @@ -0,0 +1,122 @@ +/*--------------------------------------------------------------------------------------------- + * Local image store + the meter that counts what it holds — run: node test/imageStore.test.js + *--------------------------------------------------------------------------------------------*/ +// @ts-check +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const S = require('../imageStore'); +const { estimateMsgTokens } = require('../agentMemory'); + +let n = 0; +function test(name, fn) { fn(); n++; console.log(' ok - ' + name); } + +const root = fs.mkdtempSync(path.join(os.tmpdir(), 'lc-img-')); +const slug = 'proj'; +const png = Buffer.from('89504e470d0a1a0a0000000d49484452', 'hex').toString('base64'); + +test('ROUND TRIP: bytes go in, the same bytes come back', () => { + const { ref, bytes } = S.put(root, slug, png, 'image/png'); + assert.ok(S.isRef(ref), 'ref should be sha256 + extension: ' + ref); + assert.strictEqual(bytes, Buffer.from(png, 'base64').length); + assert.strictEqual(S.read(root, slug, ref), png); + assert.strictEqual(S.mediaTypeOf(ref), 'image/png'); +}); + +test('CONTENT ADDRESSED: the same screenshot twice is one file', () => { + // Exactly what happens when someone re-pastes after a send fails. + const a = S.put(root, slug, png, 'image/png'); + const b = S.put(root, slug, png, 'image/png'); + assert.strictEqual(a.ref, b.ref); + const files = fs.readdirSync(S.mediaDir(root, slug)).filter((f) => f.endsWith('.png')); + assert.strictEqual(files.length, 1, 'a duplicate paste must not write a second file'); +}); + +test('CONTENT ADDRESSED: different bytes get different refs', () => { + const other = Buffer.from('ffd8ffe000104a464946', 'hex').toString('base64'); + assert.notStrictEqual(S.put(root, slug, png, 'image/png').ref, + S.put(root, slug, other, 'image/jpeg').ref); +}); + +test('NO TMP LEFT BEHIND: a completed write leaves only the final file', () => { + assert.ok(!fs.readdirSync(S.mediaDir(root, slug)).some((f) => f.includes('.tmp')), + 'tmp+rename must not leave a .tmp file behind'); +}); + +test('REFUSED: unsupported media type, empty bytes, and oversize', () => { + assert.throws(() => S.put(root, slug, png, 'image/tiff'), /unsupported media type/); + assert.throws(() => S.put(root, slug, png, 'image/svg+xml'), /unsupported media type/); + assert.throws(() => S.put(root, slug, '', 'image/png'), /empty image/); + const huge = Buffer.alloc(S.MAX_BYTES + 1).toString('base64'); + assert.throws(() => S.put(root, slug, huge, 'image/png'), /over the/); +}); + +test('REFS ARE NOT PATHS: traversal and junk are rejected, not read', () => { + // read() takes a ref straight from a session file, which is data on disk that a user could edit. + for (const bad of ['../../etc/passwd', '../secrets.png', 'a/b.png', 'notahash.png', + 'a'.repeat(64) + '.exe', 'a'.repeat(63) + '.png', '', null, undefined]) { + assert.strictEqual(S.isRef(bad), false, 'should not look like a ref: ' + String(bad)); + assert.strictEqual(S.read(root, slug, bad), null, 'must not read: ' + String(bad)); + assert.strictEqual(S.mediaTypeOf(bad), null); + } +}); + +test('MATERIALIZE: a ref becomes a wire block only when a request is built', () => { + const { ref } = S.put(root, slug, png, 'image/png'); + const out = S.materialize(root, slug, { type: 'image', ref, w: 100, h: 50 }); + assert.deepStrictEqual(out, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: png } }); + // already-materialized blocks pass through untouched + const inline = { type: 'image', source: { type: 'url', url: 'https://x.test/a.png' } }; + assert.strictEqual(S.materialize(root, slug, inline), inline); + assert.deepStrictEqual(S.materialize(root, slug, { type: 'text', text: 'hi' }), { type: 'text', text: 'hi' }); +}); + +test('MATERIALIZE: a missing file throws rather than sending a request without its subject', () => { + assert.throws( + () => S.materialize(root, slug, { type: 'image', ref: 'b'.repeat(64) + '.png' }), + /missing from disk/, + 'a silently dropped image is the exact failure this feature exists to avoid' + ); +}); + +test('REFS IN: the keep-set sees every attached image and nothing else', () => { + const r1 = 'a'.repeat(64) + '.png', r2 = 'c'.repeat(64) + '.webp'; + const got = S.refsIn([ + { role: 'user', content: [{ type: 'image', ref: r1 }, { type: 'text', text: 'x' }] }, + { role: 'user', content: 'a plain string turn' }, + { role: 'user', content: [{ type: 'image', ref: r2 }, { type: 'image', ref: '../evil' }] } + ]); + assert.deepStrictEqual([...got].sort(), [r1, r2].sort()); +}); + +test('METER: the storage shape does not change the number', () => { + // This is the point of the whole design. An image costs what it costs; whether the bytes are + // inline or on disk behind a ref must not move the meter. + const big = 'A'.repeat(1_400_000); + const inline = [{ role: 'user', content: [ + { type: 'image', source: { type: 'base64', media_type: 'image/png', data: big } }, + { type: 'text', text: 'why?' }] }]; + const ref = [{ role: 'user', content: [ + { type: 'image', ref: 'a'.repeat(64) + '.png', w: 3840, h: 2160 }, + { type: 'text', text: 'why?' }] }]; + + const a = estimateMsgTokens(inline, 'claude-opus-5'); + const b = estimateMsgTokens(ref, 'claude-opus-5'); + assert.strictEqual(a, b, 'inline and ref must cost the same'); + assert.ok(a < 6000, 'a 1MB image must not book six figures of tokens — got ' + a); + assert.ok(a > 4000, 'nor may it be under-counted as a short string — got ' + a); + assert.ok(estimateMsgTokens(ref, 'gpt-4o') < b, 'the standard tier costs less than high-res'); +}); + +test('METER: text-only messages are unchanged by any of this', () => { + const msgs = [{ role: 'user', content: 'hello there' }, { role: 'assistant', content: 'hi' }]; + assert.strictEqual(estimateMsgTokens(msgs), Math.round( + msgs.reduce((n2, m) => n2 + JSON.stringify(m).length, 0) / 4), + 'the old heuristic must still hold exactly for text'); +}); + +try { fs.rmSync(root, { recursive: true, force: true }); } catch { /* best effort */ } +console.log('\nimageStore: ' + n + ' tests passed.'); From 17d4e6e2949c1ef835b02e2a683958ba73c5f1dc Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 18:31:30 -0400 Subject: [PATCH 05/17] feat(images) I5: paste a screenshot, ask about it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paste, drop, a thumbnail chip you can take back off, and the refusal that has to happen before anything is lost. THE CAP IS 2000px ON THE LONG EDGE, AND IT IS MEASURED, NOT CHOSEN. The plan argued 1568 from first principles. Claude Code's own transcripts are on this machine, so I read them instead: 24 images, and every single one it re-encodes is exactly 2000 on the long edge. That is the threshold the vision docs name for staying clear of the stricter per-image dimension limit above 20 images per request — the largest size that is never unsafe. It also sits above both model tiers' caps, so the server does the final downscale and we never throw away fidelity it would have kept. Changed 1568 -> 2000. The same transcripts confirmed the rule I had derived rather than observed: images UNDER the cap are passed through untouched IN THEIR ORIGINAL FORMAT (their PNGs stay PNG, their JPEGs stay JPEG), and only oversize ones are resized and re-encoded to WebP. Re-encoding something that did not need resizing only stacks artifacts, and that is worst on screenshots of code, which is most of what gets pasted. Verified in a real browser against real images, not just in unit tests: 4K screenshot 3840x2160 png 764KB -> 2000x1125 webp 115KB resized retina window 3024x1964 png 654KB -> 2000x1299 webp 137KB resized small PNG 1160x480 png 121KB -> 1160x480 png 121KB PASS-THROUGH under-cap JPEG 1600x900 jpg 208KB -> 1600x900 jpg 208KB PASS-THROUGH wide panorama 2400x600 png 178KB -> 2000x500 webp 51KB resized TIFF -> refused, with what to do instead Aspect ratios come out exact (16:9 stays 16:9, 4:1 stays 4:1). THE REFUSAL RUNS AT ATTACH TIME, NOT SEND TIME. The composer clears on send, so refusing host-side would throw away what someone had written. canSeeImages travels with the model config instead — from BOTH config paths, gateway and BYOK, which a test pins, because one of them silently allowing is exactly the kind of half-gate that looks fine in review. Other decisions worth naming: - An image with no words is a valid message. "Look at this" is implied by attaching it. - The chip carries its own thumbnail. An attachment you cannot see is one you cannot check, and a screenshot is the case where the wrong one looks exactly like the right one in a filename. - Drop is bound to the whole panel, not the composer: someone dragging a screenshot aims at the conversation, which is the far bigger target. - A paste with no image falls through untouched, and the bail comes BEFORE preventDefault — reversing those two silently breaks ordinary text pasting. - withImages() copies. agentMessages persists across runs and is what recordTurn writes to the session log; materializing in place would put megabytes of base64 in both. 9 tests in imageAttach, 37 suites green. --- extensions/levelcode-ai/extension.js | 77 +++++++- extensions/levelcode-ai/media/chat.html | 172 +++++++++++++++++- extensions/levelcode-ai/sessions.js | 4 +- .../levelcode-ai/test/imageAttach.test.js | 131 +++++++++++++ 4 files changed, 371 insertions(+), 13 deletions(-) create mode 100644 extensions/levelcode-ai/test/imageAttach.test.js diff --git a/extensions/levelcode-ai/extension.js b/extensions/levelcode-ai/extension.js index 4b64c49..17a69f0 100644 --- a/extensions/levelcode-ai/extension.js +++ b/extensions/levelcode-ai/extension.js @@ -22,6 +22,8 @@ const { registerLmProvider } = require('./lmProvider'); const { registerInlineComplete } = require('./inlineComplete'); const { runAgent } = require('./agent'); const { findCompactionCut, estimateMsgTokens } = require('./agentMemory'); +const imageStore = require('./imageStore'); +const { supportsVisionForModel } = require('./providers/catalog'); const sessionStore = require('./sessionStore'); const sessionEvents = require('./sessionEvents'); const sessionMemory = require('./sessionMemory'); @@ -1642,7 +1644,7 @@ async function agentFlow(text) { dbg('verify.config', { enabled: verifyCfg.enabled, hasCommand: !!verifyCfg.command, maxRounds: verifyCfg.maxRounds, includeWarnings: verifyCfg.includeWarnings }); try { await runAgent({ - messages: agentMessages, // persists across runs → the agent remembers the session + messages: withImages(agentMessages), // persists across runs → the agent remembers the session providerId: req.providerId, // Anthropic native, or an OpenAI-shaped provider via translation (P2) baseURL: req.baseURL, // for the custom / Ollama endpoints label: req.label, // route name for error attribution — "LevelCode Cloud" on the gateway, @@ -1724,8 +1726,57 @@ async function agentFlow(text) { } } -async function handleSend(text) { - if (!text || !text.trim()) { return; } +/** + * Store what the webview normalized, and return the blocks that will ride the conversation. + * + * Bytes land in the session's own media/ directory and the message keeps only a ref. Refused + * images are reported and skipped rather than failing the whole send — someone who pasted three + * screenshots and one unreadable file should still get their question answered. + */ +function storeImages(images) { + const out = []; + if (!Array.isArray(images) || !images.length) { return out; } + const m = sessionsManager(); + const paths = m && m.mediaRoot ? m.mediaRoot() : null; + if (!paths) { vscode.window.showWarningMessage('Images need a session to attach to.'); return out; } + for (const im of images) { + try { + const { ref, bytes } = imageStore.put(paths.root, paths.slug, im.base64, im.media_type); + out.push({ type: 'image', ref, w: Number(im.w) || 0, h: Number(im.h) || 0, bytes }); + } catch (e) { + const msg = String((e && e.message) || e).replace(/^imageStore: /, ''); + vscode.window.showWarningMessage('Could not attach an image: ' + msg); + dbg('image.store.failed', { msg }); + } + } + return out; +} + +/** + * A copy of `msgs` with every stored image turned into a real wire block. + * + * A COPY, deliberately. `agentMessages` persists across runs and is what recordTurn writes to the + * session log — materializing in place would put megabytes of base64 into both. + */ +function withImages(msgs) { + if (!Array.isArray(msgs)) { return msgs; } + const m = sessionsManager(); + const paths = m && m.mediaRoot ? m.mediaRoot() : null; + if (!paths) { return msgs; } + let touched = false; + const out = msgs.map((msg) => { + if (!msg || !Array.isArray(msg.content)) { return msg; } + if (!msg.content.some((b) => b && b.type === 'image' && b.ref)) { return msg; } + touched = true; + return { ...msg, content: msg.content.map((b) => imageStore.materialize(paths.root, paths.slug, b)) }; + }); + return touched ? out : msgs; +} + +async function handleSend(text, images) { + const imageBlocks = storeImages(images); + if ((!text || !text.trim()) && !imageBlocks.length) { return; } + text = text || ''; if (ctx) { ctx.globalState.update('levelcode.ai.hasSentMessage', true); } // user engaged → stop auto-revealing the panel on launch if (agentMode) { await agentFlow(text); return; } const cfg = aiConfig(); @@ -1750,7 +1801,12 @@ async function handleSend(text) { if (pendingContext) { blocks.push(pendingContext); } const userContent = blocks.length ? (blocks.join('\n\n') + '\n\n' + text) : text; - conversation.push({ role: 'user', content: userContent }); + // Blocks only when there is an image; a text-only turn stays a plain string so every cached + // prefix keeps the bytes it already had. Images lead — the model reads them best before the + // text that asks about them. + conversation.push(imageBlocks.length + ? { role: 'user', content: [...imageBlocks, { type: 'text', text: userContent }] } + : { role: 'user', content: userContent }); post({ type: 'userMessage', text }); if (auto.names.length) { post({ type: 'autoContext', names: auto.names }); } pendingContext = null; @@ -1772,7 +1828,7 @@ async function handleSend(text) { const doStream = (r) => providers.streamChat({ providerId: r.providerId, apiKey: r.apiKey, baseURL: r.baseURL, label: r.label, model: r.model, maxTokens: r.maxTokens, system: SYSTEM_PROMPT, - messages: conversation, signal: abort.signal, onDelta + messages: withImages(conversation), signal: abort.signal, onDelta }); try { await doStream(req); @@ -2240,14 +2296,16 @@ function sendConfigToWebview() { type: 'config', provider: 'gateway', proseSize, proseWidth, model: gatewayModelLabel(model), modelId: model, providerLabel: 'LevelCode Cloud', contextLimit: contextLimitFor('openai', capsModel(model)), gateway: true, plan: cloudPlanName() || 'Free', paid: isPaidCloudPlan(cloudPlanName()), - groupActivity: groupActivity + groupActivity: groupActivity, canSeeImages: supportsVisionForModel('openai', capsModel(model)) }); return; } const providerId = currentProviderId(); const p = providers.getProvider(providerId) || providers.getProvider('claude'); // Carry the model's context window so the footer meter updates the moment the model changes. - post({ type: 'config', provider: providerId, proseSize, proseWidth, model: activeModel(cfg, providerId), providerLabel: p.label, contextLimit: currentContextLimit(), groupActivity: groupActivity }); + // canSeeImages travels with the model so the composer can refuse an attachment BEFORE anything is + // typed and lost, rather than after a send that the provider would reject. + post({ type: 'config', provider: providerId, proseSize, proseWidth, model: activeModel(cfg, providerId), providerLabel: p.label, contextLimit: currentContextLimit(), groupActivity: groupActivity, canSeeImages: supportsVisionForModel(providerId, activeModel(cfg, providerId)) }); } /** @@ -2280,7 +2338,10 @@ class ChatViewProvider { case 'ready': cloudSignedIn = !!(ctx && await ctx.secrets.get(ACCOUNT_TOKEN_KEY)); autopilot = aiConfig().get('agent.autopilot', false); sendConfigToWebview(); postActiveFile(); postContextFiles(); post({ type: 'mode', agent: agentMode }); post({ type: 'autopilot', on: autopilot }); postAccount(); buildFileIndex(); post({ type: 'contextUsage', input: 0, limit: currentContextLimit() }); if (review) { review.resync(); } postMemoryDigest(); if (pendingTranscriptReplay) { const t = pendingTranscriptReplay; pendingTranscriptReplay = ''; replayLiveTranscript(t); } break; case 'setMode': agentMode = !!msg.agent; post({ type: 'mode', agent: agentMode }); break; case 'setAutopilot': autopilot = !!msg.on; aiConfig().update('agent.autopilot', autopilot, vscode.ConfigurationTarget.Global); dbg('autopilot.set', { on: autopilot }); post({ type: 'autopilot', on: autopilot }); break; - case 'send': await handleSend(msg.text); break; + case 'send': await handleSend(msg.text, msg.images); break; + // One surface for "that could not be attached" — VS Code's own, not a second one + // invented inside the transcript. + case 'notice': if (msg.text) { vscode.window.showWarningMessage(String(msg.text)); } break; case 'stop': dbg('stop.clicked', { running: commandStops.size }); for (const [, stop] of commandStops) { try { stop(); } catch (e) { /* gone */ } } if (abort) { abort.abort(); } clearApprovals(); clearQuestions(); break; case 'stopCommand': { dbg('stopCommand', { id: msg.id }); const s = commandStops.get(msg.id); if (s) { try { s(); } catch (e) { /* gone */ } } break; } case 'approvalResponse': { diff --git a/extensions/levelcode-ai/media/chat.html b/extensions/levelcode-ai/media/chat.html index aacec4e..272ee7a 100644 --- a/extensions/levelcode-ai/media/chat.html +++ b/extensions/levelcode-ai/media/chat.html @@ -65,6 +65,27 @@ calc above already insets by --shell-x. Set margin-block in these rules, never margin. test/webviewCss.test.js pins this for the whole list. */ + /* ---- attached images ---- */ + /* The chip carries its own thumbnail: an attachment you cannot see is one you cannot check before + sending, and a screenshot is the one attachment where the wrong one looks exactly like the right + one in a filename. */ + .chip.imgchip { padding: 2px 6px 2px 2px; gap: 6px; align-items: center; } + .chip.imgchip .imgthumb { + width: 28px; height: 28px; object-fit: cover; border-radius: 3px; + display: block; background: rgba(127,127,127,.18); + } + .chip.imgchip .imgmeta { font-variant-numeric: tabular-nums; opacity: .75; } + .msgimg { + display: block; max-width: min(320px, 100%); max-height: 240px; width: auto; height: auto; + border-radius: 6px; border: 1px solid var(--border); margin: 0 0 8px; + } + /* Whole-panel drop target: the transcript is a far bigger target than the composer, and someone + dragging a screenshot aims at the conversation. */ + body.dropping::after { + content: ''; position: fixed; inset: 6px; border: 2px dashed var(--accent); + border-radius: 10px; pointer-events: none; z-index: 40; + } + /* ---- conversation log ---- */ /* The measure, the type and the inset are all declared on `body` (see above) so the composer and the status row resolve the same values the transcript does. `--shell-x` is the log's horizontal padding @@ -1544,12 +1565,25 @@ '' + esc(f.name) + '' + '✕'; } + /** An attached image: its own thumbnail, its real size, and a way to take it back off. */ + function imgChip(im){ + return '' + + '' + + '' + im.w + '×' + im.h + '' + + '×' + + ''; + } + function renderChips(){ let h = ''; if (activeFileLabel) { h += fileChip(activeFileLabel); } for (const f of ctxFiles) { h += pinnedChip(f); } if (selLabel) { h += selChip(selLabel); } + for (const im of pendingImages) { h += imgChip(im); } chipsEl.innerHTML = h; + chipsEl.querySelectorAll('.imgx').forEach(function(x){ + x.onclick = function(e){ e.stopPropagation(); removeImage(x.getAttribute('data-img')); }; + }); document.getElementById('addctx').onclick = () => vscode.postMessage({ type: 'addContext' }); chipsEl.querySelectorAll('.x').forEach(x => { x.onclick = (e) => { e.stopPropagation(); vscode.postMessage({ type: 'removeContext', id: x.getAttribute('data-id') }); }; @@ -3057,18 +3091,147 @@ function histSet(text){ input.value = text; auto(); const n = text.length; try { input.setSelectionRange(n, n); } catch (e) {} } function doSend(){ if (streaming) { flushAll = true; ensurePump(); vscode.postMessage({ type: 'stop' }); return; } - const t = input.value.trim(); if (!t) return; - cmdHistory.push(t); if (cmdHistory.length > 200) { cmdHistory.shift(); } // record for ↑/↓ recall + const t = input.value.trim(); + // An image with no words is a real message — "look at this" is implied by attaching it. + if (!t && !pendingImages.length) return; + if (t) { cmdHistory.push(t); if (cmdHistory.length > 200) { cmdHistory.shift(); } } // record for ↑/↓ recall histIdx = -1; histDraft = ''; // Slash commands — handled locally (deterministic, no LLM call). if (/^\/skills\b/i.test(t)){ add('user', esc(t)); forceStick(); input.value = ''; auto(); vscode.postMessage({ type: 'listSkills' }); return; } if (/^\/mcp\b/i.test(t)){ add('user', esc(t)); forceStick(); input.value = ''; auto(); vscode.postMessage({ type: 'listMcp' }); return; } if (/^\/sessions\b/i.test(t)){ input.value = ''; auto(); openSessions(); return; } if (/^\/rme\b/i.test(t)){ add('user', esc(t)); forceStick(); input.value = ''; auto(); addReminder(); return; } - add('user', render(t)); forceStick(); input.value = ''; auto(); - vscode.postMessage({ type: 'send', text: t }); + const imgs = pendingImages.map(function(i){ + return { media_type: i.media_type, base64: i.base64, w: i.w, h: i.h, bytes: i.bytes }; + }); + add('user', (imgs.length ? imgs.map(function(i, k){ + return 'attached image'; + }).join('') : '') + render(t)); + forceStick(); input.value = ''; auto(); clearImages(); renderChips(); + vscode.postMessage({ type: 'send', text: t, images: imgs }); } + // ── attached images ─────────────────────────────────────────────────────────────────────────── + // Paste a screenshot, ask about it. Normalization happens HERE, before the bytes cross to the + // host, because this is the only place that has the decoded pixels. + // + // THE CAP IS 2000px ON THE LONG EDGE, and it is not a guess. Claude Code ships exactly this — + // every image it re-encodes is 2000 on the long edge — and it is the threshold the vision docs + // name for staying clear of the stricter per-image dimension limit that applies once a request + // carries more than 20 images. It is also well above either model tier's own cap, so the server + // does the final downscale and we never throw away fidelity it would have kept. + // + // UNDER THE CAP, THE ORIGINAL BYTES GO THROUGH UNTOUCHED — same rule, same reason: re-encoding + // an image that did not need resizing only stacks compression artifacts, and that is worst on + // screenshots of code, which is most of what gets pasted. + // Whether the SELECTED model can read images. Pushed with the model so an attachment is refused + // before anything is typed and lost, rather than after a send the provider would reject. + let canSeeImages = false; + let canSeeImagesModel = ''; + const IMG_CAP = 2000; + const IMG_MAX_PER_TURN = 8; + const IMG_OK = { 'image/png': 1, 'image/jpeg': 1, 'image/gif': 1, 'image/webp': 1 }; + let pendingImages = []; // { id, url, w, h, media_type, base64, bytes } + let imgSeq = 0; + + /** A user-facing "that did not work" line. Routed to the host so it uses VS Code's own notice + * surface rather than inventing a second one inside the transcript. */ + function note(t){ vscode.postMessage({ type: 'notice', text: String(t || '') }); } + + function fmtKB(n){ return n >= 1024 * 1024 ? (n / 1048576).toFixed(1) + ' MB' : Math.max(1, Math.round(n / 1024)) + ' KB'; } + + /** Blob -> base64 payload, no data: prefix. */ + function blobToBase64(blob){ + return new Promise(function(res, rej){ + const r = new FileReader(); + r.onload = function(){ const s2 = String(r.result || ''); res(s2.slice(s2.indexOf(',') + 1)); }; + r.onerror = function(){ rej(new Error('could not read the image')); }; + r.readAsDataURL(blob); + }); + } + + /** Decode, downscale only if over the cap, re-encode only if we resized. Never scales up. */ + async function normalizeImage(file){ + if (!IMG_OK[file.type]) { throw new Error('that image format is not supported — use PNG, JPEG, GIF or WebP'); } + let bmp; + try { bmp = await createImageBitmap(file); } + catch (e) { throw new Error('that file could not be read as an image'); } + const w = bmp.width, h = bmp.height; + const scale = Math.min(1, IMG_CAP / Math.max(w, h)); + if (scale === 1) { + // Pass-through: the bytes we already have, in the format they arrived in. + bmp.close && bmp.close(); + return { w: w, h: h, media_type: file.type, base64: await blobToBase64(file), bytes: file.size }; + } + const tw = Math.round(w * scale), th = Math.round(h * scale); + // OffscreenCanvas keeps the decode and the draw off the layout path; a 4K decode on the main + // thread is a visible stall in a chat window. + const cv = (typeof OffscreenCanvas !== 'undefined') ? new OffscreenCanvas(tw, th) + : Object.assign(document.createElement('canvas'), { width: tw, height: th }); + const g = cv.getContext('2d'); + g.imageSmoothingEnabled = true; g.imageSmoothingQuality = 'high'; + g.drawImage(bmp, 0, 0, tw, th); + bmp.close && bmp.close(); + const blob = cv.convertToBlob ? await cv.convertToBlob({ type: 'image/webp', quality: 0.92 }) + : await new Promise(function(r){ cv.toBlob(r, 'image/webp', 0.92); }); + if (!blob) { throw new Error('the image could not be resized'); } + return { w: tw, h: th, media_type: 'image/webp', base64: await blobToBase64(blob), bytes: blob.size }; + } + + async function attachImageFiles(files){ + const list = Array.from(files || []).filter(function(f){ return f && /^image\//.test(f.type); }); + if (!list.length) { return false; } + if (!canSeeImages) { + note((canSeeImagesModel || 'This model') + ' cannot read images. Switch to a vision model — ' + + 'Claude, GPT-4o and Gemini all read them — and paste again.'); + return true; + } + for (const f of list) { + if (pendingImages.length >= IMG_MAX_PER_TURN) { + note('Up to ' + IMG_MAX_PER_TURN + ' images per message — the rest were not attached.'); + break; + } + try { + const im = await normalizeImage(f); + im.id = 'img' + (++imgSeq); + im.url = 'data:' + im.media_type + ';base64,' + im.base64; + pendingImages.push(im); + renderChips(); + } catch (e) { note(String((e && e.message) || e)); } + } + return true; + } + + function removeImage(id){ pendingImages = pendingImages.filter(function(i){ return i.id !== id; }); renderChips(); } + function clearImages(){ pendingImages = []; } + + // Paste is the primary way in — the whole interaction is screenshot, then Cmd+V. + input.addEventListener('paste', function(ev){ + const d = ev.clipboardData; if (!d) { return; } + const files = d.files && d.files.length ? d.files + : Array.from(d.items || []).filter(function(i){ return i.kind === 'file'; }).map(function(i){ return i.getAsFile(); }); + const imgs = Array.from(files || []).filter(function(f){ return f && /^image\//.test(f.type); }); + if (!imgs.length) { return; } // a normal text paste is untouched + ev.preventDefault(); + attachImageFiles(imgs); + }); + + // Drop anywhere in the panel, not just on the composer — the transcript is the bigger target. + document.addEventListener('dragover', function(ev){ + if (ev.dataTransfer && Array.from(ev.dataTransfer.types || []).indexOf('Files') >= 0) { + ev.preventDefault(); document.body.classList.add('dropping'); + } + }); + document.addEventListener('dragleave', function(ev){ if (!ev.relatedTarget) { document.body.classList.remove('dropping'); } }); + document.addEventListener('drop', function(ev){ + document.body.classList.remove('dropping'); + if (!ev.dataTransfer || !ev.dataTransfer.files || !ev.dataTransfer.files.length) { return; } + const imgs = Array.from(ev.dataTransfer.files).filter(function(f){ return /^image\//.test(f.type); }); + if (!imgs.length) { return; } + ev.preventDefault(); + attachImageFiles(imgs); + }); + sendBtn.onclick = doSend; document.getElementById('attach').onclick = () => vscode.postMessage({ type: 'addContext' }); document.getElementById('model').onclick = () => vscode.postMessage({ type: 'pickModel' }); @@ -3626,6 +3789,7 @@ mEl.title = (m.providerLabel || (m.provider === 'claude' ? 'Claude' : m.provider)) + ' · ' + m.model + ' — click to change'; // Update the footer context-window meter to the newly-selected model's real window. if (typeof m.contextLimit === 'number'){ renderContext({ limit: m.contextLimit }); } + if (typeof m.canSeeImages === 'boolean'){ canSeeImages = m.canSeeImages; canSeeImagesModel = label; } lastProvider = m.provider || ''; lastModelId = m.modelId || m.model || ''; // gateway sends modelId (the real id); BYOK model IS the id // calm-transcript toggle. Turning it OFF mid-run closes any open group first, so the timeline diff --git a/extensions/levelcode-ai/sessions.js b/extensions/levelcode-ai/sessions.js index b133591..caa4553 100644 --- a/extensions/levelcode-ai/sessions.js +++ b/extensions/levelcode-ai/sessions.js @@ -326,13 +326,15 @@ function createSessions(opts) { } /** Where this project's memory lives (for opening it — the transparency promise). */ function memoryPaths() { return { dir: memory.memoryDir(root, slug), journal: memory.journalFile(root, slug), memoryMd: memory.memoryMdFile(root, slug) }; } + /** Where attached images live: beside this project's sessions, so they are deleted with them. */ + function mediaRoot() { return { root, slug }; } /** The session index for this project (what the panel/view list). Empty (never throws) if unreadable. */ function list() { try { return store.loadIndex(root, slug).entries; } catch (e) { return []; } } function liveId() { return live ? live.id : null; } - return { ensure, recordTurn, seal, resume, fork, archive, trash, restore, setPinned, rename, autoArchiveStale, digest, consolidate, transcript, refineSummary, recall, recallFacts, memoryItems, forget, recordFacts, factsList, factAction, supersedeFact, memoryPaths, list, liveId }; + return { ensure, recordTurn, seal, resume, fork, archive, trash, restore, setPinned, rename, autoArchiveStale, digest, consolidate, transcript, refineSummary, recall, recallFacts, memoryItems, forget, recordFacts, factsList, factAction, supersedeFact, memoryPaths, mediaRoot, list, liveId }; } module.exports = { createSessions }; diff --git a/extensions/levelcode-ai/test/imageAttach.test.js b/extensions/levelcode-ai/test/imageAttach.test.js new file mode 100644 index 0000000..6deb316 --- /dev/null +++ b/extensions/levelcode-ai/test/imageAttach.test.js @@ -0,0 +1,131 @@ +/*--------------------------------------------------------------------------------------------- + * Attaching an image — the composer contract and the host seam. run: node test/imageAttach.test.js + * + * Source-extraction, in the house style: the webview's logic lives inline in chat.html and cannot + * be required, so the invariants that no DOM test would catch are pinned against the source. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const html = fs.readFileSync(path.join(__dirname, '..', 'media', 'chat.html'), 'utf8'); +const ext = fs.readFileSync(path.join(__dirname, '..', 'extension.js'), 'utf8'); + +let n = 0; +function test(name, fn) { fn(); n++; console.log(' ok - ' + name); } + +/** Balanced-brace body of a named function, so a rule is read in its own scope. */ +function fnBody(src, name) { + const i = src.indexOf('function ' + name + '('); + assert.ok(i >= 0, 'no function ' + name); + const open = src.indexOf('{', i); + let depth = 0; + for (let j = open; j < src.length; j++) { + if (src[j] === '{') { depth++; } + else if (src[j] === '}') { depth--; if (!depth) { return src.slice(open, j + 1); } } + } + assert.fail('unbalanced braces in ' + name); +} + +test('CAP: the long edge cap is 2000 — the value Claude Code ships, not a guess', () => { + // Measured from Claude Code's own transcripts: every image it re-encodes is 2000 on the long + // edge. It is also the threshold the vision docs name for staying clear of the stricter + // per-image dimension limit above 20 images per request. + const m = /const IMG_CAP = (\d+);/.exec(html); + assert.ok(m, 'IMG_CAP is gone'); + assert.strictEqual(m[1], '2000'); +}); + +test('NEVER UPSCALE, and never re-encode what did not need resizing', () => { + const body = fnBody(html, 'normalizeImage'); + assert.match(body, /Math\.min\(1,\s*IMG_CAP\s*\/\s*Math\.max\(w,\s*h\)\)/, + 'the scale must be clamped at 1 — upscaling costs bytes and tokens and adds nothing'); + assert.match(body, /if \(scale === 1\)/, 'there must be a pass-through branch'); + // The pass-through must forward the ORIGINAL file, in its original type. + const passthrough = body.slice(body.indexOf('if (scale === 1)'), body.indexOf('const tw =')); + assert.match(passthrough, /media_type:\s*file\.type/, 'pass-through must keep the source format'); + assert.match(passthrough, /blobToBase64\(file\)/, 'pass-through must forward the ORIGINAL bytes'); + assert.ok(!/canvas|drawImage|convertToBlob/i.test(passthrough), + 're-encoding an image that did not need resizing only stacks compression artifacts'); +}); + +test('FORMATS: only what Claude accepts, and the webview and the store agree', () => { + const inWebview = (/const IMG_OK = \{([^}]*)\}/.exec(html) || [, ''])[1]; + for (const t of ['image/png', 'image/jpeg', 'image/gif', 'image/webp']) { + assert.ok(inWebview.includes(t), 'webview should accept ' + t); + } + assert.ok(!/image\/svg|image\/tiff|image\/bmp/.test(inWebview), 'only the four Claude reads'); + const store = fs.readFileSync(path.join(__dirname, '..', 'imageStore.js'), 'utf8'); + for (const t of ['image/png', 'image/jpeg', 'image/gif', 'image/webp']) { + assert.ok(store.includes("'" + t + "'"), 'store should accept ' + t); + } +}); + +test('PASTE: a text paste is untouched; only an image paste is intercepted', () => { + const i = html.indexOf("input.addEventListener('paste'"); + assert.ok(i > 0, 'no paste handler'); + const body = html.slice(i, i + 900); + assert.match(body, /if \(!imgs\.length\) \{ return; \}/, + 'a paste with no image must fall through to normal text pasting'); + assert.ok(body.indexOf('if (!imgs.length) { return; }') < body.indexOf('preventDefault'), + 'preventDefault must come AFTER the no-image bail, or plain text pasting breaks'); +}); + +test('REFUSAL: a blind model refuses BEFORE anything typed is lost', () => { + // The composer clears on send, so refusing host-side would throw away what they wrote. The + // capability rides with the model instead, and the refusal happens at attach time. + const body = fnBody(html, 'attachImageFiles'); + assert.match(body, /if \(!canSeeImages\)/, 'no vision gate at attach time'); + assert.ok(body.indexOf('if (!canSeeImages)') < body.indexOf('normalizeImage'), + 'refuse before doing the decode work, not after'); + assert.match(body, /cannot read images/, 'the refusal must say what is wrong'); + assert.match(body, /Switch to a vision model/, '…and what to do about it'); + assert.match(ext, /canSeeImages: supportsVisionForModel\(/, 'the host must publish the capability'); + assert.strictEqual((ext.match(/canSeeImages: supportsVisionForModel\(/g) || []).length, 2, + 'both config paths — gateway and BYOK — must publish it, or one of them silently allows'); +}); + +test('BUDGET: a per-turn image count, enforced where images are added', () => { + const body = fnBody(html, 'attachImageFiles'); + assert.match(body, /IMG_MAX_PER_TURN/, 'no per-turn cap'); + assert.match(body, /break;/, 'the cap must stop the loop, not just warn'); + assert.match(body, /the rest were not attached/, 'silently dropping attachments is the bug pattern'); +}); + +test('SEND: an image with no words is a valid message', () => { + const body = fnBody(html, 'doSend'); + assert.match(body, /if \(!t && !pendingImages\.length\) return;/, + '"look at this" is implied by attaching — an image alone must be sendable'); + assert.match(body, /images: imgs/, 'the send payload must carry the images'); + assert.match(body, /clearImages\(\)/, 'the tray must empty on send, or the next turn re-sends them'); +}); + +test('HOST: images become refs in the conversation, and bytes only at request time', () => { + assert.match(ext, /case 'send': await handleSend\(msg\.text, msg\.images\)/, 'send must carry images'); + const store = fnBody(ext, 'storeImages'); + assert.match(store, /imageStore\.put\(/, 'bytes must go through the store'); + assert.match(store, /type: 'image', ref/, 'the conversation must keep a ref, never the base64'); + assert.ok(!/base64/.test(store.replace(/im\.base64/g, '')), 'no base64 should be retained host-side'); + + // A copy, not a mutation: agentMessages persists across runs and is what recordTurn writes. + const w = fnBody(ext, 'withImages'); + assert.match(w, /msgs\.map\(/, 'withImages must map to a new array'); + assert.match(w, /\{ \.\.\.msg, content:/, 'and copy each touched message rather than mutating it'); + assert.ok(!/msg\.content\[|\.content\s*=/.test(w), 'must not write into the caller\'s messages'); + for (const site of ['withImages(conversation)', 'withImages(agentMessages)']) { + assert.ok(ext.includes(site), 'the request must materialize at: ' + site); + } +}); + +test('CONVERSATION: blocks only when there is an image', () => { + const body = fnBody(ext, 'handleSend'); + assert.match(body, /imageBlocks\.length\s*\n?\s*\?\s*\{ role: 'user', content: \[\.\.\.imageBlocks/, + 'images lead the block array'); + assert.match(body, /:\s*\{ role: 'user', content: userContent \}/, + 'a text-only turn must stay a plain string, or every cached prefix churns'); +}); + +console.log('\nimageAttach: ' + n + ' tests passed.'); From 9e297f4dc9c3799f3cc561271dcb56df2e8f5b3a Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 18:42:27 -0400 Subject: [PATCH 06/17] fix(images): agent mode dropped every pasted image, and no-workspace refused them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs found by asking "how would you test this", which is the question I should have asked myself before opening the PR. AGENT MODE DROPPED THEM. handleSend stored the bytes, then early-returned into agentFlow(text) — which never received the blocks. Agent is the DEFAULT mode, so the common path wrote a screenshot to disk and then silently sent a text-only request. That is precisely the failure this whole feature exists to prevent, reintroduced two slices after I1 removed it from the translator. agentFlow now takes the blocks and leads the goal message with them, and keeps a plain string when there are none so cached agent prefixes do not churn. NO WORKSPACE REFUSED THEM. storeImages went through sessionsManager(), which returns null with no folder open — so a pasted screenshot got "Images need a session to attach to". v1.1.0 deliberately made the agent answer with no folder open; refusing an image in that state re-introduces the limitation that release removed. Images need a place on DISK, not a session. imageRoot() prefers the project's session dir and falls back to a shared bucket, so the feature works wherever the agent does. Bypass-verified: agent mode dropping images again; the goal message ignoring them; the no-workspace fallback removed. 11 tests in imageAttach, 37 suites green. --- extensions/levelcode-ai/extension.js | 33 ++++++++++++++----- .../levelcode-ai/test/imageAttach.test.js | 23 +++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/extensions/levelcode-ai/extension.js b/extensions/levelcode-ai/extension.js index 17a69f0..fa93209 100644 --- a/extensions/levelcode-ai/extension.js +++ b/extensions/levelcode-ai/extension.js @@ -1594,7 +1594,7 @@ async function compactAgentMemory() { } let lastAgentGoal = null; // remembered so the response bar's Retry can re-run it -async function agentFlow(text) { +async function agentFlow(text, imageBlocks) { if (text && text.trim()) { lastAgentGoal = text; } const cfg = aiConfig(); const providerId = currentProviderId(); @@ -1614,7 +1614,11 @@ async function agentFlow(text) { abort = new AbortController(); repairAgentMemory(); // Open a workspace checkpoint for this turn (before the goal is pushed) so the user can roll back here. - const goalMsg = { role: 'user', content: text }; + // Agent mode is the DEFAULT, so this is the path most pasted screenshots take. Blocks only when + // there IS an image — a text-only goal stays a plain string so cached prefixes keep their bytes. + const goalMsg = (imageBlocks && imageBlocks.length) + ? { role: 'user', content: [...imageBlocks, { type: 'text', text }] } + : { role: 'user', content: text }; currentCheckpoint = { turnId: ++checkpointSeq, label: (text || '').slice(0, 60), ts: Date.now(), goalMsg: goalMsg, files: new Map() }; checkpoints.push(currentCheckpoint); post({ type: 'checkpointOpened', turnId: currentCheckpoint.turnId }); @@ -1726,6 +1730,21 @@ async function agentFlow(text) { } } +/** + * Where attached images live. + * + * Beside the project's sessions when there is a workspace, so they are cleaned up with it. Without + * one they fall back to a shared bucket — images need a place on DISK, not a session, and v1.1.0 + * deliberately made the agent answer with no folder open. Refusing to accept a screenshot in that + * state would re-introduce exactly the limitation that release removed. + */ +function imageRoot() { + const m = sessionsManager(); + if (m && m.mediaRoot) { return m.mediaRoot(); } + try { return { root: sessionsRoot(), slug: '_no-workspace' }; } + catch (e) { dbg('image.root.failed', { msg: String((e && e.message) || e) }); return null; } +} + /** * Store what the webview normalized, and return the blocks that will ride the conversation. * @@ -1736,9 +1755,8 @@ async function agentFlow(text) { function storeImages(images) { const out = []; if (!Array.isArray(images) || !images.length) { return out; } - const m = sessionsManager(); - const paths = m && m.mediaRoot ? m.mediaRoot() : null; - if (!paths) { vscode.window.showWarningMessage('Images need a session to attach to.'); return out; } + const paths = imageRoot(); + if (!paths) { vscode.window.showWarningMessage('Nowhere to store the image — LevelCode has no storage directory.'); return out; } for (const im of images) { try { const { ref, bytes } = imageStore.put(paths.root, paths.slug, im.base64, im.media_type); @@ -1760,8 +1778,7 @@ function storeImages(images) { */ function withImages(msgs) { if (!Array.isArray(msgs)) { return msgs; } - const m = sessionsManager(); - const paths = m && m.mediaRoot ? m.mediaRoot() : null; + const paths = imageRoot(); if (!paths) { return msgs; } let touched = false; const out = msgs.map((msg) => { @@ -1778,7 +1795,7 @@ async function handleSend(text, images) { if ((!text || !text.trim()) && !imageBlocks.length) { return; } text = text || ''; if (ctx) { ctx.globalState.update('levelcode.ai.hasSentMessage', true); } // user engaged → stop auto-revealing the panel on launch - if (agentMode) { await agentFlow(text); return; } + if (agentMode) { await agentFlow(text, imageBlocks); return; } const cfg = aiConfig(); const providerId = currentProviderId(); dbg('chat.send', { provider: providerId, model: activeModel(cfg, providerId), chars: text.length, history: conversation.length }); diff --git a/extensions/levelcode-ai/test/imageAttach.test.js b/extensions/levelcode-ai/test/imageAttach.test.js index 6deb316..f3a3065 100644 --- a/extensions/levelcode-ai/test/imageAttach.test.js +++ b/extensions/levelcode-ai/test/imageAttach.test.js @@ -128,4 +128,27 @@ test('CONVERSATION: blocks only when there is an image', () => { 'a text-only turn must stay a plain string, or every cached prefix churns'); }); +test('AGENT MODE: the default path carries images too', () => { + // I shipped this broken. handleSend stored the bytes and then early-returned into agentFlow(text), + // which never saw them — so in the DEFAULT mode a pasted screenshot was written to disk and + // dropped. The exact silent-drop failure this whole feature exists to prevent. + assert.match(ext, /if \(agentMode\) \{ await agentFlow\(text, imageBlocks\); return; \}/, + 'agent mode must forward the image blocks'); + assert.match(ext, /async function agentFlow\(text, imageBlocks\)/, 'agentFlow must accept them'); + const body = fnBody(ext, 'agentFlow'); + assert.match(body, /imageBlocks && imageBlocks\.length/, 'and use them when present'); + assert.match(body, /content: \[\.\.\.imageBlocks, \{ type: 'text', text \}\]/, 'images lead the goal'); + assert.match(body, /:\s*\{ role: 'user', content: text \}/, + 'a text-only goal must stay a plain string, or every cached agent prefix churns'); +}); + +test('NO WORKSPACE: an image still has somewhere to live', () => { + // v1.1.0 deliberately made the agent answer with no folder open. Refusing a screenshot in that + // state would re-introduce the limitation that release removed. + const body = fnBody(ext, 'imageRoot'); + assert.match(body, /sessionsManager\(\)/, 'prefer the project session dir when there is one'); + assert.match(body, /_no-workspace/, 'and fall back when there is not'); + assert.ok(!/Images need a session/.test(ext), 'the old session-required refusal must be gone'); +}); + console.log('\nimageAttach: ' + n + ' tests passed.'); From 4cccc0e3bd5a7639f1cc14af3861f71b9d3794df Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Sun, 23 Aug 2026 19:00:40 -0400 Subject: [PATCH 07/17] fix(images): the CSP blocked every thumbnail, and an image with no words 400'd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs from a real run, all mine. 1. THE THUMBNAIL WAS BROKEN BECAUSE NOTHING COULD RENDER IT. The webview CSP is `default-src 'none'` with no img-src, so every image — including a data: URI built from the attachment's own bytes — was blocked. I added images without touching the policy. Now `img-src data:`, and deliberately NOT https:, so the panel still cannot fetch a remote image. 2. AN IMAGE WITH NO WORDS RETURNED A 400. Anthropic rejects an empty text block outright ("text content blocks must be non-empty"), and sending an image with nothing typed produced exactly that. Both paths now include the text block only when there IS text, and fall back to the images alone. This is the case I deliberately made valid in I5 — "look at this" is implied by attaching — so it was the one shape guaranteed to be hit. 3. DRAGGING FROM FINDER DID NOTHING. VS Code's workbench intercepts OS file drops before a webview iframe sees them, so dataTransfer.files is empty — while the PATH is still there as a uri-list. The drop handler now falls back to reading the uri-list and handing the paths to the host, which reads them off disk and sends the bytes back for the same normalizer a paste uses. Size is checked host-side BEFORE base64 crosses the message bus, so a 200MB file is refused rather than serialized first. Also I6: a visible Attach-image button next to +, using showOpenDialog. The picker is the route that works regardless of what the webview is allowed to receive; the drop fallback is best-effort on top of it. Two guards needed updating rather than adding — the empty-text ternary changed the shape they matched. That is the guards doing their job; I updated them and added one that pins the 400 itself. 14 tests in imageAttach, 37 suites green. --- extensions/levelcode-ai/extension.js | 53 ++++++++++++++++++- extensions/levelcode-ai/media/chat.html | 41 ++++++++++++-- .../levelcode-ai/test/imageAttach.test.js | 39 +++++++++++++- 3 files changed, 125 insertions(+), 8 deletions(-) diff --git a/extensions/levelcode-ai/extension.js b/extensions/levelcode-ai/extension.js index fa93209..aabbeb8 100644 --- a/extensions/levelcode-ai/extension.js +++ b/extensions/levelcode-ai/extension.js @@ -1617,7 +1617,7 @@ async function agentFlow(text, imageBlocks) { // Agent mode is the DEFAULT, so this is the path most pasted screenshots take. Blocks only when // there IS an image — a text-only goal stays a plain string so cached prefixes keep their bytes. const goalMsg = (imageBlocks && imageBlocks.length) - ? { role: 'user', content: [...imageBlocks, { type: 'text', text }] } + ? { role: 'user', content: text ? [...imageBlocks, { type: 'text', text }] : imageBlocks } : { role: 'user', content: text }; currentCheckpoint = { turnId: ++checkpointSeq, label: (text || '').slice(0, 60), ts: Date.now(), goalMsg: goalMsg, files: new Map() }; checkpoints.push(currentCheckpoint); @@ -1745,6 +1745,48 @@ function imageRoot() { catch (e) { dbg('image.root.failed', { msg: String((e && e.message) || e) }); return null; } } +/** + * Read image files from disk and hand their bytes to the webview to normalize. + * + * Two callers, one path. The picker (reliable everywhere) and a Finder drop that arrives as a + * uri-list rather than as File objects — VS Code's workbench intercepts OS file drops before a + * webview iframe sees them, so `dataTransfer.files` is often empty while the PATH is still there. + * Reading host-side covers both, and normalization still happens in the webview because that is + * the only place with a canvas. + */ +async function attachImagePaths(paths) { + const files = []; + for (const fsPath of (Array.isArray(paths) ? paths : []).slice(0, 8)) { + try { + const ext = String(path.extname(fsPath) || '').toLowerCase(); + const mt = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', + '.gif': 'image/gif', '.webp': 'image/webp' }[ext]; + if (!mt) { vscode.window.showWarningMessage(path.basename(fsPath) + ' is not an image LevelCode can read.'); continue; } + const buf = await fs.promises.readFile(fsPath); + // Guard before the bytes cross into the webview: a 200MB file would otherwise be + // base64-ed onto the message bus before anything got a chance to refuse it. + if (buf.length > 25 * 1024 * 1024) { + vscode.window.showWarningMessage(path.basename(fsPath) + ' is too large to attach.'); + continue; + } + files.push({ base64: buf.toString('base64'), media_type: mt, name: path.basename(fsPath) }); + } catch (e) { + dbg('image.read.failed', { msg: String((e && e.message) || e) }); + vscode.window.showWarningMessage('Could not read ' + path.basename(fsPath)); + } + } + if (files.length) { post({ type: 'attachImages', files }); } +} + +/** Pick images from disk — the path that works regardless of what the webview can receive. */ +async function pickImages() { + const picked = await vscode.window.showOpenDialog({ + canSelectMany: true, openLabel: 'Attach', + filters: { Images: ['png', 'jpg', 'jpeg', 'gif', 'webp'] } + }); + if (picked && picked.length) { await attachImagePaths(picked.map((u) => u.fsPath)); } +} + /** * Store what the webview normalized, and return the blocks that will ride the conversation. * @@ -1821,8 +1863,10 @@ async function handleSend(text, images) { // Blocks only when there is an image; a text-only turn stays a plain string so every cached // prefix keeps the bytes it already had. Images lead — the model reads them best before the // text that asks about them. + // An empty text block is a 400 from Anthropic ("text content blocks must be non-empty"), and an + // image sent with no words produces exactly that. Include the text block only when there is text. conversation.push(imageBlocks.length - ? { role: 'user', content: [...imageBlocks, { type: 'text', text: userContent }] } + ? { role: 'user', content: userContent ? [...imageBlocks, { type: 'text', text: userContent }] : imageBlocks } : { role: 'user', content: userContent }); post({ type: 'userMessage', text }); if (auto.names.length) { post({ type: 'autoContext', names: auto.names }); } @@ -2359,6 +2403,8 @@ class ChatViewProvider { // One surface for "that could not be attached" — VS Code's own, not a second one // invented inside the transcript. case 'notice': if (msg.text) { vscode.window.showWarningMessage(String(msg.text)); } break; + case 'pickImages': await pickImages(); break; + case 'attachImagePaths': await attachImagePaths(msg.paths); break; case 'stop': dbg('stop.clicked', { running: commandStops.size }); for (const [, stop] of commandStops) { try { stop(); } catch (e) { /* gone */ } } if (abort) { abort.abort(); } clearApprovals(); clearQuestions(); break; case 'stopCommand': { dbg('stopCommand', { id: msg.id }); const s = commandStops.get(msg.id); if (s) { try { s(); } catch (e) { /* gone */ } } break; } case 'approvalResponse': { @@ -2557,6 +2603,9 @@ function webviewCsp() { const nonce = String(Math.random()).slice(2) + String(Date.now()); return { nonce, csp: [ "default-src 'none'", + // data: only — attached screenshots are rendered from their own bytes. Deliberately NOT + // https:, so the panel still cannot reach out to the network for an image. + "img-src data:", "style-src 'unsafe-inline'", "script-src 'nonce-" + nonce + "'" ].join('; ') }; diff --git a/extensions/levelcode-ai/media/chat.html b/extensions/levelcode-ai/media/chat.html index 272ee7a..4fbfba9 100644 --- a/extensions/levelcode-ai/media/chat.html +++ b/extensions/levelcode-ai/media/chat.html @@ -65,6 +65,9 @@ calc above already insets by --shell-x. Set margin-block in these rules, never margin. test/webviewCss.test.js pins this for the whole list. */ + #attachImg .ci { width: 14px; height: 14px; display: block; } + #attachImg { display: inline-flex; align-items: center; justify-content: center; } + /* ---- attached images ---- */ /* The chip carries its own thumbnail: an attachment you cannot see is one you cannot check before sending, and a screenshot is the one attachment where the wrong one looks exactly like the right @@ -1333,6 +1336,9 @@
+