Merge pull request #90 from levelcodeai/feat/image-input-impl

feat(images): paste a screenshot, ask about it

Author: ndemianc

docs/CORE-PATCHES.md

@@ -26,7 +26,7 @@ So a full rebuild from nothing is: `bootstrap.sh` (clone → brand → extension
 To re-create the core patch after changing core files in `vscode/`:
 
 ```bash
-# STRUCTURAL patch only (14 files). Display-string rebrands are NOT here — they live in scripts/de-brand.mjs.
+# STRUCTURAL patch only (15 files). Display-string rebrands are NOT here — they live in scripts/de-brand.mjs.
 git -C vscode diff HEAD -- \
   src/vs/workbench/contrib/files/browser/files.contribution.ts \
   build/lib/extensions.ts build/lib/copilot.ts \
@@ -41,6 +41,7 @@ git -C vscode diff HEAD -- \
   src/vs/base/common/product.ts \
   src/vs/platform/dialogs/electron-browser/dialog.ts \
   src/vs/workbench/contrib/update/browser/updateTooltip.ts \
+  src/vs/workbench/browser/parts/editor/editorDropTarget.ts \
   > patches/levelcode-core.patch
 # NOTE 1: use `diff HEAD` (not plain `diff`) — bootstrap's `git apply` may leave these STAGED,
 # and plain `git diff` shows only UNSTAGED changes, silently dropping the staged patches.
@@ -64,6 +65,34 @@ grep -rn "\[LevelCode\]" vscode/src vscode/build
 
 ## Patches (structural / behavioural only)
 
+### `editorDropTarget.ts` — an image dropped on the chat is an attachment, not a file to open
+
+**Why it has to be here.** A webview iframe is never offered an OS file drop: the workbench takes the
+drop first and opens the file in a tab. Nothing inside `extensions/levelcode-ai` can recover it — the
+panel's own `drop` handler never fires, and the `text/uri-list` fallback has no event to fall back
+from. This is the one part of paste-a-screenshot that cannot be an extension change.
+
+**What it does.** In `DropOverlay.handleDrop`, immediately before the URI-transfer branch hands off to
+`ResourcesDropHandler`, `tryLevelCodeChatImageDrop` forwards the dropped paths to the extension via
+`levelcode.ai.attachImagePaths` and consumes the drop.
+
+**Kept narrow on purpose** — every condition is a reason not to change behaviour someone relies on:
+
+- only when the chat webview is the **active editor of the group being dropped on**, so a drop on any
+  other tab still opens the file;
+- only when **no split** is requested, so dragging to an edge still splits the group;
+- only when **every** dropped file is an image, so a mixed drop behaves as it always did;
+- only when the paths resolve — `getPathForFile` is native-only and returns undefined on web;
+- and if the command throws (extension not activated), it **falls through** to the normal handler,
+  because a dropped image doing nothing at all is worse than one that opens.
+
+The paths travel by command rather than a new IPC channel, so the diff stays a routing decision and
+nothing more — which is what keeps it cheap to re-apply on a rebase.
+
+**Regenerating:** this file was appended per NOTE 3, not swept in by a wholesale regen. A wholesale
+regen on a de-branded checkout pulls ~78 lines of link-strips into `files.contribution.ts`; I did that
+once while adding this entry and had to back it out.
+
 These can't be a content swap — behaviour, build logic, unregistrations. Kept small on purpose so they
 survive Code-OSS bumps. Each is tagged `[LevelCode]`. **The build is strict** (`noUnusedLocals` +
 `allowUnreachableCode: false`), so these avoid dead early-`return`s (unreachable-code error), commented-out

extensions/levelcode-ai/agent.js

@@ -633,6 +633,20 @@ async function approveMcpLaunch(ctx, server, dbg) {
 	return true;
 }
 
+/**
+ * What we last TOLD the user about a run's context (rules / memory / MCP).
+ *
+ * The context itself is rebuilt every run — that is deliberate, a run's servers are whatever is
+ * configured and reachable right now. Re-ANNOUNCING it every turn is different, and it was noise:
+ * three identical rows at the top of every single answer, saying the same thing they said last time.
+ *
+ * A signature, not a boolean, because the announcement has to come back the moment anything moves —
+ * a server dropping out, a rules file appearing, memory arriving for the first time. Silence is only
+ * correct while the picture is unchanged.
+ */
+let lastContextSig = '';
+function resetContextAnnounce() { lastContextSig = ''; }
+
 async function setupMcp(ctx, wsFolders, dbg) {
 	const empty = { tools: [], routes: null };
 	const cfg = ctx.mcp || {};
@@ -687,7 +701,9 @@ async function setupMcp(ctx, wsFolders, dbg) {
 		const perServer = toolCountsByServer(built.routes);
 		const summary = handles.map((h) => h.name + ' (' + (perServer.get(h.name) || 0) + ')').join(', ');
 		dbg('mcp.ready', { servers: handles.map((h) => h.name), tools: built.tools.length, allowed });
-		ctx.post({ type: 'agentTool', icon: 'sparkle', text: '🔌 mcp · ' + summary + ' · ' + allowed + '/' + built.tools.length + ' allow-listed' });
+		// Handed back rather than posted: runAgent decides whether the user needs to hear it again.
+		// Failures below still post immediately — a server that broke is news every time.
+		built.announce = { type: 'agentTool', icon: 'sparkle', text: '🔌 mcp · ' + summary + ' · ' + allowed + '/' + built.tools.length + ' allow-listed' };
 		return built;
 	} catch (e) {
 		dbg('mcp.failed', { error: (e && e.message) || String(e) });
@@ -739,21 +755,36 @@ async function runAgent(ctx) {
 	const systemTokensEst = Math.round(system.length / 4);
 
 	const dbg = ctx.dbg || (() => {});
+	// The run's context, COLLECTED rather than posted. Whether the user needs to see it again is a
+	// question about the whole picture, and the MCP part of that picture is not known until setupMcp
+	// has run — so nothing is announced until all three are in hand.
+	const contextChips = [];
 	if (rules.sources.length) {
 		dbg('projectRules.loaded', { sources: rules.sources });
-		// Quiet timeline chip at the top of the run so the user can see their repo rules are in effect
-		// (mirrors the skill chip). Reuses the agentTool → addAgentLine rendering — no webview change.
-		ctx.post({ type: 'agentTool', icon: 'file', text: '📋 project rules · ' + rules.sources.join(', ') });
+		contextChips.push({ type: 'agentTool', icon: 'file', text: '📋 project rules · ' + rules.sources.join(', ') });
 	}
 	if (ctx.projectMemory) {
 		dbg('projectMemory.loaded', { chars: ctx.projectMemory.length });
-		ctx.post({ type: 'agentTool', icon: 'history', text: '🧠 project memory' });
+		contextChips.push({ type: 'agentTool', icon: 'history', text: '🧠 project memory' });
 	}
 
 	// MCP (docs/MCP.md S3): the tool list becomes PER-RUN. It was a module constant only because it was
 	// the same every time; a run's servers are whatever is configured and reachable right now. Same shape
 	// as `system`/`systemTokensEst` two lines up — built once per run, then used for every turn.
 	const mcp = await setupMcp(ctx, wsFolders, dbg);
+	if (mcp.announce) { contextChips.push(mcp.announce); }
+
+	// Say it only when it CHANGED. The context is rebuilt every run by design; repeating it at the top
+	// of every answer is not the same thing, and three identical rows before each reply is noise the
+	// reference transcript does not have. A signature rather than a flag, so the announcement returns
+	// the moment a server drops, a rules file appears, or memory shows up for the first time.
+	const sig = contextChips.map((c) => c.text).join('|');
+	if (sig && sig !== lastContextSig) {
+		lastContextSig = sig;
+		for (const chip of contextChips) { ctx.post(chip); }
+	} else if (!sig) {
+		lastContextSig = '';   // nothing to say now; say it again when there is
+	}
 	ctx.mcpRoutes = mcp.routes;                                        // runTool's router reads this
 	// Rootless runs get the portable subset; MCP tools are unaffected either way.
 	const builtins = root ? TOOLS : PORTABLE_TOOLS;
@@ -1030,4 +1061,4 @@ async function runAgent(ctx) {
 	}
 }
 
-module.exports = { runAgent, makeDiff, resolveWorkspacePath };
+module.exports = { resetContextAnnounce, runAgent, makeDiff, resolveWorkspacePath };

extensions/levelcode-ai/agentMemory.js

@@ -6,6 +6,8 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
+
+const { imageBlockTokens } = require('./imageCost');
 /** A "goal boundary": a user message with plain STRING content (a fresh user turn, never a tool_result).
  *  It is the only splice point that cannot orphan a tool_use/tool_result pair — tool results always sit
  *  in the message immediately after their tool_use, so any pair is wholly on one side of such a cut. */
@@ -34,10 +36,32 @@ function findCompactionCut(msgs, keepRecent) {
 	return cut;
 }
 
-/** Rough token estimate for a message list — the house chars/4 heuristic, used only for the UI meter. */
-function estimateMsgTokens(msgs) {
+/**
+ * Rough token estimate for a message list — the house chars/4 heuristic, used only for the UI meter.
+ *
+ * Images are counted by their real visual cost, not by their JSON. chars/4 is sound for text and
+ * wrong for an image in whichever shape it takes: inline base64 books about a third of its byte
+ * count (a 1MB screenshot reads as ~333,000 tokens, more than most context windows, for something
+ * that really costs ~4,800), and a stored ref swings the other way — 64 hex characters read as ~18
+ * tokens for the same ~4,800. Both would make the meter lie about how much room is left.
+ *
+ * `modelId` picks the resolution tier; omitting it costs the standard tier, which over-counts
+ * rather than under-counts. See imageCost.js.
+ */
+function estimateMsgTokens(msgs, modelId) {
 	if (!Array.isArray(msgs)) { return 0; }
-	return Math.round(msgs.reduce((n, m) => n + JSON.stringify(m).length, 0) / 4);
+	let chars = 0;
+	let imageTokens = 0;
+	for (const m of msgs) {
+		if (!m) { continue; }
+		if (!Array.isArray(m.content)) { chars += JSON.stringify(m).length; continue; }
+		chars += 24;   // role + envelope, roughly what the object costs around its blocks
+		for (const b of m.content) {
+			if (b && b.type === 'image') { imageTokens += imageBlockTokens(b, modelId); }
+			else { chars += JSON.stringify(b).length; }
+		}
+	}
+	return Math.round(chars / 4) + imageTokens;
 }
 
 module.exports = { isGoalBoundary, findCompactionCut, estimateMsgTokens };