From 7cbc34351a1cec880c613dc7706999b98ddec2d9 Mon Sep 17 00:00:00 2001 From: Ondrej Ulehla Date: Tue, 14 Jul 2026 18:10:45 +0200 Subject: [PATCH] fix: decouple sandbox resolution from browser breakpoints The sandbox resolution was taken directly from the VNC iframe wrapper's client size, so small breakpoints (e.g. mobile) created Linux desktops at viewport-sized resolutions (~350x300), which the agent cannot operate reliably (#1). Introduce getSandboxResolution() which scales large viewports down proportionally to the configured max bounds and falls back to DEFAULT_RESOLUTION whenever the viewport is below the min bounds. The helper is applied on the client when sending a message and enforced server-side in the chat route before Sandbox.create, so the sandbox always gets a desktop-class resolution regardless of the client. The displayed stream still fits small screens: the E2B stream URL uses noVNC's resize=scale, which shrinks the framebuffer to the container while preserving aspect ratio and translating pointer coordinates. Model coordinates are unaffected since screenshots and actions both use the raw sandbox coordinate space. Fixes #1 Co-Authored-By: Claude Fable 5 --- app/api/chat/route.ts | 10 ++++++-- app/page.tsx | 29 +++++++++-------------- lib/resolution.ts | 54 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 lib/resolution.ts diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts index f98583f..1ab47a5 100644 --- a/app/api/chat/route.ts +++ b/app/api/chat/route.ts @@ -5,6 +5,7 @@ import { createStreamingResponse, } from "@/lib/streaming"; import { SANDBOX_TIMEOUT_MS } from "@/lib/config"; +import { getSandboxResolution } from "@/lib/resolution"; import { OpenAIComputerStreamer } from "@/lib/streaming/openai"; import { logError } from "@/lib/logger"; @@ -24,6 +25,11 @@ export async function POST(request: Request) { resolution, } = await request.json(); + // Never trust the client-provided resolution as-is: it is derived from the + // browser viewport, and small breakpoints (e.g. mobile) would create a + // desktop too small for the agent to operate reliably + const sandboxResolution = getSandboxResolution(resolution); + const apiKey = process.env.E2B_API_KEY; if (!apiKey) { @@ -37,7 +43,7 @@ export async function POST(request: Request) { try { if (!activeSandboxId) { const newSandbox = await Sandbox.create({ - resolution, + resolution: sandboxResolution, dpi: 96, timeoutMs: SANDBOX_TIMEOUT_MS, }); @@ -59,7 +65,7 @@ export async function POST(request: Request) { try { const streamer: ComputerInteractionStreamerFacade = - new OpenAIComputerStreamer(desktop, resolution); + new OpenAIComputerStreamer(desktop, sandboxResolution); if (!sandboxId && activeSandboxId && vncUrl) { async function* stream(): AsyncGenerator { diff --git a/app/page.tsx b/app/page.tsx index 38e955b..a06fd1d 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -25,6 +25,7 @@ import Link from "next/link"; import Logo from "@/components/logo"; import { RepoBanner } from "@/components/repo-banner"; import { SANDBOX_TIMEOUT_MS } from "@/lib/config"; +import { getSandboxResolution } from "@/lib/resolution"; import { Surfing } from "@/components/surfing"; export default function Home() { @@ -100,40 +101,32 @@ export default function Home() { } }; + const resolveSandboxResolution = (): [number, number] => { + const wrapper = iFrameWrapperRef.current; + + return getSandboxResolution( + wrapper ? [wrapper.clientWidth, wrapper.clientHeight] : undefined + ); + }; + const onSubmit = (e: React.FormEvent) => { const content = handleSubmit(e); if (content) { - const width = - iFrameWrapperRef.current?.clientWidth || - (window.innerWidth < 768 ? window.innerWidth - 32 : 1024); - const height = - iFrameWrapperRef.current?.clientHeight || - (window.innerWidth < 768 - ? Math.min(window.innerHeight * 0.4, 400) - : 768); - sendMessage({ content, sandboxId: sandboxId || undefined, environment: "linux", - resolution: [width, height], + resolution: resolveSandboxResolution(), }); } }; const handleExampleClick = (prompt: string) => { - const width = - iFrameWrapperRef.current?.clientWidth || - (window.innerWidth < 768 ? window.innerWidth - 32 : 1024); - const height = - iFrameWrapperRef.current?.clientHeight || - (window.innerWidth < 768 ? Math.min(window.innerHeight * 0.4, 400) : 768); - sendMessage({ content: prompt, sandboxId: sandboxId || undefined, environment: "linux", - resolution: [width, height], + resolution: resolveSandboxResolution(), }); }; diff --git a/lib/resolution.ts b/lib/resolution.ts new file mode 100644 index 0000000..4f70a3e --- /dev/null +++ b/lib/resolution.ts @@ -0,0 +1,54 @@ +import { + MAX_RESOLUTION_WIDTH, + MAX_RESOLUTION_HEIGHT, + MIN_RESOLUTION_WIDTH, + MIN_RESOLUTION_HEIGHT, + DEFAULT_RESOLUTION, +} from "@/lib/config"; + +/** + * Derives a sane sandbox resolution from a measured viewport size. + * + * The sandbox resolution must stay decoupled from browser breakpoints: + * Linux desktops (and the agent operating them) are not usable at mobile + * viewport sizes. Large viewports are scaled down proportionally to fit the + * maximum bounds, while viewports below the minimum bounds fall back to + * DEFAULT_RESOLUTION entirely - the VNC stream scales the display down to + * fit the container instead (noVNC "resize=scale" preserves aspect ratio + * and maps pointer coordinates itself). + * + * @param measured - The measured container size [width, height], if any + * @returns A resolution within the configured min/max bounds + */ +export function getSandboxResolution( + measured?: [number, number] +): [number, number] { + if ( + !measured || + !Number.isFinite(measured[0]) || + !Number.isFinite(measured[1]) || + measured[0] <= 0 || + measured[1] <= 0 + ) { + return DEFAULT_RESOLUTION; + } + + // Scale down proportionally so both dimensions fit the maximum bounds + const scaleFactor = Math.min( + MAX_RESOLUTION_WIDTH / measured[0], + MAX_RESOLUTION_HEIGHT / measured[1], + 1 + ); + + const width = Math.round(measured[0] * scaleFactor); + const height = Math.round(measured[1] * scaleFactor); + + // Viewports below the minimum bounds (e.g. mobile breakpoints) would + // produce a desktop too small for the agent - use the default resolution + // and let the stream scale the display down instead + if (width < MIN_RESOLUTION_WIDTH || height < MIN_RESOLUTION_HEIGHT) { + return DEFAULT_RESOLUTION; + } + + return [width, height]; +}