From bb1d3e31b121c69c7d87c4655a07af0003d65370 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Mon, 20 Jul 2026 04:36:06 +0000 Subject: [PATCH 01/16] feat: stage native ChatGPT artifacts privately --- .env.example | 6 + README.md | 1 + docs/artifact-exchange.md | 125 +++++ docs/configuration.md | 31 ++ docs/security.md | 50 ++ package.json | 2 +- src/artifact-tools.ts | 327 ++++++++++++ src/artifacts.test.ts | 473 +++++++++++++++++ src/artifacts.ts | 903 +++++++++++++++++++++++++++++++++ src/config.test.ts | 58 ++- src/config.ts | 49 +- src/db/migrations.ts | 63 +++ src/db/schema.ts | 57 +++ src/incoming-artifacts.test.ts | 387 ++++++++++++++ src/incoming-artifacts.ts | 523 +++++++++++++++++++ src/oauth-store.test.ts | 1 + src/server.ts | 67 ++- src/user-config.ts | 5 + 18 files changed, 3120 insertions(+), 8 deletions(-) create mode 100644 docs/artifact-exchange.md create mode 100644 src/artifact-tools.ts create mode 100644 src/artifacts.test.ts create mode 100644 src/artifacts.ts create mode 100644 src/incoming-artifacts.test.ts create mode 100644 src/incoming-artifacts.ts diff --git a/.env.example b/.env.example index d4a49662..8284bd00 100644 --- a/.env.example +++ b/.env.example @@ -27,6 +27,12 @@ DEVSPACE_LOG_SHELL_COMMANDS=0 # DEVSPACE_TRUST_PROXY=1 # DEVSPACE_STATE_DIR=/home/waishnav/.local/share/devspace # DEVSPACE_WORKTREE_ROOT=/home/waishnav/.devspace/worktrees +# Private native artifact staging is opt-in. Stored objects remain outside repositories. +# DEVSPACE_ARTIFACTS=1 +# DEVSPACE_ARTIFACT_ROOT=$HOME/.local/share/devspace/artifacts +# DEVSPACE_ARTIFACT_MAX_FILE_BYTES=104857600 +# DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES=1073741824 +# DEVSPACE_ARTIFACT_TTL_HOURS=24 # DEVSPACE_AUTO_LOAD_AGENTS_MD=1 # Skills are enabled by default. Set DEVSPACE_SKILLS=0 to hide them. # DEVSPACE_SKILLS=0 diff --git a/README.md b/README.md index 16c8817d..0bf7afa7 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ devspace doctor - [Setup Guide](https://github.com/Waishnav/devspace/blob/main/docs/setup.md) - [ChatGPT Coding Workflow](https://github.com/Waishnav/devspace/blob/main/docs/chatgpt-coding-workflow.md) - [Configuration Reference](https://github.com/Waishnav/devspace/blob/main/docs/configuration.md) +- [Native Artifact Staging](https://github.com/Waishnav/devspace/blob/main/docs/artifact-exchange.md) - [Security Model](https://github.com/Waishnav/devspace/blob/main/docs/security.md) - [Troubleshooting Gotchas](https://github.com/Waishnav/devspace/blob/main/docs/gotchas.md) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md new file mode 100644 index 00000000..039f70e2 --- /dev/null +++ b/docs/artifact-exchange.md @@ -0,0 +1,125 @@ +# Native Artifact Staging + +DevSpace can accept a file attached or generated by ChatGPT even though that +file does not yet exist on the DevSpace host. The opt-in `stage_artifact` MCP +tool streams the connector-provided file into private local storage and returns +a host path that local commands can consume. + +This is deliberately a narrow handoff seam. It does not copy files into +workspaces, write repositories, accept generic uploads, or expose a public +download route. + +## Enable + +```bash +DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve +``` + +The default private root is: + +```text +~/.local/share/devspace/artifacts +``` + +SQLite stores the owner-scoped record. File content is stored outside approved +workspace roots and repositories. + +## MCP workflow + +ChatGPT calls `stage_artifact` with its native top-level file value: + +```json +{ + "file": "", + "workspaceId": "optional metadata association", + "expectedSha256": "optional sha256:...", + "ttlHours": 24, + "pin": false +} +``` + +The tool declares `_meta["openai/fileParams"] = ["file"]`, allowing the +connector to expand the selected file into its authorized native object. +`workspaceId` is metadata only; staging never writes to that workspace. + +A successful result contains metadata, not file content: + +```json +{ + "artifactId": "art_...", + "name": "generated-image.png", + "mimeType": "image/png", + "size": 2302372, + "sha256": "...", + "hostPath": "/home/user/.local/share/devspace/artifacts/objects/...", + "expiresAt": "...", + "instruction": "Pass hostPath to a local command..." +} +``` + +Use `artifact_stat` to re-read owner-scoped metadata and `artifact_delete` to +remove the record. Local commands may consume `hostPath`; agents must never +invent paths under the artifact root. + +## Supported connector shape + +The production adapter accepts only an object containing these known keys: + +```ts +{ + download_url: string; + file_id: string; + mime_type?: string | null; + file_name?: string | null; + name?: string | null; + size?: number | null; +} +``` + +Required fields are `download_url` and `file_id`. Unknown keys, extra credential +fields, plain URL strings, filesystem paths, arrays, and malformed values fail +closed. + +Download URLs must use HTTPS on one of the exact reviewed hosts: + +- `files.oaiusercontent.com` +- `oaisdmntprcentralus.blob.core.windows.net` + +Userinfo, fragments, non-HTTPS ports, and redirects outside those hosts are +rejected. Redirects are followed manually and revalidated at each hop. Signed +URLs and opaque file IDs are never logged or returned. + +Generated-image IDs are treated as opaque bounded metadata. They are never used +as a path component. If an ID is not filename-safe, DevSpace derives a fixed +`chatgpt-file` basename plus an extension from the MIME type. + +## Storage and lifecycle + +The existing artifact store enforces: + +- 100 MiB per artifact by default +- 1 GiB total private storage by default +- server-computed SHA-256 +- optional caller-supplied digest verification +- mode `0700` storage directories and mode `0600` files +- server-selected paths and normalized non-hidden basenames +- symlink, non-regular-file, and containment checks +- atomic promotion into immutable content-addressed storage +- OAuth-client ownership for records +- a 24-hour default TTL for unpinned records +- bounded cleanup at startup and every 15 minutes + +The staging implementation reuses the store's private in-progress upload +pipeline internally so limits, hashing, quota enforcement, cleanup, and atomic +commit behavior remain identical. Those internal chunk operations are not MCP +tools in this focused change. + +## Logging + +`stage_artifact` logs only bounded structural diagnostics, the validated +download hostname, metadata options, resulting artifact ID, size, digest, and +status. It never logs raw file objects, signed URLs, file IDs, credentials, +content, or base64 data. + +MIME types and filenames are hints only. DevSpace does not execute, unpack, +publish, or automatically copy staged content into a workspace. diff --git a/docs/configuration.md b/docs/configuration.md index 4c02eb1a..525196ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,6 +39,36 @@ npx @waishnav/devspace config set publicBaseUrl https://devspace.example.com | `DEVSPACE_WORKTREE_ROOT` | Directory for managed Git worktrees. Defaults to `~/.devspace/worktrees`. | | `DEVSPACE_STATE_DIR` | Directory for SQLite state. Defaults to `~/.local/share/devspace`. | +## Native Artifact Staging + +Private native-file staging is disabled by default. Enable it when ChatGPT needs +to hand an attached or generated file to commands running on the DevSpace host: + +```bash +DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve +``` + +| Variable | Default | Purpose | +| --- | --- | --- | +| `DEVSPACE_ARTIFACTS` | `0` | Expose `stage_artifact`, `artifact_stat`, and `artifact_delete`. | +| `DEVSPACE_ARTIFACT_ROOT` | `~/.local/share/devspace/artifacts` | Private storage root outside repositories and worktrees. | +| `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` | `104857600` | Maximum decoded size of one artifact (100 MiB). | +| `DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES` | `1073741824` | Maximum combined stored-object and in-progress bytes (1 GiB). | +| `DEVSPACE_ARTIFACT_TTL_HOURS` | `24` | Default lifetime of an unpinned staged artifact. | + +The same settings may be persisted in `~/.devspace/config.json` as +`artifactsEnabled`, `artifactRoot`, `artifactMaxFileBytes`, +`artifactMaxTotalBytes`, and `artifactDefaultTtlHours`. + +`stage_artifact` accepts only the native file object supplied by the ChatGPT +connector. It does not accept arbitrary URL strings, paths, embedded credentials, +or extra object fields. Files are streamed through bounded private storage and +never copied into a workspace or repository. Cleanup runs at startup and +periodically, with a bounded number of records processed per pass. + +See [Native Artifact Staging](artifact-exchange.md) for the supported connector +shape and security boundaries. + ## OAuth DevSpace uses a single-user OAuth approval flow. @@ -158,6 +188,7 @@ DEVSPACE_OAUTH_OWNER_TOKEN="$(openssl rand -base64 32)" \ DEVSPACE_ALLOWED_ROOTS="$HOME/personal,$HOME/work" \ DEVSPACE_PUBLIC_BASE_URL="https://devspace.example.com" \ DEVSPACE_WORKTREE_ROOT="$HOME/.devspace/worktrees" \ +DEVSPACE_ARTIFACTS="1" \ DEVSPACE_TOOL_MODE="minimal" \ DEVSPACE_WIDGETS="full" \ npx @waishnav/devspace serve diff --git a/docs/security.md b/docs/security.md index 53fe027b..e022d58a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -92,9 +92,59 @@ Managed worktrees reduce accidental edits to your active checkout, but they are not a security boundary. They are a workflow boundary for isolated coding sessions. +## Artifact Exchange + +The Artifact Exchange is an opt-in private byte-transfer seam. Its root is +separate from allowed workspace roots, repositories, worktrees, temporary +folders, and public static assets. + +DevSpace selects every storage path. Artifact names are metadata, not path +components. Names are Unicode-normalized and must be a single non-hidden +basename without separators, NUL bytes, or control characters. The store uses +content-addressed immutable objects, mode `0700` directories, mode `0600` +object and partial files, atomic promotion, realpath containment checks, +regular-file enforcement, and no-follow file opens where the platform supports +them. + +Artifact records are scoped to the authenticated OAuth client. The first +version remains under the existing owner-only `devspace` scope. It does not add +an unauthenticated download route or expose the artifact root with +`express.static`. + +Native staging is adapter-gated. The production server declares ChatGPT's +top-level `openai/fileParams` contract and accepts only the documented +`download_url`, `file_id`, optional MIME/filename aliases, and optional size. +Downloads use HTTPS on two exact reviewed hosts: +`files.oaiusercontent.com` and +`oaisdmntprcentralus.blob.core.windows.net`. Credentials, fragments, alternate +ports, arbitrary sibling hosts, malformed IDs, extra fields, and redirects +outside that boundary fail closed. Opaque IDs are bounded metadata and are never +used as filenames or path components. + +The staging seam deliberately does not: + +- fetch arbitrary URLs +- expose a generic upload API +- copy content into a workspace or repository +- extract archives +- execute transferred content +- expand workspace allowlists +- preserve executable permissions +- publish or permanently retain content + +MIME types are hints only. SHA-256 and byte counts are computed and enforced by +the server. Pinned records require explicit deletion; unpinned records and +failed in-progress transfers remain subject to cleanup. + ## Logs By default, DevSpace logs requests and tool calls. Shell command previews are disabled unless `DEVSPACE_LOG_SHELL_COMMANDS=1`. Do not enable shell command logging if commands may contain secrets. + +Artifact tool logs contain identifiers, names, MIME hints, byte counts, hashes, +and status fields. `stage_artifact` logs only whether a file value and expected +digest were supplied plus non-sensitive options; it does not log the opaque +file value. Raw content, connector references, bearer credentials, presigned +URLs, and base64 chunks are never included in tool logs or tool results. diff --git a/package.json b/package.json index 25f21059..57029fb7 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dev": "node scripts/dev-server.mjs", "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", - "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", + "test": "tsx src/config.test.ts && tsx src/artifacts.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts new file mode 100644 index 00000000..eff1e307 --- /dev/null +++ b/src/artifact-tools.ts @@ -0,0 +1,327 @@ +import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import * as z from "zod/v4"; +import type { ServerConfig } from "./config.js"; +import { + ARTIFACT_CHUNK_BYTES, + ArtifactError, + type ArtifactRecord, + type ArtifactStore, +} from "./artifacts.js"; +import { + describeIncomingArtifactValue, + IncomingArtifactAdapterRegistry, + type IncomingArtifactAdapter, +} from "./incoming-artifacts.js"; +import { logEvent } from "./logger.js"; + +const ARTIFACT_WRITE_ANNOTATIONS = { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: false, + openWorldHint: false, +}; + +const openAIFileReferenceInputSchema = z.object({ + download_url: z.string(), + file_id: z.string(), + mime_type: z.string().nullable().optional(), + file_name: z.string().nullable().optional(), + name: z.string().nullable().optional(), + size: z.number().int().nonnegative().nullable().optional(), +}); + +const artifactRecordOutputSchema = { + artifactId: z.string(), + name: z.string(), + mimeType: z.string().optional(), + size: z.number().int().nonnegative(), + sha256: z.string(), + hostPath: z.string(), + source: z.string(), + workspaceId: z.string().optional(), + createdAt: z.string(), + expiresAt: z.string().optional(), + pinned: z.boolean(), +}; + +type ArtifactToolName = "stage_artifact" | "artifact_stat" | "artifact_delete"; + +export interface ArtifactToolRegistrationOptions { + config: ServerConfig; + store: ArtifactStore; + clientId: string; + incomingArtifactAdapters?: readonly IncomingArtifactAdapter[]; +} + +export interface StageArtifactInput { + file: unknown; + workspaceId?: string; + expectedSha256?: string; + ttlHours?: number; + pin?: boolean; +} + +export interface StageArtifactResult { + artifactId: string; + name: string; + mimeType?: string; + size: number; + sha256: string; + hostPath: string; + expiresAt?: string; + instruction: string; +} + +export function registerArtifactTools( + server: McpServer, + { + config, + store, + clientId, + incomingArtifactAdapters = [], + }: ArtifactToolRegistrationOptions, +): void { + const incomingRegistry = new IncomingArtifactAdapterRegistry(incomingArtifactAdapters); + + registerAppTool( + server, + "stage_artifact", + { + title: "Stage attached or generated file", + description: + "Stage one ChatGPT-provided native file reference into private DevSpace storage. DevSpace downloads only the exact file object authorized by ChatGPT; arbitrary URLs and paths fail closed. A workspace ID is association metadata, not a write destination.", + inputSchema: { + file: openAIFileReferenceInputSchema.describe("Native file value authorized and supplied by ChatGPT."), + workspaceId: z.string().min(1).optional().describe("Optional workspace association; never a write destination."), + expectedSha256: z.string().optional().describe("Optional expected SHA-256, with or without a sha256: prefix."), + ttlHours: z.number().int().min(1).max(24 * 365).optional().describe("Artifact lifetime after staging."), + pin: z.boolean().optional().describe("Preserve the artifact until explicitly deleted."), + }, + outputSchema: { + artifactId: z.string(), + name: z.string(), + mimeType: z.string().optional(), + size: z.number().int().nonnegative(), + sha256: z.string(), + hostPath: z.string(), + expiresAt: z.string().optional(), + instruction: z.string(), + }, + _meta: { "openai/fileParams": ["file"] }, + annotations: ARTIFACT_WRITE_ANNOTATIONS, + }, + async (input) => executeArtifactTool(config, "stage_artifact", input, async () => { + return stageIncomingArtifact({ + store, + clientId, + registry: incomingRegistry, + input, + }); + }), + ); + + registerAppTool( + server, + "artifact_stat", + { + title: "Inspect artifact", + description: "Return private artifact metadata by ID without returning its content.", + inputSchema: { + artifactId: z.string(), + }, + outputSchema: artifactRecordOutputSchema, + _meta: {}, + annotations: { readOnlyHint: true, openWorldHint: false }, + }, + async ({ artifactId }) => executeArtifactTool( + config, + "artifact_stat", + { artifactId }, + async () => store.statArtifact(clientId, artifactId), + ), + ); + + registerAppTool( + server, + "artifact_delete", + { + title: "Delete artifact", + description: + "Delete an artifact record. Its immutable object is removed only when no other live record references it.", + inputSchema: { + artifactId: z.string(), + }, + outputSchema: { + artifactId: z.string(), + deleted: z.boolean(), + objectDeleted: z.boolean(), + }, + _meta: {}, + annotations: ARTIFACT_WRITE_ANNOTATIONS, + }, + async ({ artifactId }) => executeArtifactTool( + config, + "artifact_delete", + { artifactId }, + async () => store.deleteArtifact(clientId, artifactId), + ), + ); +} + +export async function stageIncomingArtifact({ + store, + clientId, + registry, + input, +}: { + store: ArtifactStore; + clientId: string; + registry: IncomingArtifactAdapterRegistry; + input: StageArtifactInput; +}): Promise { + const opened = await registry.open(input.file); + let uploadId: string | undefined; + + try { + const upload = await store.beginUpload(clientId, { + filename: opened.name, + mimeType: opened.mimeType, + size: opened.size, + sha256: input.expectedSha256, + workspaceId: input.workspaceId, + ttlHours: input.ttlHours, + }); + uploadId = upload.uploadId; + + let offset = 0; + for await (const value of opened.stream) { + const bytes = incomingStreamChunk(value); + for (let chunkOffset = 0; chunkOffset < bytes.length; chunkOffset += ARTIFACT_CHUNK_BYTES) { + const chunk = bytes.subarray(chunkOffset, chunkOffset + ARTIFACT_CHUNK_BYTES); + await store.uploadChunk(clientId, { + uploadId, + offset, + dataBase64: chunk.toString("base64"), + }); + offset += chunk.length; + } + } + + const artifact = await store.commitUpload(clientId, uploadId, { + source: `incoming:${opened.adapterId}`, + pinned: input.pin === true, + }); + uploadId = undefined; + return { + artifactId: artifact.artifactId, + name: artifact.name, + mimeType: artifact.mimeType, + size: artifact.size, + sha256: artifact.sha256, + hostPath: artifact.hostPath, + expiresAt: artifact.expiresAt, + instruction: + "Pass hostPath to a local command. stage_artifact never writes into a workspace or repository.", + }; + } catch (error) { + opened.stream.destroy(); + if (uploadId) { + await store.abortUpload(clientId, uploadId).catch(() => undefined); + } + throw error; + } +} + +function incomingFileDownloadHostname(value: unknown): string | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + const rawUrl = (value as Record).download_url; + if (typeof rawUrl !== "string") return undefined; + try { + const hostname = new URL(rawUrl).hostname.toLowerCase(); + return hostname.length > 0 && hostname.length <= 253 ? hostname : undefined; + } catch { + return undefined; + } +} + +function incomingStreamChunk(value: unknown): Buffer { + if (Buffer.isBuffer(value)) return value; + if (typeof value === "string") return Buffer.from(value); + if (value instanceof Uint8Array) { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength); + } + throw new ArtifactError( + "invalid_incoming_artifact_chunk", + "Incoming artifact stream yielded a value that is not bytes or text.", + ); +} + +export function artifactToolLogFields( + tool: ArtifactToolName, + input: Record, +): Record { + if (tool === "stage_artifact") { + return { + fileProvided: input.file !== undefined, + fileReferenceShape: describeIncomingArtifactValue(input.file), + downloadUrlHostname: incomingFileDownloadHostname(input.file), + workspaceId: input.workspaceId, + expectedSha256Present: typeof input.expectedSha256 === "string", + ttlHours: input.ttlHours, + pin: input.pin === true, + }; + } + return { artifactId: input.artifactId }; +} + +async function executeArtifactTool( + config: ServerConfig, + tool: ArtifactToolName, + input: Record, + operation: () => Promise | T, +) { + const startedAt = performance.now(); + try { + const result = await operation(); + if (config.logging.toolCalls) { + logEvent(config.logging, "info", "artifact_tool_call", { + tool, + ...artifactToolLogFields(tool, input), + ...artifactResultLogFields(result as Record), + success: true, + durationMs: Math.round(performance.now() - startedAt), + }); + } + return artifactToolResponse(result); + } catch (error) { + if (config.logging.toolCalls) { + logEvent(config.logging, "warn", "artifact_tool_call", { + tool, + ...artifactToolLogFields(tool, input), + success: false, + errorCode: error instanceof ArtifactError ? error.code : "internal_error", + durationMs: Math.round(performance.now() - startedAt), + }); + } + throw error; + } +} + +function artifactToolResponse(result: T) { + return { + content: [{ type: "text" as const, text: JSON.stringify(result) }], + structuredContent: result as Record, + }; +} + +function artifactResultLogFields(result: Record): Record { + const artifact = result as Partial; + return { + artifactId: artifact.artifactId, + size: artifact.size, + sha256: artifact.sha256, + source: artifact.source, + objectDeleted: result.objectDeleted, + }; +} diff --git a/src/artifacts.test.ts b/src/artifacts.test.ts new file mode 100644 index 00000000..35413eb6 --- /dev/null +++ b/src/artifacts.test.ts @@ -0,0 +1,473 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { + chmod, + lstat, + mkdir, + mkdtemp, + readFile, + rm, + stat, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + ARTIFACT_CHUNK_BYTES, + ArtifactError, + ArtifactStore, + type ArtifactRecord, +} from "./artifacts.js"; +import { openDatabase } from "./db/client.js"; + +const root = await mkdtemp(join(tmpdir(), "devspace-artifacts-test-")); + +try { + await testBinaryRoundTrip(join(root, "binary")); + await testRestartSafeUpload(join(root, "restart")); + await testValidationAndLimits(join(root, "limits")); + await testReferenceAwareDeletion(join(root, "references")); + await testExpirationAndPinning(join(root, "expiration")); + await testBoundedCleanup(join(root, "bounded-cleanup")); + if (process.platform !== "win32") { + await testContainmentAndSymlinks(join(root, "containment")); + } +} finally { + await rm(root, { recursive: true, force: true }); +} + +async function testBinaryRoundTrip(testRoot: string): Promise { + const store = createStore(testRoot); + try { + const bytes = Buffer.alloc(ARTIFACT_CHUNK_BYTES + 137); + for (let index = 0; index < bytes.length; index += 1) { + bytes[index] = (index * 31) % 256; + } + const digest = createHash("sha256").update(bytes).digest("hex"); + const begin = await store.beginUpload("client-a", { + filename: "payload.bin", + mimeType: "application/octet-stream", + size: bytes.length, + sha256: `sha256:${digest}`, + workspaceId: "ws_123", + }); + assert.equal(begin.chunkBytes, ARTIFACT_CHUNK_BYTES); + assert.equal(begin.nextOffset, 0); + + if (process.platform !== "win32") { + const database = openDatabase(join(testRoot, "state")); + let partialPath: string; + try { + partialPath = String( + database.sqlite.prepare("select temp_path from artifact_uploads where id = ?").pluck().get(begin.uploadId), + ); + } finally { + database.close(); + } + assert.equal((await stat(partialPath)).mode & 0o777, 0o600); + assert.equal((await stat(dirname(partialPath))).mode & 0o777, 0o700); + } + + const first = bytes.subarray(0, ARTIFACT_CHUNK_BYTES); + const firstChunk = await store.uploadChunk("client-a", { + uploadId: begin.uploadId, + offset: 0, + dataBase64: first.toString("base64"), + }); + assert.equal(firstChunk.receivedBytes, ARTIFACT_CHUNK_BYTES); + assert.equal(firstChunk.retry, false); + + const retry = await store.uploadChunk("client-a", { + uploadId: begin.uploadId, + offset: 0, + dataBase64: first.toString("base64"), + }); + assert.equal(retry.receivedBytes, ARTIFACT_CHUNK_BYTES); + assert.equal(retry.retry, true); + + const conflicting = Buffer.from(first); + conflicting[0] ^= 0xff; + await expectArtifactError( + store.uploadChunk("client-a", { + uploadId: begin.uploadId, + offset: 0, + dataBase64: conflicting.toString("base64"), + }), + "conflicting_retry", + ); + await expectArtifactError( + store.uploadChunk("client-a", { + uploadId: begin.uploadId, + offset: ARTIFACT_CHUNK_BYTES + 1, + dataBase64: Buffer.from("x").toString("base64"), + }), + "out_of_order_chunk", + ); + + await store.uploadChunk("client-a", { + uploadId: begin.uploadId, + offset: ARTIFACT_CHUNK_BYTES, + dataBase64: bytes.subarray(ARTIFACT_CHUNK_BYTES).toString("base64"), + }); + const artifact = await store.commitUpload("client-a", begin.uploadId); + assert.equal(artifact.name, "payload.bin"); + assert.equal(artifact.mimeType, "application/octet-stream"); + assert.equal(artifact.size, bytes.length); + assert.equal(artifact.sha256, `sha256:${digest}`); + assert.equal(artifact.workspaceId, "ws_123"); + assert.deepEqual(await readFile(artifact.hostPath), bytes); + + const inspected = store.statArtifact("client-a", artifact.artifactId); + assert.deepEqual(inspected, artifact); + await expectArtifactError( + Promise.resolve().then(() => store.statArtifact("client-b", artifact.artifactId)), + "artifact_not_found", + ); + + const health = store.health(); + assert.equal(health.storedBytes, bytes.length); + assert.equal(health.pendingUploads, 0); + + if (process.platform !== "win32") { + assert.equal((await stat(join(testRoot, "artifacts"))).mode & 0o777, 0o700); + assert.equal((await stat(join(testRoot, "artifacts", "objects"))).mode & 0o777, 0o700); + assert.equal((await stat(dirname(dirname(artifact.hostPath)))).mode & 0o777, 0o700); + assert.equal((await stat(dirname(artifact.hostPath))).mode & 0o777, 0o700); + assert.equal((await stat(artifact.hostPath)).mode & 0o777, 0o600); + } + + const deleted = await store.deleteArtifact("client-a", artifact.artifactId); + assert.equal(deleted.objectDeleted, true); + await assert.rejects(lstat(artifact.hostPath), { code: "ENOENT" }); + } finally { + store.close(); + } +} + +async function testRestartSafeUpload(testRoot: string): Promise { + const bytes = Buffer.from("restart-safe-artifact-upload"); + const firstChunk = bytes.subarray(0, 10); + const initialStore = createStore(testRoot); + const upload = await initialStore.beginUpload("client-a", { + filename: "restart.bin", + size: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + await initialStore.uploadChunk("client-a", { + uploadId: upload.uploadId, + offset: 0, + dataBase64: firstChunk.toString("base64"), + }); + initialStore.close(); + + const resumedStore = createStore(testRoot); + try { + const retry = await resumedStore.uploadChunk("client-a", { + uploadId: upload.uploadId, + offset: 0, + dataBase64: firstChunk.toString("base64"), + }); + assert.equal(retry.retry, true); + await resumedStore.uploadChunk("client-a", { + uploadId: upload.uploadId, + offset: firstChunk.length, + dataBase64: bytes.subarray(firstChunk.length).toString("base64"), + }); + const artifact = await resumedStore.commitUpload("client-a", upload.uploadId); + assert.deepEqual(await readFile(artifact.hostPath), bytes); + } finally { + resumedStore.close(); + } +} + +async function testValidationAndLimits(testRoot: string): Promise { + const store = createStore(testRoot, { + artifactMaxFileBytes: 8, + artifactMaxTotalBytes: 10, + }); + try { + for (const filename of ["../escape", "nested/file", "nested\\file", ".hidden", ".", "..", "bad\0name"]) { + await expectArtifactError( + store.beginUpload("client-a", { filename }), + "invalid_filename", + ); + } + + await expectArtifactError( + store.beginUpload("client-a", { filename: "large.bin", size: 9 }), + "file_too_large", + ); + + const invalidBase64 = await store.beginUpload("client-a", { filename: "invalid.bin" }); + await expectArtifactError( + store.uploadChunk("client-a", { + uploadId: invalidBase64.uploadId, + offset: 0, + dataBase64: "not-base64", + }), + "invalid_base64", + ); + await store.abortUpload("client-a", invalidBase64.uploadId); + + const oversize = await store.beginUpload("client-a", { filename: "oversize.bin" }); + await expectArtifactError( + store.uploadChunk("client-a", { + uploadId: oversize.uploadId, + offset: 0, + dataBase64: Buffer.alloc(9).toString("base64"), + }), + "file_too_large", + ); + await store.abortUpload("client-a", oversize.uploadId); + + const first = await stage(store, "client-a", "first.bin", Buffer.alloc(6, 1)); + assert.equal(store.health().storedBytes, 6); + await expectArtifactError( + store.beginUpload("client-a", { filename: "quota.bin", size: 5 }), + "artifact_quota_exceeded", + ); + await store.deleteArtifact("client-a", first.artifactId); + + const mismatchBytes = Buffer.from("hash"); + const mismatch = await store.beginUpload("client-a", { + filename: "mismatch.bin", + size: mismatchBytes.length, + sha256: "0".repeat(64), + }); + await store.uploadChunk("client-a", { + uploadId: mismatch.uploadId, + offset: 0, + dataBase64: mismatchBytes.toString("base64"), + }); + await expectArtifactError( + store.commitUpload("client-a", mismatch.uploadId), + "sha256_mismatch", + ); + await store.abortUpload("client-a", mismatch.uploadId); + + const empty = await store.beginUpload("client-a", { + filename: "empty.bin", + size: 0, + sha256: createHash("sha256").digest("hex"), + }); + const emptyArtifact = await store.commitUpload("client-a", empty.uploadId); + assert.equal(emptyArtifact.size, 0); + assert.deepEqual(await readFile(emptyArtifact.hostPath), Buffer.alloc(0)); + } finally { + store.close(); + } +} + +async function testReferenceAwareDeletion(testRoot: string): Promise { + const store = createStore(testRoot); + try { + const bytes = Buffer.from("deduplicated-object"); + const first = await stage(store, "client-a", "first.txt", bytes); + const second = await stage(store, "client-a", "second.txt", bytes); + assert.equal(first.hostPath, second.hostPath); + assert.equal(store.health().storedBytes, bytes.length); + + const firstDelete = await store.deleteArtifact("client-a", first.artifactId); + assert.equal(firstDelete.objectDeleted, false); + assert.deepEqual(await readFile(second.hostPath), bytes); + + const secondDelete = await store.deleteArtifact("client-a", second.artifactId); + assert.equal(secondDelete.objectDeleted, true); + await assert.rejects(lstat(second.hostPath), { code: "ENOENT" }); + } finally { + store.close(); + } +} + +async function testExpirationAndPinning(testRoot: string): Promise { + let current = new Date("2026-07-18T12:00:00.000Z"); + const store = createStore(testRoot, {}, () => new Date(current)); + try { + const abandoned = await store.beginUpload("client-a", { filename: "abandoned.bin" }); + current = new Date("2026-07-18T14:00:00.000Z"); + const uploadCleanup = await store.cleanupExpired(); + assert.equal(uploadCleanup.uploadsDeleted, 1); + await expectArtifactError( + store.abortUpload("client-a", abandoned.uploadId), + "upload_not_found", + ); + + current = new Date("2026-07-18T12:00:00.000Z"); + const shared = Buffer.from("shared-expiry-object"); + const expiring = await stage(store, "client-a", "expiring.txt", shared, 1); + const live = await stage(store, "client-a", "live.txt", shared, 4); + const pinned = await stage(store, "client-a", "pinned.txt", Buffer.from("pinned"), 1); + + const database = openDatabase(join(testRoot, "state")); + try { + database.sqlite.prepare("update artifacts set pinned = 1 where id = ?").run(pinned.artifactId); + } finally { + database.close(); + } + + current = new Date("2026-07-18T14:00:00.000Z"); + const cleanup = await store.cleanupExpired(); + assert.equal(cleanup.artifactsDeleted, 1); + assert.equal(cleanup.objectsDeleted, 0); + await expectArtifactError( + Promise.resolve().then(() => store.statArtifact("client-a", expiring.artifactId)), + "artifact_not_found", + ); + assert.deepEqual(await readFile(live.hostPath), shared); + assert.equal(store.statArtifact("client-a", live.artifactId).artifactId, live.artifactId); + assert.equal(store.statArtifact("client-a", pinned.artifactId).pinned, true); + } finally { + store.close(); + } +} + +async function testBoundedCleanup(testRoot: string): Promise { + let current = new Date("2026-07-18T12:00:00.000Z"); + const store = createStore(testRoot, {}, () => new Date(current), 1); + try { + await store.beginUpload("client-a", { filename: "one.bin" }); + await store.beginUpload("client-a", { filename: "two.bin" }); + current = new Date("2026-07-18T14:00:00.000Z"); + const first = await store.cleanupExpired(); + assert.equal(first.uploadsDeleted, 1); + assert.equal(store.health().pendingUploads, 1); + const second = await store.cleanupExpired(); + assert.equal(second.uploadsDeleted, 1); + assert.equal(store.health().pendingUploads, 0); + } finally { + store.close(); + } +} + +async function testContainmentAndSymlinks(testRoot: string): Promise { + const store = createStore(testRoot); + try { + const outside = join(testRoot, "outside.txt"); + await writeFile(outside, "outside-safe"); + + const upload = await store.beginUpload("client-a", { filename: "link.bin" }); + const database = openDatabase(join(testRoot, "state")); + let tempPath: string; + try { + tempPath = String( + database.sqlite.prepare("select temp_path from artifact_uploads where id = ?").pluck().get(upload.uploadId), + ); + } finally { + database.close(); + } + await rm(tempPath); + await symlink(outside, tempPath); + await assert.rejects( + store.uploadChunk("client-a", { + uploadId: upload.uploadId, + offset: 0, + dataBase64: Buffer.from("attack").toString("base64"), + }), + ); + assert.equal(await readFile(outside, "utf8"), "outside-safe"); + await assert.rejects(store.abortUpload("client-a", upload.uploadId)); + assert.equal(await readFile(outside, "utf8"), "outside-safe"); + + const collisionBytes = Buffer.from("collision"); + const collisionDigest = createHash("sha256").update(collisionBytes).digest("hex"); + const collisionUpload = await store.beginUpload("client-a", { + filename: "collision.bin", + size: collisionBytes.length, + }); + await store.uploadChunk("client-a", { + uploadId: collisionUpload.uploadId, + offset: 0, + dataBase64: collisionBytes.toString("base64"), + }); + const objectPath = join( + testRoot, + "artifacts", + "objects", + collisionDigest.slice(0, 2), + collisionDigest.slice(2, 4), + collisionDigest, + ); + await mkdir(dirname(objectPath), { recursive: true, mode: 0o700 }); + await chmod(dirname(objectPath), 0o700); + await symlink(outside, objectPath); + await expectArtifactError( + store.commitUpload("client-a", collisionUpload.uploadId), + "unsafe_object", + ); + assert.equal(await readFile(outside, "utf8"), "outside-safe"); + + const realRoot = join(testRoot, "real-artifact-root"); + const aliasRoot = join(testRoot, "artifact-root-alias"); + await mkdir(realRoot, { recursive: true }); + await symlink(realRoot, aliasRoot); + assert.throws( + () => new ArtifactStore({ + stateDir: join(testRoot, "alias-state"), + artifactRoot: aliasRoot, + artifactMaxFileBytes: 1024, + artifactMaxTotalBytes: 4096, + artifactDefaultTtlHours: 24, + }), + (error: unknown) => error instanceof ArtifactError && error.code === "unsafe_artifact_root", + ); + } finally { + store.close(); + } +} + +function createStore( + testRoot: string, + overrides: Partial<{ + artifactMaxFileBytes: number; + artifactMaxTotalBytes: number; + artifactDefaultTtlHours: number; + }> = {}, + now?: () => Date, + cleanupLimit?: number, +): ArtifactStore { + return new ArtifactStore( + { + stateDir: join(testRoot, "state"), + artifactRoot: join(testRoot, "artifacts"), + artifactMaxFileBytes: overrides.artifactMaxFileBytes ?? 100 * 1024 * 1024, + artifactMaxTotalBytes: overrides.artifactMaxTotalBytes ?? 1024 * 1024 * 1024, + artifactDefaultTtlHours: overrides.artifactDefaultTtlHours ?? 24, + }, + { now, cleanupLimit }, + ); +} + +async function stage( + store: ArtifactStore, + clientId: string, + filename: string, + bytes: Buffer, + ttlHours?: number, +): Promise { + const upload = await store.beginUpload(clientId, { + filename, + size: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + ttlHours, + }); + for (let offset = 0; offset < bytes.length; offset += ARTIFACT_CHUNK_BYTES) { + const chunk = bytes.subarray(offset, offset + ARTIFACT_CHUNK_BYTES); + await store.uploadChunk(clientId, { + uploadId: upload.uploadId, + offset, + dataBase64: chunk.toString("base64"), + }); + } + return store.commitUpload(clientId, upload.uploadId); +} + +async function expectArtifactError( + promise: Promise, + code: string, +): Promise { + await assert.rejects( + promise, + (error: unknown) => error instanceof ArtifactError && error.code === code, + ); +} diff --git a/src/artifacts.ts b/src/artifacts.ts new file mode 100644 index 00000000..aee77fa2 --- /dev/null +++ b/src/artifacts.ts @@ -0,0 +1,903 @@ +import { createHash, randomUUID, type Hash } from "node:crypto"; +import { + constants as fsConstants, + createReadStream, + chmodSync, + lstatSync, + mkdirSync, + realpathSync, +} from "node:fs"; +import { + chmod, + lstat, + mkdir, + open, + realpath, + rename, + unlink, +} from "node:fs/promises"; +import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { openDatabase, type DatabaseHandle } from "./db/client.js"; +import { MAX_ARTIFACT_TTL_HOURS, type ServerConfig } from "./config.js"; + +export const ARTIFACT_CHUNK_BYTES = 48 * 1024; +export const ARTIFACT_UPLOAD_TTL_HOURS = 1; +export const ARTIFACT_CLEANUP_INTERVAL_MS = 15 * 60 * 1_000; +export const ARTIFACT_CLEANUP_LIMIT = 100; + +const MAX_FILENAME_BYTES = 255; +const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; + +export class ArtifactError extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + this.name = "ArtifactError"; + } +} + +export interface ArtifactUploadBeginInput { + filename: string; + mimeType?: string; + size?: number; + sha256?: string; + workspaceId?: string; + ttlHours?: number; +} + +export interface ArtifactUploadBeginResult { + uploadId: string; + chunkBytes: number; + expiresAt: string; + nextOffset: number; +} + +export interface ArtifactUploadChunkInput { + uploadId: string; + offset: number; + dataBase64: string; +} + +export interface ArtifactUploadChunkResult { + uploadId: string; + receivedBytes: number; + nextOffset: number; + retry: boolean; +} + +export interface ArtifactRecord { + artifactId: string; + name: string; + mimeType?: string; + size: number; + sha256: string; + hostPath: string; + source: string; + workspaceId?: string; + createdAt: string; + expiresAt?: string; + pinned: boolean; +} + +export interface ArtifactCommitOptions { + source?: string; + pinned?: boolean; +} + +export interface ArtifactDeleteResult { + artifactId: string; + deleted: boolean; + objectDeleted: boolean; +} + +export interface ArtifactCleanupResult { + uploadsDeleted: number; + artifactsDeleted: number; + objectsDeleted: number; + skippedUnsafePaths: number; +} + +export interface ArtifactStorageHealth { + root: string; + storedBytes: number; + maxTotalBytes: number; + pendingUploads: number; + expiredArtifacts: number; +} + +interface ArtifactStoreOptions { + now?: () => Date; + cleanupLimit?: number; +} + +interface UploadRow { + id: string; + client_id: string | null; + workspace_id: string | null; + original_name: string; + mime_type: string | null; + expected_size: number | null; + expected_sha256: string | null; + received_size: number; + temp_path: string; + status: string; + artifact_ttl_hours: number; + last_chunk_offset: number | null; + last_chunk_size: number | null; + last_chunk_sha256: string | null; + created_at: string; + expires_at: string; +} + +interface ArtifactRow { + id: string; + client_id: string | null; + workspace_id: string | null; + original_name: string; + mime_type: string | null; + size: number; + sha256: string; + storage_path: string; + source: string; + status: string; + created_at: string; + expires_at: string | null; + pinned: number; + last_used_at: string; +} + +interface IncrementalHashState { + hash: Hash; + receivedSize: number; +} + +export class ArtifactStore { + private readonly database: DatabaseHandle; + private readonly root: string; + private readonly rootRealPath: string; + private readonly objectsRoot: string; + private readonly uploadsRoot: string; + private readonly now: () => Date; + private readonly cleanupLimit: number; + private readonly incrementalHashes = new Map(); + private mutationQueue: Promise = Promise.resolve(); + + constructor( + private readonly config: Pick< + ServerConfig, + | "stateDir" + | "artifactRoot" + | "artifactMaxFileBytes" + | "artifactMaxTotalBytes" + | "artifactDefaultTtlHours" + >, + options: ArtifactStoreOptions = {}, + ) { + this.root = resolve(config.artifactRoot); + this.rootRealPath = ensureSecureDirectorySync(this.root); + this.objectsRoot = join(this.root, "objects"); + this.uploadsRoot = join(this.root, "uploads"); + ensureSecureDirectorySync(this.objectsRoot, this.rootRealPath); + ensureSecureDirectorySync(this.uploadsRoot, this.rootRealPath); + this.database = openDatabase(config.stateDir); + this.now = options.now ?? (() => new Date()); + this.cleanupLimit = options.cleanupLimit ?? ARTIFACT_CLEANUP_LIMIT; + } + + close(): void { + this.database.close(); + } + + beginUpload(clientId: string, input: ArtifactUploadBeginInput): Promise { + return this.withMutation(async () => { + const name = normalizeArtifactFilename(input.filename); + const mimeType = normalizeMimeType(input.mimeType); + const expectedSize = normalizeExpectedSize(input.size, this.config.artifactMaxFileBytes); + const expectedSha256 = normalizeSha256(input.sha256); + const ttlHours = normalizeTtlHours(input.ttlHours, this.config.artifactDefaultTtlHours); + + if (expectedSize !== undefined) { + this.assertQuotaAvailable(expectedSize); + } + + const uploadId = `upl_${randomUUID()}`; + const tempPath = join(this.uploadsRoot, `${uploadId}.partial`); + assertLexicalContainment(this.uploadsRoot, tempPath); + const handle = await open( + tempPath, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | NO_FOLLOW, + 0o600, + ); + try { + const fileStat = await handle.stat(); + if (!fileStat.isFile()) { + throw new ArtifactError("unsafe_partial", "Upload partial is not a regular file."); + } + await handle.sync(); + } finally { + await handle.close(); + } + await chmod(tempPath, 0o600); + await assertExistingFileContained(tempPath, this.uploadsRoot, this.rootRealPath); + + const now = this.now(); + const expiresAt = addHours(now, ARTIFACT_UPLOAD_TTL_HOURS).toISOString(); + this.database.sqlite.prepare(` + insert into artifact_uploads ( + id, client_id, workspace_id, original_name, mime_type, + expected_size, expected_sha256, received_size, temp_path, + status, artifact_ttl_hours, last_chunk_offset, last_chunk_size, + last_chunk_sha256, created_at, expires_at + ) values (?, ?, ?, ?, ?, ?, ?, 0, ?, 'active', ?, null, null, null, ?, ?) + `).run( + uploadId, + clientId, + input.workspaceId ?? null, + name, + mimeType ?? null, + expectedSize ?? null, + expectedSha256 ?? null, + tempPath, + ttlHours, + now.toISOString(), + expiresAt, + ); + this.incrementalHashes.set(uploadId, { + hash: createHash("sha256"), + receivedSize: 0, + }); + + return { + uploadId, + chunkBytes: ARTIFACT_CHUNK_BYTES, + expiresAt, + nextOffset: 0, + }; + }); + } + + uploadChunk(clientId: string, input: ArtifactUploadChunkInput): Promise { + return this.withMutation(async () => { + const row = this.requireUpload(clientId, input.uploadId); + this.assertUploadActive(row); + this.assertUploadNotExpired(row); + const data = decodeBase64Strict(input.dataBase64); + if (data.length === 0) { + throw new ArtifactError("empty_chunk", "Upload chunks must contain at least one decoded byte."); + } + if (data.length > ARTIFACT_CHUNK_BYTES) { + throw new ArtifactError( + "chunk_too_large", + `Decoded chunk exceeds the ${ARTIFACT_CHUNK_BYTES}-byte limit.`, + ); + } + if (!Number.isSafeInteger(input.offset) || input.offset < 0) { + throw new ArtifactError("invalid_offset", "Chunk offset must be a non-negative safe integer."); + } + + const chunkSha256 = createHash("sha256").update(data).digest("hex"); + if (input.offset < row.received_size) { + const identicalRetry = + row.last_chunk_offset === input.offset + && row.last_chunk_size === data.length + && row.last_chunk_sha256 === chunkSha256 + && input.offset + data.length === row.received_size; + if (!identicalRetry) { + throw new ArtifactError( + "conflicting_retry", + "Chunk offset has already been committed with different data.", + ); + } + return { + uploadId: row.id, + receivedBytes: row.received_size, + nextOffset: row.received_size, + retry: true, + }; + } + if (input.offset !== row.received_size) { + throw new ArtifactError( + "out_of_order_chunk", + `Expected chunk offset ${row.received_size}, received ${input.offset}.`, + ); + } + + const nextSize = row.received_size + data.length; + if (nextSize > this.config.artifactMaxFileBytes) { + throw new ArtifactError( + "file_too_large", + `Artifact exceeds the ${this.config.artifactMaxFileBytes}-byte file limit.`, + ); + } + if (row.expected_size !== null && nextSize > row.expected_size) { + throw new ArtifactError( + "declared_size_exceeded", + `Chunk would exceed the declared ${row.expected_size}-byte size.`, + ); + } + this.assertQuotaAvailable(data.length); + + const handle = await open(row.temp_path, fsConstants.O_WRONLY | NO_FOLLOW); + try { + const fileStat = await handle.stat(); + if (!fileStat.isFile()) { + throw new ArtifactError("unsafe_partial", "Upload partial is not a regular file."); + } + const resolvedPath = await realpath(row.temp_path); + assertRealContainment(this.rootRealPath, resolvedPath); + assertLexicalContainment(this.uploadsRoot, row.temp_path); + const { bytesWritten } = await handle.write(data, 0, data.length, input.offset); + if (bytesWritten !== data.length) { + throw new ArtifactError("short_write", "The artifact chunk was not fully written."); + } + await handle.sync(); + } finally { + await handle.close(); + } + + this.database.sqlite.prepare(` + update artifact_uploads + set received_size = ?, last_chunk_offset = ?, last_chunk_size = ?, last_chunk_sha256 = ? + where id = ? and client_id = ? and status = 'active' + `).run(nextSize, input.offset, data.length, chunkSha256, row.id, clientId); + + const hashState = this.incrementalHashes.get(row.id); + if (hashState && hashState.receivedSize === row.received_size) { + hashState.hash.update(data); + hashState.receivedSize = nextSize; + } else { + this.incrementalHashes.delete(row.id); + } + + return { + uploadId: row.id, + receivedBytes: nextSize, + nextOffset: nextSize, + retry: false, + }; + }); + } + + commitUpload( + clientId: string, + uploadId: string, + options: ArtifactCommitOptions = {}, + ): Promise { + return this.withMutation(async () => { + const row = this.requireUpload(clientId, uploadId); + this.assertUploadActive(row); + this.assertUploadNotExpired(row); + const source = normalizeArtifactSource(options.source); + const pinned = options.pinned === true; + const partialStat = await assertExistingFileContained( + row.temp_path, + this.uploadsRoot, + this.rootRealPath, + ); + if (partialStat.size !== row.received_size) { + throw new ArtifactError( + "partial_size_mismatch", + `Partial file size ${partialStat.size} does not match recorded size ${row.received_size}.`, + ); + } + if (row.expected_size !== null && row.received_size !== row.expected_size) { + throw new ArtifactError( + "size_mismatch", + `Received ${row.received_size} bytes, expected ${row.expected_size}.`, + ); + } + if (row.received_size > this.config.artifactMaxFileBytes) { + throw new ArtifactError("file_too_large", "Artifact exceeds the configured file limit."); + } + + const digest = await this.uploadDigest(row); + if (row.expected_sha256 && digest !== row.expected_sha256) { + throw new ArtifactError( + "sha256_mismatch", + `Artifact SHA-256 does not match the declared digest.`, + ); + } + + const objectDirectory = join(this.objectsRoot, digest.slice(0, 2), digest.slice(2, 4)); + await ensureSecureDirectory(objectDirectory, this.rootRealPath); + const objectPath = join(objectDirectory, digest); + assertLexicalContainment(this.objectsRoot, objectPath); + + let createdObject = false; + const existingObject = await lstatOrUndefined(objectPath); + if (existingObject) { + if (existingObject.isSymbolicLink() || !existingObject.isFile()) { + throw new ArtifactError("unsafe_object", "Artifact object path is not a regular file."); + } + await assertExistingFileContained(objectPath, this.objectsRoot, this.rootRealPath); + if (existingObject.size !== row.received_size || await hashFile(objectPath) !== digest) { + throw new ArtifactError("object_collision", "Existing artifact object failed digest verification."); + } + await unlink(row.temp_path); + } else { + await rename(row.temp_path, objectPath); + createdObject = true; + await chmod(objectPath, 0o600); + await assertExistingFileContained(objectPath, this.objectsRoot, this.rootRealPath); + } + + const now = this.now(); + const artifactId = `art_${randomUUID()}`; + const expiresAt = addHours(now, row.artifact_ttl_hours).toISOString(); + const insert = this.database.sqlite.transaction(() => { + this.database.sqlite.prepare(` + insert into artifacts ( + id, client_id, workspace_id, original_name, mime_type, + size, sha256, storage_path, source, status, + created_at, expires_at, pinned, last_used_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, 'available', ?, ?, ?, ?) + `).run( + artifactId, + clientId, + row.workspace_id, + row.original_name, + row.mime_type, + row.received_size, + digest, + objectPath, + source, + now.toISOString(), + expiresAt, + pinned ? 1 : 0, + now.toISOString(), + ); + this.database.sqlite.prepare("delete from artifact_uploads where id = ? and client_id = ?") + .run(row.id, clientId); + }); + + try { + insert.immediate(); + } catch (error) { + if (createdObject) { + await unlink(objectPath).catch(() => undefined); + } + throw error; + } finally { + this.incrementalHashes.delete(row.id); + } + + return { + artifactId, + name: row.original_name, + mimeType: row.mime_type ?? undefined, + size: row.received_size, + sha256: `sha256:${digest}`, + hostPath: objectPath, + source, + workspaceId: row.workspace_id ?? undefined, + createdAt: now.toISOString(), + expiresAt, + pinned, + }; + }); + } + + abortUpload(clientId: string, uploadId: string): Promise<{ uploadId: string; aborted: boolean }> { + return this.withMutation(async () => { + const row = this.requireUpload(clientId, uploadId); + let unsafePathError: unknown; + try { + await removeManagedFile(row.temp_path, this.uploadsRoot, this.rootRealPath); + } catch (error) { + unsafePathError = error; + } + this.database.sqlite.prepare("delete from artifact_uploads where id = ? and client_id = ?") + .run(row.id, clientId); + this.incrementalHashes.delete(row.id); + if (unsafePathError) throw unsafePathError; + return { uploadId: row.id, aborted: true }; + }); + } + + statArtifact(clientId: string, artifactId: string): ArtifactRecord { + const row = this.requireArtifact(clientId, artifactId); + if (row.status !== "available") { + throw new ArtifactError("artifact_unavailable", "Artifact is not available."); + } + const now = this.now().toISOString(); + this.database.sqlite.prepare( + "update artifacts set last_used_at = ? where id = ? and client_id = ?", + ).run(now, artifactId, clientId); + return artifactRowToRecord(row); + } + + deleteArtifact(clientId: string, artifactId: string): Promise { + return this.withMutation(async () => { + const row = this.requireArtifact(clientId, artifactId); + this.database.sqlite.prepare("delete from artifacts where id = ? and client_id = ?") + .run(row.id, clientId); + const objectDeleted = await this.deleteObjectIfUnreferenced(row); + return { artifactId: row.id, deleted: true, objectDeleted }; + }); + } + + cleanupExpired(): Promise { + return this.withMutation(async () => { + const now = this.now().toISOString(); + let remaining = this.cleanupLimit; + let uploadsDeleted = 0; + let artifactsDeleted = 0; + let objectsDeleted = 0; + let skippedUnsafePaths = 0; + + const expiredUploads = this.database.sqlite.prepare(` + select * from artifact_uploads + where status = 'active' and expires_at <= ? + order by expires_at asc + limit ? + `).all(now, remaining) as UploadRow[]; + for (const row of expiredUploads) { + try { + await removeManagedFile(row.temp_path, this.uploadsRoot, this.rootRealPath); + } catch { + skippedUnsafePaths += 1; + } + this.database.sqlite.prepare("delete from artifact_uploads where id = ?").run(row.id); + this.incrementalHashes.delete(row.id); + uploadsDeleted += 1; + remaining -= 1; + } + + if (remaining > 0) { + const expiredArtifacts = this.database.sqlite.prepare(` + select * from artifacts + where status = 'available' and pinned = 0 and expires_at is not null and expires_at <= ? + order by expires_at asc + limit ? + `).all(now, remaining) as ArtifactRow[]; + for (const row of expiredArtifacts) { + this.database.sqlite.prepare("delete from artifacts where id = ?").run(row.id); + artifactsDeleted += 1; + try { + if (await this.deleteObjectIfUnreferenced(row)) objectsDeleted += 1; + } catch { + skippedUnsafePaths += 1; + } + } + } + + return { + uploadsDeleted, + artifactsDeleted, + objectsDeleted, + skippedUnsafePaths, + }; + }); + } + + health(): ArtifactStorageHealth { + const storedBytes = this.objectBytes(); + const pendingUploads = Number( + this.database.sqlite.prepare( + "select count(*) from artifact_uploads where status = 'active'", + ).pluck().get() ?? 0, + ); + const expiredArtifacts = Number( + this.database.sqlite.prepare(` + select count(*) from artifacts + where status = 'available' and pinned = 0 + and expires_at is not null and expires_at <= ? + `).pluck().get(this.now().toISOString()) ?? 0, + ); + return { + root: this.root, + storedBytes, + maxTotalBytes: this.config.artifactMaxTotalBytes, + pendingUploads, + expiredArtifacts, + }; + } + + private requireUpload(clientId: string, uploadId: string): UploadRow { + const row = this.database.sqlite.prepare( + "select * from artifact_uploads where id = ? and client_id = ?", + ).get(uploadId, clientId) as UploadRow | undefined; + if (!row) { + throw new ArtifactError("upload_not_found", "Artifact upload was not found."); + } + return row; + } + + private requireArtifact(clientId: string, artifactId: string): ArtifactRow { + const row = this.database.sqlite.prepare( + "select * from artifacts where id = ? and client_id = ?", + ).get(artifactId, clientId) as ArtifactRow | undefined; + if (!row) { + throw new ArtifactError("artifact_not_found", "Artifact was not found."); + } + return row; + } + + private assertUploadActive(row: UploadRow): void { + if (row.status !== "active") { + throw new ArtifactError("upload_inactive", "Artifact upload is not active."); + } + } + + private assertUploadNotExpired(row: UploadRow): void { + if (row.expires_at <= this.now().toISOString()) { + throw new ArtifactError("upload_expired", "Artifact upload has expired."); + } + } + + private assertQuotaAvailable(additionalBytes: number): void { + const used = this.objectBytes() + this.pendingUploadBytes(); + if (used + additionalBytes > this.config.artifactMaxTotalBytes) { + throw new ArtifactError( + "artifact_quota_exceeded", + `Artifact storage would exceed the ${this.config.artifactMaxTotalBytes}-byte total limit.`, + ); + } + } + + private objectBytes(): number { + return Number( + this.database.sqlite.prepare(` + select coalesce(sum(size), 0) + from ( + select storage_path, max(size) as size + from artifacts + where status = 'available' + group by storage_path + ) + `).pluck().get() ?? 0, + ); + } + + private pendingUploadBytes(): number { + return Number( + this.database.sqlite.prepare(` + select coalesce(sum(received_size), 0) + from artifact_uploads + where status = 'active' + `).pluck().get() ?? 0, + ); + } + + private async uploadDigest(row: UploadRow): Promise { + const state = this.incrementalHashes.get(row.id); + if (state && state.receivedSize === row.received_size) { + return state.hash.copy().digest("hex"); + } + return hashFile(row.temp_path); + } + + private async deleteObjectIfUnreferenced(row: ArtifactRow): Promise { + const referenceCount = Number( + this.database.sqlite.prepare(` + select count(*) from artifacts + where status = 'available' and storage_path = ? + `).pluck().get(row.storage_path) ?? 0, + ); + if (referenceCount > 0) return false; + await removeManagedFile(row.storage_path, this.objectsRoot, this.rootRealPath); + return true; + } + + private withMutation(operation: () => Promise): Promise { + const result = this.mutationQueue.then(operation, operation); + this.mutationQueue = result.then( + () => undefined, + () => undefined, + ); + return result; + } +} + +export function normalizeArtifactFilename(value: string): string { + if (typeof value !== "string") { + throw new ArtifactError("invalid_filename", "Artifact filename must be a string."); + } + const normalized = value.normalize("NFC"); + if (!normalized || normalized === "." || normalized === "..") { + throw new ArtifactError("invalid_filename", "Artifact filename is empty or reserved."); + } + if (normalized.startsWith(".")) { + throw new ArtifactError("invalid_filename", "Artifact filename must not start with a dot."); + } + if (/[\\/]/u.test(normalized) || basename(normalized) !== normalized) { + throw new ArtifactError("invalid_filename", "Artifact filename must contain one basename only."); + } + if(/[\u0000-\u001f\u007f-\u009f]/u.test(normalized)) { + throw new ArtifactError("invalid_filename", "Artifact filename contains control characters."); + } + if (Buffer.byteLength(normalized, "utf8") > MAX_FILENAME_BYTES) { + throw new ArtifactError( + "invalid_filename", + `Artifact filename exceeds ${MAX_FILENAME_BYTES} UTF-8 bytes.`, + ); + } + return normalized; +} + +function normalizeArtifactSource(value: string | undefined): string { + const source = value ?? "chunked"; + if (!/^[a-z0-9][a-z0-9._:-]{0,63}$/u.test(source)) { + throw new ArtifactError( + "invalid_artifact_source", + "Artifact source must be a short lowercase identifier.", + ); + } + return source; +} + +export function normalizeSha256(value: string | undefined): string | undefined { + if (value === undefined) return undefined; + const normalized = value.trim().toLowerCase().replace(/^sha256:/u, ""); + if (!/^[0-9a-f]{64}$/u.test(normalized)) { + throw new ArtifactError("invalid_sha256", "SHA-256 must contain exactly 64 hexadecimal characters."); + } + return normalized; +} + +export function decodeBase64Strict(value: string): Buffer { + if (typeof value !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value)) { + throw new ArtifactError("invalid_base64", "Chunk data is not valid canonical base64."); + } + const decoded = Buffer.from(value, "base64"); + if (decoded.toString("base64") !== value) { + throw new ArtifactError("invalid_base64", "Chunk data is not valid canonical base64."); + } + return decoded; +} + +export function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const units = ["KiB", "MiB", "GiB", "TiB"]; + let value = bytes; + let unit = -1; + do { + value /= 1024; + unit += 1; + } while (value >= 1024 && unit < units.length - 1); + const digits = value >= 10 ? 1 : 2; + return `${value.toFixed(digits).replace(/\.0+$/u, "")} ${units[unit]}`; +} + +function normalizeMimeType(value: string | undefined): string | undefined { + const mimeType = value?.trim(); + if (!mimeType) return undefined; + if (mimeType.length > 255 || /[\u0000-\u001f\u007f]/u.test(mimeType)) { + throw new ArtifactError("invalid_mime_type", "Artifact MIME type is invalid."); + } + return mimeType; +} + +function normalizeExpectedSize(value: number | undefined, maxFileBytes: number): number | undefined { + if (value === undefined) return undefined; + if (!Number.isSafeInteger(value) || value < 0) { + throw new ArtifactError("invalid_size", "Artifact size must be a non-negative safe integer."); + } + if (value > maxFileBytes) { + throw new ArtifactError("file_too_large", `Artifact exceeds the ${maxFileBytes}-byte file limit.`); + } + return value; +} + +function normalizeTtlHours(value: number | undefined, fallback: number): number { + const ttlHours = value ?? fallback; + if ( + !Number.isSafeInteger(ttlHours) + || ttlHours < 1 + || ttlHours > MAX_ARTIFACT_TTL_HOURS + ) { + throw new ArtifactError( + "invalid_ttl", + `Artifact TTL must be an integer between 1 and ${MAX_ARTIFACT_TTL_HOURS} hours.`, + ); + } + return ttlHours; +} + +function artifactRowToRecord(row: ArtifactRow): ArtifactRecord { + return { + artifactId: row.id, + name: row.original_name, + mimeType: row.mime_type ?? undefined, + size: row.size, + sha256: `sha256:${row.sha256}`, + hostPath: row.storage_path, + source: row.source, + workspaceId: row.workspace_id ?? undefined, + createdAt: row.created_at, + expiresAt: row.expires_at ?? undefined, + pinned: row.pinned === 1, + }; +} + +function addHours(date: Date, hours: number): Date { + return new Date(date.getTime() + hours * 60 * 60 * 1_000); +} + +function ensureSecureDirectorySync(path: string, containingRootRealPath?: string): string { + mkdirSync(path, { recursive: true, mode: 0o700 }); + const entry = lstatSync(path); + if (entry.isSymbolicLink() || !entry.isDirectory()) { + throw new ArtifactError("unsafe_artifact_root", `Artifact directory is not a real directory: ${path}`); + } + chmodSync(path, 0o700); + const realPath = realpathSync(path); + if (containingRootRealPath) assertRealContainment(containingRootRealPath, realPath); + return realPath; +} + +async function ensureSecureDirectory(path: string, rootRealPath: string): Promise { + await mkdir(path, { recursive: true, mode: 0o700 }); + const entry = await lstat(path); + if (entry.isSymbolicLink() || !entry.isDirectory()) { + throw new ArtifactError("unsafe_artifact_path", `Artifact directory is not a real directory: ${path}`); + } + await chmod(path, 0o700); + assertRealContainment(rootRealPath, await realpath(path)); +} + +async function assertExistingFileContained( + path: string, + lexicalRoot: string, + rootRealPath: string, +) { + assertLexicalContainment(lexicalRoot, path); + const entry = await lstat(path); + if (entry.isSymbolicLink() || !entry.isFile()) { + throw new ArtifactError("unsafe_artifact_path", "Artifact path is not a regular file."); + } + assertRealContainment(rootRealPath, await realpath(path)); + return entry; +} + +async function removeManagedFile( + path: string, + lexicalRoot: string, + rootRealPath: string, +): Promise { + assertLexicalContainment(lexicalRoot, path); + const entry = await lstatOrUndefined(path); + if (!entry) return; + if (entry.isSymbolicLink() || !entry.isFile()) { + throw new ArtifactError("unsafe_artifact_path", "Refusing to delete a non-regular artifact path."); + } + assertRealContainment(rootRealPath, await realpath(path)); + await unlink(path); +} + +function assertLexicalContainment(root: string, candidate: string): void { + const relativePath = relative(resolve(root), resolve(candidate)); + if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) { + throw new ArtifactError("artifact_path_escape", "Artifact path escapes the configured storage root."); + } +} + +function assertRealContainment(rootRealPath: string, candidateRealPath: string): void { + const relativePath = relative(rootRealPath, candidateRealPath); + if (relativePath.startsWith("..") || isAbsolute(relativePath)) { + throw new ArtifactError("artifact_path_escape", "Artifact path resolves outside the configured storage root."); + } +} + +async function lstatOrUndefined(path: string) { + try { + return await lstat(path); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return undefined; + throw error; + } +} + +async function hashFile(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) { + hash.update(chunk as Buffer); + } + return hash.digest("hex"); +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; +} diff --git a/src/config.test.ts b/src/config.test.ts index 0b4f99a8..7a6563a5 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -1,8 +1,8 @@ import assert from "node:assert/strict"; import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; -import { loadConfig } from "./config.js"; +import { loadConfig, MAX_ARTIFACT_TTL_HOURS } from "./config.js"; import { ensureDevspaceDefaultSkills, resolveSubagentsFlag } from "./user-config.js"; const emptyConfigDir = mkdtempSync(join(tmpdir(), "devspace-empty-config-test-")); @@ -26,6 +26,31 @@ assert.equal(loadConfig(baseEnv).skillsEnabled, true); assert.equal(loadConfig(baseEnv).devspaceSkillsDir, join(emptyConfigDir, "skills")); assert.equal(loadConfig(baseEnv).devspaceAgentsDir, join(emptyConfigDir, "agents")); assert.equal(loadConfig(baseEnv).subagents, false); +assert.equal(loadConfig(baseEnv).artifactsEnabled, false); +assert.equal( + loadConfig(baseEnv).artifactRoot, + join(homedir(), ".local", "share", "devspace", "artifacts"), +); +assert.equal(loadConfig(baseEnv).artifactMaxFileBytes, 100 * 1024 * 1024); +assert.equal(loadConfig(baseEnv).artifactMaxTotalBytes, 1024 * 1024 * 1024); +assert.equal(loadConfig(baseEnv).artifactDefaultTtlHours, 24); +assert.equal(loadConfig({ ...baseEnv, DEVSPACE_ARTIFACTS: "1" }).artifactsEnabled, true); +assert.equal( + loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_ROOT: "~/private-artifacts" }).artifactRoot, + join(homedir(), "private-artifacts"), +); +assert.equal( + loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_FILE_BYTES: "123" }).artifactMaxFileBytes, + 123, +); +assert.equal( + loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES: "456" }).artifactMaxTotalBytes, + 456, +); +assert.equal( + loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_TTL_HOURS: "48" }).artifactDefaultTtlHours, + 48, +); assert.equal(loadConfig({ ...baseEnv, DEVSPACE_SKILLS: "0" }).skillsEnabled, false); assert.equal(loadConfig({ ...baseEnv, DEVSPACE_SKILLS: "1" }).skillsEnabled, true); assert.equal( @@ -138,6 +163,25 @@ assert.throws( () => loadConfig({ ...baseEnv, DEVSPACE_OAUTH_ACCESS_TOKEN_TTL_SECONDS: "0" }), /Invalid DEVSPACE_OAUTH_ACCESS_TOKEN_TTL_SECONDS: 0/, ); +assert.throws( + () => loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_FILE_BYTES: "0" }), + /Invalid DEVSPACE_ARTIFACT_MAX_FILE_BYTES: 0/, +); +assert.throws( + () => loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES: "nope" }), + /Invalid DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES: nope/, +); +assert.throws( + () => loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_TTL_HOURS: "-1" }), + /Invalid DEVSPACE_ARTIFACT_TTL_HOURS: -1/, +); +assert.throws( + () => loadConfig({ + ...baseEnv, + DEVSPACE_ARTIFACT_TTL_HOURS: String(MAX_ARTIFACT_TTL_HOURS + 1), + }), + /Invalid DEVSPACE_ARTIFACT_TTL_HOURS/, +); assert.equal(loadConfig(baseEnv).publicBaseUrl, "http://127.0.0.1:7676"); assert.deepEqual(loadConfig(baseEnv).allowedHosts, ["localhost", "127.0.0.1", "::1"]); @@ -163,6 +207,11 @@ writeFileSync( allowedRoots: [process.cwd()], publicBaseUrl: "https://devspace.example.com", subagents: true, + artifactsEnabled: true, + artifactRoot: join(configDir, "persisted-artifacts"), + artifactMaxFileBytes: 321, + artifactMaxTotalBytes: 654, + artifactDefaultTtlHours: 72, }), ); writeFileSync( @@ -177,6 +226,11 @@ assert.equal(fileConfig.port, 8787); assert.equal(fileConfig.oauth.ownerToken, "persisted-owner-token-long-enough"); assert.equal(fileConfig.publicBaseUrl, "https://devspace.example.com"); assert.equal(fileConfig.subagents, true); +assert.equal(fileConfig.artifactsEnabled, true); +assert.equal(fileConfig.artifactRoot, join(configDir, "persisted-artifacts")); +assert.equal(fileConfig.artifactMaxFileBytes, 321); +assert.equal(fileConfig.artifactMaxTotalBytes, 654); +assert.equal(fileConfig.artifactDefaultTtlHours, 72); assert.deepEqual(fileConfig.allowedHosts, [ "localhost", "127.0.0.1", diff --git a/src/config.ts b/src/config.ts index 4fc1bcbb..55fb286d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,6 +9,10 @@ export type ToolMode = "minimal" | "full" | "codex"; export type WidgetMode = "off" | "changes" | "full"; const DEFAULT_OAUTH_ACCESS_TOKEN_TTL_SECONDS = 60 * 60; const DEFAULT_OAUTH_REFRESH_TOKEN_TTL_SECONDS = 30 * 24 * 60 * 60; +const DEFAULT_ARTIFACT_MAX_FILE_BYTES = 100 * 1024 * 1024; +const DEFAULT_ARTIFACT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024; +const DEFAULT_ARTIFACT_TTL_HOURS = 24; +export const MAX_ARTIFACT_TTL_HOURS = 24 * 365; export interface ServerConfig { host: string; @@ -21,6 +25,11 @@ export interface ServerConfig { widgets: WidgetMode; stateDir: string; worktreeRoot: string; + artifactsEnabled: boolean; + artifactRoot: string; + artifactMaxFileBytes: number; + artifactMaxTotalBytes: number; + artifactDefaultTtlHours: number; skillsEnabled: boolean; skillPaths: string[]; devspaceSkillsDir: string; @@ -124,11 +133,16 @@ function parseStringList(value: string | undefined, fallback: string[]): string[ return entries && entries.length > 0 ? entries : fallback; } -function parsePositiveInteger(value: string | undefined, fallback: number, name: string): number { +function parsePositiveInteger( + value: string | undefined, + fallback: number, + name: string, + max = Number.MAX_SAFE_INTEGER, +): number { if (!value) return fallback; const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 1) { + if (!Number.isInteger(parsed) || parsed < 1 || parsed > max) { throw new Error(`Invalid ${name}: ${value}`); } @@ -195,6 +209,10 @@ function defaultWorktreeRoot(): string { return join(homedir(), ".devspace", "worktrees"); } +function defaultArtifactRoot(): string { + return join(homedir(), ".local", "share", "devspace", "artifacts"); +} + function defaultAgentDir(): string { return join(homedir(), ".codex"); } @@ -226,6 +244,29 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): ServerConfig { widgets: parseWidgetMode(env.DEVSPACE_WIDGETS), stateDir: resolve(expandHomePath(env.DEVSPACE_STATE_DIR ?? files.config.stateDir ?? defaultStateDir())), worktreeRoot: resolve(expandHomePath(env.DEVSPACE_WORKTREE_ROOT ?? files.config.worktreeRoot ?? defaultWorktreeRoot())), + artifactsEnabled: + env.DEVSPACE_ARTIFACTS === undefined + ? files.config.artifactsEnabled === true + : parseBoolean(env.DEVSPACE_ARTIFACTS), + artifactRoot: resolve(expandHomePath( + env.DEVSPACE_ARTIFACT_ROOT ?? files.config.artifactRoot ?? defaultArtifactRoot(), + )), + artifactMaxFileBytes: parsePositiveInteger( + env.DEVSPACE_ARTIFACT_MAX_FILE_BYTES ?? numberConfigValue(files.config.artifactMaxFileBytes), + DEFAULT_ARTIFACT_MAX_FILE_BYTES, + "DEVSPACE_ARTIFACT_MAX_FILE_BYTES", + ), + artifactMaxTotalBytes: parsePositiveInteger( + env.DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES ?? numberConfigValue(files.config.artifactMaxTotalBytes), + DEFAULT_ARTIFACT_MAX_TOTAL_BYTES, + "DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES", + ), + artifactDefaultTtlHours: parsePositiveInteger( + env.DEVSPACE_ARTIFACT_TTL_HOURS ?? numberConfigValue(files.config.artifactDefaultTtlHours), + DEFAULT_ARTIFACT_TTL_HOURS, + "DEVSPACE_ARTIFACT_TTL_HOURS", + MAX_ARTIFACT_TTL_HOURS, + ), skillsEnabled: env.DEVSPACE_SKILLS === undefined ? true : parseBoolean(env.DEVSPACE_SKILLS), skillPaths: parsePathList(env.DEVSPACE_SKILL_PATHS), devspaceSkillsDir: devspaceSkillsDir(env), @@ -239,6 +280,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): ServerConfig { }; } +function numberConfigValue(value: number | undefined): string | undefined { + return value === undefined ? undefined : String(value); +} + function parsePublicBaseUrl(value: string): string { const parsed = new URL(value); parsed.hash = ""; diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 2bda20c2..5ad5ec52 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -22,6 +22,11 @@ const migrations: Migration[] = [ name: "local-agent-sessions", up: migrateLocalAgentSessions, }, + { + version: 4, + name: "artifact-exchange", + up: migrateArtifactExchange, + }, ]; export function migrateDatabase(sqlite: Database.Database): void { @@ -174,6 +179,64 @@ function migrateLocalAgentSessions(sqlite: Database.Database): void { addColumnIfMissing(sqlite, "local_agent_sessions", "thinking", "text"); } +function migrateArtifactExchange(sqlite: Database.Database): void { + sqlite.exec(` + create table if not exists artifacts ( + id text primary key, + client_id text, + workspace_id text, + original_name text not null, + mime_type text, + size integer not null, + sha256 text not null, + storage_path text not null, + source text not null, + status text not null, + created_at text not null, + expires_at text, + pinned integer not null default 0, + last_used_at text not null + ); + + create index if not exists artifacts_sha256_storage_idx + on artifacts(sha256, storage_path); + + create index if not exists artifacts_expiry_idx + on artifacts(status, pinned, expires_at); + + create index if not exists artifacts_workspace_idx + on artifacts(workspace_id, last_used_at desc); + + create index if not exists artifacts_client_idx + on artifacts(client_id, last_used_at desc); + + create table if not exists artifact_uploads ( + id text primary key, + client_id text, + workspace_id text, + original_name text not null, + mime_type text, + expected_size integer, + expected_sha256 text, + received_size integer not null default 0, + temp_path text not null, + status text not null, + artifact_ttl_hours integer not null, + last_chunk_offset integer, + last_chunk_size integer, + last_chunk_sha256 text, + created_at text not null, + expires_at text not null + ); + + create index if not exists artifact_uploads_expiry_idx + on artifact_uploads(status, expires_at); + + create index if not exists artifact_uploads_client_idx + on artifact_uploads(client_id, created_at desc); + `); +} + function addColumnIfMissing( sqlite: Database.Database, table: "workspace_sessions" | "local_agent_sessions", diff --git a/src/db/schema.ts b/src/db/schema.ts index 01d13fac..7f994f14 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,3 +1,4 @@ +import { desc } from "drizzle-orm"; import { index, integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core"; export const workspaceSessions = sqliteTable( @@ -73,6 +74,58 @@ export const oauthRefreshTokens = sqliteTable( }, ); +export const artifacts = sqliteTable( + "artifacts", + { + id: text("id").primaryKey(), + clientId: text("client_id"), + workspaceId: text("workspace_id"), + originalName: text("original_name").notNull(), + mimeType: text("mime_type"), + size: integer("size").notNull(), + sha256: text("sha256").notNull(), + storagePath: text("storage_path").notNull(), + source: text("source").notNull(), + status: text("status").notNull(), + createdAt: text("created_at").notNull(), + expiresAt: text("expires_at"), + pinned: integer("pinned").notNull().default(0), + lastUsedAt: text("last_used_at").notNull(), + }, + (table) => [ + index("artifacts_sha256_storage_idx").on(table.sha256, table.storagePath), + index("artifacts_expiry_idx").on(table.status, table.pinned, table.expiresAt), + index("artifacts_workspace_idx").on(table.workspaceId, desc(table.lastUsedAt)), + index("artifacts_client_idx").on(table.clientId, desc(table.lastUsedAt)), + ], +); + +export const artifactUploads = sqliteTable( + "artifact_uploads", + { + id: text("id").primaryKey(), + clientId: text("client_id"), + workspaceId: text("workspace_id"), + originalName: text("original_name").notNull(), + mimeType: text("mime_type"), + expectedSize: integer("expected_size"), + expectedSha256: text("expected_sha256"), + receivedSize: integer("received_size").notNull().default(0), + tempPath: text("temp_path").notNull(), + status: text("status").notNull(), + artifactTtlHours: integer("artifact_ttl_hours").notNull(), + lastChunkOffset: integer("last_chunk_offset"), + lastChunkSize: integer("last_chunk_size"), + lastChunkSha256: text("last_chunk_sha256"), + createdAt: text("created_at").notNull(), + expiresAt: text("expires_at").notNull(), + }, + (table) => [ + index("artifact_uploads_expiry_idx").on(table.status, table.expiresAt), + index("artifact_uploads_client_idx").on(table.clientId, desc(table.createdAt)), + ], +); + export const localAgentSessions = sqliteTable( "local_agent_sessions", { @@ -101,5 +154,9 @@ export type WorkspaceSessionRow = typeof workspaceSessions.$inferSelect; export type NewWorkspaceSessionRow = typeof workspaceSessions.$inferInsert; export type LoadedAgentFileRow = typeof loadedAgentFiles.$inferSelect; export type NewLoadedAgentFileRow = typeof loadedAgentFiles.$inferInsert; +export type ArtifactRow = typeof artifacts.$inferSelect; +export type NewArtifactRow = typeof artifacts.$inferInsert; +export type ArtifactUploadRow = typeof artifactUploads.$inferSelect; +export type NewArtifactUploadRow = typeof artifactUploads.$inferInsert; export type LocalAgentSessionRow = typeof localAgentSessions.$inferSelect; export type NewLocalAgentSessionRow = typeof localAgentSessions.$inferInsert; diff --git a/src/incoming-artifacts.test.ts b/src/incoming-artifacts.test.ts new file mode 100644 index 00000000..862d4e2c --- /dev/null +++ b/src/incoming-artifacts.test.ts @@ -0,0 +1,387 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import * as z from "zod/v4"; +import { + artifactToolLogFields, + registerArtifactTools, + stageIncomingArtifact, +} from "./artifact-tools.js"; +import { ArtifactError, ArtifactStore } from "./artifacts.js"; +import { + IncomingArtifactAdapterRegistry, + createOpenAIIncomingArtifactAdapter, + type IncomingArtifactAdapter, +} from "./incoming-artifacts.js"; + +const root = await mkdtemp(join(tmpdir(), "devspace-incoming-artifacts-test-")); + +try { + await testRegistryFailsClosed(); + testChatGPTFileDescriptor(); + await testOpenAIFileAdapter(join(root, "openai")); + await testStageArtifact(join(root, "stage")); + await testFailedStageCleanup(join(root, "failed-stage")); + testStageLogRedaction(); +} finally { + await rm(root, { recursive: true, force: true }); +} + +function testChatGPTFileDescriptor(): void { + const registered = new Map>(); + const server = { + registerTool( + name: string, + descriptor: Record, + _callback: unknown, + ) { + registered.set(name, descriptor); + return {}; + }, + }; + + registerArtifactTools(server as never, { + config: {} as never, + store: {} as never, + clientId: "client-a", + }); + + assert.deepEqual([...registered.keys()], [ + "stage_artifact", + "artifact_stat", + "artifact_delete", + ]); + const descriptor = registered.get("stage_artifact"); + assert.ok(descriptor); + assert.deepEqual(descriptor._meta, { "openai/fileParams": ["file"] }); + + const inputSchema = descriptor.inputSchema as z.ZodRawShape; + const fileSchema = inputSchema.file as z.ZodType; + const valid = { + download_url: "https://files.oaiusercontent.com/file_123/download?sig=secret", + file_id: "file_123", + mime_type: "image/png", + file_name: "generated.png", + }; + assert.deepEqual(fileSchema.parse(valid), valid); + const generated = { + ...valid, + mime_type: null, + file_name: null, + name: "/mnt/data/generated.png", + size: 123, + }; + assert.deepEqual(fileSchema.parse(generated), generated); + assert.throws(() => fileSchema.parse({ file_id: "file_123" })); + assert.throws(() => fileSchema.parse({ ...valid, mime_type: 123 })); + + const jsonSchema = z.toJSONSchema(fileSchema) as { + additionalProperties?: boolean; + properties?: Record; + required?: string[]; + }; + assert.equal(jsonSchema.additionalProperties, false); + assert.deepEqual(Object.keys(jsonSchema.properties ?? {}).sort(), [ + "download_url", + "file_id", + "file_name", + "mime_type", + "name", + "size", + ]); + assert.deepEqual([...(jsonSchema.required ?? [])].sort(), ["download_url", "file_id"]); +} + +async function testOpenAIFileAdapter(testRoot: string): Promise { + const bytes = Buffer.from("chatgpt generated image bytes"); + const requested: string[] = []; + const fetchImpl: typeof fetch = async (input, init) => { + requested.push(String(input)); + assert.equal(init?.redirect, "manual"); + return new Response(bytes, { + status: 200, + headers: { + "content-length": String(bytes.length), + "content-type": "image/png", + }, + }); + }; + const registry = new IncomingArtifactAdapterRegistry([ + createOpenAIIncomingArtifactAdapter({ fetch: fetchImpl }), + ]); + const reference = { + download_url: "https://files.oaiusercontent.com/file_123/download?sig=secret", + file_id: "file_123", + mime_type: "image/png", + file_name: "generated.png", + }; + const generatedReference = { + download_url: "https://oaisdmntprcentralus.blob.core.windows.net/chatgpt-file/generated-image.png?sig=secret", + file_id: "file-service://generated+opaque/abc123", + mime_type: null, + file_name: null, + name: "/mnt/data/generated-image.png", + size: bytes.length, + }; + + const opened = await registry.open(reference); + assert.equal(opened.adapterId, "openai-file"); + assert.equal(opened.name, "generated.png"); + assert.equal(opened.mimeType, "image/png"); + assert.equal(opened.size, bytes.length); + assert.deepEqual(await collect(opened.stream), bytes); + + const generatedOpened = await registry.open(generatedReference); + assert.equal(generatedOpened.name, "generated-image.png"); + assert.equal(generatedOpened.mimeType, "image/png"); + assert.deepEqual(await collect(generatedOpened.stream), bytes); + + const store = createStore(testRoot); + try { + const staged = await stageIncomingArtifact({ + store, + clientId: "client-a", + registry, + input: { file: generatedReference }, + }); + assert.equal(staged.name, "generated-image.png"); + assert.equal(staged.mimeType, "image/png"); + assert.deepEqual(await readFile(staged.hostPath), bytes); + assert.equal( + store.statArtifact("client-a", staged.artifactId).source, + "incoming:openai-file", + ); + } finally { + store.close(); + } + + const fallbackOpened = await registry.open({ + ...generatedReference, + file_name: null, + name: null, + }); + assert.equal(fallbackOpened.name, "chatgpt-file.png"); + fallbackOpened.stream.destroy(); + + await expectArtifactError( + registry.open({ + ...generatedReference, + file_name: "first.png", + name: "second.png", + }), + "ambiguous_openai_file_name", + ); + await expectArtifactError( + registry.open({ ...generatedReference, size: bytes.length + 1 }), + "openai_file_size_mismatch", + ); + await expectArtifactError( + registry.open({ ...reference, download_url: "https://example.test/private.png" }), + "unsafe_openai_file_reference", + ); + await expectArtifactError( + registry.open({ + ...reference, + download_url: "https://attacker.blob.core.windows.net/private.png", + }), + "unsafe_openai_file_reference", + ); + await expectArtifactError( + registry.open({ ...reference, download_url: "http://files.oaiusercontent.com/file_123" }), + "unsafe_openai_file_reference", + ); + await expectArtifactError( + registry.open({ ...reference, file_id: "file_\u0000secret" }), + "invalid_openai_file_reference", + ); + await expectArtifactError( + registry.open({ ...reference, file_id: "x".repeat(513) }), + "invalid_openai_file_reference", + ); + await expectArtifactError( + registry.open({ ...reference, bearer: "secret" }), + "unsupported_incoming_artifact", + ); + + let redirectRequests = 0; + const redirectingRegistry = new IncomingArtifactAdapterRegistry([ + createOpenAIIncomingArtifactAdapter({ + fetch: async () => { + redirectRequests += 1; + return new Response(null, { + status: 302, + headers: { location: "http://127.0.0.1/private" }, + }); + }, + }), + ]); + await expectArtifactError( + redirectingRegistry.open(reference), + "unsafe_openai_file_reference", + ); + assert.equal(redirectRequests, 1); + assert.deepEqual(requested, [ + reference.download_url, + generatedReference.download_url, + generatedReference.download_url, + generatedReference.download_url, + generatedReference.download_url, + ]); +} + +async function testRegistryFailsClosed(): Promise { + const empty = new IncomingArtifactAdapterRegistry(); + await expectArtifactError( + empty.open("https://example.test/file.txt?token=secret"), + "unsupported_incoming_artifact", + ); + await expectArtifactError(empty.open("/tmp/arbitrary.txt"), "unsupported_incoming_artifact"); + await expectArtifactError( + empty.open({ url: "https://example.test/file.txt" }), + "unsupported_incoming_artifact", + ); + + const first = memoryAdapter("first", Buffer.from("first")); + const second = memoryAdapter("second", Buffer.from("second")); + await expectArtifactError( + new IncomingArtifactAdapterRegistry([first, second]).open({ kind: "memory" }), + "ambiguous_incoming_artifact", + ); + assert.throws( + () => new IncomingArtifactAdapterRegistry([{ ...first, id: "Bad Adapter" }]), + (error: unknown) => error instanceof ArtifactError && error.code === "invalid_incoming_adapter", + ); +} + +async function testStageArtifact(testRoot: string): Promise { + const bytes = Buffer.alloc(80 * 1024 + 17); + for (let index = 0; index < bytes.length; index += 1) bytes[index] = (index * 17) % 256; + const store = createStore(testRoot); + try { + const digest = createHash("sha256").update(bytes).digest("hex"); + const registry = new IncomingArtifactAdapterRegistry([ + memoryAdapter("memory-test", bytes), + ]); + const result = await stageIncomingArtifact({ + store, + clientId: "client-a", + registry, + input: { + file: { kind: "memory" }, + workspaceId: "ws_association_only", + expectedSha256: `sha256:${digest}`, + ttlHours: 12, + pin: true, + }, + }); + + assert.equal(result.name, "payload.bin"); + assert.equal(result.mimeType, "application/octet-stream"); + assert.equal(result.size, bytes.length); + assert.equal(result.sha256, `sha256:${digest}`); + assert.match(result.instruction, /never writes into a workspace or repository/); + assert.deepEqual(await readFile(result.hostPath), bytes); + + const record = store.statArtifact("client-a", result.artifactId); + assert.equal(record.source, "incoming:memory-test"); + assert.equal(record.workspaceId, "ws_association_only"); + assert.equal(record.pinned, true); + assert.equal(store.health().pendingUploads, 0); + } finally { + store.close(); + } +} + +async function testFailedStageCleanup(testRoot: string): Promise { + const store = createStore(testRoot); + try { + const registry = new IncomingArtifactAdapterRegistry([ + memoryAdapter("memory-test", Buffer.from("digest mismatch")), + ]); + await expectArtifactError( + stageIncomingArtifact({ + store, + clientId: "client-a", + registry, + input: { + file: { kind: "memory" }, + expectedSha256: "0".repeat(64), + }, + }), + "sha256_mismatch", + ); + assert.equal(store.health().pendingUploads, 0); + assert.equal(store.health().storedBytes, 0); + } finally { + store.close(); + } +} + +function testStageLogRedaction(): void { + const secret = "https://files.example.test/download?token=super-secret"; + const fields = artifactToolLogFields("stage_artifact", { + file: { download_url: secret, href: secret, bearer: "Bearer secret" }, + workspaceId: "ws_123", + expectedSha256: "f".repeat(64), + ttlHours: 24, + pin: true, + }); + const serialized = JSON.stringify(fields); + assert.equal("file" in fields, false); + assert.equal(serialized.includes(secret), false); + assert.equal(serialized.includes("Bearer secret"), false); + assert.equal(fields.fileProvided, true); + assert.equal(fields.downloadUrlHostname, "files.example.test"); + assert.equal(fields.expectedSha256Present, true); +} + +function memoryAdapter(id: string, bytes: Buffer): IncomingArtifactAdapter { + return { + id, + canHandle: (value) => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && (value as { kind?: unknown }).kind === "memory" + ), + async open() { + return { + name: "payload.bin", + mimeType: "application/octet-stream", + size: bytes.length, + stream: Readable.from(bytes), + }; + }, + }; +} + +function createStore(testRoot: string): ArtifactStore { + return new ArtifactStore({ + stateDir: join(testRoot, "state"), + artifactRoot: join(testRoot, "artifacts"), + artifactMaxFileBytes: 100 * 1024 * 1024, + artifactMaxTotalBytes: 1024 * 1024 * 1024, + artifactDefaultTtlHours: 24, + }); +} + +async function collect(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +async function expectArtifactError( + promise: Promise, + code: string, +): Promise { + await assert.rejects( + promise, + (error: unknown) => error instanceof ArtifactError && error.code === code, + ); +} diff --git a/src/incoming-artifacts.ts b/src/incoming-artifacts.ts new file mode 100644 index 00000000..4502cfa2 --- /dev/null +++ b/src/incoming-artifacts.ts @@ -0,0 +1,523 @@ +import { basename, isAbsolute } from "node:path"; +import { Readable } from "node:stream"; +import type { ReadableStream as NodeReadableStream } from "node:stream/web"; +import { ArtifactError } from "./artifacts.js"; + +const ADAPTER_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,63}$/u; +const OPENAI_FILE_HOSTS = new Set([ + "files.oaiusercontent.com", + "oaisdmntprcentralus.blob.core.windows.net", +]); +const OPENAI_FILENAME_SAFE_FILE_ID_PATTERN = /^file[-_][A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u; +const OPENAI_FILE_ID_MAX_LENGTH = 512; +const OPENAI_FILE_ID_CONTROL_PATTERN = /[\u0000-\u001F\u007F]/u; +const OPENAI_FILE_KEYS = new Set([ + "download_url", + "file_id", + "mime_type", + "file_name", + "name", + "size", +]); +const OPENAI_FILE_REDIRECT_LIMIT = 3; +const OPENAI_FILE_DOWNLOAD_TIMEOUT_MS = 30_000; + +export interface IncomingArtifactSource { + name: string; + mimeType?: string; + size?: number; + stream: Readable; +} + +export interface IncomingArtifactAdapter { + readonly id: string; + canHandle(value: unknown): boolean; + open(value: unknown): Promise; +} + +export interface OpenedIncomingArtifact extends IncomingArtifactSource { + adapterId: string; +} + +export class IncomingArtifactAdapterRegistry { + private readonly adapters: readonly IncomingArtifactAdapter[]; + + constructor(adapters: readonly IncomingArtifactAdapter[] = []) { + const ids = new Set(); + for (const adapter of adapters) { + if (!ADAPTER_ID_PATTERN.test(adapter.id)) { + throw new ArtifactError( + "invalid_incoming_adapter", + "Incoming artifact adapter IDs must be short lowercase identifiers.", + ); + } + if (ids.has(adapter.id)) { + throw new ArtifactError( + "duplicate_incoming_adapter", + `Incoming artifact adapter '${adapter.id}' is registered more than once.`, + ); + } + ids.add(adapter.id); + } + this.adapters = [...adapters]; + } + + async open(value: unknown): Promise { + const matching: IncomingArtifactAdapter[] = []; + for (const adapter of this.adapters) { + let handles = false; + try { + handles = adapter.canHandle(value); + } catch { + throw new ArtifactError( + "incoming_artifact_adapter_failed", + `Incoming artifact adapter '${adapter.id}' failed during recognition.`, + ); + } + if (handles) matching.push(adapter); + } + + if (matching.length === 0) { + throw new ArtifactError( + "unsupported_incoming_artifact", + "No trusted incoming artifact adapter recognized this file reference.", + ); + } + if (matching.length > 1) { + throw new ArtifactError( + "ambiguous_incoming_artifact", + "More than one trusted incoming artifact adapter recognized this file reference.", + ); + } + + const adapter = matching[0]; + let source: IncomingArtifactSource; + try { + source = await adapter.open(value); + } catch (error) { + if (error instanceof ArtifactError) throw error; + throw new ArtifactError( + "incoming_artifact_open_failed", + `Incoming artifact adapter '${adapter.id}' could not open the file reference.`, + ); + } + try { + validateIncomingArtifactSource(source); + } catch (error) { + source?.stream?.destroy?.(); + throw error; + } + return { ...source, adapterId: adapter.id }; + } +} + +export interface OpenAIFileReference { + download_url: string; + file_id: string; + mime_type?: string; + file_name?: string; + size?: number; +} + +export interface OpenAIIncomingArtifactAdapterOptions { + fetch?: typeof fetch; + timeoutMs?: number; +} + +export function createOpenAIIncomingArtifactAdapter( + options: OpenAIIncomingArtifactAdapterOptions = {}, +): IncomingArtifactAdapter { + const fetchFile = options.fetch ?? globalThis.fetch; + const timeoutMs = options.timeoutMs ?? OPENAI_FILE_DOWNLOAD_TIMEOUT_MS; + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new ArtifactError( + "invalid_openai_file_adapter", + "OpenAI file download timeout must be a positive integer.", + ); + } + + return { + id: "openai-file", + canHandle: isOpenAIFileReferenceCandidate, + async open(value: unknown): Promise { + const reference = normalizeOpenAIFileReference(value); + + let downloadUrl = validateOpenAIFileUrl(reference.download_url); + let response: Response | undefined; + for (let redirect = 0; redirect <= OPENAI_FILE_REDIRECT_LIMIT; redirect += 1) { + try { + response = await fetchFile(downloadUrl, { + redirect: "manual", + signal: AbortSignal.timeout(timeoutMs), + }); + } catch { + throw new ArtifactError( + "openai_file_download_failed", + "ChatGPT file could not be downloaded.", + ); + } + + if (!isRedirectStatus(response.status)) break; + const location = response.headers.get("location"); + await response.body?.cancel().catch(() => undefined); + if (!location || redirect === OPENAI_FILE_REDIRECT_LIMIT) { + throw new ArtifactError( + "openai_file_download_failed", + "ChatGPT file download returned an invalid redirect.", + ); + } + downloadUrl = validateOpenAIFileUrl(new URL(location, downloadUrl).toString()); + } + + if (!response?.ok || !response.body) { + await response?.body?.cancel().catch(() => undefined); + throw new ArtifactError( + "openai_file_download_failed", + "ChatGPT file download did not return file content.", + ); + } + + const responseSize = responseContentLength(response); + if ( + reference.size !== undefined + && responseSize !== undefined + && reference.size !== responseSize + ) { + await response.body.cancel().catch(() => undefined); + throw new ArtifactError( + "openai_file_size_mismatch", + "ChatGPT file metadata did not match the downloaded content.", + ); + } + const mimeType = reference.mime_type ?? responseMimeType(response); + + return { + name: normalizeOpenAIFileName(reference.file_name, reference.file_id, mimeType), + mimeType, + size: responseSize ?? reference.size, + stream: Readable.fromWeb(response.body as unknown as NodeReadableStream), + }; + }, + }; +} + +export type IncomingArtifactValueShape = + | { type: "null" } + | { type: "undefined" } + | { type: "boolean" } + | { type: "number"; finite: boolean } + | { type: "bigint" } + | { type: "string"; kind: "absolute-path" | "url" | "data-url" | "text"; length: number } + | { type: "array"; length: number; items: IncomingArtifactValueShape[]; truncated: boolean } + | { + type: "object"; + constructor?: string; + entries: Record; + truncated: boolean; + } + | { type: "function" | "symbol" } + | { type: "cycle" }; + +export function describeIncomingArtifactValue( + value: unknown, + maxDepth = 4, + maxEntries = 20, +): IncomingArtifactValueShape { + const seen = new WeakSet(); + + const describe = (current: unknown, depth: number): IncomingArtifactValueShape => { + if (current === null) return { type: "null" }; + if (current === undefined) return { type: "undefined" }; + if (typeof current === "boolean") return { type: "boolean" }; + if (typeof current === "number") return { type: "number", finite: Number.isFinite(current) }; + if (typeof current === "bigint") return { type: "bigint" }; + if (typeof current === "function") return { type: "function" }; + if (typeof current === "symbol") return { type: "symbol" }; + if (typeof current === "string") { + return { + type: "string", + kind: classifyValueString(current), + length: current.length, + }; + } + if (seen.has(current)) return { type: "cycle" }; + seen.add(current); + + if (Array.isArray(current)) { + if (depth >= maxDepth) { + return { type: "array", length: current.length, items: [], truncated: current.length > 0 }; + } + const items = current.slice(0, maxEntries).map((item) => describe(item, depth + 1)); + return { + type: "array", + length: current.length, + items, + truncated: current.length > items.length, + }; + } + + const keys = Object.keys(current).sort(); + if (depth >= maxDepth) { + return { + type: "object", + constructor: safeConstructorName(current), + entries: {}, + truncated: keys.length > 0, + }; + } + const entries: Record = {}; + for (const [index, key] of keys.slice(0, maxEntries).entries()) { + let entryValue: unknown; + try { + entryValue = (current as Record)[key]; + } catch { + entryValue = undefined; + } + entries[safeValueEntryKey(key, index)] = describe(entryValue, depth + 1); + } + return { + type: "object", + constructor: safeConstructorName(current), + entries, + truncated: keys.length > Object.keys(entries).length, + }; + }; + + return describe(value, 0); +} + +function validateIncomingArtifactSource(source: IncomingArtifactSource): void { + if (!source || typeof source !== "object") { + throw new ArtifactError( + "invalid_incoming_artifact_source", + "Incoming artifact adapter returned an invalid source.", + ); + } + if (typeof source.name !== "string" || source.name.length === 0) { + throw new ArtifactError( + "invalid_incoming_artifact_source", + "Incoming artifact adapter must provide a filename.", + ); + } + if (source.mimeType !== undefined && typeof source.mimeType !== "string") { + throw new ArtifactError( + "invalid_incoming_artifact_source", + "Incoming artifact adapter returned an invalid MIME hint.", + ); + } + if ( + source.size !== undefined + && (!Number.isSafeInteger(source.size) || source.size < 0) + ) { + throw new ArtifactError( + "invalid_incoming_artifact_source", + "Incoming artifact adapter returned an invalid byte size.", + ); + } + const stream = source.stream as Partial | undefined; + if (!stream || typeof stream[Symbol.asyncIterator] !== "function") { + throw new ArtifactError( + "invalid_incoming_artifact_source", + "Incoming artifact adapter must provide an async-readable stream.", + ); + } +} + +function isOpenAIFileReferenceCandidate(value: unknown): value is Record { + if (!isRecord(value)) return false; + const keys = Object.keys(value); + return keys.length >= 2 + && keys.every((key) => OPENAI_FILE_KEYS.has(key)) + && Object.hasOwn(value, "download_url") + && Object.hasOwn(value, "file_id"); +} + +function normalizeOpenAIFileReference(value: unknown): OpenAIFileReference { + if (!isOpenAIFileReferenceCandidate(value)) { + throw new ArtifactError( + "invalid_openai_file_reference", + "ChatGPT file reference is malformed.", + ); + } + + const downloadUrl = value.download_url; + const fileId = value.file_id; + if ( + typeof downloadUrl !== "string" + || typeof fileId !== "string" + || !isValidOpenAIFileId(fileId) + ) { + throw new ArtifactError( + "invalid_openai_file_reference", + "ChatGPT file reference is malformed.", + ); + } + + const mimeType = nullableString(value.mime_type); + const fileName = nullableString(value.file_name); + const nameAlias = nullableString(value.name); + if (mimeType === null || fileName === null || nameAlias === null) { + throw new ArtifactError( + "invalid_openai_file_reference", + "ChatGPT file reference is malformed.", + ); + } + const normalizedFileName = normalizeSuppliedOpenAIFileName(fileName); + const normalizedNameAlias = normalizeSuppliedOpenAIFileName(nameAlias); + if ( + normalizedFileName + && normalizedNameAlias + && normalizedFileName !== normalizedNameAlias + ) { + throw new ArtifactError( + "ambiguous_openai_file_name", + "ChatGPT file reference contained conflicting filenames.", + ); + } + + let size: number | undefined; + const rawSize = value.size; + if (rawSize !== undefined && rawSize !== null) { + if (typeof rawSize !== "number" || !Number.isSafeInteger(rawSize) || rawSize < 0) { + throw new ArtifactError( + "invalid_openai_file_reference", + "ChatGPT file reference is malformed.", + ); + } + size = rawSize; + } + + return { + download_url: downloadUrl, + file_id: fileId, + mime_type: mimeType, + file_name: normalizedFileName ?? normalizedNameAlias, + size, + }; +} + +function nullableString(value: unknown): string | undefined | null { + if (value === undefined || value === null) return undefined; + return typeof value === "string" ? value : null; +} + +function normalizeOpenAIFileName( + suppliedName: string | undefined, + fileId: string, + mimeType: string | undefined, +): string { + if (suppliedName) return suppliedName; + const safeBaseName = OPENAI_FILENAME_SAFE_FILE_ID_PATTERN.test(fileId) + ? fileId + : "chatgpt-file"; + return `${safeBaseName}${extensionForMimeType(mimeType) ?? ".bin"}`; +} + +function isValidOpenAIFileId(value: string): boolean { + return value.length > 0 + && value.length <= OPENAI_FILE_ID_MAX_LENGTH + && !OPENAI_FILE_ID_CONTROL_PATTERN.test(value); +} + +function normalizeSuppliedOpenAIFileName(value: string | undefined): string | undefined { + if (!value) return undefined; + const normalized = value.replaceAll("\\", "/"); + const candidate = basename(normalized).trim(); + if (!candidate || candidate === "." || candidate === ".." || candidate.startsWith(".")) { + return undefined; + } + return candidate; +} + +function extensionForMimeType(mimeType: string | undefined): string | undefined { + switch (mimeType?.toLowerCase()) { + case "image/png": return ".png"; + case "image/jpeg": return ".jpg"; + case "image/webp": return ".webp"; + case "image/gif": return ".gif"; + case "application/pdf": return ".pdf"; + case "text/plain": return ".txt"; + case "application/zip": return ".zip"; + case "application/vnd.openxmlformats-officedocument.wordprocessingml.document": return ".docx"; + default: return undefined; + } +} + +function validateOpenAIFileUrl(value: string): string { + let url: URL; + try { + url = new URL(value); + } catch { + throw new ArtifactError( + "unsafe_openai_file_reference", + "ChatGPT file download URL is invalid.", + ); + } + if ( + url.protocol !== "https:" + || !OPENAI_FILE_HOSTS.has(url.hostname) + || (url.port !== "" && url.port !== "443") + || url.username !== "" + || url.password !== "" + || url.hash !== "" + ) { + throw new ArtifactError( + "unsafe_openai_file_reference", + "ChatGPT file download URL is outside the trusted file host.", + ); + } + return url.toString(); +} + +function isRedirectStatus(status: number): boolean { + return status === 301 + || status === 302 + || status === 303 + || status === 307 + || status === 308; +} + +function responseMimeType(response: Response): string | undefined { + const value = response.headers.get("content-type")?.split(";", 1)[0]?.trim(); + return value || undefined; +} + +function responseContentLength(response: Response): number | undefined { + const value = response.headers.get("content-length"); + if (!value || !/^\d+$/u.test(value)) return undefined; + const size = Number(value); + return Number.isSafeInteger(size) ? size : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function classifyValueString( + value: string, +): "absolute-path" | "url" | "data-url" | "text" { + if (value.startsWith("data:")) return "data-url"; + if (isAbsolute(value)) return "absolute-path"; + try { + const parsed = new URL(value); + if (parsed.protocol === "http:" || parsed.protocol === "https:") return "url"; + } catch { + // Non-URL strings are summarized only by type and length. + } + return "text"; +} + +function safeValueEntryKey(value: string, index: number): string { + return /^[A-Za-z_][A-Za-z0-9_.-]{0,79}$/u.test(value) + ? value + : ``; +} + +function safeConstructorName(value: object): string | undefined { + try { + const name = value.constructor?.name; + return typeof name === "string" && name.length <= 80 ? name : undefined; + } catch { + return undefined; + } +} diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index e1c00338..0b4abb19 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -44,6 +44,7 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 1, name: "workspace-state" }, { version: 2, name: "oauth-state" }, { version: 3, name: "local-agent-sessions" }, + { version: 4, name: "artifact-exchange" }, ]); } finally { database.close(); diff --git a/src/server.ts b/src/server.ts index 7dd9629e..5cdb3a9f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -18,7 +18,16 @@ import express from "express"; import type { Request, Response } from "express"; import * as z from "zod/v4"; import { applyPatch } from "./apply-patch.js"; +import { registerArtifactTools } from "./artifact-tools.js"; +import { + ARTIFACT_CLEANUP_INTERVAL_MS, + ArtifactStore, +} from "./artifacts.js"; import { loadConfig, type ServerConfig, type WidgetMode } from "./config.js"; +import { + createOpenAIIncomingArtifactAdapter, + type IncomingArtifactAdapter, +} from "./incoming-artifacts.js"; import { logEvent, requestIp, @@ -179,13 +188,16 @@ interface ToolLogFields { } function serverInstructions(config: ServerConfig): string { + const artifactInstruction = config.artifactsEnabled + ? " When the user supplies or generates a file that is not present on the DevSpace host, pass its native file value to stage_artifact instead of recreating it through write/edit calls. Paths returned by artifact tools may be passed to local commands. Only use host paths returned by artifact tools; never invent artifact-store paths or place artifact content, signed URLs, or base64 in shell commands or logs. stage_artifact stores privately and never writes into a workspace or repository." + : ""; const showChangesInstruction = config.widgets === "changes" ? " If the turn successfully modifies files by creating, editing, overwriting, deleting, moving, or applying patches, call show_changes exactly once for that workspace after the final related file change and before your final response so the user can inspect the aggregate diff for that turn. Do not call it after every individual file change; do not skip it because individual file-change tools already returned diffs." : ""; if (config.toolMode === "codex") { - return `Use DevSpace as a local coding workspace. Call ${toolNames.openWorkspace} once per project folder or worktree and reuse its workspaceId. Use ${toolNames.read} for direct file reads, apply_patch for all file modifications, exec_command for inspection, tests, builds, and other commands, and write_stdin to poll or interact with running processes. Follow instructions returned by ${toolNames.openWorkspace}; read applicable instruction and skill files before working in their scope.${showChangesInstruction}`; + return `Use DevSpace as a local coding workspace. Call ${toolNames.openWorkspace} once per project folder or worktree and reuse its workspaceId. Use ${toolNames.read} for direct file reads, apply_patch for all file modifications, exec_command for inspection, tests, builds, and other commands, and write_stdin to poll or interact with running processes. Follow instructions returned by ${toolNames.openWorkspace}; read applicable instruction and skill files before working in their scope.${artifactInstruction}${showChangesInstruction}`; } const inspection = config.toolMode !== "full" @@ -198,7 +210,7 @@ function serverInstructions(config: ServerConfig): string { const agentsMd = `Follow instructions returned by ${toolNames.openWorkspace}. Before working under a path listed in availableAgentsFiles, use ${toolNames.read} to inspect that instruction file and follow it. `; - return `Use DevSpace as a local coding workspace. Call ${toolNames.openWorkspace} once per project folder or worktree to obtain a workspaceId. Reuse that same workspaceId for all later file, search, edit, write, show-changes, and shell tools in that folder; do not call ${toolNames.openWorkspace} again unless switching folders/worktrees, changing checkout/worktree mode, the workspaceId is rejected as unknown, or the user explicitly asks to reopen. ${agentsMd}${skills}${inspection}Prefer ${toolNames.edit} for targeted modifications, ${toolNames.write} only for new files or complete rewrites, and ${toolNames.shell} for tests, builds, git inspection, package scripts, and commands that are better executed by the shell. Do not create or modify files with ${toolNames.shell}; avoid shell redirection, heredocs, tee, sed -i, perl -i, node/python/ruby scripts, or any command whose purpose is to write project files.${showChangesInstruction}`; + return `Use DevSpace as a local coding workspace. Call ${toolNames.openWorkspace} once per project folder or worktree to obtain a workspaceId. Reuse that same workspaceId for all later file, search, edit, write, show-changes, and shell tools in that folder; do not call ${toolNames.openWorkspace} again unless switching folders/worktrees, changing checkout/worktree mode, the workspaceId is rejected as unknown, or the user explicitly asks to reopen. ${agentsMd}${skills}${inspection}Prefer ${toolNames.edit} for targeted modifications, ${toolNames.write} only for new files or complete rewrites, and ${toolNames.shell} for tests, builds, git inspection, package scripts, and commands that are better executed by the shell. Do not create or modify files with ${toolNames.shell}; avoid shell redirection, heredocs, tee, sed -i, perl -i, node/python/ruby scripts, or any command whose purpose is to write project files.${artifactInstruction}${showChangesInstruction}`; } function formatVisibleAgent(agent: { @@ -688,6 +700,9 @@ function createMcpServer( reviewCheckpoints: ReturnType, processSessions: ProcessSessionManager, localAgentProviders: LocalAgentProviderAvailability[], + artifactStore: ArtifactStore | undefined, + clientId: string, + incomingArtifactAdapters: readonly IncomingArtifactAdapter[], ): McpServer { const server = new McpServer( { @@ -1594,10 +1609,28 @@ function createMcpServer( registerCodexProcessTools(server, config, workspaces, processSessions); } + if (artifactStore) { + registerArtifactTools(server, { + config, + store: artifactStore, + clientId, + incomingArtifactAdapters, + }); + } + return server; } -export function createServer(config = loadConfig()): RunningServer { +export interface CreateServerOptions { + incomingArtifactAdapters?: readonly IncomingArtifactAdapter[]; +} + +export function createServer( + config = loadConfig(), + options: CreateServerOptions = {}, +): RunningServer { + const incomingArtifactAdapters = options.incomingArtifactAdapters + ?? [createOpenAIIncomingArtifactAdapter()]; const allowedHosts = config.allowedHosts.includes("*") ? undefined : Array.from(new Set([config.host, ...config.allowedHosts])); @@ -1615,6 +1648,7 @@ export function createServer(config = loadConfig()): RunningServer { resourceMetadataUrl: getOAuthProtectedResourceMetadataUrl(resourceServerUrl), }); const workspaceStore = createWorkspaceStore(config.stateDir); + const artifactStore = config.artifactsEnabled ? new ArtifactStore(config) : undefined; const workspaces = new WorkspaceRegistry(config, workspaceStore); const reviewCheckpoints = createReviewCheckpointManager(); const processSessions = new ProcessSessionManager(); @@ -1653,6 +1687,24 @@ export function createServer(config = loadConfig()): RunningServer { }, MCP_SESSION_CLEANUP_INTERVAL_MS); sessionCleanupTimer.unref(); + const runArtifactCleanup = () => { + if (!artifactStore) return; + void artifactStore.cleanupExpired() + .then((result) => { + logEvent(config.logging, "debug", "artifact_cleanup", { ...result }); + }) + .catch((error) => { + logEvent(config.logging, "warn", "artifact_cleanup_failed", { + error: error instanceof Error ? error.message : String(error), + }); + }); + }; + runArtifactCleanup(); + const artifactCleanupTimer = artifactStore + ? setInterval(runArtifactCleanup, ARTIFACT_CLEANUP_INTERVAL_MS) + : undefined; + artifactCleanupTimer?.unref(); + if (config.logging.trustProxy) { app.set("trust proxy", true); } @@ -1781,6 +1833,9 @@ export function createServer(config = loadConfig()): RunningServer { reviewCheckpoints, processSessions, localAgentProviders, + artifactStore, + req.auth.clientId, + incomingArtifactAdapters, ); await server.connect(transport); } else { @@ -1808,10 +1863,12 @@ export function createServer(config = loadConfig()): RunningServer { close: () => { closePromise ??= (async () => { clearInterval(sessionCleanupTimer); + if (artifactCleanupTimer) clearInterval(artifactCleanupTimer); const results = await transports.closeAll(); logSessionCloseResults("server_shutdown", results); processSessions.shutdown(); oauthProvider.close(); + artifactStore?.close(); workspaceStore.close?.(); })(); return closePromise; @@ -1839,6 +1896,10 @@ if (await isMainModule()) { console.log(`request logging: ${config.logging.requests ? "enabled" : "disabled"}`); console.log(`asset logging: ${config.logging.assets ? "enabled" : "disabled"}`); console.log(`trust proxy: ${config.logging.trustProxy ? "enabled" : "disabled"}`); + console.log(`native artifact staging: ${config.artifactsEnabled ? "enabled" : "disabled"}`); + if (config.artifactsEnabled) { + console.log(`artifact root: ${config.artifactRoot}`); + } if (config.subagents) { console.log(`subagent providers: ${formatLocalAgentProviderAvailabilitySummary(localAgentProviders)}`); } diff --git a/src/user-config.ts b/src/user-config.ts index c8da90b5..ce1e3386 100644 --- a/src/user-config.ts +++ b/src/user-config.ts @@ -17,6 +17,11 @@ export interface DevspaceUserConfig { allowedHosts?: string[]; stateDir?: string; worktreeRoot?: string; + artifactsEnabled?: boolean; + artifactRoot?: string; + artifactMaxFileBytes?: number; + artifactMaxTotalBytes?: number; + artifactDefaultTtlHours?: number; agentDir?: string; subagents?: boolean; } From b72df5ebee2ddc490ea292364142156d8d105fad Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:34:31 +0000 Subject: [PATCH 02/16] feat: materialize staged artifacts in workspaces --- docs/artifact-exchange.md | 18 ++- docs/configuration.md | 10 +- docs/security.md | 19 ++- package.json | 2 +- src/artifact-tools.ts | 58 ++++++++- src/artifact-workspace.test.ts | 58 +++++++++ src/artifact-workspace.ts | 231 +++++++++++++++++++++++++++++++++ src/incoming-artifacts.test.ts | 4 +- src/incoming-artifacts.ts | 1 + src/server.ts | 3 +- 10 files changed, 384 insertions(+), 20 deletions(-) create mode 100644 src/artifact-workspace.test.ts create mode 100644 src/artifact-workspace.ts diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 039f70e2..3ce7541d 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -5,9 +5,9 @@ file does not yet exist on the DevSpace host. The opt-in `stage_artifact` MCP tool streams the connector-provided file into private local storage and returns a host path that local commands can consume. -This is deliberately a narrow handoff seam. It does not copy files into -workspaces, write repositories, accept generic uploads, or expose a public -download route. +This is deliberately a narrow handoff seam. It supports explicit copy of a +staged artifact into an already-open workspace, but does not automatically write +repositories, accept generic uploads, or expose a public download route. ## Enable @@ -58,8 +58,12 @@ A successful result contains metadata, not file content: ``` Use `artifact_stat` to re-read owner-scoped metadata and `artifact_delete` to -remove the record. Local commands may consume `hostPath`; agents must never -invent paths under the artifact root. +remove the record. To materialize an image or document, use +`artifact_copy_to_workspace` with its `artifactId`, an existing `workspaceId`, a +relative `destination`, and explicit `onConflict` mode (`error`, `rename`, or +`replace`). The copy verifies bytes and SHA-256, rejects workspace escapes and +symlink paths, and returns the final workspace path. Agents must never invent +paths under the artifact root. ## Supported connector shape @@ -84,6 +88,7 @@ Download URLs must use HTTPS on one of the exact reviewed hosts: - `files.oaiusercontent.com` - `oaisdmntprcentralus.blob.core.windows.net` +- `oaisdmntprwestcentralus.blob.core.windows.net` Userinfo, fragments, non-HTTPS ports, and redirects outside those hosts are rejected. Redirects are followed manually and revalidated at each hop. Signed @@ -122,4 +127,5 @@ status. It never logs raw file objects, signed URLs, file IDs, credentials, content, or base64 data. MIME types and filenames are hints only. DevSpace does not execute, unpack, -publish, or automatically copy staged content into a workspace. +publish, or automatically copy staged content into a workspace; materialization +requires the explicit workspace-copy tool. diff --git a/docs/configuration.md b/docs/configuration.md index 525196ca..e45fd1a6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,7 +50,7 @@ DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve | Variable | Default | Purpose | | --- | --- | --- | -| `DEVSPACE_ARTIFACTS` | `0` | Expose `stage_artifact`, `artifact_stat`, and `artifact_delete`. | +| `DEVSPACE_ARTIFACTS` | `0` | Expose `stage_artifact`, `artifact_stat`, `artifact_copy_to_workspace`, and `artifact_delete`. | | `DEVSPACE_ARTIFACT_ROOT` | `~/.local/share/devspace/artifacts` | Private storage root outside repositories and worktrees. | | `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` | `104857600` | Maximum decoded size of one artifact (100 MiB). | | `DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES` | `1073741824` | Maximum combined stored-object and in-progress bytes (1 GiB). | @@ -62,9 +62,11 @@ The same settings may be persisted in `~/.devspace/config.json` as `stage_artifact` accepts only the native file object supplied by the ChatGPT connector. It does not accept arbitrary URL strings, paths, embedded credentials, -or extra object fields. Files are streamed through bounded private storage and -never copied into a workspace or repository. Cleanup runs at startup and -periodically, with a bounded number of records processed per pass. +or extra object fields. Files are streamed through bounded private storage. +`artifact_copy_to_workspace` can then materialize one owner-scoped artifact into +an already-open workspace at an explicit relative destination and conflict mode. +Cleanup runs at startup and periodically, with a bounded number of records +processed per pass. See [Native Artifact Staging](artifact-exchange.md) for the supported connector shape and security boundaries. diff --git a/docs/security.md b/docs/security.md index e022d58a..eb2fd818 100644 --- a/docs/security.md +++ b/docs/security.md @@ -114,24 +114,31 @@ an unauthenticated download route or expose the artifact root with Native staging is adapter-gated. The production server declares ChatGPT's top-level `openai/fileParams` contract and accepts only the documented `download_url`, `file_id`, optional MIME/filename aliases, and optional size. -Downloads use HTTPS on two exact reviewed hosts: +Downloads use HTTPS on exact reviewed hosts: `files.oaiusercontent.com` and -`oaisdmntprcentralus.blob.core.windows.net`. Credentials, fragments, alternate -ports, arbitrary sibling hosts, malformed IDs, extra fields, and redirects -outside that boundary fail closed. Opaque IDs are bounded metadata and are never -used as filenames or path components. +`oaisdmntprcentralus.blob.core.windows.net` and +`oaisdmntprwestcentralus.blob.core.windows.net`. Credentials, fragments, +alternate ports, arbitrary sibling hosts, malformed IDs, extra fields, and +redirects outside that boundary fail closed. Opaque IDs are bounded metadata and +are never used as filenames or path components. The staging seam deliberately does not: - fetch arbitrary URLs - expose a generic upload API -- copy content into a workspace or repository +- automatically copy content into a workspace or repository - extract archives - execute transferred content - expand workspace allowlists - preserve executable permissions - publish or permanently retain content +`artifact_copy_to_workspace` is the explicit exception: it reads only an +owner-scoped staged artifact and writes only within an already-open allowed +workspace. It requires a relative destination and an explicit conflict mode, +creates only real contained parent directories, rejects symlink/non-regular-file +paths, and verifies the final size and SHA-256. + MIME types are hints only. SHA-256 and byte counts are computed and enforced by the server. Pinned records require explicit deletion; unpinned records and failed in-progress transfers remain subject to cleanup. diff --git a/package.json b/package.json index 57029fb7..ecf3a529 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dev": "node scripts/dev-server.mjs", "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", - "test": "tsx src/config.test.ts && tsx src/artifacts.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", + "test": "tsx src/config.test.ts && tsx src/artifacts.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/artifact-workspace.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index eff1e307..4fe06873 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -2,6 +2,10 @@ import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import * as z from "zod/v4"; import type { ServerConfig } from "./config.js"; +import { + copyArtifactToWorkspace, + type ArtifactCopyConflictMode, +} from "./artifact-workspace.js"; import { ARTIFACT_CHUNK_BYTES, ArtifactError, @@ -14,6 +18,7 @@ import { type IncomingArtifactAdapter, } from "./incoming-artifacts.js"; import { logEvent } from "./logger.js"; +import type { WorkspaceRegistry } from "./workspaces.js"; const ARTIFACT_WRITE_ANNOTATIONS = { readOnlyHint: false, @@ -45,11 +50,16 @@ const artifactRecordOutputSchema = { pinned: z.boolean(), }; -type ArtifactToolName = "stage_artifact" | "artifact_stat" | "artifact_delete"; +type ArtifactToolName = + | "stage_artifact" + | "artifact_stat" + | "artifact_copy_to_workspace" + | "artifact_delete"; export interface ArtifactToolRegistrationOptions { config: ServerConfig; store: ArtifactStore; + workspaces: WorkspaceRegistry; clientId: string; incomingArtifactAdapters?: readonly IncomingArtifactAdapter[]; } @@ -78,6 +88,7 @@ export function registerArtifactTools( { config, store, + workspaces, clientId, incomingArtifactAdapters = [], }: ArtifactToolRegistrationOptions, @@ -142,6 +153,51 @@ export function registerArtifactTools( ), ); + registerAppTool( + server, + "artifact_copy_to_workspace", + { + title: "Copy artifact into workspace", + description: + "Copy one owner-scoped private artifact into a path inside an already-open workspace. The destination must stay within that workspace; choose rename or replace explicitly when a file exists.", + inputSchema: { + artifactId: z.string(), + workspaceId: z.string().min(1), + destination: z.string().min(1).describe("Relative path inside the selected workspace."), + onConflict: z.enum(["error", "rename", "replace"]).default("error"), + }, + outputSchema: { + artifactId: z.string(), + workspaceId: z.string(), + path: z.string(), + size: z.number().int().nonnegative(), + sha256: z.string(), + onConflict: z.enum(["error", "rename", "replace"]), + renamed: z.boolean(), + }, + _meta: {}, + annotations: ARTIFACT_WRITE_ANNOTATIONS, + }, + async (input) => executeArtifactTool( + config, + "artifact_copy_to_workspace", + input, + async () => { + const workspace = workspaces.getWorkspace(input.workspaceId); + const destination = workspaces.resolvePath(workspace, input.destination); + return copyArtifactToWorkspace({ + store, + clientId, + workspaceId: workspace.id, + workspaceRoot: workspace.root, + artifactId: input.artifactId, + destination, + onConflict: input.onConflict as ArtifactCopyConflictMode, + }); + }, + ), + ); + registerAppTool( server, "artifact_delete", diff --git a/src/artifact-workspace.test.ts b/src/artifact-workspace.test.ts new file mode 100644 index 00000000..7ed40297 --- /dev/null +++ b/src/artifact-workspace.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { copyArtifactToWorkspace } from "./artifact-workspace.js"; +import { ArtifactStore } from "./artifacts.js"; + +const root = await mkdtemp(join(tmpdir(), "devspace-artifact-workspace-test-")); + +try { + const workspaceRoot = join(root, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + const store = new ArtifactStore({ + stateDir: join(root, "state"), + artifactRoot: join(root, "artifacts"), + artifactMaxFileBytes: 1024 * 1024, + artifactMaxTotalBytes: 4 * 1024 * 1024, + artifactDefaultTtlHours: 24, + }); + try { + const bytes = Buffer.from("native ChatGPT artifact bytes\u0000\xff", "latin1"); + const upload = await store.beginUpload("client-a", { + filename: "generated.png", + mimeType: "image/png", + size: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + await store.uploadChunk("client-a", { + uploadId: upload.uploadId, + offset: 0, + dataBase64: bytes.toString("base64"), + }); + const artifact = await store.commitUpload("client-a", upload.uploadId, { + source: "incoming:openai-file", + }); + + const destination = join(workspaceRoot, "assets", "generated.png"); + const copied = await copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination, + onConflict: "error", + }); + + assert.deepEqual(await readFile(destination), bytes); + assert.equal(copied.path, destination); + assert.equal(copied.sha256, artifact.sha256); + assert.equal(copied.renamed, false); + } finally { + store.close(); + } +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/src/artifact-workspace.ts b/src/artifact-workspace.ts new file mode 100644 index 00000000..f6630fb0 --- /dev/null +++ b/src/artifact-workspace.ts @@ -0,0 +1,231 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants as fsConstants, createReadStream } from "node:fs"; +import { chmod, link, lstat, mkdir, open, realpath, rename, unlink } from "node:fs/promises"; +import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { ArtifactError, type ArtifactStore } from "./artifacts.js"; + +const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; +const MAX_RENAME_ATTEMPTS = 10_000; + +export type ArtifactCopyConflictMode = "error" | "rename" | "replace"; + +export interface ArtifactCopyToWorkspaceInput { + store: ArtifactStore; + clientId: string; + workspaceId: string; + workspaceRoot: string; + artifactId: string; + destination: string; + onConflict: ArtifactCopyConflictMode; +} + +export interface ArtifactCopyToWorkspaceResult { + artifactId: string; + workspaceId: string; + path: string; + size: number; + sha256: string; + onConflict: ArtifactCopyConflictMode; + renamed: boolean; +} + +/** Copy one owner-scoped private artifact into one already-open allowed workspace. */ +export async function copyArtifactToWorkspace( + input: ArtifactCopyToWorkspaceInput, +): Promise { + assertLexicalWorkspaceContainment(input.workspaceRoot, input.destination); + if (resolve(input.destination) === resolve(input.workspaceRoot)) { + throw new ArtifactError("workspace_destination_invalid", "Artifact destination must name a file inside the workspace."); + } + + const artifact = input.store.statArtifact(input.clientId, input.artifactId); + const parent = dirname(input.destination); + await ensureContainedWorkspaceDirectory(input.workspaceRoot, parent); + if (input.onConflict === "error" && await lstatOrUndefined(input.destination)) { + throw new ArtifactError("workspace_destination_exists", "Artifact destination already exists. Select rename or replace explicitly."); + } + + const tempPath = join(parent, `.devspace-artifact-${randomUUID()}.partial`); + let finalPath = input.destination; + try { + const copied = await copyVerifiedArtifactToTemp(artifact.hostPath, tempPath, artifact.size, artifact.sha256); + await assertContainedWorkspaceRegularFile(input.workspaceRoot, tempPath); + + if (input.onConflict === "replace") { + const existing = await lstatOrUndefined(finalPath); + if (existing && (existing.isSymbolicLink() || !existing.isFile())) { + throw new ArtifactError("workspace_destination_unsafe", "Replace is allowed only for an existing regular file."); + } + await ensureContainedWorkspaceDirectory(input.workspaceRoot, parent); + await rename(tempPath, finalPath); + } else { + let linked = false; + const attempts = input.onConflict === "rename" ? MAX_RENAME_ATTEMPTS : 1; + for (let attempt = 0; attempt < attempts; attempt += 1) { + finalPath = attempt === 0 ? input.destination : renamedDestination(input.destination, attempt); + assertLexicalWorkspaceContainment(input.workspaceRoot, finalPath); + try { + await link(tempPath, finalPath); + linked = true; + break; + } catch (error) { + if (isNodeError(error) && error.code === "EEXIST" && input.onConflict === "rename") continue; + if (isNodeError(error) && error.code === "EEXIST") { + throw new ArtifactError("workspace_destination_exists", "Artifact destination already exists. Select rename or replace explicitly."); + } + throw error; + } + } + if (!linked) { + throw new ArtifactError("workspace_rename_exhausted", "Could not find an available destination filename."); + } + await unlink(tempPath); + } + + await chmod(finalPath, 0o644); + const finalEntry = await assertContainedWorkspaceRegularFile(input.workspaceRoot, finalPath); + const digest = await hashFile(finalPath); + if (finalEntry.size !== copied.size || `sha256:${digest}` !== copied.sha256) { + throw new ArtifactError("workspace_copy_integrity_failed", "Workspace copy failed size or SHA-256 verification."); + } + return { + artifactId: artifact.artifactId, + workspaceId: input.workspaceId, + path: finalPath, + size: finalEntry.size, + sha256: `sha256:${digest}`, + onConflict: input.onConflict, + renamed: finalPath !== input.destination, + }; + } catch (error) { + await unlink(tempPath).catch(() => undefined); + throw error; + } +} + +async function assertContainedWorkspaceRegularFile(workspaceRoot: string, path: string) { + assertLexicalWorkspaceContainment(workspaceRoot, path); + const entry = await lstat(path); + if (entry.isSymbolicLink() || !entry.isFile()) { + throw new ArtifactError("workspace_destination_unsafe", "Workspace destination is not a regular non-symlink file."); + } + const rootRealPath = await realpath(workspaceRoot); + assertRealWorkspaceContainment(rootRealPath, await realpath(path)); + return entry; +} + +async function ensureContainedWorkspaceDirectory(workspaceRoot: string, directory: string): Promise { + assertLexicalWorkspaceContainment(workspaceRoot, directory); + const root = resolve(workspaceRoot); + const rootRealPath = await realpath(root); + const relationship = relative(root, resolve(directory)); + const parts = relationship === "" ? [] : relationship.split(sep); + let current = root; + for (const part of parts) { + if (!part || part === "." || part === "..") { + throw new ArtifactError("workspace_path_escape", "Workspace destination escapes the workspace root."); + } + current = join(current, part); + let entry = await lstatOrUndefined(current); + if (!entry) { + try { + await mkdir(current, { mode: 0o755 }); + } catch (error) { + if (!isNodeError(error) || error.code !== "EEXIST") throw error; + } + entry = await lstat(current); + } + if (entry.isSymbolicLink() || !entry.isDirectory()) { + throw new ArtifactError("workspace_parent_unsafe", "Workspace destination parent is not a real directory."); + } + assertRealWorkspaceContainment(rootRealPath, await realpath(current)); + } +} + +async function copyVerifiedArtifactToTemp( + sourcePath: string, + tempPath: string, + expectedSize: number, + expectedSha256: string, +): Promise<{ size: number; sha256: string }> { + const sourceEntry = await lstat(sourcePath); + if (sourceEntry.isSymbolicLink() || !sourceEntry.isFile() || sourceEntry.size !== expectedSize) { + throw new ArtifactError("artifact_integrity_failed", "Artifact source is not a regular file with the recorded size."); + } + const source = await open(sourcePath, fsConstants.O_RDONLY | NO_FOLLOW); + let destination: Awaited> | undefined; + const hash = createHash("sha256"); + let copied = 0; + try { + destination = await open(tempPath, fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | NO_FOLLOW, 0o600); + const buffer = Buffer.allocUnsafe(64 * 1024); + while (true) { + const { bytesRead } = await source.read(buffer, 0, buffer.length, copied); + if (bytesRead === 0) break; + const chunk = buffer.subarray(0, bytesRead); + await writeAll(destination, chunk, copied); + hash.update(chunk); + copied += bytesRead; + } + await destination.sync(); + } finally { + await source.close().catch(() => undefined); + await destination?.close().catch(() => undefined); + } + const sha256 = `sha256:${hash.digest("hex")}`; + if (copied !== expectedSize || sha256 !== expectedSha256) { + await unlink(tempPath).catch(() => undefined); + throw new ArtifactError("artifact_integrity_failed", "Artifact source failed size or SHA-256 verification during copy."); + } + await chmod(tempPath, 0o644); + return { size: copied, sha256 }; +} + +async function writeAll(handle: Awaited>, buffer: Buffer, position: number): Promise { + let offset = 0; + while (offset < buffer.length) { + const { bytesWritten } = await handle.write(buffer, offset, buffer.length - offset, position + offset); + if (bytesWritten <= 0) throw new ArtifactError("short_write", "Artifact workspace copy was not fully written."); + offset += bytesWritten; + } +} + +function renamedDestination(path: string, attempt: number): string { + const extension = extname(path); + const filename = basename(path); + const stem = extension ? filename.slice(0, -extension.length) : filename; + return join(dirname(path), `${stem} (${attempt})${extension}`); +} + +function assertLexicalWorkspaceContainment(workspaceRoot: string, candidate: string): void { + const relationship = relative(resolve(workspaceRoot), resolve(candidate)); + if (relationship.startsWith("..") || isAbsolute(relationship)) { + throw new ArtifactError("workspace_path_escape", "Workspace path escapes the workspace root."); + } +} + +function assertRealWorkspaceContainment(rootRealPath: string, candidateRealPath: string): void { + const relationship = relative(rootRealPath, candidateRealPath); + if (relationship.startsWith("..") || isAbsolute(relationship)) { + throw new ArtifactError("workspace_path_escape", "Workspace path resolves outside the workspace root."); + } +} + +async function hashFile(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); + return hash.digest("hex"); +} + +async function lstatOrUndefined(path: string) { + try { + return await lstat(path); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return undefined; + throw error; + } +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; +} diff --git a/src/incoming-artifacts.test.ts b/src/incoming-artifacts.test.ts index 862d4e2c..25a3dd61 100644 --- a/src/incoming-artifacts.test.ts +++ b/src/incoming-artifacts.test.ts @@ -46,12 +46,14 @@ function testChatGPTFileDescriptor(): void { registerArtifactTools(server as never, { config: {} as never, store: {} as never, + workspaces: {} as never, clientId: "client-a", }); assert.deepEqual([...registered.keys()], [ "stage_artifact", "artifact_stat", + "artifact_copy_to_workspace", "artifact_delete", ]); const descriptor = registered.get("stage_artifact"); @@ -119,7 +121,7 @@ async function testOpenAIFileAdapter(testRoot: string): Promise { file_name: "generated.png", }; const generatedReference = { - download_url: "https://oaisdmntprcentralus.blob.core.windows.net/chatgpt-file/generated-image.png?sig=secret", + download_url: "https://oaisdmntprwestcentralus.blob.core.windows.net/chatgpt-file/generated-image.png?sig=secret", file_id: "file-service://generated+opaque/abc123", mime_type: null, file_name: null, diff --git a/src/incoming-artifacts.ts b/src/incoming-artifacts.ts index 4502cfa2..7b3d0d43 100644 --- a/src/incoming-artifacts.ts +++ b/src/incoming-artifacts.ts @@ -7,6 +7,7 @@ const ADAPTER_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,63}$/u; const OPENAI_FILE_HOSTS = new Set([ "files.oaiusercontent.com", "oaisdmntprcentralus.blob.core.windows.net", + "oaisdmntprwestcentralus.blob.core.windows.net", ]); const OPENAI_FILENAME_SAFE_FILE_ID_PATTERN = /^file[-_][A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u; const OPENAI_FILE_ID_MAX_LENGTH = 512; diff --git a/src/server.ts b/src/server.ts index 5cdb3a9f..cf9868c0 100644 --- a/src/server.ts +++ b/src/server.ts @@ -189,7 +189,7 @@ interface ToolLogFields { function serverInstructions(config: ServerConfig): string { const artifactInstruction = config.artifactsEnabled - ? " When the user supplies or generates a file that is not present on the DevSpace host, pass its native file value to stage_artifact instead of recreating it through write/edit calls. Paths returned by artifact tools may be passed to local commands. Only use host paths returned by artifact tools; never invent artifact-store paths or place artifact content, signed URLs, or base64 in shell commands or logs. stage_artifact stores privately and never writes into a workspace or repository." + ? " When the user supplies or generates a file that is not present on the DevSpace host, pass its native file value to stage_artifact instead of recreating it through write/edit calls. To place a staged artifact in an approved workspace, use artifact_copy_to_workspace with that artifact ID, the existing workspace ID, a relative destination, and an explicit conflict mode. Only use host paths returned by artifact tools; never invent artifact-store paths or place artifact content, signed URLs, or base64 in shell commands or logs. stage_artifact stores privately and never writes into a workspace or repository; artifact_copy_to_workspace is the explicit materialization step and may dirty a repository." : ""; const showChangesInstruction = config.widgets === "changes" @@ -1613,6 +1613,7 @@ function createMcpServer( registerArtifactTools(server, { config, store: artifactStore, + workspaces, clientId, incomingArtifactAdapters, }); From 172b71bb8a6af3a8d92bfa9be0c68a97bc5093e0 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:54:03 +0000 Subject: [PATCH 03/16] fix: accept regional ChatGPT artifact hosts --- docs/artifact-exchange.md | 6 +++--- docs/security.md | 15 ++++++++------- src/incoming-artifacts.test.ts | 2 +- src/incoming-artifacts.ts | 11 ++++++++--- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 3ce7541d..6c8af1a4 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -84,11 +84,11 @@ Required fields are `download_url` and `file_id`. Unknown keys, extra credential fields, plain URL strings, filesystem paths, arrays, and malformed values fail closed. -Download URLs must use HTTPS on one of the exact reviewed hosts: +Download URLs must use HTTPS on one of these reviewed OpenAI delivery origins: - `files.oaiusercontent.com` -- `oaisdmntprcentralus.blob.core.windows.net` -- `oaisdmntprwestcentralus.blob.core.windows.net` +- `oaisdmntpr.blob.core.windows.net`, where `` is lowercase + alphanumeric (for example `centralus`, `westcentralus`, or `centralindia`) Userinfo, fragments, non-HTTPS ports, and redirects outside those hosts are rejected. Redirects are followed manually and revalidated at each hop. Signed diff --git a/docs/security.md b/docs/security.md index eb2fd818..8e8108df 100644 --- a/docs/security.md +++ b/docs/security.md @@ -114,13 +114,14 @@ an unauthenticated download route or expose the artifact root with Native staging is adapter-gated. The production server declares ChatGPT's top-level `openai/fileParams` contract and accepts only the documented `download_url`, `file_id`, optional MIME/filename aliases, and optional size. -Downloads use HTTPS on exact reviewed hosts: -`files.oaiusercontent.com` and -`oaisdmntprcentralus.blob.core.windows.net` and -`oaisdmntprwestcentralus.blob.core.windows.net`. Credentials, fragments, -alternate ports, arbitrary sibling hosts, malformed IDs, extra fields, and -redirects outside that boundary fail closed. Opaque IDs are bounded metadata and -are never used as filenames or path components. +Downloads use HTTPS on `files.oaiusercontent.com` or the constrained regional +OpenAI Azure account family +`oaisdmntpr.blob.core.windows.net`, where `` is lowercase +alphanumeric. This permits observed OpenAI regional accounts such as +`centralus`, `westcentralus`, and `centralindia`, but not arbitrary Azure Blob +accounts. Credentials, fragments, alternate ports, malformed IDs, extra fields, +and redirects outside that boundary fail closed. Opaque IDs are bounded metadata +and are never used as filenames or path components. The staging seam deliberately does not: diff --git a/src/incoming-artifacts.test.ts b/src/incoming-artifacts.test.ts index 25a3dd61..5b90e6c5 100644 --- a/src/incoming-artifacts.test.ts +++ b/src/incoming-artifacts.test.ts @@ -121,7 +121,7 @@ async function testOpenAIFileAdapter(testRoot: string): Promise { file_name: "generated.png", }; const generatedReference = { - download_url: "https://oaisdmntprwestcentralus.blob.core.windows.net/chatgpt-file/generated-image.png?sig=secret", + download_url: "https://oaisdmntprcentralindia.blob.core.windows.net/chatgpt-file/generated-image.png?sig=secret", file_id: "file-service://generated+opaque/abc123", mime_type: null, file_name: null, diff --git a/src/incoming-artifacts.ts b/src/incoming-artifacts.ts index 7b3d0d43..f2497bcd 100644 --- a/src/incoming-artifacts.ts +++ b/src/incoming-artifacts.ts @@ -6,9 +6,10 @@ import { ArtifactError } from "./artifacts.js"; const ADAPTER_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,63}$/u; const OPENAI_FILE_HOSTS = new Set([ "files.oaiusercontent.com", - "oaisdmntprcentralus.blob.core.windows.net", - "oaisdmntprwestcentralus.blob.core.windows.net", ]); +// ChatGPT-generated files are served from regional OpenAI-managed Azure storage +// accounts. Accept that account family only, never arbitrary Azure Blob hosts. +const OPENAI_REGIONAL_BLOB_HOST_PATTERN = /^oaisdmntpr[a-z0-9]+\.blob\.core\.windows\.net$/u; const OPENAI_FILENAME_SAFE_FILE_ID_PATTERN = /^file[-_][A-Za-z0-9][A-Za-z0-9._-]{0,255}$/u; const OPENAI_FILE_ID_MAX_LENGTH = 512; const OPENAI_FILE_ID_CONTROL_PATTERN = /[\u0000-\u001F\u007F]/u; @@ -456,7 +457,7 @@ function validateOpenAIFileUrl(value: string): string { } if ( url.protocol !== "https:" - || !OPENAI_FILE_HOSTS.has(url.hostname) + || !isTrustedOpenAIFileHost(url.hostname) || (url.port !== "" && url.port !== "443") || url.username !== "" || url.password !== "" @@ -470,6 +471,10 @@ function validateOpenAIFileUrl(value: string): string { return url.toString(); } +function isTrustedOpenAIFileHost(hostname: string): boolean { + return OPENAI_FILE_HOSTS.has(hostname) || OPENAI_REGIONAL_BLOB_HOST_PATTERN.test(hostname); +} + function isRedirectStatus(status: number): boolean { return status === 301 || status === 302 From 5edc5a4d15870ffa88bccfb809e24ab2bf315c15 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:37:38 +0000 Subject: [PATCH 04/16] test: harden artifact workspace materialization --- package.json | 2 +- src/artifact-workspace.test.ts | 96 +++++++++++++++++++++++++++++++++- src/artifact-workspace.ts | 6 +++ 3 files changed, 102 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index ecf3a529..3fb223d8 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dev": "node scripts/dev-server.mjs", "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", - "test": "tsx src/config.test.ts && tsx src/artifacts.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/artifact-workspace.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", + "test": "tsx src/config.test.ts && tsx src/artifacts.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/artifact-workspace.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/artifact-workspace.test.ts b/src/artifact-workspace.test.ts index 7ed40297..db4269bd 100644 --- a/src/artifact-workspace.test.ts +++ b/src/artifact-workspace.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { copyArtifactToWorkspace } from "./artifact-workspace.js"; @@ -50,9 +50,103 @@ try { assert.equal(copied.path, destination); assert.equal(copied.sha256, artifact.sha256); assert.equal(copied.renamed, false); + + await writeFile(destination, "existing"); + await expectArtifactError( + copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination, + onConflict: "error", + }), + "workspace_destination_exists", + ); + assert.equal(await readFile(destination, "utf8"), "existing"); + + const renamed = await copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination, + onConflict: "rename", + }); + assert.equal(renamed.path, join(workspaceRoot, "assets", "generated (1).png")); + assert.equal(renamed.renamed, true); + assert.deepEqual(await readFile(renamed.path), bytes); + + const replaced = await copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination, + onConflict: "replace", + }); + assert.equal(replaced.path, destination); + assert.equal(replaced.renamed, false); + assert.deepEqual(await readFile(destination), bytes); + + await expectArtifactError( + copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination: join(workspaceRoot, "..", "outside.png"), + onConflict: "error", + }), + "workspace_path_escape", + ); + + if (process.platform !== "win32") { + const outsideRoot = join(root, "outside"); + await mkdir(outsideRoot, { recursive: true }); + const linkedParent = join(workspaceRoot, "linked-parent"); + await symlink(outsideRoot, linkedParent, "dir"); + await expectArtifactError( + copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination: join(linkedParent, "escaped.png"), + onConflict: "error", + }), + "workspace_parent_unsafe", + ); + const linkedDestination = join(workspaceRoot, "linked.png"); + await symlink(join(outsideRoot, "outside.png"), linkedDestination); + await expectArtifactError( + copyArtifactToWorkspace({ + store, + clientId: "client-a", + workspaceId: "ws_test", + workspaceRoot, + artifactId: artifact.artifactId, + destination: linkedDestination, + onConflict: "replace", + }), + "workspace_destination_unsafe", + ); + } } finally { store.close(); } } finally { await rm(root, { recursive: true, force: true }); } + +async function expectArtifactError(promise: Promise, code: string): Promise { + await assert.rejects( + promise, + (error: unknown) => error instanceof Error && "code" in error && error.code === code, + ); +} diff --git a/src/artifact-workspace.ts b/src/artifact-workspace.ts index f6630fb0..7340d24f 100644 --- a/src/artifact-workspace.ts +++ b/src/artifact-workspace.ts @@ -47,6 +47,7 @@ export async function copyArtifactToWorkspace( const tempPath = join(parent, `.devspace-artifact-${randomUUID()}.partial`); let finalPath = input.destination; + let materialized = false; try { const copied = await copyVerifiedArtifactToTemp(artifact.hostPath, tempPath, artifact.size, artifact.sha256); await assertContainedWorkspaceRegularFile(input.workspaceRoot, tempPath); @@ -58,6 +59,7 @@ export async function copyArtifactToWorkspace( } await ensureContainedWorkspaceDirectory(input.workspaceRoot, parent); await rename(tempPath, finalPath); + materialized = true; } else { let linked = false; const attempts = input.onConflict === "rename" ? MAX_RENAME_ATTEMPTS : 1; @@ -79,6 +81,7 @@ export async function copyArtifactToWorkspace( if (!linked) { throw new ArtifactError("workspace_rename_exhausted", "Could not find an available destination filename."); } + materialized = true; await unlink(tempPath); } @@ -99,6 +102,9 @@ export async function copyArtifactToWorkspace( }; } catch (error) { await unlink(tempPath).catch(() => undefined); + if (materialized) { + await unlink(finalPath).catch(() => undefined); + } throw error; } } From 89ba6840a4488065a3562ac1626bda531bffebf6 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Tue, 21 Jul 2026 01:43:49 +0000 Subject: [PATCH 05/16] feat: materialize native artifacts in one call --- .env.example | 2 +- docs/artifact-exchange.md | 137 ++++++------------ docs/configuration.md | 15 +- docs/security.md | 26 ++-- src/artifact-tools.ts | 247 ++++++++++++++------------------- src/artifact-workspace.ts | 13 ++ src/incoming-artifacts.test.ts | 58 ++++++-- src/server.ts | 2 +- 8 files changed, 227 insertions(+), 273 deletions(-) diff --git a/.env.example b/.env.example index 8284bd00..0ac70b97 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,7 @@ DEVSPACE_LOG_SHELL_COMMANDS=0 # DEVSPACE_TRUST_PROXY=1 # DEVSPACE_STATE_DIR=/home/waishnav/.local/share/devspace # DEVSPACE_WORKTREE_ROOT=/home/waishnav/.devspace/worktrees -# Private native artifact staging is opt-in. Stored objects remain outside repositories. +# Native-file materialization is opt-in. Private internal storage remains outside repositories. # DEVSPACE_ARTIFACTS=1 # DEVSPACE_ARTIFACT_ROOT=$HOME/.local/share/devspace/artifacts # DEVSPACE_ARTIFACT_MAX_FILE_BYTES=104857600 diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 6c8af1a4..b1932cb0 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -1,131 +1,72 @@ -# Native Artifact Staging +# Native file materialization -DevSpace can accept a file attached or generated by ChatGPT even though that -file does not yet exist on the DevSpace host. The opt-in `stage_artifact` MCP -tool streams the connector-provided file into private local storage and returns -a host path that local commands can consume. +DevSpace can save a file attached or generated by an MCP host (such as ChatGPT) directly into an already-open workspace. Enable this opt-in feature with `DEVSPACE_ARTIFACTS=1`. -This is deliberately a narrow handoff seam. It supports explicit copy of a -staged artifact into an already-open workspace, but does not automatically write -repositories, accept generic uploads, or expose a public download route. - -## Enable - -```bash -DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve -``` - -The default private root is: +## Workflow ```text -~/.local/share/devspace/artifacts +open_workspace +materialize_artifact ``` -SQLite stores the owner-scoped record. File content is stored outside approved -workspace roots and repositories. - -## MCP workflow - -ChatGPT calls `stage_artifact` with its native top-level file value: +`materialize_artifact` is the only artifact-specific MCP tool exposed by default. It accepts the native file value authorized by the MCP host, a workspace ID returned by `open_workspace`, a relative destination, and an explicit conflict policy. ```json { - "file": "", - "workspaceId": "optional metadata association", - "expectedSha256": "optional sha256:...", - "ttlHours": 24, - "pin": false + "file": { + "download_url": "", + "file_id": "", + "mime_type": "image/png", + "file_name": "generated-image.png" + }, + "workspaceId": "ws_123", + "destination": "assets/generated-image.png", + "onConflict": "rename" } ``` -The tool declares `_meta["openai/fileParams"] = ["file"]`, allowing the -connector to expand the selected file into its authorized native object. -`workspaceId` is metadata only; staging never writes to that workspace. +The `file` value must be passed natively by the MCP host. DevSpace rejects arbitrary URL strings, local filesystem paths, extra credential-bearing fields, and untrusted download origins. -A successful result contains metadata, not file content: +A successful call returns only workspace-facing metadata: ```json { - "artifactId": "art_...", + "workspaceId": "ws_123", + "path": "assets/generated-image.png", "name": "generated-image.png", "mimeType": "image/png", "size": 2302372, - "sha256": "...", - "hostPath": "/home/user/.local/share/devspace/artifacts/objects/...", - "expiresAt": "...", - "instruction": "Pass hostPath to a local command..." + "sha256": "sha256:...", + "onConflict": "rename", + "renamed": false } ``` -Use `artifact_stat` to re-read owner-scoped metadata and `artifact_delete` to -remove the record. To materialize an image or document, use -`artifact_copy_to_workspace` with its `artifactId`, an existing `workspaceId`, a -relative `destination`, and explicit `onConflict` mode (`error`, `rename`, or -`replace`). The copy verifies bytes and SHA-256, rejects workspace escapes and -symlink paths, and returns the final workspace path. Agents must never invent -paths under the artifact root. +It never returns a signed URL, native file ID, private host path, temporary path, or internal storage record. -## Supported connector shape +## Safety model -The production adapter accepts only an object containing these known keys: +DevSpace validates the native file reference and redirects before downloading. It streams the file through private server-controlled storage, enforces configured size limits, computes and verifies SHA-256, and then uses the workspace materializer to: -```ts -{ - download_url: string; - file_id: string; - mime_type?: string | null; - file_name?: string | null; - name?: string | null; - size?: number | null; -} -``` - -Required fields are `download_url` and `file_id`. Unknown keys, extra credential -fields, plain URL strings, filesystem paths, arrays, and malformed values fail -closed. - -Download URLs must use HTTPS on one of these reviewed OpenAI delivery origins: - -- `files.oaiusercontent.com` -- `oaisdmntpr.blob.core.windows.net`, where `` is lowercase - alphanumeric (for example `centralus`, `westcentralus`, or `centralindia`) +- reject destinations outside the selected workspace; +- reject symlinked parents, destinations, and non-regular files; +- verify the final file's size and SHA-256; +- clean up internal staging and workspace partials on failure. -Userinfo, fragments, non-HTTPS ports, and redirects outside those hosts are -rejected. Redirects are followed manually and revalidated at each hop. Signed -URLs and opaque file IDs are never logged or returned. +The file is never buffered as model-visible base64, and DevSpace never runs shell `cp`, `mv`, or `rm` based on a file reference. -Generated-image IDs are treated as opaque bounded metadata. They are never used -as a path component. If an ID is not filename-safe, DevSpace derives a fixed -`chatgpt-file` basename plus an extension from the MIME type. +## Conflicts -## Storage and lifecycle +`onConflict` is explicit: -The existing artifact store enforces: - -- 100 MiB per artifact by default -- 1 GiB total private storage by default -- server-computed SHA-256 -- optional caller-supplied digest verification -- mode `0700` storage directories and mode `0600` files -- server-selected paths and normalized non-hidden basenames -- symlink, non-regular-file, and containment checks -- atomic promotion into immutable content-addressed storage -- OAuth-client ownership for records -- a 24-hour default TTL for unpinned records -- bounded cleanup at startup and every 15 minutes - -The staging implementation reuses the store's private in-progress upload -pipeline internally so limits, hashing, quota enforcement, cleanup, and atomic -commit behavior remain identical. Those internal chunk operations are not MCP -tools in this focused change. +- `error` (default): fail without changing an existing destination. +- `rename`: choose `name (1).ext`, `name (2).ext`, and so on, up to a bounded limit. +- `replace`: overwrite an existing regular non-symlink file. Use only when replacement is intended. ## Logging -`stage_artifact` logs only bounded structural diagnostics, the validated -download hostname, metadata options, resulting artifact ID, size, digest, and -status. It never logs raw file objects, signed URLs, file IDs, credentials, -content, or base64 data. +Tool logs contain bounded metadata such as workspace ID, relative destination, conflict mode, validated hostname, byte count, SHA-256, duration, and stable error code. They never contain native file objects, signed URLs, opaque file IDs, credentials, file contents, base64 data, or private temporary/storage paths. + +## Internal storage -MIME types and filenames are hints only. DevSpace does not execute, unpack, -publish, or automatically copy staged content into a workspace; materialization -requires the explicit workspace-copy tool. +DevSpace may use private, server-owned staging internally while materializing a file. This is an implementation detail: the staged record is removed automatically after the operation and is not exposed to the model as a reusable library or lifecycle API. diff --git a/docs/configuration.md b/docs/configuration.md index e45fd1a6..cc761e68 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -50,7 +50,7 @@ DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve | Variable | Default | Purpose | | --- | --- | --- | -| `DEVSPACE_ARTIFACTS` | `0` | Expose `stage_artifact`, `artifact_stat`, `artifact_copy_to_workspace`, and `artifact_delete`. | +| `DEVSPACE_ARTIFACTS` | `0` | Expose `materialize_artifact` for trusted native files. | | `DEVSPACE_ARTIFACT_ROOT` | `~/.local/share/devspace/artifacts` | Private storage root outside repositories and worktrees. | | `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` | `104857600` | Maximum decoded size of one artifact (100 MiB). | | `DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES` | `1073741824` | Maximum combined stored-object and in-progress bytes (1 GiB). | @@ -60,13 +60,12 @@ The same settings may be persisted in `~/.devspace/config.json` as `artifactsEnabled`, `artifactRoot`, `artifactMaxFileBytes`, `artifactMaxTotalBytes`, and `artifactDefaultTtlHours`. -`stage_artifact` accepts only the native file object supplied by the ChatGPT -connector. It does not accept arbitrary URL strings, paths, embedded credentials, -or extra object fields. Files are streamed through bounded private storage. -`artifact_copy_to_workspace` can then materialize one owner-scoped artifact into -an already-open workspace at an explicit relative destination and conflict mode. -Cleanup runs at startup and periodically, with a bounded number of records -processed per pass. +`materialize_artifact` accepts only the native file object supplied by the MCP +connector and writes it to a relative destination inside an already-open workspace. +It does not accept arbitrary URL strings, paths, embedded credentials, or extra +object fields. Files are streamed through bounded private storage, then verified +and automatically cleaned up after materialization. Startup and periodic cleanup +also process stale internal state with bounded work per pass. See [Native Artifact Staging](artifact-exchange.md) for the supported connector shape and security boundaries. diff --git a/docs/security.md b/docs/security.md index 8e8108df..a437f739 100644 --- a/docs/security.md +++ b/docs/security.md @@ -123,26 +123,23 @@ accounts. Credentials, fragments, alternate ports, malformed IDs, extra fields, and redirects outside that boundary fail closed. Opaque IDs are bounded metadata and are never used as filenames or path components. -The staging seam deliberately does not: +The native materialization seam deliberately does not: - fetch arbitrary URLs - expose a generic upload API -- automatically copy content into a workspace or repository +- expose persistent artifact IDs or a user-facing artifact library - extract archives - execute transferred content - expand workspace allowlists - preserve executable permissions - publish or permanently retain content -`artifact_copy_to_workspace` is the explicit exception: it reads only an -owner-scoped staged artifact and writes only within an already-open allowed -workspace. It requires a relative destination and an explicit conflict mode, +`materialize_artifact` is the explicit, bounded write path: it accepts only a +native MCP-host file value and writes only within an already-open allowed +workspace. It requires a relative destination and explicit conflict mode, creates only real contained parent directories, rejects symlink/non-regular-file -paths, and verifies the final size and SHA-256. - -MIME types are hints only. SHA-256 and byte counts are computed and enforced by -the server. Pinned records require explicit deletion; unpinned records and -failed in-progress transfers remain subject to cleanup. +paths, and verifies the final size and SHA-256. Any private staging is server +implementation detail and is cleaned up automatically. ## Logs @@ -151,8 +148,7 @@ disabled unless `DEVSPACE_LOG_SHELL_COMMANDS=1`. Do not enable shell command logging if commands may contain secrets. -Artifact tool logs contain identifiers, names, MIME hints, byte counts, hashes, -and status fields. `stage_artifact` logs only whether a file value and expected -digest were supplied plus non-sensitive options; it does not log the opaque -file value. Raw content, connector references, bearer credentials, presigned -URLs, and base64 chunks are never included in tool logs or tool results. +Artifact tool logs contain bounded workspace, destination, conflict-mode, +hostname, byte-count, hash, and status metadata. `materialize_artifact` does not +log the opaque file value. Raw content, connector references, bearer credentials, +presigned URLs, and base64 chunks are never included in tool logs or tool results. diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index 4fe06873..51e0c524 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -1,5 +1,6 @@ import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { basename, isAbsolute, relative } from "node:path"; import * as z from "zod/v4"; import type { ServerConfig } from "./config.js"; import { @@ -9,7 +10,6 @@ import { import { ARTIFACT_CHUNK_BYTES, ArtifactError, - type ArtifactRecord, type ArtifactStore, } from "./artifacts.js"; import { @@ -36,25 +36,7 @@ const openAIFileReferenceInputSchema = z.object({ size: z.number().int().nonnegative().nullable().optional(), }); -const artifactRecordOutputSchema = { - artifactId: z.string(), - name: z.string(), - mimeType: z.string().optional(), - size: z.number().int().nonnegative(), - sha256: z.string(), - hostPath: z.string(), - source: z.string(), - workspaceId: z.string().optional(), - createdAt: z.string(), - expiresAt: z.string().optional(), - pinned: z.boolean(), -}; - -type ArtifactToolName = - | "stage_artifact" - | "artifact_stat" - | "artifact_copy_to_workspace" - | "artifact_delete"; +type ArtifactToolName = "materialize_artifact"; export interface ArtifactToolRegistrationOptions { config: ServerConfig; @@ -83,6 +65,25 @@ export interface StageArtifactResult { instruction: string; } +export interface MaterializeArtifactInput { + file: unknown; + workspaceId: string; + destination: string; + onConflict: ArtifactCopyConflictMode; + expectedSha256?: string; +} + +export interface MaterializeArtifactResult { + workspaceId: string; + path: string; + name: string; + mimeType?: string; + size: number; + sha256: string; + onConflict: ArtifactCopyConflictMode; + renamed: boolean; +} + export function registerArtifactTools( server: McpServer, { @@ -97,132 +98,104 @@ export function registerArtifactTools( registerAppTool( server, - "stage_artifact", + "materialize_artifact", { - title: "Stage attached or generated file", + title: "Materialize attached or generated file", description: - "Stage one ChatGPT-provided native file reference into private DevSpace storage. DevSpace downloads only the exact file object authorized by ChatGPT; arbitrary URLs and paths fail closed. A workspace ID is association metadata, not a write destination.", + "Save one MCP-host-provided native file into an already-open workspace. DevSpace validates the file reference, streams and verifies the bytes, and atomically writes the destination. Arbitrary URLs and host paths are rejected.", inputSchema: { - file: openAIFileReferenceInputSchema.describe("Native file value authorized and supplied by ChatGPT."), - workspaceId: z.string().min(1).optional().describe("Optional workspace association; never a write destination."), + file: openAIFileReferenceInputSchema.describe("Native file value authorized and supplied by the MCP host."), + workspaceId: z.string().min(1).describe("Workspace identifier returned by open_workspace."), + destination: z.string().min(1).describe("Relative file path inside the selected workspace."), + onConflict: z.enum(["error", "rename", "replace"]).default("error").describe("How to handle an existing destination."), expectedSha256: z.string().optional().describe("Optional expected SHA-256, with or without a sha256: prefix."), - ttlHours: z.number().int().min(1).max(24 * 365).optional().describe("Artifact lifetime after staging."), - pin: z.boolean().optional().describe("Preserve the artifact until explicitly deleted."), }, outputSchema: { - artifactId: z.string(), + workspaceId: z.string(), + path: z.string(), name: z.string(), mimeType: z.string().optional(), size: z.number().int().nonnegative(), sha256: z.string(), - hostPath: z.string(), - expiresAt: z.string().optional(), - instruction: z.string(), + onConflict: z.enum(["error", "rename", "replace"]), + renamed: z.boolean(), }, _meta: { "openai/fileParams": ["file"] }, annotations: ARTIFACT_WRITE_ANNOTATIONS, }, - async (input) => executeArtifactTool(config, "stage_artifact", input, async () => { - return stageIncomingArtifact({ + async (input) => executeArtifactTool(config, "materialize_artifact", input, async () => { + if (isAbsolute(input.destination)) { + throw new ArtifactError("workspace_destination_invalid", "Artifact destination must be a relative workspace path."); + } + const workspace = workspaces.getWorkspace(input.workspaceId); + const destination = workspaces.resolvePath(workspace, input.destination); + return materializeIncomingArtifact({ store, clientId, registry: incomingRegistry, + workspaceId: workspace.id, + workspaceRoot: workspace.root, + destination, input, }); }), ); - registerAppTool( - server, - "artifact_stat", - { - title: "Inspect artifact", - description: "Return private artifact metadata by ID without returning its content.", - inputSchema: { - artifactId: z.string(), - }, - outputSchema: artifactRecordOutputSchema, - _meta: {}, - annotations: { readOnlyHint: true, openWorldHint: false }, - }, - async ({ artifactId }) => executeArtifactTool( - config, - "artifact_stat", - { artifactId }, - async () => store.statArtifact(clientId, artifactId), - ), - ); +} - registerAppTool( - server, - "artifact_copy_to_workspace", - { - title: "Copy artifact into workspace", - description: - "Copy one owner-scoped private artifact into a path inside an already-open workspace. The destination must stay within that workspace; choose rename or replace explicitly when a file exists.", - inputSchema: { - artifactId: z.string(), - workspaceId: z.string().min(1), - destination: z.string().min(1).describe("Relative path inside the selected workspace."), - onConflict: z.enum(["error", "rename", "replace"]).default("error"), - }, - outputSchema: { - artifactId: z.string(), - workspaceId: z.string(), - path: z.string(), - size: z.number().int().nonnegative(), - sha256: z.string(), - onConflict: z.enum(["error", "rename", "replace"]), - renamed: z.boolean(), - }, - _meta: {}, - annotations: ARTIFACT_WRITE_ANNOTATIONS, +/** Materialize a native file through the private store, then remove its internal record. */ +export async function materializeIncomingArtifact({ + store, + clientId, + registry, + workspaceId, + workspaceRoot, + destination, + input, +}: { + store: ArtifactStore; + clientId: string; + registry: IncomingArtifactAdapterRegistry; + workspaceId: string; + workspaceRoot: string; + destination: string; + input: MaterializeArtifactInput; +}): Promise { + const staged = await stageIncomingArtifact({ + store, + clientId, + registry, + input: { + file: input.file, + workspaceId, + expectedSha256: input.expectedSha256, }, - async (input) => executeArtifactTool( - config, - "artifact_copy_to_workspace", - input, - async () => { - const workspace = workspaces.getWorkspace(input.workspaceId); - const destination = workspaces.resolvePath(workspace, input.destination); - return copyArtifactToWorkspace({ - store, - clientId, - workspaceId: workspace.id, - workspaceRoot: workspace.root, - artifactId: input.artifactId, - destination, - onConflict: input.onConflict as ArtifactCopyConflictMode, - }); - }, - ), - ); + }); - registerAppTool( - server, - "artifact_delete", - { - title: "Delete artifact", - description: - "Delete an artifact record. Its immutable object is removed only when no other live record references it.", - inputSchema: { - artifactId: z.string(), - }, - outputSchema: { - artifactId: z.string(), - deleted: z.boolean(), - objectDeleted: z.boolean(), - }, - _meta: {}, - annotations: ARTIFACT_WRITE_ANNOTATIONS, - }, - async ({ artifactId }) => executeArtifactTool( - config, - "artifact_delete", - { artifactId }, - async () => store.deleteArtifact(clientId, artifactId), - ), - ); + try { + const copied = await copyArtifactToWorkspace({ + store, + clientId, + workspaceId, + workspaceRoot, + artifactId: staged.artifactId, + destination, + onConflict: input.onConflict, + }); + const path = relative(workspaceRoot, copied.path); + return { + workspaceId, + path, + name: basename(path), + mimeType: staged.mimeType, + size: copied.size, + sha256: copied.sha256, + onConflict: copied.onConflict, + renamed: copied.renamed, + }; + } finally { + await store.deleteArtifact(clientId, staged.artifactId).catch(() => undefined); + } } export async function stageIncomingArtifact({ @@ -317,18 +290,15 @@ export function artifactToolLogFields( tool: ArtifactToolName, input: Record, ): Record { - if (tool === "stage_artifact") { - return { - fileProvided: input.file !== undefined, - fileReferenceShape: describeIncomingArtifactValue(input.file), - downloadUrlHostname: incomingFileDownloadHostname(input.file), - workspaceId: input.workspaceId, - expectedSha256Present: typeof input.expectedSha256 === "string", - ttlHours: input.ttlHours, - pin: input.pin === true, - }; - } - return { artifactId: input.artifactId }; + return { + fileProvided: input.file !== undefined, + fileReferenceShape: describeIncomingArtifactValue(input.file), + downloadUrlHostname: incomingFileDownloadHostname(input.file), + workspaceId: input.workspaceId, + destination: input.destination, + expectedSha256Present: typeof input.expectedSha256 === "string", + onConflict: input.onConflict, + }; } async function executeArtifactTool( @@ -372,12 +342,11 @@ function artifactToolResponse(result: T) { } function artifactResultLogFields(result: Record): Record { - const artifact = result as Partial; return { - artifactId: artifact.artifactId, - size: artifact.size, - sha256: artifact.sha256, - source: artifact.source, - objectDeleted: result.objectDeleted, + workspaceId: result.workspaceId, + path: result.path, + size: result.size, + sha256: result.sha256, + renamed: result.renamed, }; } diff --git a/src/artifact-workspace.ts b/src/artifact-workspace.ts index 7340d24f..574226cd 100644 --- a/src/artifact-workspace.ts +++ b/src/artifact-workspace.ts @@ -48,6 +48,7 @@ export async function copyArtifactToWorkspace( const tempPath = join(parent, `.devspace-artifact-${randomUUID()}.partial`); let finalPath = input.destination; let materialized = false; + let replaceBackupPath: string | undefined; try { const copied = await copyVerifiedArtifactToTemp(artifact.hostPath, tempPath, artifact.size, artifact.sha256); await assertContainedWorkspaceRegularFile(input.workspaceRoot, tempPath); @@ -58,6 +59,11 @@ export async function copyArtifactToWorkspace( throw new ArtifactError("workspace_destination_unsafe", "Replace is allowed only for an existing regular file."); } await ensureContainedWorkspaceDirectory(input.workspaceRoot, parent); + if (existing) { + replaceBackupPath = join(parent, `.devspace-artifact-${randomUUID()}.backup`); + await rename(finalPath, replaceBackupPath); + await assertContainedWorkspaceRegularFile(input.workspaceRoot, replaceBackupPath); + } await rename(tempPath, finalPath); materialized = true; } else { @@ -91,6 +97,10 @@ export async function copyArtifactToWorkspace( if (finalEntry.size !== copied.size || `sha256:${digest}` !== copied.sha256) { throw new ArtifactError("workspace_copy_integrity_failed", "Workspace copy failed size or SHA-256 verification."); } + if (replaceBackupPath) { + await unlink(replaceBackupPath).catch(() => undefined); + replaceBackupPath = undefined; + } return { artifactId: artifact.artifactId, workspaceId: input.workspaceId, @@ -105,6 +115,9 @@ export async function copyArtifactToWorkspace( if (materialized) { await unlink(finalPath).catch(() => undefined); } + if (replaceBackupPath) { + await rename(replaceBackupPath, finalPath).catch(() => undefined); + } throw error; } } diff --git a/src/incoming-artifacts.test.ts b/src/incoming-artifacts.test.ts index 5b90e6c5..2fdcde0a 100644 --- a/src/incoming-artifacts.test.ts +++ b/src/incoming-artifacts.test.ts @@ -1,12 +1,13 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Readable } from "node:stream"; import * as z from "zod/v4"; import { artifactToolLogFields, + materializeIncomingArtifact, registerArtifactTools, stageIncomingArtifact, } from "./artifact-tools.js"; @@ -24,6 +25,7 @@ try { testChatGPTFileDescriptor(); await testOpenAIFileAdapter(join(root, "openai")); await testStageArtifact(join(root, "stage")); + await testMaterializeArtifact(join(root, "materialize")); await testFailedStageCleanup(join(root, "failed-stage")); testStageLogRedaction(); } finally { @@ -50,13 +52,8 @@ function testChatGPTFileDescriptor(): void { clientId: "client-a", }); - assert.deepEqual([...registered.keys()], [ - "stage_artifact", - "artifact_stat", - "artifact_copy_to_workspace", - "artifact_delete", - ]); - const descriptor = registered.get("stage_artifact"); + assert.deepEqual([...registered.keys()], ["materialize_artifact"]); + const descriptor = registered.get("materialize_artifact"); assert.ok(descriptor); assert.deepEqual(descriptor._meta, { "openai/fileParams": ["file"] }); @@ -297,6 +294,45 @@ async function testStageArtifact(testRoot: string): Promise { } } +async function testMaterializeArtifact(testRoot: string): Promise { + const bytes = Buffer.from("native artifact bytes"); + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + const store = createStore(testRoot); + try { + const result = await materializeIncomingArtifact({ + store, + clientId: "client-a", + registry: new IncomingArtifactAdapterRegistry([memoryAdapter("memory-test", bytes)]), + workspaceId: "ws_materialize", + workspaceRoot, + destination: join(workspaceRoot, "assets", "generated.bin"), + input: { + file: { kind: "memory" }, + workspaceId: "ws_materialize", + destination: "assets/generated.bin", + onConflict: "error", + }, + }); + assert.deepEqual(await readFile(join(workspaceRoot, result.path)), bytes); + assert.deepEqual(result, { + workspaceId: "ws_materialize", + path: "assets/generated.bin", + name: "generated.bin", + mimeType: "application/octet-stream", + size: bytes.length, + sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + onConflict: "error", + renamed: false, + }); + assert.equal("artifactId" in result, false); + assert.equal("hostPath" in result, false); + assert.equal(store.health().storedBytes, 0); + } finally { + store.close(); + } +} + async function testFailedStageCleanup(testRoot: string): Promise { const store = createStore(testRoot); try { @@ -324,12 +360,12 @@ async function testFailedStageCleanup(testRoot: string): Promise { function testStageLogRedaction(): void { const secret = "https://files.example.test/download?token=super-secret"; - const fields = artifactToolLogFields("stage_artifact", { + const fields = artifactToolLogFields("materialize_artifact", { file: { download_url: secret, href: secret, bearer: "Bearer secret" }, workspaceId: "ws_123", + destination: "assets/generated.png", expectedSha256: "f".repeat(64), - ttlHours: 24, - pin: true, + onConflict: "error", }); const serialized = JSON.stringify(fields); assert.equal("file" in fields, false); diff --git a/src/server.ts b/src/server.ts index cf9868c0..888babe2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -189,7 +189,7 @@ interface ToolLogFields { function serverInstructions(config: ServerConfig): string { const artifactInstruction = config.artifactsEnabled - ? " When the user supplies or generates a file that is not present on the DevSpace host, pass its native file value to stage_artifact instead of recreating it through write/edit calls. To place a staged artifact in an approved workspace, use artifact_copy_to_workspace with that artifact ID, the existing workspace ID, a relative destination, and an explicit conflict mode. Only use host paths returned by artifact tools; never invent artifact-store paths or place artifact content, signed URLs, or base64 in shell commands or logs. stage_artifact stores privately and never writes into a workspace or repository; artifact_copy_to_workspace is the explicit materialization step and may dirty a repository." + ? " When the user supplies or generates a file that is not present on the DevSpace host, use materialize_artifact with its native file value, the existing workspace ID, a relative destination, and an explicit conflict mode. Do not recreate binary files with write/edit calls. Do not place signed URLs, native file objects, base64 content, or invented artifact-storage paths in shell commands or logs. materialize_artifact validates, streams, verifies, and writes the file into the selected workspace; it may dirty a repository." : ""; const showChangesInstruction = config.widgets === "changes" From 1cbf8d876c0e0ebfe70fdd0d8af9e7b120e35c84 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:24:45 +0000 Subject: [PATCH 06/16] refactor: use one-shot native artifact downloads --- .env.example | 6 +- README.md | 2 +- docs/artifact-exchange.md | 75 +-- docs/configuration.md | 37 +- docs/security.md | 103 ++-- package.json | 2 +- src/artifact-download.test.ts | 261 ++++++++++ src/artifact-error.ts | 9 + src/artifact-tools.ts | 667 ++++++++++++++++-------- src/artifact-workspace.test.ts | 154 +----- src/artifact-workspace.ts | 253 +-------- src/artifacts.test.ts | 475 +---------------- src/artifacts.ts | 907 +-------------------------------- src/config.test.ts | 43 +- src/config.ts | 24 - src/db/migrations.ts | 65 +-- src/db/schema.ts | 57 --- src/incoming-artifacts.test.ts | 403 ++++----------- src/incoming-artifacts.ts | 2 +- src/oauth-store.test.ts | 7 +- src/server.ts | 40 +- src/user-config.ts | 3 - 22 files changed, 963 insertions(+), 2632 deletions(-) create mode 100644 src/artifact-download.test.ts create mode 100644 src/artifact-error.ts diff --git a/.env.example b/.env.example index 0ac70b97..7f4a48bf 100644 --- a/.env.example +++ b/.env.example @@ -27,12 +27,10 @@ DEVSPACE_LOG_SHELL_COMMANDS=0 # DEVSPACE_TRUST_PROXY=1 # DEVSPACE_STATE_DIR=/home/waishnav/.local/share/devspace # DEVSPACE_WORKTREE_ROOT=/home/waishnav/.devspace/worktrees -# Native-file materialization is opt-in. Private internal storage remains outside repositories. +# Native-file download is opt-in. Files stream directly into the selected +# workspace's .devspace/incoming directory. # DEVSPACE_ARTIFACTS=1 -# DEVSPACE_ARTIFACT_ROOT=$HOME/.local/share/devspace/artifacts # DEVSPACE_ARTIFACT_MAX_FILE_BYTES=104857600 -# DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES=1073741824 -# DEVSPACE_ARTIFACT_TTL_HOURS=24 # DEVSPACE_AUTO_LOAD_AGENTS_MD=1 # Skills are enabled by default. Set DEVSPACE_SKILLS=0 to hide them. # DEVSPACE_SKILLS=0 diff --git a/README.md b/README.md index 0bf7afa7..5342573a 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ devspace doctor - [Setup Guide](https://github.com/Waishnav/devspace/blob/main/docs/setup.md) - [ChatGPT Coding Workflow](https://github.com/Waishnav/devspace/blob/main/docs/chatgpt-coding-workflow.md) - [Configuration Reference](https://github.com/Waishnav/devspace/blob/main/docs/configuration.md) -- [Native Artifact Staging](https://github.com/Waishnav/devspace/blob/main/docs/artifact-exchange.md) +- [Native File Download](https://github.com/Waishnav/devspace/blob/main/docs/artifact-exchange.md) - [Security Model](https://github.com/Waishnav/devspace/blob/main/docs/security.md) - [Troubleshooting Gotchas](https://github.com/Waishnav/devspace/blob/main/docs/gotchas.md) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index b1932cb0..e0cd46d2 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -1,15 +1,19 @@ -# Native file materialization +# Native file download -DevSpace can save a file attached or generated by an MCP host (such as ChatGPT) directly into an already-open workspace. Enable this opt-in feature with `DEVSPACE_ARTIFACTS=1`. +DevSpace can stream a file attached or generated by an MCP host (such as ChatGPT) into an already-open workspace. Enable this opt-in feature with `DEVSPACE_ARTIFACTS=1`. ## Workflow ```text open_workspace -materialize_artifact + -> download_artifact({ file, workspaceId }) + -> { path } ``` -`materialize_artifact` is the only artifact-specific MCP tool exposed by default. It accepts the native file value authorized by the MCP host, a workspace ID returned by `open_workspace`, a relative destination, and an explicit conflict policy. +`download_artifact` is the only artifact-specific MCP tool. It accepts exactly two inputs: + +- the native `file` value authorized by the MCP host; +- a `workspaceId` returned by `open_workspace`. ```json { @@ -19,54 +23,57 @@ materialize_artifact "mime_type": "image/png", "file_name": "generated-image.png" }, - "workspaceId": "ws_123", - "destination": "assets/generated-image.png", - "onConflict": "rename" + "workspaceId": "ws_123" } ``` -The `file` value must be passed natively by the MCP host. DevSpace rejects arbitrary URL strings, local filesystem paths, extra credential-bearing fields, and untrusted download origins. - -A successful call returns only workspace-facing metadata: +DevSpace chooses a normalized, collision-free filename below the selected workspace's `.devspace/incoming/` directory. A successful call returns only the workspace-relative path: ```json { - "workspaceId": "ws_123", - "path": "assets/generated-image.png", - "name": "generated-image.png", - "mimeType": "image/png", - "size": 2302372, - "sha256": "sha256:...", - "onConflict": "rename", - "renamed": false + "path": ".devspace/incoming/generated-image.png" } ``` -It never returns a signed URL, native file ID, private host path, temporary path, or internal storage record. +When that name already exists, DevSpace chooses `generated-image (1).png`, `generated-image (2).png`, and so on. Use the normal DevSpace filesystem tools for any later move, rename, replacement, or deletion. + +## Accepted file values + +The `file` value must be passed natively by the MCP host. DevSpace validates the complete object shape and rejects: + +- arbitrary URL strings; +- local filesystem paths; +- malformed or ambiguous objects; +- extra credential-bearing fields; +- untrusted hosts or redirect targets; +- unsupported adapter values. + +The tool retains the MCP metadata declaration `openai/fileParams: ["file"]` so compatible hosts treat `file` as a native file parameter. -## Safety model +## Streaming and publication -DevSpace validates the native file reference and redirects before downloading. It streams the file through private server-controlled storage, enforces configured size limits, computes and verifies SHA-256, and then uses the workspace materializer to: +The download is never buffered wholesale in memory. DevSpace: -- reject destinations outside the selected workspace; -- reject symlinked parents, destinations, and non-regular files; -- verify the final file's size and SHA-256; -- clean up internal staging and workspace partials on failure. +1. validates the trusted native reference and redirect chain; +2. opens a private exclusive partial below `.devspace/incoming/`; +3. streams bytes under `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` while computing SHA-256; +4. verifies any supplied size hint; +5. chmods and fsyncs through the still-open file descriptor; +6. publishes the verified inode with an atomic hard link to a collision-free final name; +7. removes the private partial. -The file is never buffered as model-visible base64, and DevSpace never runs shell `cp`, `mv`, or `rm` based on a file reference. +There is no persistent artifact database, completed-object store, upload/chunk API, quota ledger, TTL, pinning, artifact ID, signed download route, or reusable artifact lifecycle. -## Conflicts +## Directory safety -`onConflict` is explicit: +DevSpace opens the selected workspace without following symlinks, then creates or opens `.devspace` and `incoming` through already-open parent directory descriptors. All partial creation, cleanup, and final publication remain anchored to the open `incoming` descriptor, so replacing a pathname cannot redirect writes outside the selected workspace. Symlinked parents, non-directories, and group/world-writable incoming directories fail closed. Existing directories are inspected but never chmodded as a startup side effect. -- `error` (default): fail without changing an existing destination. -- `rename`: choose `name (1).ext`, `name (2).ext`, and so on, up to a bounded limit. -- `replace`: overwrite an existing regular non-symlink file. Use only when replacement is intended. +Published files are not path-chmodded or path-hashed after publication. Temporary partial cleanup is bounded, only considers DevSpace partial names, and ignores symlinks and non-regular files. ## Logging -Tool logs contain bounded metadata such as workspace ID, relative destination, conflict mode, validated hostname, byte count, SHA-256, duration, and stable error code. They never contain native file objects, signed URLs, opaque file IDs, credentials, file contents, base64 data, or private temporary/storage paths. +Tool logs contain bounded metadata such as workspace ID, validated download hostname, workspace-relative output path, byte count, SHA-256, duration, and stable error code. They never contain signed URLs, native file IDs, credentials, file contents, base64 data, host paths, or temporary paths. -## Internal storage +## Migration behavior -DevSpace may use private, server-owned staging internally while materializing a file. This is an implementation detail: the staged record is removed automatically after the operation and is not exposed to the model as a reusable library or lifecycle API. +Older DevSpace builds may have created artifact tables or private artifact bytes. The one-shot implementation does not read, mutate, or delete that legacy data. Existing users can remove it manually after confirming they no longer need it; DevSpace does not opportunistically delete user data during migration. diff --git a/docs/configuration.md b/docs/configuration.md index cc761e68..360de695 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,10 +39,10 @@ npx @waishnav/devspace config set publicBaseUrl https://devspace.example.com | `DEVSPACE_WORKTREE_ROOT` | Directory for managed Git worktrees. Defaults to `~/.devspace/worktrees`. | | `DEVSPACE_STATE_DIR` | Directory for SQLite state. Defaults to `~/.local/share/devspace`. | -## Native Artifact Staging +## Native Artifact Download -Private native-file staging is disabled by default. Enable it when ChatGPT needs -to hand an attached or generated file to commands running on the DevSpace host: +Native-file download is disabled by default. Enable it when ChatGPT needs to hand +an attached or generated file into an already-open workspace: ```bash DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve @@ -50,25 +50,22 @@ DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve | Variable | Default | Purpose | | --- | --- | --- | -| `DEVSPACE_ARTIFACTS` | `0` | Expose `materialize_artifact` for trusted native files. | -| `DEVSPACE_ARTIFACT_ROOT` | `~/.local/share/devspace/artifacts` | Private storage root outside repositories and worktrees. | -| `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` | `104857600` | Maximum decoded size of one artifact (100 MiB). | -| `DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES` | `1073741824` | Maximum combined stored-object and in-progress bytes (1 GiB). | -| `DEVSPACE_ARTIFACT_TTL_HOURS` | `24` | Default lifetime of an unpinned staged artifact. | +| `DEVSPACE_ARTIFACTS` | `0` | Expose `download_artifact` for trusted native files. | +| `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` | `104857600` | Maximum streamed size of one file (100 MiB). | The same settings may be persisted in `~/.devspace/config.json` as -`artifactsEnabled`, `artifactRoot`, `artifactMaxFileBytes`, -`artifactMaxTotalBytes`, and `artifactDefaultTtlHours`. - -`materialize_artifact` accepts only the native file object supplied by the MCP -connector and writes it to a relative destination inside an already-open workspace. -It does not accept arbitrary URL strings, paths, embedded credentials, or extra -object fields. Files are streamed through bounded private storage, then verified -and automatically cleaned up after materialization. Startup and periodic cleanup -also process stale internal state with bounded work per pass. - -See [Native Artifact Staging](artifact-exchange.md) for the supported connector -shape and security boundaries. +`artifactsEnabled` and `artifactMaxFileBytes`. + +`download_artifact` accepts only the native file object supplied by the MCP +connector plus a `workspaceId` returned by `open_workspace`. DevSpace chooses a +collision-free path under `.devspace/incoming/` and returns only that +workspace-relative path. It does not accept destinations, conflict modes, +expected hashes, arbitrary URL strings, local paths, embedded credentials, or +extra object fields. + +There is no artifact root, total quota, TTL, pinning, persistent database record, +or background artifact cleanup service. See [Native File Download](artifact-exchange.md) +for the supported connector shape and security boundaries. ## OAuth diff --git a/docs/security.md b/docs/security.md index a437f739..e4d6ab41 100644 --- a/docs/security.md +++ b/docs/security.md @@ -92,54 +92,51 @@ Managed worktrees reduce accidental edits to your active checkout, but they are not a security boundary. They are a workflow boundary for isolated coding sessions. -## Artifact Exchange - -The Artifact Exchange is an opt-in private byte-transfer seam. Its root is -separate from allowed workspace roots, repositories, worktrees, temporary -folders, and public static assets. - -DevSpace selects every storage path. Artifact names are metadata, not path -components. Names are Unicode-normalized and must be a single non-hidden -basename without separators, NUL bytes, or control characters. The store uses -content-addressed immutable objects, mode `0700` directories, mode `0600` -object and partial files, atomic promotion, realpath containment checks, -regular-file enforcement, and no-follow file opens where the platform supports -them. - -Artifact records are scoped to the authenticated OAuth client. The first -version remains under the existing owner-only `devspace` scope. It does not add -an unauthenticated download route or expose the artifact root with -`express.static`. - -Native staging is adapter-gated. The production server declares ChatGPT's -top-level `openai/fileParams` contract and accepts only the documented -`download_url`, `file_id`, optional MIME/filename aliases, and optional size. -Downloads use HTTPS on `files.oaiusercontent.com` or the constrained regional -OpenAI Azure account family -`oaisdmntpr.blob.core.windows.net`, where `` is lowercase -alphanumeric. This permits observed OpenAI regional accounts such as -`centralus`, `westcentralus`, and `centralindia`, but not arbitrary Azure Blob -accounts. Credentials, fragments, alternate ports, malformed IDs, extra fields, -and redirects outside that boundary fail closed. Opaque IDs are bounded metadata -and are never used as filenames or path components. - -The native materialization seam deliberately does not: - -- fetch arbitrary URLs -- expose a generic upload API -- expose persistent artifact IDs or a user-facing artifact library -- extract archives -- execute transferred content -- expand workspace allowlists -- preserve executable permissions -- publish or permanently retain content - -`materialize_artifact` is the explicit, bounded write path: it accepts only a -native MCP-host file value and writes only within an already-open allowed -workspace. It requires a relative destination and explicit conflict mode, -creates only real contained parent directories, rejects symlink/non-regular-file -paths, and verifies the final size and SHA-256. Any private staging is server -implementation detail and is cleaned up automatically. +## Native File Download + +Native file download is an opt-in, one-shot byte-transfer seam. It has no +persistent artifact root, object database, upload lifecycle, reusable artifact +ID, public download route, TTL, pinning, or quota ledger. + +The production server declares ChatGPT's top-level `openai/fileParams` contract +and accepts only the documented `download_url`, `file_id`, optional +MIME/filename aliases, and optional size. Downloads use HTTPS on +`files.oaiusercontent.com` or the constrained regional OpenAI Azure account +family `oaisdmntpr.blob.core.windows.net`, where `` is lowercase +alphanumeric. Arbitrary Azure Blob accounts, alternate ports, credentials, +fragments, malformed IDs, extra fields, and redirects outside that boundary fail +closed. Opaque IDs are bounded metadata and are never used as filenames or path +components. + +`download_artifact` accepts only the native file value plus a `workspaceId` +returned by `open_workspace`. DevSpace chooses a normalized, collision-free path +below `.devspace/incoming/`; callers cannot supply a destination, conflict mode, +expected hash, host path, or storage policy. + +The selected workspace is opened without following symlinks. DevSpace then +creates or opens `.devspace` and `incoming` through already-open parent directory +descriptors, and keeps partial creation, cleanup, and final publication anchored +to the open `incoming` descriptor. Replacing a pathname therefore cannot redirect +writes outside the selected workspace. Symlinked components, non-directories, +and group/world-writable incoming directories fail closed. Existing directories +are inspected but are never chmodded as a startup side effect. + +Bytes stream into an exclusive mode-`0600` partial under the configured per-file +limit. DevSpace computes SHA-256 while writing, verifies any size hint, chmods +and fsyncs through the still-open descriptor, then publishes the verified inode +with an atomic hard link. It does not path-chmod or path-hash the published file. +Partials are removed on success or failure; crash-leftover cleanup is bounded and +only considers owned, regular DevSpace partial files. + +The native download seam deliberately does not: + +- fetch arbitrary URLs or local paths; +- expose generic upload/chunk/stat/delete/copy tools; +- expose artifact IDs, signed URLs, host paths, temp paths, or raw content; +- extract archives or execute transferred content; +- expand workspace allowlists; +- preserve executable permissions; +- opportunistically delete legacy artifact tables or bytes. ## Logs @@ -148,7 +145,9 @@ disabled unless `DEVSPACE_LOG_SHELL_COMMANDS=1`. Do not enable shell command logging if commands may contain secrets. -Artifact tool logs contain bounded workspace, destination, conflict-mode, -hostname, byte-count, hash, and status metadata. `materialize_artifact` does not -log the opaque file value. Raw content, connector references, bearer credentials, -presigned URLs, and base64 chunks are never included in tool logs or tool results. +Artifact tool logs contain bounded workspace ID, validated hostname, +workspace-relative output path, byte count, hash, duration, and status metadata. +`download_artifact` does not log the opaque file value. Raw content, connector +references, native file IDs, bearer credentials, presigned URLs, host paths, +temporary paths, and base64 chunks are never included in tool logs or tool +results. diff --git a/package.json b/package.json index 3fb223d8..2a564f40 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "dev": "node scripts/dev-server.mjs", "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", - "test": "tsx src/config.test.ts && tsx src/artifacts.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/artifact-workspace.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", + "test": "tsx src/config.test.ts && tsx src/incoming-artifacts.test.ts && tsx src/artifact-download.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts new file mode 100644 index 00000000..0d8de1ac --- /dev/null +++ b/src/artifact-download.test.ts @@ -0,0 +1,261 @@ +import assert from "node:assert/strict"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + symlink, + utimes, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import * as z from "zod/v4"; +import { + artifactToolLogFields, + downloadIncomingArtifact, + registerArtifactTools, +} from "./artifact-tools.js"; +import { ArtifactError } from "./artifact-error.js"; +import { + IncomingArtifactAdapterRegistry, + type IncomingArtifactAdapter, +} from "./incoming-artifacts.js"; + +const root = await mkdtemp(join(tmpdir(), "devspace-artifact-download-test-")); + +try { + testOneToolContract(); + await testSafeDownloadAndCollision(join(root, "downloads")); + await testSizeLimitAndCleanup(join(root, "size-limit")); + await testCrashLeftoverCleanup(join(root, "stale-partials")); + await testSymlinkRejection(join(root, "symlinks")); + testLogRedaction(); +} finally { + await rm(root, { recursive: true, force: true }); +} + +function testOneToolContract(): void { + const registered = new Map; callback: (input: never) => unknown }>(); + const server = { + registerTool( + name: string, + descriptor: Record, + callback: (input: never) => unknown, + ) { + registered.set(name, { descriptor, callback }); + return {}; + }, + }; + + registerArtifactTools(server as never, { + config: { + artifactMaxFileBytes: 1024, + logging: { toolCalls: false }, + } as never, + workspaces: {} as never, + }); + + assert.deepEqual([...registered.keys()], ["download_artifact"]); + const descriptor = registered.get("download_artifact")?.descriptor; + assert.ok(descriptor); + assert.deepEqual(descriptor._meta, { "openai/fileParams": ["file"] }); + assert.deepEqual(Object.keys(descriptor.inputSchema as object).sort(), ["file", "workspaceId"]); + assert.deepEqual(Object.keys(descriptor.outputSchema as object), ["path"]); + + const fileSchema = (descriptor.inputSchema as z.ZodRawShape).file as z.ZodType; + const valid = { + download_url: "https://files.oaiusercontent.com/file_123/download?sig=secret", + file_id: "file_123", + mime_type: "image/png", + file_name: "generated.png", + }; + assert.deepEqual(fileSchema.parse(valid), valid); + assert.throws(() => fileSchema.parse({ file_id: "file_123" })); +} + +async function testSafeDownloadAndCollision(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + const bytes = Buffer.from("native artifact bytes\u0000\xff", "latin1"); + const registry = registryFor({ + name: "../../generated.png", + size: bytes.length, + stream: Readable.from([bytes]), + }); + + const first = await downloadIncomingArtifact({ + registry, + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + }); + assert.equal(first.path, ".devspace/incoming/generated.png"); + assert.deepEqual(await readFile(join(workspaceRoot, first.path)), bytes); + + const second = await downloadIncomingArtifact({ + registry: registryFor({ + name: "generated.png", + size: bytes.length, + stream: Readable.from([bytes]), + }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + }); + assert.equal(second.path, ".devspace/incoming/generated (1).png"); + assert.deepEqual(await readFile(join(workspaceRoot, second.path)), bytes); + + const entries = await readdir(join(workspaceRoot, ".devspace", "incoming")); + assert.deepEqual(entries.sort(), ["generated (1).png", "generated.png"]); +} + +async function testSizeLimitAndCleanup(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ + name: "too-large.bin", + size: 5, + stream: Readable.from([Buffer.from("12345")]), + }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 4, + file: { native: true }, + }), + "artifact_file_too_large", + ); + + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ + name: "stream-too-large.bin", + stream: Readable.from([Buffer.from("123"), Buffer.from("45")]), + }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 4, + file: { native: true }, + }), + "artifact_file_too_large", + ); + + const incoming = join(workspaceRoot, ".devspace", "incoming"); + assert.deepEqual(await readdir(incoming), []); +} + +async function testCrashLeftoverCleanup(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + await downloadIncomingArtifact({ + registry: registryFor({ name: "first.txt", stream: Readable.from(["first"]) }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + }); + + const incoming = join(workspaceRoot, ".devspace", "incoming"); + const stalePartial = join(incoming, ".devspace-download-stale.partial"); + const recentPartial = join(incoming, ".devspace-download-recent.partial"); + const unrelated = join(incoming, "keep-me.partial"); + await writeFile(stalePartial, "stale"); + await writeFile(recentPartial, "recent"); + await writeFile(unrelated, "unrelated"); + const old = new Date(Date.now() - (48 * 60 * 60 * 1_000)); + await utimes(stalePartial, old, old); + + await downloadIncomingArtifact({ + registry: registryFor({ name: "second.txt", stream: Readable.from(["second"]) }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + }); + + const entries = await readdir(incoming); + assert.equal(entries.includes(".devspace-download-stale.partial"), false); + assert.equal(entries.includes(".devspace-download-recent.partial"), true); + assert.equal(entries.includes("keep-me.partial"), true); +} + +async function testSymlinkRejection(testRoot: string): Promise { + if (process.platform === "win32") return; + + const outside = join(testRoot, "outside"); + await mkdir(outside, { recursive: true }); + + const linkedDevspaceRoot = join(testRoot, "linked-devspace-workspace"); + await mkdir(linkedDevspaceRoot, { recursive: true }); + await symlink(outside, join(linkedDevspaceRoot, ".devspace"), "dir"); + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), + workspaceId: "ws_test", + workspaceRoot: linkedDevspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + }), + "artifact_directory_unsafe", + ); + + const linkedIncomingRoot = join(testRoot, "linked-incoming-workspace"); + await mkdir(join(linkedIncomingRoot, ".devspace"), { recursive: true }); + await symlink(outside, join(linkedIncomingRoot, ".devspace", "incoming"), "dir"); + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), + workspaceId: "ws_test", + workspaceRoot: linkedIncomingRoot, + maxFileBytes: 1024, + file: { native: true }, + }), + "artifact_directory_unsafe", + ); +} + +function testLogRedaction(): void { + const fields = artifactToolLogFields({ + file: { + download_url: "https://files.oaiusercontent.com/file_123/download?sig=super-secret", + file_id: "file_secret", + file_name: "generated.png", + }, + workspaceId: "ws_secret", + }); + const serialized = JSON.stringify(fields); + assert.equal(serialized.includes("super-secret"), false); + assert.equal(serialized.includes("file_secret"), false); + assert.equal(serialized.includes("ws_secret"), true); + assert.equal(serialized.includes("files.oaiusercontent.com"), true); +} + +function registryFor(source: { + name: string; + mimeType?: string; + size?: number; + stream: Readable; +}): IncomingArtifactAdapterRegistry { + const adapter: IncomingArtifactAdapter = { + id: "test-native", + canHandle: () => true, + async open() { + return source; + }, + }; + return new IncomingArtifactAdapterRegistry([adapter]); +} + +async function expectArtifactError(promise: Promise, code: string): Promise { + await assert.rejects( + promise, + (error: unknown) => error instanceof ArtifactError && error.code === code, + ); +} diff --git a/src/artifact-error.ts b/src/artifact-error.ts new file mode 100644 index 00000000..a6076f30 --- /dev/null +++ b/src/artifact-error.ts @@ -0,0 +1,9 @@ +export class ArtifactError extends Error { + readonly code: string; + + constructor(code: string, message: string) { + super(message); + this.name = "ArtifactError"; + this.code = code; + } +} diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index 51e0c524..702aafd6 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -1,17 +1,20 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants as fsConstants } from "node:fs"; +import { + link, + lstat, + mkdir, + open, + readdir, + unlink, + type FileHandle, +} from "node:fs/promises"; +import { basename, extname, join } from "node:path"; import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; -import { basename, isAbsolute, relative } from "node:path"; import * as z from "zod/v4"; +import { ArtifactError } from "./artifact-error.js"; import type { ServerConfig } from "./config.js"; -import { - copyArtifactToWorkspace, - type ArtifactCopyConflictMode, -} from "./artifact-workspace.js"; -import { - ARTIFACT_CHUNK_BYTES, - ArtifactError, - type ArtifactStore, -} from "./artifacts.js"; import { describeIncomingArtifactValue, IncomingArtifactAdapterRegistry, @@ -24,8 +27,17 @@ const ARTIFACT_WRITE_ANNOTATIONS = { readOnlyHint: false, destructiveHint: true, idempotentHint: false, - openWorldHint: false, + openWorldHint: true, }; +const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; +const DIRECTORY_FLAGS = fsConstants.O_RDONLY | (fsConstants.O_DIRECTORY ?? 0) | NO_FOLLOW; +const MAX_RENAME_ATTEMPTS = 10_000; +const MAX_SAFE_FILENAME_BYTES = 180; +const MAX_SAFE_EXTENSION_BYTES = 32; +const PARTIAL_PREFIX = ".devspace-download-"; +const PARTIAL_SUFFIX = ".partial"; +const STALE_PARTIAL_AGE_MS = 24 * 60 * 60 * 1_000; +const MAX_STALE_PARTIAL_CLEANUP = 32; const openAIFileReferenceInputSchema = z.object({ download_url: z.string(), @@ -36,61 +48,36 @@ const openAIFileReferenceInputSchema = z.object({ size: z.number().int().nonnegative().nullable().optional(), }); -type ArtifactToolName = "materialize_artifact"; - export interface ArtifactToolRegistrationOptions { config: ServerConfig; - store: ArtifactStore; workspaces: WorkspaceRegistry; - clientId: string; incomingArtifactAdapters?: readonly IncomingArtifactAdapter[]; } -export interface StageArtifactInput { - file: unknown; - workspaceId?: string; - expectedSha256?: string; - ttlHours?: number; - pin?: boolean; -} - -export interface StageArtifactResult { - artifactId: string; - name: string; - mimeType?: string; - size: number; - sha256: string; - hostPath: string; - expiresAt?: string; - instruction: string; -} - -export interface MaterializeArtifactInput { +export interface DownloadIncomingArtifactInput { file: unknown; workspaceId: string; - destination: string; - onConflict: ArtifactCopyConflictMode; - expectedSha256?: string; } -export interface MaterializeArtifactResult { - workspaceId: string; +export interface DownloadIncomingArtifactResult { path: string; - name: string; - mimeType?: string; size: number; sha256: string; - onConflict: ArtifactCopyConflictMode; - renamed: boolean; +} + +interface SecureIncomingDirectory { + rootHandle: FileHandle; + devspaceHandle: FileHandle; + incomingHandle: FileHandle; + anchorPath: string; + close(): Promise; } export function registerArtifactTools( server: McpServer, { config, - store, workspaces, - clientId, incomingArtifactAdapters = [], }: ArtifactToolRegistrationOptions, ): void { @@ -98,196 +85,175 @@ export function registerArtifactTools( registerAppTool( server, - "materialize_artifact", + "download_artifact", { - title: "Materialize attached or generated file", + title: "Download attached or generated file", description: - "Save one MCP-host-provided native file into an already-open workspace. DevSpace validates the file reference, streams and verifies the bytes, and atomically writes the destination. Arbitrary URLs and host paths are rejected.", + "Stream one MCP-host-provided native file into the selected workspace's .devspace/incoming directory. DevSpace chooses a collision-free filename and returns its workspace-relative path. Arbitrary URLs, local paths, and malformed file objects are rejected.", inputSchema: { - file: openAIFileReferenceInputSchema.describe("Native file value authorized and supplied by the MCP host."), - workspaceId: z.string().min(1).describe("Workspace identifier returned by open_workspace."), - destination: z.string().min(1).describe("Relative file path inside the selected workspace."), - onConflict: z.enum(["error", "rename", "replace"]).default("error").describe("How to handle an existing destination."), - expectedSha256: z.string().optional().describe("Optional expected SHA-256, with or without a sha256: prefix."), + file: openAIFileReferenceInputSchema.describe( + "Native file value authorized and supplied by the MCP host.", + ), + workspaceId: z.string().min(1).describe( + "Workspace identifier returned by open_workspace.", + ), }, outputSchema: { - workspaceId: z.string(), path: z.string(), - name: z.string(), - mimeType: z.string().optional(), - size: z.number().int().nonnegative(), - sha256: z.string(), - onConflict: z.enum(["error", "rename", "replace"]), - renamed: z.boolean(), }, _meta: { "openai/fileParams": ["file"] }, annotations: ARTIFACT_WRITE_ANNOTATIONS, }, - async (input) => executeArtifactTool(config, "materialize_artifact", input, async () => { - if (isAbsolute(input.destination)) { - throw new ArtifactError("workspace_destination_invalid", "Artifact destination must be a relative workspace path."); - } + async (input) => executeArtifactTool(config, input, async () => { const workspace = workspaces.getWorkspace(input.workspaceId); - const destination = workspaces.resolvePath(workspace, input.destination); - return materializeIncomingArtifact({ - store, - clientId, + const downloaded = await downloadIncomingArtifact({ registry: incomingRegistry, workspaceId: workspace.id, workspaceRoot: workspace.root, - destination, - input, + maxFileBytes: config.artifactMaxFileBytes, + file: input.file, }); + return { + publicResult: { path: downloaded.path }, + logResult: downloaded, + }; }), ); - } -/** Materialize a native file through the private store, then remove its internal record. */ -export async function materializeIncomingArtifact({ - store, - clientId, +/** + * Stream a trusted native file directly into one already-open workspace. + * + * Bytes are written to an exclusive private partial, hashed and size-checked, + * chmodded through the still-open file descriptor, fsynced, and only then + * published with an atomic hard link. No path-based chmod/hash/verification is + * performed after publication. + */ +export async function downloadIncomingArtifact({ registry, workspaceId, workspaceRoot, - destination, - input, + maxFileBytes, + file, }: { - store: ArtifactStore; - clientId: string; registry: IncomingArtifactAdapterRegistry; workspaceId: string; workspaceRoot: string; - destination: string; - input: MaterializeArtifactInput; -}): Promise { - const staged = await stageIncomingArtifact({ - store, - clientId, - registry, - input: { - file: input.file, - workspaceId, - expectedSha256: input.expectedSha256, - }, - }); - - try { - const copied = await copyArtifactToWorkspace({ - store, - clientId, - workspaceId, - workspaceRoot, - artifactId: staged.artifactId, - destination, - onConflict: input.onConflict, - }); - const path = relative(workspaceRoot, copied.path); - return { - workspaceId, - path, - name: basename(path), - mimeType: staged.mimeType, - size: copied.size, - sha256: copied.sha256, - onConflict: copied.onConflict, - renamed: copied.renamed, - }; - } finally { - await store.deleteArtifact(clientId, staged.artifactId).catch(() => undefined); + maxFileBytes: number; + file: unknown; +}): Promise { + if (!Number.isSafeInteger(maxFileBytes) || maxFileBytes < 1) { + throw new ArtifactError( + "artifact_limit_invalid", + "Artifact file-size limit must be a positive integer.", + ); + } + if (!workspaceId) { + throw new ArtifactError( + "artifact_workspace_invalid", + "A selected workspace is required for native file download.", + ); } -} -export async function stageIncomingArtifact({ - store, - clientId, - registry, - input, -}: { - store: ArtifactStore; - clientId: string; - registry: IncomingArtifactAdapterRegistry; - input: StageArtifactInput; -}): Promise { - const opened = await registry.open(input.file); - let uploadId: string | undefined; + const opened = await registry.open(file); + let secureDirectory: SecureIncomingDirectory | undefined; + let partialPath: string | undefined; + let handle: FileHandle | undefined; try { - const upload = await store.beginUpload(clientId, { - filename: opened.name, - mimeType: opened.mimeType, - size: opened.size, - sha256: input.expectedSha256, - workspaceId: input.workspaceId, - ttlHours: input.ttlHours, - }); - uploadId = upload.uploadId; - - let offset = 0; + if (opened.size !== undefined && opened.size > maxFileBytes) { + throw new ArtifactError( + "artifact_file_too_large", + "Native file exceeds the configured per-file limit.", + ); + } + + secureDirectory = await prepareIncomingDirectory(workspaceRoot); + await cleanupStalePartials(secureDirectory); + await assertPrivateDirectoryHandle(secureDirectory.incomingHandle); + + partialPath = join( + secureDirectory.anchorPath, + `${PARTIAL_PREFIX}${randomUUID()}${PARTIAL_SUFFIX}`, + ); + handle = await open( + partialPath, + fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | NO_FOLLOW, + 0o600, + ); + + const hash = createHash("sha256"); + let size = 0; for await (const value of opened.stream) { - const bytes = incomingStreamChunk(value); - for (let chunkOffset = 0; chunkOffset < bytes.length; chunkOffset += ARTIFACT_CHUNK_BYTES) { - const chunk = bytes.subarray(chunkOffset, chunkOffset + ARTIFACT_CHUNK_BYTES); - await store.uploadChunk(clientId, { - uploadId, - offset, - dataBase64: chunk.toString("base64"), - }); - offset += chunk.length; + const chunk = incomingStreamChunk(value); + if (size + chunk.length > maxFileBytes) { + throw new ArtifactError( + "artifact_file_too_large", + "Native file exceeds the configured per-file limit.", + ); } + await writeAll(handle, chunk, size); + hash.update(chunk); + size += chunk.length; + } + + if (opened.size !== undefined && opened.size !== size) { + throw new ArtifactError( + "artifact_file_size_mismatch", + "Native file metadata did not match the downloaded content.", + ); } - const artifact = await store.commitUpload(clientId, uploadId, { - source: `incoming:${opened.adapterId}`, - pinned: input.pin === true, - }); - uploadId = undefined; + await handle.chmod(0o644); + await handle.sync(); + const writtenEntry = await handle.stat(); + if (!writtenEntry.isFile() || writtenEntry.size !== size) { + throw new ArtifactError( + "artifact_write_integrity_failed", + "Native file could not be verified before publication.", + ); + } + + await assertPrivateDirectoryHandle(secureDirectory.incomingHandle); + const partialEntry = await lstat(partialPath); + if ( + partialEntry.isSymbolicLink() + || !partialEntry.isFile() + || partialEntry.dev !== writtenEntry.dev + || partialEntry.ino !== writtenEntry.ino + || partialEntry.size !== writtenEntry.size + ) { + throw new ArtifactError( + "artifact_partial_unsafe", + "Native file partial changed before publication.", + ); + } + + const safeName = normalizeArtifactFilename(opened.name); + const finalName = await publishCollisionFree( + secureDirectory, + partialPath, + safeName, + ); + await unlink(partialPath); + partialPath = undefined; + return { - artifactId: artifact.artifactId, - name: artifact.name, - mimeType: artifact.mimeType, - size: artifact.size, - sha256: artifact.sha256, - hostPath: artifact.hostPath, - expiresAt: artifact.expiresAt, - instruction: - "Pass hostPath to a local command. stage_artifact never writes into a workspace or repository.", + path: `.devspace/incoming/${finalName}`, + size, + sha256: `sha256:${hash.digest("hex")}`, }; } catch (error) { opened.stream.destroy(); - if (uploadId) { - await store.abortUpload(clientId, uploadId).catch(() => undefined); - } throw error; + } finally { + await handle?.close().catch(() => undefined); + if (partialPath) await unlink(partialPath).catch(() => undefined); + await secureDirectory?.close().catch(() => undefined); } } -function incomingFileDownloadHostname(value: unknown): string | undefined { - if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; - const rawUrl = (value as Record).download_url; - if (typeof rawUrl !== "string") return undefined; - try { - const hostname = new URL(rawUrl).hostname.toLowerCase(); - return hostname.length > 0 && hostname.length <= 253 ? hostname : undefined; - } catch { - return undefined; - } -} - -function incomingStreamChunk(value: unknown): Buffer { - if (Buffer.isBuffer(value)) return value; - if (typeof value === "string") return Buffer.from(value); - if (value instanceof Uint8Array) { - return Buffer.from(value.buffer, value.byteOffset, value.byteLength); - } - throw new ArtifactError( - "invalid_incoming_artifact_chunk", - "Incoming artifact stream yielded a value that is not bytes or text.", - ); -} - export function artifactToolLogFields( - tool: ArtifactToolName, input: Record, ): Record { return { @@ -295,36 +261,37 @@ export function artifactToolLogFields( fileReferenceShape: describeIncomingArtifactValue(input.file), downloadUrlHostname: incomingFileDownloadHostname(input.file), workspaceId: input.workspaceId, - destination: input.destination, - expectedSha256Present: typeof input.expectedSha256 === "string", - onConflict: input.onConflict, }; } -async function executeArtifactTool( +async function executeArtifactTool( config: ServerConfig, - tool: ArtifactToolName, input: Record, - operation: () => Promise | T, + operation: () => Promise<{ + publicResult: { path: string }; + logResult: DownloadIncomingArtifactResult; + }>, ) { const startedAt = performance.now(); try { - const result = await operation(); + const { publicResult, logResult } = await operation(); if (config.logging.toolCalls) { logEvent(config.logging, "info", "artifact_tool_call", { - tool, - ...artifactToolLogFields(tool, input), - ...artifactResultLogFields(result as Record), + tool: "download_artifact", + ...artifactToolLogFields(input), + path: logResult.path, + size: logResult.size, + sha256: logResult.sha256, success: true, durationMs: Math.round(performance.now() - startedAt), }); } - return artifactToolResponse(result); + return artifactToolResponse(publicResult); } catch (error) { if (config.logging.toolCalls) { logEvent(config.logging, "warn", "artifact_tool_call", { - tool, - ...artifactToolLogFields(tool, input), + tool: "download_artifact", + ...artifactToolLogFields(input), success: false, errorCode: error instanceof ArtifactError ? error.code : "internal_error", durationMs: Math.round(performance.now() - startedAt), @@ -334,19 +301,287 @@ async function executeArtifactTool( } } -function artifactToolResponse(result: T) { +function artifactToolResponse(result: { path: string }) { return { content: [{ type: "text" as const, text: JSON.stringify(result) }], - structuredContent: result as Record, + structuredContent: result, }; } -function artifactResultLogFields(result: Record): Record { - return { - workspaceId: result.workspaceId, - path: result.path, - size: result.size, - sha256: result.sha256, - renamed: result.renamed, - }; +async function prepareIncomingDirectory( + workspaceRoot: string, +): Promise { + let rootHandle: FileHandle | undefined; + let devspaceHandle: FileHandle | undefined; + let incomingHandle: FileHandle | undefined; + + try { + rootHandle = await openDirectoryNoFollow( + workspaceRoot, + "artifact_workspace_unsafe", + "Selected workspace root is not a real directory.", + ); + const rootAnchor = descriptorDirectoryPath(rootHandle); + devspaceHandle = await ensureChildDirectory(rootHandle, rootAnchor, ".devspace"); + const devspaceAnchor = descriptorDirectoryPath(devspaceHandle); + incomingHandle = await ensureChildDirectory( + devspaceHandle, + devspaceAnchor, + "incoming", + ); + const anchorPath = descriptorDirectoryPath(incomingHandle); + + return { + rootHandle, + devspaceHandle, + incomingHandle, + anchorPath, + async close() { + await incomingHandle?.close().catch(() => undefined); + await devspaceHandle?.close().catch(() => undefined); + await rootHandle?.close().catch(() => undefined); + }, + }; + } catch (error) { + await incomingHandle?.close().catch(() => undefined); + await devspaceHandle?.close().catch(() => undefined); + await rootHandle?.close().catch(() => undefined); + throw error; + } +} + +async function ensureChildDirectory( + parentHandle: FileHandle, + parentAnchor: string, + name: string, +): Promise { + await assertDirectoryHandle(parentHandle); + const path = join(parentAnchor, name); + try { + await mkdir(path, { mode: 0o700 }); + } catch (error) { + if (!isNodeError(error) || error.code !== "EEXIST") throw error; + } + + const child = await openDirectoryNoFollow( + path, + "artifact_directory_unsafe", + "Artifact destination parent is not a real directory.", + ); + try { + await assertPrivateDirectoryHandle(child); + return child; + } catch (error) { + await child.close().catch(() => undefined); + throw error; + } +} + +async function openDirectoryNoFollow( + path: string, + code: string, + message: string, +): Promise { + let handle: FileHandle | undefined; + try { + handle = await open(path, DIRECTORY_FLAGS); + await assertDirectoryHandle(handle); + return handle; + } catch (error) { + await handle?.close().catch(() => undefined); + if (error instanceof ArtifactError) throw error; + throw new ArtifactError(code, message); + } +} + +async function assertDirectoryHandle(handle: FileHandle): Promise { + const entry = await handle.stat(); + if (!entry.isDirectory()) { + throw new ArtifactError( + "artifact_directory_unsafe", + "Artifact destination parent is not a directory.", + ); + } +} + +async function assertPrivateDirectoryHandle(handle: FileHandle): Promise { + const entry = await handle.stat(); + if (!entry.isDirectory()) { + throw new ArtifactError( + "artifact_directory_unsafe", + "Artifact destination parent is not a directory.", + ); + } + if (process.platform === "win32") return; + const currentUid = process.getuid?.(); + if ( + (currentUid !== undefined && entry.uid !== currentUid) + || (Number(entry.mode) & 0o022) !== 0 + ) { + throw new ArtifactError( + "artifact_directory_permissions_unsafe", + "Artifact destination directory must be owned by the current user and not group/world writable.", + ); + } +} + +function descriptorDirectoryPath(handle: FileHandle): string { + if (process.platform === "linux") return `/proc/self/fd/${handle.fd}`; + if (["darwin", "freebsd", "openbsd", "netbsd"].includes(process.platform)) { + return `/dev/fd/${handle.fd}`; + } + throw new ArtifactError( + "artifact_platform_unsupported", + "Native file download requires descriptor-anchored directory operations on this platform.", + ); +} + +async function publishCollisionFree( + directory: SecureIncomingDirectory, + partialPath: string, + filename: string, +): Promise { + for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt += 1) { + await assertPrivateDirectoryHandle(directory.incomingHandle); + const candidateName = attempt === 0 + ? filename + : renamedFilename(filename, attempt); + const candidate = join(directory.anchorPath, candidateName); + try { + await link(partialPath, candidate); + return candidateName; + } catch (error) { + if (isNodeError(error) && error.code === "EEXIST") continue; + throw error; + } + } + throw new ArtifactError( + "artifact_filename_exhausted", + "Could not choose an available incoming filename.", + ); +} + +export function normalizeArtifactFilename(value: string): string { + const flattened = value.replaceAll("\\", "/"); + let candidate = basename(flattened) + .replace(/[\u0000-\u001F\u007F]/gu, "") + .trim(); + if (!candidate || candidate === "." || candidate === ".." || candidate.startsWith(".")) { + candidate = "download.bin"; + } + + const extension = extname(candidate); + const safeExtension = Buffer.byteLength(extension, "utf8") <= MAX_SAFE_EXTENSION_BYTES + ? extension + : ""; + const stemCharacters = Array.from( + safeExtension ? candidate.slice(0, -safeExtension.length) : candidate, + ); + while ( + stemCharacters.length > 1 + && Buffer.byteLength(`${stemCharacters.join("")}${safeExtension}`, "utf8") + > MAX_SAFE_FILENAME_BYTES + ) { + stemCharacters.pop(); + } + candidate = `${stemCharacters.join("")}${safeExtension}`; + return Buffer.byteLength(candidate, "utf8") <= MAX_SAFE_FILENAME_BYTES + ? candidate + : "download.bin"; +} + +function renamedFilename(filename: string, attempt: number): string { + const extension = extname(filename); + const stem = extension ? filename.slice(0, -extension.length) : filename; + return `${stem} (${attempt})${extension}`; +} + +async function cleanupStalePartials( + directory: SecureIncomingDirectory, +): Promise { + await assertPrivateDirectoryHandle(directory.incomingHandle); + const entries = await readdir(directory.anchorPath, { withFileTypes: true }); + let inspected = 0; + const cutoff = Date.now() - STALE_PARTIAL_AGE_MS; + for (const entry of entries) { + if (inspected >= MAX_STALE_PARTIAL_CLEANUP) break; + if ( + !entry.name.startsWith(PARTIAL_PREFIX) + || !entry.name.endsWith(PARTIAL_SUFFIX) + ) continue; + inspected += 1; + + const path = join(directory.anchorPath, entry.name); + const metadata = await lstatOrUndefined(path); + if ( + !metadata + || metadata.isSymbolicLink() + || !metadata.isFile() + || metadata.mtimeMs >= cutoff + || (process.getuid?.() !== undefined && metadata.uid !== process.getuid?.()) + ) continue; + await unlink(path).catch(() => undefined); + } +} + +async function writeAll( + handle: FileHandle, + buffer: Buffer, + position: number, +): Promise { + let offset = 0; + while (offset < buffer.length) { + const { bytesWritten } = await handle.write( + buffer, + offset, + buffer.length - offset, + position + offset, + ); + if (bytesWritten <= 0) { + throw new ArtifactError( + "artifact_short_write", + "Native file was not fully written.", + ); + } + offset += bytesWritten; + } +} + +function incomingFileDownloadHostname(value: unknown): string | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return undefined; + } + const rawUrl = (value as Record).download_url; + if (typeof rawUrl !== "string") return undefined; + try { + const hostname = new URL(rawUrl).hostname.toLowerCase(); + return hostname.length > 0 && hostname.length <= 253 ? hostname : undefined; + } catch { + return undefined; + } +} + +function incomingStreamChunk(value: unknown): Buffer { + if (Buffer.isBuffer(value)) return value; + if (typeof value === "string") return Buffer.from(value); + if (value instanceof Uint8Array) { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength); + } + throw new ArtifactError( + "invalid_incoming_artifact_chunk", + "Incoming artifact stream yielded a value that is not bytes or text.", + ); +} + +async function lstatOrUndefined(path: string) { + try { + return await lstat(path); + } catch (error) { + if (isNodeError(error) && error.code === "ENOENT") return undefined; + throw error; + } +} + +function isNodeError(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; } diff --git a/src/artifact-workspace.test.ts b/src/artifact-workspace.test.ts index db4269bd..ae2e2ae0 100644 --- a/src/artifact-workspace.test.ts +++ b/src/artifact-workspace.test.ts @@ -1,152 +1,2 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { copyArtifactToWorkspace } from "./artifact-workspace.js"; -import { ArtifactStore } from "./artifacts.js"; - -const root = await mkdtemp(join(tmpdir(), "devspace-artifact-workspace-test-")); - -try { - const workspaceRoot = join(root, "workspace"); - await mkdir(workspaceRoot, { recursive: true }); - const store = new ArtifactStore({ - stateDir: join(root, "state"), - artifactRoot: join(root, "artifacts"), - artifactMaxFileBytes: 1024 * 1024, - artifactMaxTotalBytes: 4 * 1024 * 1024, - artifactDefaultTtlHours: 24, - }); - try { - const bytes = Buffer.from("native ChatGPT artifact bytes\u0000\xff", "latin1"); - const upload = await store.beginUpload("client-a", { - filename: "generated.png", - mimeType: "image/png", - size: bytes.length, - sha256: createHash("sha256").update(bytes).digest("hex"), - }); - await store.uploadChunk("client-a", { - uploadId: upload.uploadId, - offset: 0, - dataBase64: bytes.toString("base64"), - }); - const artifact = await store.commitUpload("client-a", upload.uploadId, { - source: "incoming:openai-file", - }); - - const destination = join(workspaceRoot, "assets", "generated.png"); - const copied = await copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination, - onConflict: "error", - }); - - assert.deepEqual(await readFile(destination), bytes); - assert.equal(copied.path, destination); - assert.equal(copied.sha256, artifact.sha256); - assert.equal(copied.renamed, false); - - await writeFile(destination, "existing"); - await expectArtifactError( - copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination, - onConflict: "error", - }), - "workspace_destination_exists", - ); - assert.equal(await readFile(destination, "utf8"), "existing"); - - const renamed = await copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination, - onConflict: "rename", - }); - assert.equal(renamed.path, join(workspaceRoot, "assets", "generated (1).png")); - assert.equal(renamed.renamed, true); - assert.deepEqual(await readFile(renamed.path), bytes); - - const replaced = await copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination, - onConflict: "replace", - }); - assert.equal(replaced.path, destination); - assert.equal(replaced.renamed, false); - assert.deepEqual(await readFile(destination), bytes); - - await expectArtifactError( - copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination: join(workspaceRoot, "..", "outside.png"), - onConflict: "error", - }), - "workspace_path_escape", - ); - - if (process.platform !== "win32") { - const outsideRoot = join(root, "outside"); - await mkdir(outsideRoot, { recursive: true }); - const linkedParent = join(workspaceRoot, "linked-parent"); - await symlink(outsideRoot, linkedParent, "dir"); - await expectArtifactError( - copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination: join(linkedParent, "escaped.png"), - onConflict: "error", - }), - "workspace_parent_unsafe", - ); - const linkedDestination = join(workspaceRoot, "linked.png"); - await symlink(join(outsideRoot, "outside.png"), linkedDestination); - await expectArtifactError( - copyArtifactToWorkspace({ - store, - clientId: "client-a", - workspaceId: "ws_test", - workspaceRoot, - artifactId: artifact.artifactId, - destination: linkedDestination, - onConflict: "replace", - }), - "workspace_destination_unsafe", - ); - } - } finally { - store.close(); - } -} finally { - await rm(root, { recursive: true, force: true }); -} - -async function expectArtifactError(promise: Promise, code: string): Promise { - await assert.rejects( - promise, - (error: unknown) => error instanceof Error && "code" in error && error.code === code, - ); -} +// Workspace materializer coverage was replaced by artifact-download.test.ts. +export {}; diff --git a/src/artifact-workspace.ts b/src/artifact-workspace.ts index 574226cd..d769053f 100644 --- a/src/artifact-workspace.ts +++ b/src/artifact-workspace.ts @@ -1,250 +1,3 @@ -import { createHash, randomUUID } from "node:crypto"; -import { constants as fsConstants, createReadStream } from "node:fs"; -import { chmod, link, lstat, mkdir, open, realpath, rename, unlink } from "node:fs/promises"; -import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path"; -import { ArtifactError, type ArtifactStore } from "./artifacts.js"; - -const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; -const MAX_RENAME_ATTEMPTS = 10_000; - -export type ArtifactCopyConflictMode = "error" | "rename" | "replace"; - -export interface ArtifactCopyToWorkspaceInput { - store: ArtifactStore; - clientId: string; - workspaceId: string; - workspaceRoot: string; - artifactId: string; - destination: string; - onConflict: ArtifactCopyConflictMode; -} - -export interface ArtifactCopyToWorkspaceResult { - artifactId: string; - workspaceId: string; - path: string; - size: number; - sha256: string; - onConflict: ArtifactCopyConflictMode; - renamed: boolean; -} - -/** Copy one owner-scoped private artifact into one already-open allowed workspace. */ -export async function copyArtifactToWorkspace( - input: ArtifactCopyToWorkspaceInput, -): Promise { - assertLexicalWorkspaceContainment(input.workspaceRoot, input.destination); - if (resolve(input.destination) === resolve(input.workspaceRoot)) { - throw new ArtifactError("workspace_destination_invalid", "Artifact destination must name a file inside the workspace."); - } - - const artifact = input.store.statArtifact(input.clientId, input.artifactId); - const parent = dirname(input.destination); - await ensureContainedWorkspaceDirectory(input.workspaceRoot, parent); - if (input.onConflict === "error" && await lstatOrUndefined(input.destination)) { - throw new ArtifactError("workspace_destination_exists", "Artifact destination already exists. Select rename or replace explicitly."); - } - - const tempPath = join(parent, `.devspace-artifact-${randomUUID()}.partial`); - let finalPath = input.destination; - let materialized = false; - let replaceBackupPath: string | undefined; - try { - const copied = await copyVerifiedArtifactToTemp(artifact.hostPath, tempPath, artifact.size, artifact.sha256); - await assertContainedWorkspaceRegularFile(input.workspaceRoot, tempPath); - - if (input.onConflict === "replace") { - const existing = await lstatOrUndefined(finalPath); - if (existing && (existing.isSymbolicLink() || !existing.isFile())) { - throw new ArtifactError("workspace_destination_unsafe", "Replace is allowed only for an existing regular file."); - } - await ensureContainedWorkspaceDirectory(input.workspaceRoot, parent); - if (existing) { - replaceBackupPath = join(parent, `.devspace-artifact-${randomUUID()}.backup`); - await rename(finalPath, replaceBackupPath); - await assertContainedWorkspaceRegularFile(input.workspaceRoot, replaceBackupPath); - } - await rename(tempPath, finalPath); - materialized = true; - } else { - let linked = false; - const attempts = input.onConflict === "rename" ? MAX_RENAME_ATTEMPTS : 1; - for (let attempt = 0; attempt < attempts; attempt += 1) { - finalPath = attempt === 0 ? input.destination : renamedDestination(input.destination, attempt); - assertLexicalWorkspaceContainment(input.workspaceRoot, finalPath); - try { - await link(tempPath, finalPath); - linked = true; - break; - } catch (error) { - if (isNodeError(error) && error.code === "EEXIST" && input.onConflict === "rename") continue; - if (isNodeError(error) && error.code === "EEXIST") { - throw new ArtifactError("workspace_destination_exists", "Artifact destination already exists. Select rename or replace explicitly."); - } - throw error; - } - } - if (!linked) { - throw new ArtifactError("workspace_rename_exhausted", "Could not find an available destination filename."); - } - materialized = true; - await unlink(tempPath); - } - - await chmod(finalPath, 0o644); - const finalEntry = await assertContainedWorkspaceRegularFile(input.workspaceRoot, finalPath); - const digest = await hashFile(finalPath); - if (finalEntry.size !== copied.size || `sha256:${digest}` !== copied.sha256) { - throw new ArtifactError("workspace_copy_integrity_failed", "Workspace copy failed size or SHA-256 verification."); - } - if (replaceBackupPath) { - await unlink(replaceBackupPath).catch(() => undefined); - replaceBackupPath = undefined; - } - return { - artifactId: artifact.artifactId, - workspaceId: input.workspaceId, - path: finalPath, - size: finalEntry.size, - sha256: `sha256:${digest}`, - onConflict: input.onConflict, - renamed: finalPath !== input.destination, - }; - } catch (error) { - await unlink(tempPath).catch(() => undefined); - if (materialized) { - await unlink(finalPath).catch(() => undefined); - } - if (replaceBackupPath) { - await rename(replaceBackupPath, finalPath).catch(() => undefined); - } - throw error; - } -} - -async function assertContainedWorkspaceRegularFile(workspaceRoot: string, path: string) { - assertLexicalWorkspaceContainment(workspaceRoot, path); - const entry = await lstat(path); - if (entry.isSymbolicLink() || !entry.isFile()) { - throw new ArtifactError("workspace_destination_unsafe", "Workspace destination is not a regular non-symlink file."); - } - const rootRealPath = await realpath(workspaceRoot); - assertRealWorkspaceContainment(rootRealPath, await realpath(path)); - return entry; -} - -async function ensureContainedWorkspaceDirectory(workspaceRoot: string, directory: string): Promise { - assertLexicalWorkspaceContainment(workspaceRoot, directory); - const root = resolve(workspaceRoot); - const rootRealPath = await realpath(root); - const relationship = relative(root, resolve(directory)); - const parts = relationship === "" ? [] : relationship.split(sep); - let current = root; - for (const part of parts) { - if (!part || part === "." || part === "..") { - throw new ArtifactError("workspace_path_escape", "Workspace destination escapes the workspace root."); - } - current = join(current, part); - let entry = await lstatOrUndefined(current); - if (!entry) { - try { - await mkdir(current, { mode: 0o755 }); - } catch (error) { - if (!isNodeError(error) || error.code !== "EEXIST") throw error; - } - entry = await lstat(current); - } - if (entry.isSymbolicLink() || !entry.isDirectory()) { - throw new ArtifactError("workspace_parent_unsafe", "Workspace destination parent is not a real directory."); - } - assertRealWorkspaceContainment(rootRealPath, await realpath(current)); - } -} - -async function copyVerifiedArtifactToTemp( - sourcePath: string, - tempPath: string, - expectedSize: number, - expectedSha256: string, -): Promise<{ size: number; sha256: string }> { - const sourceEntry = await lstat(sourcePath); - if (sourceEntry.isSymbolicLink() || !sourceEntry.isFile() || sourceEntry.size !== expectedSize) { - throw new ArtifactError("artifact_integrity_failed", "Artifact source is not a regular file with the recorded size."); - } - const source = await open(sourcePath, fsConstants.O_RDONLY | NO_FOLLOW); - let destination: Awaited> | undefined; - const hash = createHash("sha256"); - let copied = 0; - try { - destination = await open(tempPath, fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | NO_FOLLOW, 0o600); - const buffer = Buffer.allocUnsafe(64 * 1024); - while (true) { - const { bytesRead } = await source.read(buffer, 0, buffer.length, copied); - if (bytesRead === 0) break; - const chunk = buffer.subarray(0, bytesRead); - await writeAll(destination, chunk, copied); - hash.update(chunk); - copied += bytesRead; - } - await destination.sync(); - } finally { - await source.close().catch(() => undefined); - await destination?.close().catch(() => undefined); - } - const sha256 = `sha256:${hash.digest("hex")}`; - if (copied !== expectedSize || sha256 !== expectedSha256) { - await unlink(tempPath).catch(() => undefined); - throw new ArtifactError("artifact_integrity_failed", "Artifact source failed size or SHA-256 verification during copy."); - } - await chmod(tempPath, 0o644); - return { size: copied, sha256 }; -} - -async function writeAll(handle: Awaited>, buffer: Buffer, position: number): Promise { - let offset = 0; - while (offset < buffer.length) { - const { bytesWritten } = await handle.write(buffer, offset, buffer.length - offset, position + offset); - if (bytesWritten <= 0) throw new ArtifactError("short_write", "Artifact workspace copy was not fully written."); - offset += bytesWritten; - } -} - -function renamedDestination(path: string, attempt: number): string { - const extension = extname(path); - const filename = basename(path); - const stem = extension ? filename.slice(0, -extension.length) : filename; - return join(dirname(path), `${stem} (${attempt})${extension}`); -} - -function assertLexicalWorkspaceContainment(workspaceRoot: string, candidate: string): void { - const relationship = relative(resolve(workspaceRoot), resolve(candidate)); - if (relationship.startsWith("..") || isAbsolute(relationship)) { - throw new ArtifactError("workspace_path_escape", "Workspace path escapes the workspace root."); - } -} - -function assertRealWorkspaceContainment(rootRealPath: string, candidateRealPath: string): void { - const relationship = relative(rootRealPath, candidateRealPath); - if (relationship.startsWith("..") || isAbsolute(relationship)) { - throw new ArtifactError("workspace_path_escape", "Workspace path resolves outside the workspace root."); - } -} - -async function hashFile(path: string): Promise { - const hash = createHash("sha256"); - for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); - return hash.digest("hex"); -} - -async function lstatOrUndefined(path: string) { - try { - return await lstat(path); - } catch (error) { - if (isNodeError(error) && error.code === "ENOENT") return undefined; - throw error; - } -} - -function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && "code" in error; -} +// The persistent-store workspace materializer was removed. Native files now +// stream directly into .devspace/incoming through download_artifact. +export {}; diff --git a/src/artifacts.test.ts b/src/artifacts.test.ts index 35413eb6..adf921ef 100644 --- a/src/artifacts.test.ts +++ b/src/artifacts.test.ts @@ -1,473 +1,2 @@ -import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { - chmod, - lstat, - mkdir, - mkdtemp, - readFile, - rm, - stat, - symlink, - writeFile, -} from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { - ARTIFACT_CHUNK_BYTES, - ArtifactError, - ArtifactStore, - type ArtifactRecord, -} from "./artifacts.js"; -import { openDatabase } from "./db/client.js"; - -const root = await mkdtemp(join(tmpdir(), "devspace-artifacts-test-")); - -try { - await testBinaryRoundTrip(join(root, "binary")); - await testRestartSafeUpload(join(root, "restart")); - await testValidationAndLimits(join(root, "limits")); - await testReferenceAwareDeletion(join(root, "references")); - await testExpirationAndPinning(join(root, "expiration")); - await testBoundedCleanup(join(root, "bounded-cleanup")); - if (process.platform !== "win32") { - await testContainmentAndSymlinks(join(root, "containment")); - } -} finally { - await rm(root, { recursive: true, force: true }); -} - -async function testBinaryRoundTrip(testRoot: string): Promise { - const store = createStore(testRoot); - try { - const bytes = Buffer.alloc(ARTIFACT_CHUNK_BYTES + 137); - for (let index = 0; index < bytes.length; index += 1) { - bytes[index] = (index * 31) % 256; - } - const digest = createHash("sha256").update(bytes).digest("hex"); - const begin = await store.beginUpload("client-a", { - filename: "payload.bin", - mimeType: "application/octet-stream", - size: bytes.length, - sha256: `sha256:${digest}`, - workspaceId: "ws_123", - }); - assert.equal(begin.chunkBytes, ARTIFACT_CHUNK_BYTES); - assert.equal(begin.nextOffset, 0); - - if (process.platform !== "win32") { - const database = openDatabase(join(testRoot, "state")); - let partialPath: string; - try { - partialPath = String( - database.sqlite.prepare("select temp_path from artifact_uploads where id = ?").pluck().get(begin.uploadId), - ); - } finally { - database.close(); - } - assert.equal((await stat(partialPath)).mode & 0o777, 0o600); - assert.equal((await stat(dirname(partialPath))).mode & 0o777, 0o700); - } - - const first = bytes.subarray(0, ARTIFACT_CHUNK_BYTES); - const firstChunk = await store.uploadChunk("client-a", { - uploadId: begin.uploadId, - offset: 0, - dataBase64: first.toString("base64"), - }); - assert.equal(firstChunk.receivedBytes, ARTIFACT_CHUNK_BYTES); - assert.equal(firstChunk.retry, false); - - const retry = await store.uploadChunk("client-a", { - uploadId: begin.uploadId, - offset: 0, - dataBase64: first.toString("base64"), - }); - assert.equal(retry.receivedBytes, ARTIFACT_CHUNK_BYTES); - assert.equal(retry.retry, true); - - const conflicting = Buffer.from(first); - conflicting[0] ^= 0xff; - await expectArtifactError( - store.uploadChunk("client-a", { - uploadId: begin.uploadId, - offset: 0, - dataBase64: conflicting.toString("base64"), - }), - "conflicting_retry", - ); - await expectArtifactError( - store.uploadChunk("client-a", { - uploadId: begin.uploadId, - offset: ARTIFACT_CHUNK_BYTES + 1, - dataBase64: Buffer.from("x").toString("base64"), - }), - "out_of_order_chunk", - ); - - await store.uploadChunk("client-a", { - uploadId: begin.uploadId, - offset: ARTIFACT_CHUNK_BYTES, - dataBase64: bytes.subarray(ARTIFACT_CHUNK_BYTES).toString("base64"), - }); - const artifact = await store.commitUpload("client-a", begin.uploadId); - assert.equal(artifact.name, "payload.bin"); - assert.equal(artifact.mimeType, "application/octet-stream"); - assert.equal(artifact.size, bytes.length); - assert.equal(artifact.sha256, `sha256:${digest}`); - assert.equal(artifact.workspaceId, "ws_123"); - assert.deepEqual(await readFile(artifact.hostPath), bytes); - - const inspected = store.statArtifact("client-a", artifact.artifactId); - assert.deepEqual(inspected, artifact); - await expectArtifactError( - Promise.resolve().then(() => store.statArtifact("client-b", artifact.artifactId)), - "artifact_not_found", - ); - - const health = store.health(); - assert.equal(health.storedBytes, bytes.length); - assert.equal(health.pendingUploads, 0); - - if (process.platform !== "win32") { - assert.equal((await stat(join(testRoot, "artifacts"))).mode & 0o777, 0o700); - assert.equal((await stat(join(testRoot, "artifacts", "objects"))).mode & 0o777, 0o700); - assert.equal((await stat(dirname(dirname(artifact.hostPath)))).mode & 0o777, 0o700); - assert.equal((await stat(dirname(artifact.hostPath))).mode & 0o777, 0o700); - assert.equal((await stat(artifact.hostPath)).mode & 0o777, 0o600); - } - - const deleted = await store.deleteArtifact("client-a", artifact.artifactId); - assert.equal(deleted.objectDeleted, true); - await assert.rejects(lstat(artifact.hostPath), { code: "ENOENT" }); - } finally { - store.close(); - } -} - -async function testRestartSafeUpload(testRoot: string): Promise { - const bytes = Buffer.from("restart-safe-artifact-upload"); - const firstChunk = bytes.subarray(0, 10); - const initialStore = createStore(testRoot); - const upload = await initialStore.beginUpload("client-a", { - filename: "restart.bin", - size: bytes.length, - sha256: createHash("sha256").update(bytes).digest("hex"), - }); - await initialStore.uploadChunk("client-a", { - uploadId: upload.uploadId, - offset: 0, - dataBase64: firstChunk.toString("base64"), - }); - initialStore.close(); - - const resumedStore = createStore(testRoot); - try { - const retry = await resumedStore.uploadChunk("client-a", { - uploadId: upload.uploadId, - offset: 0, - dataBase64: firstChunk.toString("base64"), - }); - assert.equal(retry.retry, true); - await resumedStore.uploadChunk("client-a", { - uploadId: upload.uploadId, - offset: firstChunk.length, - dataBase64: bytes.subarray(firstChunk.length).toString("base64"), - }); - const artifact = await resumedStore.commitUpload("client-a", upload.uploadId); - assert.deepEqual(await readFile(artifact.hostPath), bytes); - } finally { - resumedStore.close(); - } -} - -async function testValidationAndLimits(testRoot: string): Promise { - const store = createStore(testRoot, { - artifactMaxFileBytes: 8, - artifactMaxTotalBytes: 10, - }); - try { - for (const filename of ["../escape", "nested/file", "nested\\file", ".hidden", ".", "..", "bad\0name"]) { - await expectArtifactError( - store.beginUpload("client-a", { filename }), - "invalid_filename", - ); - } - - await expectArtifactError( - store.beginUpload("client-a", { filename: "large.bin", size: 9 }), - "file_too_large", - ); - - const invalidBase64 = await store.beginUpload("client-a", { filename: "invalid.bin" }); - await expectArtifactError( - store.uploadChunk("client-a", { - uploadId: invalidBase64.uploadId, - offset: 0, - dataBase64: "not-base64", - }), - "invalid_base64", - ); - await store.abortUpload("client-a", invalidBase64.uploadId); - - const oversize = await store.beginUpload("client-a", { filename: "oversize.bin" }); - await expectArtifactError( - store.uploadChunk("client-a", { - uploadId: oversize.uploadId, - offset: 0, - dataBase64: Buffer.alloc(9).toString("base64"), - }), - "file_too_large", - ); - await store.abortUpload("client-a", oversize.uploadId); - - const first = await stage(store, "client-a", "first.bin", Buffer.alloc(6, 1)); - assert.equal(store.health().storedBytes, 6); - await expectArtifactError( - store.beginUpload("client-a", { filename: "quota.bin", size: 5 }), - "artifact_quota_exceeded", - ); - await store.deleteArtifact("client-a", first.artifactId); - - const mismatchBytes = Buffer.from("hash"); - const mismatch = await store.beginUpload("client-a", { - filename: "mismatch.bin", - size: mismatchBytes.length, - sha256: "0".repeat(64), - }); - await store.uploadChunk("client-a", { - uploadId: mismatch.uploadId, - offset: 0, - dataBase64: mismatchBytes.toString("base64"), - }); - await expectArtifactError( - store.commitUpload("client-a", mismatch.uploadId), - "sha256_mismatch", - ); - await store.abortUpload("client-a", mismatch.uploadId); - - const empty = await store.beginUpload("client-a", { - filename: "empty.bin", - size: 0, - sha256: createHash("sha256").digest("hex"), - }); - const emptyArtifact = await store.commitUpload("client-a", empty.uploadId); - assert.equal(emptyArtifact.size, 0); - assert.deepEqual(await readFile(emptyArtifact.hostPath), Buffer.alloc(0)); - } finally { - store.close(); - } -} - -async function testReferenceAwareDeletion(testRoot: string): Promise { - const store = createStore(testRoot); - try { - const bytes = Buffer.from("deduplicated-object"); - const first = await stage(store, "client-a", "first.txt", bytes); - const second = await stage(store, "client-a", "second.txt", bytes); - assert.equal(first.hostPath, second.hostPath); - assert.equal(store.health().storedBytes, bytes.length); - - const firstDelete = await store.deleteArtifact("client-a", first.artifactId); - assert.equal(firstDelete.objectDeleted, false); - assert.deepEqual(await readFile(second.hostPath), bytes); - - const secondDelete = await store.deleteArtifact("client-a", second.artifactId); - assert.equal(secondDelete.objectDeleted, true); - await assert.rejects(lstat(second.hostPath), { code: "ENOENT" }); - } finally { - store.close(); - } -} - -async function testExpirationAndPinning(testRoot: string): Promise { - let current = new Date("2026-07-18T12:00:00.000Z"); - const store = createStore(testRoot, {}, () => new Date(current)); - try { - const abandoned = await store.beginUpload("client-a", { filename: "abandoned.bin" }); - current = new Date("2026-07-18T14:00:00.000Z"); - const uploadCleanup = await store.cleanupExpired(); - assert.equal(uploadCleanup.uploadsDeleted, 1); - await expectArtifactError( - store.abortUpload("client-a", abandoned.uploadId), - "upload_not_found", - ); - - current = new Date("2026-07-18T12:00:00.000Z"); - const shared = Buffer.from("shared-expiry-object"); - const expiring = await stage(store, "client-a", "expiring.txt", shared, 1); - const live = await stage(store, "client-a", "live.txt", shared, 4); - const pinned = await stage(store, "client-a", "pinned.txt", Buffer.from("pinned"), 1); - - const database = openDatabase(join(testRoot, "state")); - try { - database.sqlite.prepare("update artifacts set pinned = 1 where id = ?").run(pinned.artifactId); - } finally { - database.close(); - } - - current = new Date("2026-07-18T14:00:00.000Z"); - const cleanup = await store.cleanupExpired(); - assert.equal(cleanup.artifactsDeleted, 1); - assert.equal(cleanup.objectsDeleted, 0); - await expectArtifactError( - Promise.resolve().then(() => store.statArtifact("client-a", expiring.artifactId)), - "artifact_not_found", - ); - assert.deepEqual(await readFile(live.hostPath), shared); - assert.equal(store.statArtifact("client-a", live.artifactId).artifactId, live.artifactId); - assert.equal(store.statArtifact("client-a", pinned.artifactId).pinned, true); - } finally { - store.close(); - } -} - -async function testBoundedCleanup(testRoot: string): Promise { - let current = new Date("2026-07-18T12:00:00.000Z"); - const store = createStore(testRoot, {}, () => new Date(current), 1); - try { - await store.beginUpload("client-a", { filename: "one.bin" }); - await store.beginUpload("client-a", { filename: "two.bin" }); - current = new Date("2026-07-18T14:00:00.000Z"); - const first = await store.cleanupExpired(); - assert.equal(first.uploadsDeleted, 1); - assert.equal(store.health().pendingUploads, 1); - const second = await store.cleanupExpired(); - assert.equal(second.uploadsDeleted, 1); - assert.equal(store.health().pendingUploads, 0); - } finally { - store.close(); - } -} - -async function testContainmentAndSymlinks(testRoot: string): Promise { - const store = createStore(testRoot); - try { - const outside = join(testRoot, "outside.txt"); - await writeFile(outside, "outside-safe"); - - const upload = await store.beginUpload("client-a", { filename: "link.bin" }); - const database = openDatabase(join(testRoot, "state")); - let tempPath: string; - try { - tempPath = String( - database.sqlite.prepare("select temp_path from artifact_uploads where id = ?").pluck().get(upload.uploadId), - ); - } finally { - database.close(); - } - await rm(tempPath); - await symlink(outside, tempPath); - await assert.rejects( - store.uploadChunk("client-a", { - uploadId: upload.uploadId, - offset: 0, - dataBase64: Buffer.from("attack").toString("base64"), - }), - ); - assert.equal(await readFile(outside, "utf8"), "outside-safe"); - await assert.rejects(store.abortUpload("client-a", upload.uploadId)); - assert.equal(await readFile(outside, "utf8"), "outside-safe"); - - const collisionBytes = Buffer.from("collision"); - const collisionDigest = createHash("sha256").update(collisionBytes).digest("hex"); - const collisionUpload = await store.beginUpload("client-a", { - filename: "collision.bin", - size: collisionBytes.length, - }); - await store.uploadChunk("client-a", { - uploadId: collisionUpload.uploadId, - offset: 0, - dataBase64: collisionBytes.toString("base64"), - }); - const objectPath = join( - testRoot, - "artifacts", - "objects", - collisionDigest.slice(0, 2), - collisionDigest.slice(2, 4), - collisionDigest, - ); - await mkdir(dirname(objectPath), { recursive: true, mode: 0o700 }); - await chmod(dirname(objectPath), 0o700); - await symlink(outside, objectPath); - await expectArtifactError( - store.commitUpload("client-a", collisionUpload.uploadId), - "unsafe_object", - ); - assert.equal(await readFile(outside, "utf8"), "outside-safe"); - - const realRoot = join(testRoot, "real-artifact-root"); - const aliasRoot = join(testRoot, "artifact-root-alias"); - await mkdir(realRoot, { recursive: true }); - await symlink(realRoot, aliasRoot); - assert.throws( - () => new ArtifactStore({ - stateDir: join(testRoot, "alias-state"), - artifactRoot: aliasRoot, - artifactMaxFileBytes: 1024, - artifactMaxTotalBytes: 4096, - artifactDefaultTtlHours: 24, - }), - (error: unknown) => error instanceof ArtifactError && error.code === "unsafe_artifact_root", - ); - } finally { - store.close(); - } -} - -function createStore( - testRoot: string, - overrides: Partial<{ - artifactMaxFileBytes: number; - artifactMaxTotalBytes: number; - artifactDefaultTtlHours: number; - }> = {}, - now?: () => Date, - cleanupLimit?: number, -): ArtifactStore { - return new ArtifactStore( - { - stateDir: join(testRoot, "state"), - artifactRoot: join(testRoot, "artifacts"), - artifactMaxFileBytes: overrides.artifactMaxFileBytes ?? 100 * 1024 * 1024, - artifactMaxTotalBytes: overrides.artifactMaxTotalBytes ?? 1024 * 1024 * 1024, - artifactDefaultTtlHours: overrides.artifactDefaultTtlHours ?? 24, - }, - { now, cleanupLimit }, - ); -} - -async function stage( - store: ArtifactStore, - clientId: string, - filename: string, - bytes: Buffer, - ttlHours?: number, -): Promise { - const upload = await store.beginUpload(clientId, { - filename, - size: bytes.length, - sha256: createHash("sha256").update(bytes).digest("hex"), - ttlHours, - }); - for (let offset = 0; offset < bytes.length; offset += ARTIFACT_CHUNK_BYTES) { - const chunk = bytes.subarray(offset, offset + ARTIFACT_CHUNK_BYTES); - await store.uploadChunk(clientId, { - uploadId: upload.uploadId, - offset, - dataBase64: chunk.toString("base64"), - }); - } - return store.commitUpload(clientId, upload.uploadId); -} - -async function expectArtifactError( - promise: Promise, - code: string, -): Promise { - await assert.rejects( - promise, - (error: unknown) => error instanceof ArtifactError && error.code === code, - ); -} +// Persistent artifact-store coverage was replaced by artifact-download.test.ts. +export {}; diff --git a/src/artifacts.ts b/src/artifacts.ts index aee77fa2..fac7395a 100644 --- a/src/artifacts.ts +++ b/src/artifacts.ts @@ -1,903 +1,4 @@ -import { createHash, randomUUID, type Hash } from "node:crypto"; -import { - constants as fsConstants, - createReadStream, - chmodSync, - lstatSync, - mkdirSync, - realpathSync, -} from "node:fs"; -import { - chmod, - lstat, - mkdir, - open, - realpath, - rename, - unlink, -} from "node:fs/promises"; -import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; -import { openDatabase, type DatabaseHandle } from "./db/client.js"; -import { MAX_ARTIFACT_TTL_HOURS, type ServerConfig } from "./config.js"; - -export const ARTIFACT_CHUNK_BYTES = 48 * 1024; -export const ARTIFACT_UPLOAD_TTL_HOURS = 1; -export const ARTIFACT_CLEANUP_INTERVAL_MS = 15 * 60 * 1_000; -export const ARTIFACT_CLEANUP_LIMIT = 100; - -const MAX_FILENAME_BYTES = 255; -const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; - -export class ArtifactError extends Error { - constructor( - readonly code: string, - message: string, - ) { - super(message); - this.name = "ArtifactError"; - } -} - -export interface ArtifactUploadBeginInput { - filename: string; - mimeType?: string; - size?: number; - sha256?: string; - workspaceId?: string; - ttlHours?: number; -} - -export interface ArtifactUploadBeginResult { - uploadId: string; - chunkBytes: number; - expiresAt: string; - nextOffset: number; -} - -export interface ArtifactUploadChunkInput { - uploadId: string; - offset: number; - dataBase64: string; -} - -export interface ArtifactUploadChunkResult { - uploadId: string; - receivedBytes: number; - nextOffset: number; - retry: boolean; -} - -export interface ArtifactRecord { - artifactId: string; - name: string; - mimeType?: string; - size: number; - sha256: string; - hostPath: string; - source: string; - workspaceId?: string; - createdAt: string; - expiresAt?: string; - pinned: boolean; -} - -export interface ArtifactCommitOptions { - source?: string; - pinned?: boolean; -} - -export interface ArtifactDeleteResult { - artifactId: string; - deleted: boolean; - objectDeleted: boolean; -} - -export interface ArtifactCleanupResult { - uploadsDeleted: number; - artifactsDeleted: number; - objectsDeleted: number; - skippedUnsafePaths: number; -} - -export interface ArtifactStorageHealth { - root: string; - storedBytes: number; - maxTotalBytes: number; - pendingUploads: number; - expiredArtifacts: number; -} - -interface ArtifactStoreOptions { - now?: () => Date; - cleanupLimit?: number; -} - -interface UploadRow { - id: string; - client_id: string | null; - workspace_id: string | null; - original_name: string; - mime_type: string | null; - expected_size: number | null; - expected_sha256: string | null; - received_size: number; - temp_path: string; - status: string; - artifact_ttl_hours: number; - last_chunk_offset: number | null; - last_chunk_size: number | null; - last_chunk_sha256: string | null; - created_at: string; - expires_at: string; -} - -interface ArtifactRow { - id: string; - client_id: string | null; - workspace_id: string | null; - original_name: string; - mime_type: string | null; - size: number; - sha256: string; - storage_path: string; - source: string; - status: string; - created_at: string; - expires_at: string | null; - pinned: number; - last_used_at: string; -} - -interface IncrementalHashState { - hash: Hash; - receivedSize: number; -} - -export class ArtifactStore { - private readonly database: DatabaseHandle; - private readonly root: string; - private readonly rootRealPath: string; - private readonly objectsRoot: string; - private readonly uploadsRoot: string; - private readonly now: () => Date; - private readonly cleanupLimit: number; - private readonly incrementalHashes = new Map(); - private mutationQueue: Promise = Promise.resolve(); - - constructor( - private readonly config: Pick< - ServerConfig, - | "stateDir" - | "artifactRoot" - | "artifactMaxFileBytes" - | "artifactMaxTotalBytes" - | "artifactDefaultTtlHours" - >, - options: ArtifactStoreOptions = {}, - ) { - this.root = resolve(config.artifactRoot); - this.rootRealPath = ensureSecureDirectorySync(this.root); - this.objectsRoot = join(this.root, "objects"); - this.uploadsRoot = join(this.root, "uploads"); - ensureSecureDirectorySync(this.objectsRoot, this.rootRealPath); - ensureSecureDirectorySync(this.uploadsRoot, this.rootRealPath); - this.database = openDatabase(config.stateDir); - this.now = options.now ?? (() => new Date()); - this.cleanupLimit = options.cleanupLimit ?? ARTIFACT_CLEANUP_LIMIT; - } - - close(): void { - this.database.close(); - } - - beginUpload(clientId: string, input: ArtifactUploadBeginInput): Promise { - return this.withMutation(async () => { - const name = normalizeArtifactFilename(input.filename); - const mimeType = normalizeMimeType(input.mimeType); - const expectedSize = normalizeExpectedSize(input.size, this.config.artifactMaxFileBytes); - const expectedSha256 = normalizeSha256(input.sha256); - const ttlHours = normalizeTtlHours(input.ttlHours, this.config.artifactDefaultTtlHours); - - if (expectedSize !== undefined) { - this.assertQuotaAvailable(expectedSize); - } - - const uploadId = `upl_${randomUUID()}`; - const tempPath = join(this.uploadsRoot, `${uploadId}.partial`); - assertLexicalContainment(this.uploadsRoot, tempPath); - const handle = await open( - tempPath, - fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | NO_FOLLOW, - 0o600, - ); - try { - const fileStat = await handle.stat(); - if (!fileStat.isFile()) { - throw new ArtifactError("unsafe_partial", "Upload partial is not a regular file."); - } - await handle.sync(); - } finally { - await handle.close(); - } - await chmod(tempPath, 0o600); - await assertExistingFileContained(tempPath, this.uploadsRoot, this.rootRealPath); - - const now = this.now(); - const expiresAt = addHours(now, ARTIFACT_UPLOAD_TTL_HOURS).toISOString(); - this.database.sqlite.prepare(` - insert into artifact_uploads ( - id, client_id, workspace_id, original_name, mime_type, - expected_size, expected_sha256, received_size, temp_path, - status, artifact_ttl_hours, last_chunk_offset, last_chunk_size, - last_chunk_sha256, created_at, expires_at - ) values (?, ?, ?, ?, ?, ?, ?, 0, ?, 'active', ?, null, null, null, ?, ?) - `).run( - uploadId, - clientId, - input.workspaceId ?? null, - name, - mimeType ?? null, - expectedSize ?? null, - expectedSha256 ?? null, - tempPath, - ttlHours, - now.toISOString(), - expiresAt, - ); - this.incrementalHashes.set(uploadId, { - hash: createHash("sha256"), - receivedSize: 0, - }); - - return { - uploadId, - chunkBytes: ARTIFACT_CHUNK_BYTES, - expiresAt, - nextOffset: 0, - }; - }); - } - - uploadChunk(clientId: string, input: ArtifactUploadChunkInput): Promise { - return this.withMutation(async () => { - const row = this.requireUpload(clientId, input.uploadId); - this.assertUploadActive(row); - this.assertUploadNotExpired(row); - const data = decodeBase64Strict(input.dataBase64); - if (data.length === 0) { - throw new ArtifactError("empty_chunk", "Upload chunks must contain at least one decoded byte."); - } - if (data.length > ARTIFACT_CHUNK_BYTES) { - throw new ArtifactError( - "chunk_too_large", - `Decoded chunk exceeds the ${ARTIFACT_CHUNK_BYTES}-byte limit.`, - ); - } - if (!Number.isSafeInteger(input.offset) || input.offset < 0) { - throw new ArtifactError("invalid_offset", "Chunk offset must be a non-negative safe integer."); - } - - const chunkSha256 = createHash("sha256").update(data).digest("hex"); - if (input.offset < row.received_size) { - const identicalRetry = - row.last_chunk_offset === input.offset - && row.last_chunk_size === data.length - && row.last_chunk_sha256 === chunkSha256 - && input.offset + data.length === row.received_size; - if (!identicalRetry) { - throw new ArtifactError( - "conflicting_retry", - "Chunk offset has already been committed with different data.", - ); - } - return { - uploadId: row.id, - receivedBytes: row.received_size, - nextOffset: row.received_size, - retry: true, - }; - } - if (input.offset !== row.received_size) { - throw new ArtifactError( - "out_of_order_chunk", - `Expected chunk offset ${row.received_size}, received ${input.offset}.`, - ); - } - - const nextSize = row.received_size + data.length; - if (nextSize > this.config.artifactMaxFileBytes) { - throw new ArtifactError( - "file_too_large", - `Artifact exceeds the ${this.config.artifactMaxFileBytes}-byte file limit.`, - ); - } - if (row.expected_size !== null && nextSize > row.expected_size) { - throw new ArtifactError( - "declared_size_exceeded", - `Chunk would exceed the declared ${row.expected_size}-byte size.`, - ); - } - this.assertQuotaAvailable(data.length); - - const handle = await open(row.temp_path, fsConstants.O_WRONLY | NO_FOLLOW); - try { - const fileStat = await handle.stat(); - if (!fileStat.isFile()) { - throw new ArtifactError("unsafe_partial", "Upload partial is not a regular file."); - } - const resolvedPath = await realpath(row.temp_path); - assertRealContainment(this.rootRealPath, resolvedPath); - assertLexicalContainment(this.uploadsRoot, row.temp_path); - const { bytesWritten } = await handle.write(data, 0, data.length, input.offset); - if (bytesWritten !== data.length) { - throw new ArtifactError("short_write", "The artifact chunk was not fully written."); - } - await handle.sync(); - } finally { - await handle.close(); - } - - this.database.sqlite.prepare(` - update artifact_uploads - set received_size = ?, last_chunk_offset = ?, last_chunk_size = ?, last_chunk_sha256 = ? - where id = ? and client_id = ? and status = 'active' - `).run(nextSize, input.offset, data.length, chunkSha256, row.id, clientId); - - const hashState = this.incrementalHashes.get(row.id); - if (hashState && hashState.receivedSize === row.received_size) { - hashState.hash.update(data); - hashState.receivedSize = nextSize; - } else { - this.incrementalHashes.delete(row.id); - } - - return { - uploadId: row.id, - receivedBytes: nextSize, - nextOffset: nextSize, - retry: false, - }; - }); - } - - commitUpload( - clientId: string, - uploadId: string, - options: ArtifactCommitOptions = {}, - ): Promise { - return this.withMutation(async () => { - const row = this.requireUpload(clientId, uploadId); - this.assertUploadActive(row); - this.assertUploadNotExpired(row); - const source = normalizeArtifactSource(options.source); - const pinned = options.pinned === true; - const partialStat = await assertExistingFileContained( - row.temp_path, - this.uploadsRoot, - this.rootRealPath, - ); - if (partialStat.size !== row.received_size) { - throw new ArtifactError( - "partial_size_mismatch", - `Partial file size ${partialStat.size} does not match recorded size ${row.received_size}.`, - ); - } - if (row.expected_size !== null && row.received_size !== row.expected_size) { - throw new ArtifactError( - "size_mismatch", - `Received ${row.received_size} bytes, expected ${row.expected_size}.`, - ); - } - if (row.received_size > this.config.artifactMaxFileBytes) { - throw new ArtifactError("file_too_large", "Artifact exceeds the configured file limit."); - } - - const digest = await this.uploadDigest(row); - if (row.expected_sha256 && digest !== row.expected_sha256) { - throw new ArtifactError( - "sha256_mismatch", - `Artifact SHA-256 does not match the declared digest.`, - ); - } - - const objectDirectory = join(this.objectsRoot, digest.slice(0, 2), digest.slice(2, 4)); - await ensureSecureDirectory(objectDirectory, this.rootRealPath); - const objectPath = join(objectDirectory, digest); - assertLexicalContainment(this.objectsRoot, objectPath); - - let createdObject = false; - const existingObject = await lstatOrUndefined(objectPath); - if (existingObject) { - if (existingObject.isSymbolicLink() || !existingObject.isFile()) { - throw new ArtifactError("unsafe_object", "Artifact object path is not a regular file."); - } - await assertExistingFileContained(objectPath, this.objectsRoot, this.rootRealPath); - if (existingObject.size !== row.received_size || await hashFile(objectPath) !== digest) { - throw new ArtifactError("object_collision", "Existing artifact object failed digest verification."); - } - await unlink(row.temp_path); - } else { - await rename(row.temp_path, objectPath); - createdObject = true; - await chmod(objectPath, 0o600); - await assertExistingFileContained(objectPath, this.objectsRoot, this.rootRealPath); - } - - const now = this.now(); - const artifactId = `art_${randomUUID()}`; - const expiresAt = addHours(now, row.artifact_ttl_hours).toISOString(); - const insert = this.database.sqlite.transaction(() => { - this.database.sqlite.prepare(` - insert into artifacts ( - id, client_id, workspace_id, original_name, mime_type, - size, sha256, storage_path, source, status, - created_at, expires_at, pinned, last_used_at - ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, 'available', ?, ?, ?, ?) - `).run( - artifactId, - clientId, - row.workspace_id, - row.original_name, - row.mime_type, - row.received_size, - digest, - objectPath, - source, - now.toISOString(), - expiresAt, - pinned ? 1 : 0, - now.toISOString(), - ); - this.database.sqlite.prepare("delete from artifact_uploads where id = ? and client_id = ?") - .run(row.id, clientId); - }); - - try { - insert.immediate(); - } catch (error) { - if (createdObject) { - await unlink(objectPath).catch(() => undefined); - } - throw error; - } finally { - this.incrementalHashes.delete(row.id); - } - - return { - artifactId, - name: row.original_name, - mimeType: row.mime_type ?? undefined, - size: row.received_size, - sha256: `sha256:${digest}`, - hostPath: objectPath, - source, - workspaceId: row.workspace_id ?? undefined, - createdAt: now.toISOString(), - expiresAt, - pinned, - }; - }); - } - - abortUpload(clientId: string, uploadId: string): Promise<{ uploadId: string; aborted: boolean }> { - return this.withMutation(async () => { - const row = this.requireUpload(clientId, uploadId); - let unsafePathError: unknown; - try { - await removeManagedFile(row.temp_path, this.uploadsRoot, this.rootRealPath); - } catch (error) { - unsafePathError = error; - } - this.database.sqlite.prepare("delete from artifact_uploads where id = ? and client_id = ?") - .run(row.id, clientId); - this.incrementalHashes.delete(row.id); - if (unsafePathError) throw unsafePathError; - return { uploadId: row.id, aborted: true }; - }); - } - - statArtifact(clientId: string, artifactId: string): ArtifactRecord { - const row = this.requireArtifact(clientId, artifactId); - if (row.status !== "available") { - throw new ArtifactError("artifact_unavailable", "Artifact is not available."); - } - const now = this.now().toISOString(); - this.database.sqlite.prepare( - "update artifacts set last_used_at = ? where id = ? and client_id = ?", - ).run(now, artifactId, clientId); - return artifactRowToRecord(row); - } - - deleteArtifact(clientId: string, artifactId: string): Promise { - return this.withMutation(async () => { - const row = this.requireArtifact(clientId, artifactId); - this.database.sqlite.prepare("delete from artifacts where id = ? and client_id = ?") - .run(row.id, clientId); - const objectDeleted = await this.deleteObjectIfUnreferenced(row); - return { artifactId: row.id, deleted: true, objectDeleted }; - }); - } - - cleanupExpired(): Promise { - return this.withMutation(async () => { - const now = this.now().toISOString(); - let remaining = this.cleanupLimit; - let uploadsDeleted = 0; - let artifactsDeleted = 0; - let objectsDeleted = 0; - let skippedUnsafePaths = 0; - - const expiredUploads = this.database.sqlite.prepare(` - select * from artifact_uploads - where status = 'active' and expires_at <= ? - order by expires_at asc - limit ? - `).all(now, remaining) as UploadRow[]; - for (const row of expiredUploads) { - try { - await removeManagedFile(row.temp_path, this.uploadsRoot, this.rootRealPath); - } catch { - skippedUnsafePaths += 1; - } - this.database.sqlite.prepare("delete from artifact_uploads where id = ?").run(row.id); - this.incrementalHashes.delete(row.id); - uploadsDeleted += 1; - remaining -= 1; - } - - if (remaining > 0) { - const expiredArtifacts = this.database.sqlite.prepare(` - select * from artifacts - where status = 'available' and pinned = 0 and expires_at is not null and expires_at <= ? - order by expires_at asc - limit ? - `).all(now, remaining) as ArtifactRow[]; - for (const row of expiredArtifacts) { - this.database.sqlite.prepare("delete from artifacts where id = ?").run(row.id); - artifactsDeleted += 1; - try { - if (await this.deleteObjectIfUnreferenced(row)) objectsDeleted += 1; - } catch { - skippedUnsafePaths += 1; - } - } - } - - return { - uploadsDeleted, - artifactsDeleted, - objectsDeleted, - skippedUnsafePaths, - }; - }); - } - - health(): ArtifactStorageHealth { - const storedBytes = this.objectBytes(); - const pendingUploads = Number( - this.database.sqlite.prepare( - "select count(*) from artifact_uploads where status = 'active'", - ).pluck().get() ?? 0, - ); - const expiredArtifacts = Number( - this.database.sqlite.prepare(` - select count(*) from artifacts - where status = 'available' and pinned = 0 - and expires_at is not null and expires_at <= ? - `).pluck().get(this.now().toISOString()) ?? 0, - ); - return { - root: this.root, - storedBytes, - maxTotalBytes: this.config.artifactMaxTotalBytes, - pendingUploads, - expiredArtifacts, - }; - } - - private requireUpload(clientId: string, uploadId: string): UploadRow { - const row = this.database.sqlite.prepare( - "select * from artifact_uploads where id = ? and client_id = ?", - ).get(uploadId, clientId) as UploadRow | undefined; - if (!row) { - throw new ArtifactError("upload_not_found", "Artifact upload was not found."); - } - return row; - } - - private requireArtifact(clientId: string, artifactId: string): ArtifactRow { - const row = this.database.sqlite.prepare( - "select * from artifacts where id = ? and client_id = ?", - ).get(artifactId, clientId) as ArtifactRow | undefined; - if (!row) { - throw new ArtifactError("artifact_not_found", "Artifact was not found."); - } - return row; - } - - private assertUploadActive(row: UploadRow): void { - if (row.status !== "active") { - throw new ArtifactError("upload_inactive", "Artifact upload is not active."); - } - } - - private assertUploadNotExpired(row: UploadRow): void { - if (row.expires_at <= this.now().toISOString()) { - throw new ArtifactError("upload_expired", "Artifact upload has expired."); - } - } - - private assertQuotaAvailable(additionalBytes: number): void { - const used = this.objectBytes() + this.pendingUploadBytes(); - if (used + additionalBytes > this.config.artifactMaxTotalBytes) { - throw new ArtifactError( - "artifact_quota_exceeded", - `Artifact storage would exceed the ${this.config.artifactMaxTotalBytes}-byte total limit.`, - ); - } - } - - private objectBytes(): number { - return Number( - this.database.sqlite.prepare(` - select coalesce(sum(size), 0) - from ( - select storage_path, max(size) as size - from artifacts - where status = 'available' - group by storage_path - ) - `).pluck().get() ?? 0, - ); - } - - private pendingUploadBytes(): number { - return Number( - this.database.sqlite.prepare(` - select coalesce(sum(received_size), 0) - from artifact_uploads - where status = 'active' - `).pluck().get() ?? 0, - ); - } - - private async uploadDigest(row: UploadRow): Promise { - const state = this.incrementalHashes.get(row.id); - if (state && state.receivedSize === row.received_size) { - return state.hash.copy().digest("hex"); - } - return hashFile(row.temp_path); - } - - private async deleteObjectIfUnreferenced(row: ArtifactRow): Promise { - const referenceCount = Number( - this.database.sqlite.prepare(` - select count(*) from artifacts - where status = 'available' and storage_path = ? - `).pluck().get(row.storage_path) ?? 0, - ); - if (referenceCount > 0) return false; - await removeManagedFile(row.storage_path, this.objectsRoot, this.rootRealPath); - return true; - } - - private withMutation(operation: () => Promise): Promise { - const result = this.mutationQueue.then(operation, operation); - this.mutationQueue = result.then( - () => undefined, - () => undefined, - ); - return result; - } -} - -export function normalizeArtifactFilename(value: string): string { - if (typeof value !== "string") { - throw new ArtifactError("invalid_filename", "Artifact filename must be a string."); - } - const normalized = value.normalize("NFC"); - if (!normalized || normalized === "." || normalized === "..") { - throw new ArtifactError("invalid_filename", "Artifact filename is empty or reserved."); - } - if (normalized.startsWith(".")) { - throw new ArtifactError("invalid_filename", "Artifact filename must not start with a dot."); - } - if (/[\\/]/u.test(normalized) || basename(normalized) !== normalized) { - throw new ArtifactError("invalid_filename", "Artifact filename must contain one basename only."); - } - if(/[\u0000-\u001f\u007f-\u009f]/u.test(normalized)) { - throw new ArtifactError("invalid_filename", "Artifact filename contains control characters."); - } - if (Buffer.byteLength(normalized, "utf8") > MAX_FILENAME_BYTES) { - throw new ArtifactError( - "invalid_filename", - `Artifact filename exceeds ${MAX_FILENAME_BYTES} UTF-8 bytes.`, - ); - } - return normalized; -} - -function normalizeArtifactSource(value: string | undefined): string { - const source = value ?? "chunked"; - if (!/^[a-z0-9][a-z0-9._:-]{0,63}$/u.test(source)) { - throw new ArtifactError( - "invalid_artifact_source", - "Artifact source must be a short lowercase identifier.", - ); - } - return source; -} - -export function normalizeSha256(value: string | undefined): string | undefined { - if (value === undefined) return undefined; - const normalized = value.trim().toLowerCase().replace(/^sha256:/u, ""); - if (!/^[0-9a-f]{64}$/u.test(normalized)) { - throw new ArtifactError("invalid_sha256", "SHA-256 must contain exactly 64 hexadecimal characters."); - } - return normalized; -} - -export function decodeBase64Strict(value: string): Buffer { - if (typeof value !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value)) { - throw new ArtifactError("invalid_base64", "Chunk data is not valid canonical base64."); - } - const decoded = Buffer.from(value, "base64"); - if (decoded.toString("base64") !== value) { - throw new ArtifactError("invalid_base64", "Chunk data is not valid canonical base64."); - } - return decoded; -} - -export function formatBytes(bytes: number): string { - if (bytes < 1024) return `${bytes} B`; - const units = ["KiB", "MiB", "GiB", "TiB"]; - let value = bytes; - let unit = -1; - do { - value /= 1024; - unit += 1; - } while (value >= 1024 && unit < units.length - 1); - const digits = value >= 10 ? 1 : 2; - return `${value.toFixed(digits).replace(/\.0+$/u, "")} ${units[unit]}`; -} - -function normalizeMimeType(value: string | undefined): string | undefined { - const mimeType = value?.trim(); - if (!mimeType) return undefined; - if (mimeType.length > 255 || /[\u0000-\u001f\u007f]/u.test(mimeType)) { - throw new ArtifactError("invalid_mime_type", "Artifact MIME type is invalid."); - } - return mimeType; -} - -function normalizeExpectedSize(value: number | undefined, maxFileBytes: number): number | undefined { - if (value === undefined) return undefined; - if (!Number.isSafeInteger(value) || value < 0) { - throw new ArtifactError("invalid_size", "Artifact size must be a non-negative safe integer."); - } - if (value > maxFileBytes) { - throw new ArtifactError("file_too_large", `Artifact exceeds the ${maxFileBytes}-byte file limit.`); - } - return value; -} - -function normalizeTtlHours(value: number | undefined, fallback: number): number { - const ttlHours = value ?? fallback; - if ( - !Number.isSafeInteger(ttlHours) - || ttlHours < 1 - || ttlHours > MAX_ARTIFACT_TTL_HOURS - ) { - throw new ArtifactError( - "invalid_ttl", - `Artifact TTL must be an integer between 1 and ${MAX_ARTIFACT_TTL_HOURS} hours.`, - ); - } - return ttlHours; -} - -function artifactRowToRecord(row: ArtifactRow): ArtifactRecord { - return { - artifactId: row.id, - name: row.original_name, - mimeType: row.mime_type ?? undefined, - size: row.size, - sha256: `sha256:${row.sha256}`, - hostPath: row.storage_path, - source: row.source, - workspaceId: row.workspace_id ?? undefined, - createdAt: row.created_at, - expiresAt: row.expires_at ?? undefined, - pinned: row.pinned === 1, - }; -} - -function addHours(date: Date, hours: number): Date { - return new Date(date.getTime() + hours * 60 * 60 * 1_000); -} - -function ensureSecureDirectorySync(path: string, containingRootRealPath?: string): string { - mkdirSync(path, { recursive: true, mode: 0o700 }); - const entry = lstatSync(path); - if (entry.isSymbolicLink() || !entry.isDirectory()) { - throw new ArtifactError("unsafe_artifact_root", `Artifact directory is not a real directory: ${path}`); - } - chmodSync(path, 0o700); - const realPath = realpathSync(path); - if (containingRootRealPath) assertRealContainment(containingRootRealPath, realPath); - return realPath; -} - -async function ensureSecureDirectory(path: string, rootRealPath: string): Promise { - await mkdir(path, { recursive: true, mode: 0o700 }); - const entry = await lstat(path); - if (entry.isSymbolicLink() || !entry.isDirectory()) { - throw new ArtifactError("unsafe_artifact_path", `Artifact directory is not a real directory: ${path}`); - } - await chmod(path, 0o700); - assertRealContainment(rootRealPath, await realpath(path)); -} - -async function assertExistingFileContained( - path: string, - lexicalRoot: string, - rootRealPath: string, -) { - assertLexicalContainment(lexicalRoot, path); - const entry = await lstat(path); - if (entry.isSymbolicLink() || !entry.isFile()) { - throw new ArtifactError("unsafe_artifact_path", "Artifact path is not a regular file."); - } - assertRealContainment(rootRealPath, await realpath(path)); - return entry; -} - -async function removeManagedFile( - path: string, - lexicalRoot: string, - rootRealPath: string, -): Promise { - assertLexicalContainment(lexicalRoot, path); - const entry = await lstatOrUndefined(path); - if (!entry) return; - if (entry.isSymbolicLink() || !entry.isFile()) { - throw new ArtifactError("unsafe_artifact_path", "Refusing to delete a non-regular artifact path."); - } - assertRealContainment(rootRealPath, await realpath(path)); - await unlink(path); -} - -function assertLexicalContainment(root: string, candidate: string): void { - const relativePath = relative(resolve(root), resolve(candidate)); - if (relativePath === "" || relativePath.startsWith("..") || isAbsolute(relativePath)) { - throw new ArtifactError("artifact_path_escape", "Artifact path escapes the configured storage root."); - } -} - -function assertRealContainment(rootRealPath: string, candidateRealPath: string): void { - const relativePath = relative(rootRealPath, candidateRealPath); - if (relativePath.startsWith("..") || isAbsolute(relativePath)) { - throw new ArtifactError("artifact_path_escape", "Artifact path resolves outside the configured storage root."); - } -} - -async function lstatOrUndefined(path: string) { - try { - return await lstat(path); - } catch (error) { - if (isNodeError(error) && error.code === "ENOENT") return undefined; - throw error; - } -} - -async function hashFile(path: string): Promise { - const hash = createHash("sha256"); - for await (const chunk of createReadStream(path)) { - hash.update(chunk as Buffer); - } - return hash.digest("hex"); -} - -function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && "code" in error; -} +// Persistent artifact storage was removed in favor of the one-shot +// download_artifact workspace handoff. Keep this re-export temporarily so +// downstream source imports of ArtifactError have a deliberate migration path. +export { ArtifactError } from "./artifact-error.js"; diff --git a/src/config.test.ts b/src/config.test.ts index 7a6563a5..9bc8b4c9 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -1,8 +1,8 @@ import assert from "node:assert/strict"; import { existsSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; -import { homedir, tmpdir } from "node:os"; +import { tmpdir } from "node:os"; import { join } from "node:path"; -import { loadConfig, MAX_ARTIFACT_TTL_HOURS } from "./config.js"; +import { loadConfig } from "./config.js"; import { ensureDevspaceDefaultSkills, resolveSubagentsFlag } from "./user-config.js"; const emptyConfigDir = mkdtempSync(join(tmpdir(), "devspace-empty-config-test-")); @@ -27,30 +27,12 @@ assert.equal(loadConfig(baseEnv).devspaceSkillsDir, join(emptyConfigDir, "skills assert.equal(loadConfig(baseEnv).devspaceAgentsDir, join(emptyConfigDir, "agents")); assert.equal(loadConfig(baseEnv).subagents, false); assert.equal(loadConfig(baseEnv).artifactsEnabled, false); -assert.equal( - loadConfig(baseEnv).artifactRoot, - join(homedir(), ".local", "share", "devspace", "artifacts"), -); assert.equal(loadConfig(baseEnv).artifactMaxFileBytes, 100 * 1024 * 1024); -assert.equal(loadConfig(baseEnv).artifactMaxTotalBytes, 1024 * 1024 * 1024); -assert.equal(loadConfig(baseEnv).artifactDefaultTtlHours, 24); assert.equal(loadConfig({ ...baseEnv, DEVSPACE_ARTIFACTS: "1" }).artifactsEnabled, true); -assert.equal( - loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_ROOT: "~/private-artifacts" }).artifactRoot, - join(homedir(), "private-artifacts"), -); assert.equal( loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_FILE_BYTES: "123" }).artifactMaxFileBytes, 123, ); -assert.equal( - loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES: "456" }).artifactMaxTotalBytes, - 456, -); -assert.equal( - loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_TTL_HOURS: "48" }).artifactDefaultTtlHours, - 48, -); assert.equal(loadConfig({ ...baseEnv, DEVSPACE_SKILLS: "0" }).skillsEnabled, false); assert.equal(loadConfig({ ...baseEnv, DEVSPACE_SKILLS: "1" }).skillsEnabled, true); assert.equal( @@ -167,21 +149,6 @@ assert.throws( () => loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_FILE_BYTES: "0" }), /Invalid DEVSPACE_ARTIFACT_MAX_FILE_BYTES: 0/, ); -assert.throws( - () => loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES: "nope" }), - /Invalid DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES: nope/, -); -assert.throws( - () => loadConfig({ ...baseEnv, DEVSPACE_ARTIFACT_TTL_HOURS: "-1" }), - /Invalid DEVSPACE_ARTIFACT_TTL_HOURS: -1/, -); -assert.throws( - () => loadConfig({ - ...baseEnv, - DEVSPACE_ARTIFACT_TTL_HOURS: String(MAX_ARTIFACT_TTL_HOURS + 1), - }), - /Invalid DEVSPACE_ARTIFACT_TTL_HOURS/, -); assert.equal(loadConfig(baseEnv).publicBaseUrl, "http://127.0.0.1:7676"); assert.deepEqual(loadConfig(baseEnv).allowedHosts, ["localhost", "127.0.0.1", "::1"]); @@ -208,10 +175,7 @@ writeFileSync( publicBaseUrl: "https://devspace.example.com", subagents: true, artifactsEnabled: true, - artifactRoot: join(configDir, "persisted-artifacts"), artifactMaxFileBytes: 321, - artifactMaxTotalBytes: 654, - artifactDefaultTtlHours: 72, }), ); writeFileSync( @@ -227,10 +191,7 @@ assert.equal(fileConfig.oauth.ownerToken, "persisted-owner-token-long-enough"); assert.equal(fileConfig.publicBaseUrl, "https://devspace.example.com"); assert.equal(fileConfig.subagents, true); assert.equal(fileConfig.artifactsEnabled, true); -assert.equal(fileConfig.artifactRoot, join(configDir, "persisted-artifacts")); assert.equal(fileConfig.artifactMaxFileBytes, 321); -assert.equal(fileConfig.artifactMaxTotalBytes, 654); -assert.equal(fileConfig.artifactDefaultTtlHours, 72); assert.deepEqual(fileConfig.allowedHosts, [ "localhost", "127.0.0.1", diff --git a/src/config.ts b/src/config.ts index 55fb286d..f8c8b995 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,9 +10,6 @@ export type WidgetMode = "off" | "changes" | "full"; const DEFAULT_OAUTH_ACCESS_TOKEN_TTL_SECONDS = 60 * 60; const DEFAULT_OAUTH_REFRESH_TOKEN_TTL_SECONDS = 30 * 24 * 60 * 60; const DEFAULT_ARTIFACT_MAX_FILE_BYTES = 100 * 1024 * 1024; -const DEFAULT_ARTIFACT_MAX_TOTAL_BYTES = 1024 * 1024 * 1024; -const DEFAULT_ARTIFACT_TTL_HOURS = 24; -export const MAX_ARTIFACT_TTL_HOURS = 24 * 365; export interface ServerConfig { host: string; @@ -26,10 +23,7 @@ export interface ServerConfig { stateDir: string; worktreeRoot: string; artifactsEnabled: boolean; - artifactRoot: string; artifactMaxFileBytes: number; - artifactMaxTotalBytes: number; - artifactDefaultTtlHours: number; skillsEnabled: boolean; skillPaths: string[]; devspaceSkillsDir: string; @@ -209,10 +203,6 @@ function defaultWorktreeRoot(): string { return join(homedir(), ".devspace", "worktrees"); } -function defaultArtifactRoot(): string { - return join(homedir(), ".local", "share", "devspace", "artifacts"); -} - function defaultAgentDir(): string { return join(homedir(), ".codex"); } @@ -248,25 +238,11 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): ServerConfig { env.DEVSPACE_ARTIFACTS === undefined ? files.config.artifactsEnabled === true : parseBoolean(env.DEVSPACE_ARTIFACTS), - artifactRoot: resolve(expandHomePath( - env.DEVSPACE_ARTIFACT_ROOT ?? files.config.artifactRoot ?? defaultArtifactRoot(), - )), artifactMaxFileBytes: parsePositiveInteger( env.DEVSPACE_ARTIFACT_MAX_FILE_BYTES ?? numberConfigValue(files.config.artifactMaxFileBytes), DEFAULT_ARTIFACT_MAX_FILE_BYTES, "DEVSPACE_ARTIFACT_MAX_FILE_BYTES", ), - artifactMaxTotalBytes: parsePositiveInteger( - env.DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES ?? numberConfigValue(files.config.artifactMaxTotalBytes), - DEFAULT_ARTIFACT_MAX_TOTAL_BYTES, - "DEVSPACE_ARTIFACT_MAX_TOTAL_BYTES", - ), - artifactDefaultTtlHours: parsePositiveInteger( - env.DEVSPACE_ARTIFACT_TTL_HOURS ?? numberConfigValue(files.config.artifactDefaultTtlHours), - DEFAULT_ARTIFACT_TTL_HOURS, - "DEVSPACE_ARTIFACT_TTL_HOURS", - MAX_ARTIFACT_TTL_HOURS, - ), skillsEnabled: env.DEVSPACE_SKILLS === undefined ? true : parseBoolean(env.DEVSPACE_SKILLS), skillPaths: parsePathList(env.DEVSPACE_SKILL_PATHS), devspaceSkillsDir: devspaceSkillsDir(env), diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 5ad5ec52..c122068e 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -24,8 +24,11 @@ const migrations: Migration[] = [ }, { version: 4, - name: "artifact-exchange", - up: migrateArtifactExchange, + name: "artifact-exchange-retired", + // Preserve the historical migration version without creating new + // persistent artifact tables. Existing user tables and bytes are left + // untouched; the one-shot download path does not read or mutate them. + up: () => undefined, }, ]; @@ -179,64 +182,6 @@ function migrateLocalAgentSessions(sqlite: Database.Database): void { addColumnIfMissing(sqlite, "local_agent_sessions", "thinking", "text"); } -function migrateArtifactExchange(sqlite: Database.Database): void { - sqlite.exec(` - create table if not exists artifacts ( - id text primary key, - client_id text, - workspace_id text, - original_name text not null, - mime_type text, - size integer not null, - sha256 text not null, - storage_path text not null, - source text not null, - status text not null, - created_at text not null, - expires_at text, - pinned integer not null default 0, - last_used_at text not null - ); - - create index if not exists artifacts_sha256_storage_idx - on artifacts(sha256, storage_path); - - create index if not exists artifacts_expiry_idx - on artifacts(status, pinned, expires_at); - - create index if not exists artifacts_workspace_idx - on artifacts(workspace_id, last_used_at desc); - - create index if not exists artifacts_client_idx - on artifacts(client_id, last_used_at desc); - - create table if not exists artifact_uploads ( - id text primary key, - client_id text, - workspace_id text, - original_name text not null, - mime_type text, - expected_size integer, - expected_sha256 text, - received_size integer not null default 0, - temp_path text not null, - status text not null, - artifact_ttl_hours integer not null, - last_chunk_offset integer, - last_chunk_size integer, - last_chunk_sha256 text, - created_at text not null, - expires_at text not null - ); - - create index if not exists artifact_uploads_expiry_idx - on artifact_uploads(status, expires_at); - - create index if not exists artifact_uploads_client_idx - on artifact_uploads(client_id, created_at desc); - `); -} - function addColumnIfMissing( sqlite: Database.Database, table: "workspace_sessions" | "local_agent_sessions", diff --git a/src/db/schema.ts b/src/db/schema.ts index 7f994f14..01d13fac 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,3 @@ -import { desc } from "drizzle-orm"; import { index, integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core"; export const workspaceSessions = sqliteTable( @@ -74,58 +73,6 @@ export const oauthRefreshTokens = sqliteTable( }, ); -export const artifacts = sqliteTable( - "artifacts", - { - id: text("id").primaryKey(), - clientId: text("client_id"), - workspaceId: text("workspace_id"), - originalName: text("original_name").notNull(), - mimeType: text("mime_type"), - size: integer("size").notNull(), - sha256: text("sha256").notNull(), - storagePath: text("storage_path").notNull(), - source: text("source").notNull(), - status: text("status").notNull(), - createdAt: text("created_at").notNull(), - expiresAt: text("expires_at"), - pinned: integer("pinned").notNull().default(0), - lastUsedAt: text("last_used_at").notNull(), - }, - (table) => [ - index("artifacts_sha256_storage_idx").on(table.sha256, table.storagePath), - index("artifacts_expiry_idx").on(table.status, table.pinned, table.expiresAt), - index("artifacts_workspace_idx").on(table.workspaceId, desc(table.lastUsedAt)), - index("artifacts_client_idx").on(table.clientId, desc(table.lastUsedAt)), - ], -); - -export const artifactUploads = sqliteTable( - "artifact_uploads", - { - id: text("id").primaryKey(), - clientId: text("client_id"), - workspaceId: text("workspace_id"), - originalName: text("original_name").notNull(), - mimeType: text("mime_type"), - expectedSize: integer("expected_size"), - expectedSha256: text("expected_sha256"), - receivedSize: integer("received_size").notNull().default(0), - tempPath: text("temp_path").notNull(), - status: text("status").notNull(), - artifactTtlHours: integer("artifact_ttl_hours").notNull(), - lastChunkOffset: integer("last_chunk_offset"), - lastChunkSize: integer("last_chunk_size"), - lastChunkSha256: text("last_chunk_sha256"), - createdAt: text("created_at").notNull(), - expiresAt: text("expires_at").notNull(), - }, - (table) => [ - index("artifact_uploads_expiry_idx").on(table.status, table.expiresAt), - index("artifact_uploads_client_idx").on(table.clientId, desc(table.createdAt)), - ], -); - export const localAgentSessions = sqliteTable( "local_agent_sessions", { @@ -154,9 +101,5 @@ export type WorkspaceSessionRow = typeof workspaceSessions.$inferSelect; export type NewWorkspaceSessionRow = typeof workspaceSessions.$inferInsert; export type LoadedAgentFileRow = typeof loadedAgentFiles.$inferSelect; export type NewLoadedAgentFileRow = typeof loadedAgentFiles.$inferInsert; -export type ArtifactRow = typeof artifacts.$inferSelect; -export type NewArtifactRow = typeof artifacts.$inferInsert; -export type ArtifactUploadRow = typeof artifactUploads.$inferSelect; -export type NewArtifactUploadRow = typeof artifactUploads.$inferInsert; export type LocalAgentSessionRow = typeof localAgentSessions.$inferSelect; export type NewLocalAgentSessionRow = typeof localAgentSessions.$inferInsert; diff --git a/src/incoming-artifacts.test.ts b/src/incoming-artifacts.test.ts index 2fdcde0a..7f126676 100644 --- a/src/incoming-artifacts.test.ts +++ b/src/incoming-artifacts.test.ts @@ -1,100 +1,59 @@ import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { Readable } from "node:stream"; -import * as z from "zod/v4"; -import { - artifactToolLogFields, - materializeIncomingArtifact, - registerArtifactTools, - stageIncomingArtifact, -} from "./artifact-tools.js"; -import { ArtifactError, ArtifactStore } from "./artifacts.js"; +import { ArtifactError } from "./artifact-error.js"; import { IncomingArtifactAdapterRegistry, createOpenAIIncomingArtifactAdapter, + describeIncomingArtifactValue, type IncomingArtifactAdapter, } from "./incoming-artifacts.js"; -const root = await mkdtemp(join(tmpdir(), "devspace-incoming-artifacts-test-")); +await testRegistryFailsClosed(); +await testOpenAIFileAdapter(); +testLogShapeRedaction(); -try { - await testRegistryFailsClosed(); - testChatGPTFileDescriptor(); - await testOpenAIFileAdapter(join(root, "openai")); - await testStageArtifact(join(root, "stage")); - await testMaterializeArtifact(join(root, "materialize")); - await testFailedStageCleanup(join(root, "failed-stage")); - testStageLogRedaction(); -} finally { - await rm(root, { recursive: true, force: true }); -} +async function testRegistryFailsClosed(): Promise { + const registry = new IncomingArtifactAdapterRegistry(); + await expectArtifactError( + registry.open("https://example.com/untrusted.bin"), + "unsupported_incoming_artifact", + ); + await expectArtifactError( + registry.open("/tmp/local-file.bin"), + "unsupported_incoming_artifact", + ); + await expectArtifactError( + registry.open(Buffer.from("raw bytes")), + "unsupported_incoming_artifact", + ); -function testChatGPTFileDescriptor(): void { - const registered = new Map>(); - const server = { - registerTool( - name: string, - descriptor: Record, - _callback: unknown, - ) { - registered.set(name, descriptor); - return {}; + const ambiguous: IncomingArtifactAdapter = { + id: "ambiguous", + canHandle: () => true, + async open() { + return { name: "a.bin", stream: Readable.from(["a"]) }; }, }; - - registerArtifactTools(server as never, { - config: {} as never, - store: {} as never, - workspaces: {} as never, - clientId: "client-a", - }); - - assert.deepEqual([...registered.keys()], ["materialize_artifact"]); - const descriptor = registered.get("materialize_artifact"); - assert.ok(descriptor); - assert.deepEqual(descriptor._meta, { "openai/fileParams": ["file"] }); - - const inputSchema = descriptor.inputSchema as z.ZodRawShape; - const fileSchema = inputSchema.file as z.ZodType; - const valid = { - download_url: "https://files.oaiusercontent.com/file_123/download?sig=secret", - file_id: "file_123", - mime_type: "image/png", - file_name: "generated.png", - }; - assert.deepEqual(fileSchema.parse(valid), valid); - const generated = { - ...valid, - mime_type: null, - file_name: null, - name: "/mnt/data/generated.png", - size: 123, - }; - assert.deepEqual(fileSchema.parse(generated), generated); - assert.throws(() => fileSchema.parse({ file_id: "file_123" })); - assert.throws(() => fileSchema.parse({ ...valid, mime_type: 123 })); - - const jsonSchema = z.toJSONSchema(fileSchema) as { - additionalProperties?: boolean; - properties?: Record; - required?: string[]; - }; - assert.equal(jsonSchema.additionalProperties, false); - assert.deepEqual(Object.keys(jsonSchema.properties ?? {}).sort(), [ - "download_url", - "file_id", - "file_name", - "mime_type", - "name", - "size", + const ambiguousRegistry = new IncomingArtifactAdapterRegistry([ + ambiguous, + { ...ambiguous, id: "ambiguous-two" }, ]); - assert.deepEqual([...(jsonSchema.required ?? [])].sort(), ["download_url", "file_id"]); + await expectArtifactError( + ambiguousRegistry.open({ native: true }), + "ambiguous_incoming_artifact", + ); + + assert.throws( + () => new IncomingArtifactAdapterRegistry([{ ...ambiguous, id: "UPPER" }]), + (error: unknown) => error instanceof ArtifactError && error.code === "invalid_incoming_adapter", + ); + assert.throws( + () => new IncomingArtifactAdapterRegistry([ambiguous, ambiguous]), + (error: unknown) => error instanceof ArtifactError && error.code === "duplicate_incoming_adapter", + ); } -async function testOpenAIFileAdapter(testRoot: string): Promise { +async function testOpenAIFileAdapter(): Promise { const bytes = Buffer.from("chatgpt generated image bytes"); const requested: string[] = []; const fetchImpl: typeof fetch = async (input, init) => { @@ -137,25 +96,7 @@ async function testOpenAIFileAdapter(testRoot: string): Promise { assert.equal(generatedOpened.name, "generated-image.png"); assert.equal(generatedOpened.mimeType, "image/png"); assert.deepEqual(await collect(generatedOpened.stream), bytes); - - const store = createStore(testRoot); - try { - const staged = await stageIncomingArtifact({ - store, - clientId: "client-a", - registry, - input: { file: generatedReference }, - }); - assert.equal(staged.name, "generated-image.png"); - assert.equal(staged.mimeType, "image/png"); - assert.deepEqual(await readFile(staged.hostPath), bytes); - assert.equal( - store.statArtifact("client-a", staged.artifactId).source, - "incoming:openai-file", - ); - } finally { - store.close(); - } + assert.equal(requested.length, 2); const fallbackOpened = await registry.open({ ...generatedReference, @@ -178,232 +119,93 @@ async function testOpenAIFileAdapter(testRoot: string): Promise { "openai_file_size_mismatch", ); await expectArtifactError( - registry.open({ ...reference, download_url: "https://example.test/private.png" }), + registry.open({ + ...reference, + download_url: "http://files.oaiusercontent.com/file_123/download", + }), "unsafe_openai_file_reference", ); await expectArtifactError( registry.open({ ...reference, - download_url: "https://attacker.blob.core.windows.net/private.png", + download_url: "https://example.com/file_123/download", }), "unsafe_openai_file_reference", ); await expectArtifactError( - registry.open({ ...reference, download_url: "http://files.oaiusercontent.com/file_123" }), + registry.open({ + ...reference, + download_url: "https://arbitrary.blob.core.windows.net/container/file.png", + }), "unsafe_openai_file_reference", ); await expectArtifactError( - registry.open({ ...reference, file_id: "file_\u0000secret" }), - "invalid_openai_file_reference", - ); - await expectArtifactError( - registry.open({ ...reference, file_id: "x".repeat(513) }), - "invalid_openai_file_reference", - ); - await expectArtifactError( - registry.open({ ...reference, bearer: "secret" }), + registry.open({ + ...reference, + extra: "unexpected", + }), "unsupported_incoming_artifact", ); - let redirectRequests = 0; - const redirectingRegistry = new IncomingArtifactAdapterRegistry([ + const redirectRegistry = new IncomingArtifactAdapterRegistry([ createOpenAIIncomingArtifactAdapter({ - fetch: async () => { - redirectRequests += 1; - return new Response(null, { - status: 302, - headers: { location: "http://127.0.0.1/private" }, - }); + fetch: async (input) => { + if (String(input).includes("first")) { + return new Response(null, { + status: 302, + headers: { + location: "https://files.oaiusercontent.com/second?sig=next", + }, + }); + } + return new Response(bytes, { status: 200 }); }, }), ]); - await expectArtifactError( - redirectingRegistry.open(reference), - "unsafe_openai_file_reference", - ); - assert.equal(redirectRequests, 1); - assert.deepEqual(requested, [ - reference.download_url, - generatedReference.download_url, - generatedReference.download_url, - generatedReference.download_url, - generatedReference.download_url, - ]); -} + const redirected = await redirectRegistry.open({ + ...reference, + download_url: "https://files.oaiusercontent.com/first?sig=initial", + }); + assert.deepEqual(await collect(redirected.stream), bytes); -async function testRegistryFailsClosed(): Promise { - const empty = new IncomingArtifactAdapterRegistry(); - await expectArtifactError( - empty.open("https://example.test/file.txt?token=secret"), - "unsupported_incoming_artifact", - ); - await expectArtifactError(empty.open("/tmp/arbitrary.txt"), "unsupported_incoming_artifact"); + const unsafeRedirectRegistry = new IncomingArtifactAdapterRegistry([ + createOpenAIIncomingArtifactAdapter({ + fetch: async () => new Response(null, { + status: 302, + headers: { location: "https://example.com/stolen" }, + }), + }), + ]); await expectArtifactError( - empty.open({ url: "https://example.test/file.txt" }), - "unsupported_incoming_artifact", + unsafeRedirectRegistry.open(reference), + "unsafe_openai_file_reference", ); - const first = memoryAdapter("first", Buffer.from("first")); - const second = memoryAdapter("second", Buffer.from("second")); + const failedDownloadRegistry = new IncomingArtifactAdapterRegistry([ + createOpenAIIncomingArtifactAdapter({ + fetch: async () => new Response("nope", { status: 500 }), + }), + ]); await expectArtifactError( - new IncomingArtifactAdapterRegistry([first, second]).open({ kind: "memory" }), - "ambiguous_incoming_artifact", - ); - assert.throws( - () => new IncomingArtifactAdapterRegistry([{ ...first, id: "Bad Adapter" }]), - (error: unknown) => error instanceof ArtifactError && error.code === "invalid_incoming_adapter", + failedDownloadRegistry.open(reference), + "openai_file_download_failed", ); } -async function testStageArtifact(testRoot: string): Promise { - const bytes = Buffer.alloc(80 * 1024 + 17); - for (let index = 0; index < bytes.length; index += 1) bytes[index] = (index * 17) % 256; - const store = createStore(testRoot); - try { - const digest = createHash("sha256").update(bytes).digest("hex"); - const registry = new IncomingArtifactAdapterRegistry([ - memoryAdapter("memory-test", bytes), - ]); - const result = await stageIncomingArtifact({ - store, - clientId: "client-a", - registry, - input: { - file: { kind: "memory" }, - workspaceId: "ws_association_only", - expectedSha256: `sha256:${digest}`, - ttlHours: 12, - pin: true, - }, - }); - - assert.equal(result.name, "payload.bin"); - assert.equal(result.mimeType, "application/octet-stream"); - assert.equal(result.size, bytes.length); - assert.equal(result.sha256, `sha256:${digest}`); - assert.match(result.instruction, /never writes into a workspace or repository/); - assert.deepEqual(await readFile(result.hostPath), bytes); - - const record = store.statArtifact("client-a", result.artifactId); - assert.equal(record.source, "incoming:memory-test"); - assert.equal(record.workspaceId, "ws_association_only"); - assert.equal(record.pinned, true); - assert.equal(store.health().pendingUploads, 0); - } finally { - store.close(); - } -} - -async function testMaterializeArtifact(testRoot: string): Promise { - const bytes = Buffer.from("native artifact bytes"); - const workspaceRoot = join(testRoot, "workspace"); - await mkdir(workspaceRoot, { recursive: true }); - const store = createStore(testRoot); - try { - const result = await materializeIncomingArtifact({ - store, - clientId: "client-a", - registry: new IncomingArtifactAdapterRegistry([memoryAdapter("memory-test", bytes)]), - workspaceId: "ws_materialize", - workspaceRoot, - destination: join(workspaceRoot, "assets", "generated.bin"), - input: { - file: { kind: "memory" }, - workspaceId: "ws_materialize", - destination: "assets/generated.bin", - onConflict: "error", - }, - }); - assert.deepEqual(await readFile(join(workspaceRoot, result.path)), bytes); - assert.deepEqual(result, { - workspaceId: "ws_materialize", - path: "assets/generated.bin", - name: "generated.bin", - mimeType: "application/octet-stream", - size: bytes.length, - sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, - onConflict: "error", - renamed: false, - }); - assert.equal("artifactId" in result, false); - assert.equal("hostPath" in result, false); - assert.equal(store.health().storedBytes, 0); - } finally { - store.close(); - } -} - -async function testFailedStageCleanup(testRoot: string): Promise { - const store = createStore(testRoot); - try { - const registry = new IncomingArtifactAdapterRegistry([ - memoryAdapter("memory-test", Buffer.from("digest mismatch")), - ]); - await expectArtifactError( - stageIncomingArtifact({ - store, - clientId: "client-a", - registry, - input: { - file: { kind: "memory" }, - expectedSha256: "0".repeat(64), - }, - }), - "sha256_mismatch", - ); - assert.equal(store.health().pendingUploads, 0); - assert.equal(store.health().storedBytes, 0); - } finally { - store.close(); - } -} - -function testStageLogRedaction(): void { - const secret = "https://files.example.test/download?token=super-secret"; - const fields = artifactToolLogFields("materialize_artifact", { - file: { download_url: secret, href: secret, bearer: "Bearer secret" }, - workspaceId: "ws_123", - destination: "assets/generated.png", - expectedSha256: "f".repeat(64), - onConflict: "error", - }); - const serialized = JSON.stringify(fields); - assert.equal("file" in fields, false); - assert.equal(serialized.includes(secret), false); - assert.equal(serialized.includes("Bearer secret"), false); - assert.equal(fields.fileProvided, true); - assert.equal(fields.downloadUrlHostname, "files.example.test"); - assert.equal(fields.expectedSha256Present, true); -} - -function memoryAdapter(id: string, bytes: Buffer): IncomingArtifactAdapter { - return { - id, - canHandle: (value) => ( - typeof value === "object" - && value !== null - && !Array.isArray(value) - && (value as { kind?: unknown }).kind === "memory" - ), - async open() { - return { - name: "payload.bin", - mimeType: "application/octet-stream", - size: bytes.length, - stream: Readable.from(bytes), - }; +function testLogShapeRedaction(): void { + const value = { + download_url: "https://files.oaiusercontent.com/file_123/download?sig=super-secret", + file_id: "file_secret", + nested: { + arbitrary: "private-value", }, }; -} - -function createStore(testRoot: string): ArtifactStore { - return new ArtifactStore({ - stateDir: join(testRoot, "state"), - artifactRoot: join(testRoot, "artifacts"), - artifactMaxFileBytes: 100 * 1024 * 1024, - artifactMaxTotalBytes: 1024 * 1024 * 1024, - artifactDefaultTtlHours: 24, - }); + const serialized = JSON.stringify(describeIncomingArtifactValue(value)); + assert.equal(serialized.includes("super-secret"), false); + assert.equal(serialized.includes("file_secret"), false); + assert.equal(serialized.includes("private-value"), false); + assert.equal(serialized.includes("download_url"), true); + assert.equal(serialized.includes("file_id"), true); } async function collect(stream: Readable): Promise { @@ -414,10 +216,7 @@ async function collect(stream: Readable): Promise { return Buffer.concat(chunks); } -async function expectArtifactError( - promise: Promise, - code: string, -): Promise { +async function expectArtifactError(promise: Promise, code: string): Promise { await assert.rejects( promise, (error: unknown) => error instanceof ArtifactError && error.code === code, diff --git a/src/incoming-artifacts.ts b/src/incoming-artifacts.ts index f2497bcd..0f926085 100644 --- a/src/incoming-artifacts.ts +++ b/src/incoming-artifacts.ts @@ -1,7 +1,7 @@ import { basename, isAbsolute } from "node:path"; import { Readable } from "node:stream"; import type { ReadableStream as NodeReadableStream } from "node:stream/web"; -import { ArtifactError } from "./artifacts.js"; +import { ArtifactError } from "./artifact-error.js"; const ADAPTER_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,63}$/u; const OPENAI_FILE_HOSTS = new Set([ diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index 0b4abb19..74e6ea8b 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -44,8 +44,13 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 1, name: "workspace-state" }, { version: 2, name: "oauth-state" }, { version: 3, name: "local-agent-sessions" }, - { version: 4, name: "artifact-exchange" }, + { version: 4, name: "artifact-exchange-retired" }, ]); + const artifactTables = database.sqlite + .prepare("select name from sqlite_master where type = 'table' and name in ('artifacts', 'artifact_uploads')") + .pluck() + .all(); + assert.deepEqual(artifactTables, []); } finally { database.close(); } diff --git a/src/server.ts b/src/server.ts index 888babe2..1045ad2a 100644 --- a/src/server.ts +++ b/src/server.ts @@ -19,10 +19,6 @@ import type { Request, Response } from "express"; import * as z from "zod/v4"; import { applyPatch } from "./apply-patch.js"; import { registerArtifactTools } from "./artifact-tools.js"; -import { - ARTIFACT_CLEANUP_INTERVAL_MS, - ArtifactStore, -} from "./artifacts.js"; import { loadConfig, type ServerConfig, type WidgetMode } from "./config.js"; import { createOpenAIIncomingArtifactAdapter, @@ -189,7 +185,7 @@ interface ToolLogFields { function serverInstructions(config: ServerConfig): string { const artifactInstruction = config.artifactsEnabled - ? " When the user supplies or generates a file that is not present on the DevSpace host, use materialize_artifact with its native file value, the existing workspace ID, a relative destination, and an explicit conflict mode. Do not recreate binary files with write/edit calls. Do not place signed URLs, native file objects, base64 content, or invented artifact-storage paths in shell commands or logs. materialize_artifact validates, streams, verifies, and writes the file into the selected workspace; it may dirty a repository." + ? " When the user supplies or generates a file that is not present on the DevSpace host, use download_artifact with its native file value and the existing workspace ID. DevSpace chooses a collision-free path under .devspace/incoming and returns that workspace-relative path. Use the normal workspace tools for any later move, rename, or deletion. Do not recreate binary files with write/edit calls or place signed URLs, native file objects, base64 content, or invented host paths in shell commands or logs." : ""; const showChangesInstruction = config.widgets === "changes" @@ -700,8 +696,6 @@ function createMcpServer( reviewCheckpoints: ReturnType, processSessions: ProcessSessionManager, localAgentProviders: LocalAgentProviderAvailability[], - artifactStore: ArtifactStore | undefined, - clientId: string, incomingArtifactAdapters: readonly IncomingArtifactAdapter[], ): McpServer { const server = new McpServer( @@ -1609,12 +1603,10 @@ function createMcpServer( registerCodexProcessTools(server, config, workspaces, processSessions); } - if (artifactStore) { + if (config.artifactsEnabled) { registerArtifactTools(server, { config, - store: artifactStore, workspaces, - clientId, incomingArtifactAdapters, }); } @@ -1649,7 +1641,6 @@ export function createServer( resourceMetadataUrl: getOAuthProtectedResourceMetadataUrl(resourceServerUrl), }); const workspaceStore = createWorkspaceStore(config.stateDir); - const artifactStore = config.artifactsEnabled ? new ArtifactStore(config) : undefined; const workspaces = new WorkspaceRegistry(config, workspaceStore); const reviewCheckpoints = createReviewCheckpointManager(); const processSessions = new ProcessSessionManager(); @@ -1688,24 +1679,6 @@ export function createServer( }, MCP_SESSION_CLEANUP_INTERVAL_MS); sessionCleanupTimer.unref(); - const runArtifactCleanup = () => { - if (!artifactStore) return; - void artifactStore.cleanupExpired() - .then((result) => { - logEvent(config.logging, "debug", "artifact_cleanup", { ...result }); - }) - .catch((error) => { - logEvent(config.logging, "warn", "artifact_cleanup_failed", { - error: error instanceof Error ? error.message : String(error), - }); - }); - }; - runArtifactCleanup(); - const artifactCleanupTimer = artifactStore - ? setInterval(runArtifactCleanup, ARTIFACT_CLEANUP_INTERVAL_MS) - : undefined; - artifactCleanupTimer?.unref(); - if (config.logging.trustProxy) { app.set("trust proxy", true); } @@ -1834,8 +1807,6 @@ export function createServer( reviewCheckpoints, processSessions, localAgentProviders, - artifactStore, - req.auth.clientId, incomingArtifactAdapters, ); await server.connect(transport); @@ -1864,12 +1835,10 @@ export function createServer( close: () => { closePromise ??= (async () => { clearInterval(sessionCleanupTimer); - if (artifactCleanupTimer) clearInterval(artifactCleanupTimer); const results = await transports.closeAll(); logSessionCloseResults("server_shutdown", results); processSessions.shutdown(); oauthProvider.close(); - artifactStore?.close(); workspaceStore.close?.(); })(); return closePromise; @@ -1897,10 +1866,7 @@ if (await isMainModule()) { console.log(`request logging: ${config.logging.requests ? "enabled" : "disabled"}`); console.log(`asset logging: ${config.logging.assets ? "enabled" : "disabled"}`); console.log(`trust proxy: ${config.logging.trustProxy ? "enabled" : "disabled"}`); - console.log(`native artifact staging: ${config.artifactsEnabled ? "enabled" : "disabled"}`); - if (config.artifactsEnabled) { - console.log(`artifact root: ${config.artifactRoot}`); - } + console.log(`native artifact download: ${config.artifactsEnabled ? "enabled" : "disabled"}`); if (config.subagents) { console.log(`subagent providers: ${formatLocalAgentProviderAvailabilitySummary(localAgentProviders)}`); } diff --git a/src/user-config.ts b/src/user-config.ts index ce1e3386..5dd793ef 100644 --- a/src/user-config.ts +++ b/src/user-config.ts @@ -18,10 +18,7 @@ export interface DevspaceUserConfig { stateDir?: string; worktreeRoot?: string; artifactsEnabled?: boolean; - artifactRoot?: string; artifactMaxFileBytes?: number; - artifactMaxTotalBytes?: number; - artifactDefaultTtlHours?: number; agentDir?: string; subagents?: boolean; } From 511217865fea80c7f9bd94b2d503b639e47cf8f4 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:43:40 +0000 Subject: [PATCH 07/16] test: make artifact security fixtures umask-independent --- src/artifact-download.test.ts | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index 0d8de1ac..ecc6d829 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { + chmod, mkdir, mkdtemp, readFile, @@ -31,6 +32,7 @@ try { await testSafeDownloadAndCollision(join(root, "downloads")); await testSizeLimitAndCleanup(join(root, "size-limit")); await testCrashLeftoverCleanup(join(root, "stale-partials")); + await testUnsafeDirectoryPermissions(join(root, "unsafe-permissions")); await testSymlinkRejection(join(root, "symlinks")); testLogRedaction(); } finally { @@ -186,14 +188,34 @@ async function testCrashLeftoverCleanup(testRoot: string): Promise { assert.equal(entries.includes("keep-me.partial"), true); } +async function testUnsafeDirectoryPermissions(testRoot: string): Promise { + if (process.platform === "win32") return; + + const workspaceRoot = join(testRoot, "workspace"); + const devspaceDirectory = join(workspaceRoot, ".devspace"); + await mkdir(devspaceDirectory, { recursive: true, mode: 0o700 }); + await chmod(devspaceDirectory, 0o777); + + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + }), + "artifact_directory_permissions_unsafe", + ); +} + async function testSymlinkRejection(testRoot: string): Promise { if (process.platform === "win32") return; const outside = join(testRoot, "outside"); - await mkdir(outside, { recursive: true }); + await mkdir(outside, { recursive: true, mode: 0o700 }); const linkedDevspaceRoot = join(testRoot, "linked-devspace-workspace"); - await mkdir(linkedDevspaceRoot, { recursive: true }); + await mkdir(linkedDevspaceRoot, { recursive: true, mode: 0o700 }); await symlink(outside, join(linkedDevspaceRoot, ".devspace"), "dir"); await expectArtifactError( downloadIncomingArtifact({ @@ -207,7 +229,10 @@ async function testSymlinkRejection(testRoot: string): Promise { ); const linkedIncomingRoot = join(testRoot, "linked-incoming-workspace"); - await mkdir(join(linkedIncomingRoot, ".devspace"), { recursive: true }); + await mkdir(join(linkedIncomingRoot, ".devspace"), { + recursive: true, + mode: 0o700, + }); await symlink(outside, join(linkedIncomingRoot, ".devspace", "incoming"), "dir"); await expectArtifactError( downloadIncomingArtifact({ From ed1bff2d923f88008853f759dce4dcd33ad2f4c5 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 18:33:05 +0530 Subject: [PATCH 08/16] feat: download artifacts to requested paths --- src/artifact-download.test.ts | 103 ++++++++++--- src/artifact-tools.ts | 273 +++++++++++++++++++++++++--------- src/server.ts | 2 +- 3 files changed, 292 insertions(+), 86 deletions(-) diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index ecc6d829..36df6d5f 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -29,11 +29,13 @@ const root = await mkdtemp(join(tmpdir(), "devspace-artifact-download-test-")); try { testOneToolContract(); - await testSafeDownloadAndCollision(join(root, "downloads")); + await testSafeDownloadAndConflict(join(root, "downloads")); + await testDestinationValidation(join(root, "destinations")); await testSizeLimitAndCleanup(join(root, "size-limit")); await testCrashLeftoverCleanup(join(root, "stale-partials")); await testUnsafeDirectoryPermissions(join(root, "unsafe-permissions")); await testSymlinkRejection(join(root, "symlinks")); + await testPrivateStagingPermissions(join(root, "permissions")); testLogRedaction(); } finally { await rm(root, { recursive: true, force: true }); @@ -64,8 +66,9 @@ function testOneToolContract(): void { const descriptor = registered.get("download_artifact")?.descriptor; assert.ok(descriptor); assert.deepEqual(descriptor._meta, { "openai/fileParams": ["file"] }); - assert.deepEqual(Object.keys(descriptor.inputSchema as object).sort(), ["file", "workspaceId"]); + assert.deepEqual(Object.keys(descriptor.inputSchema as object).sort(), ["file", "path", "workspaceId"]); assert.deepEqual(Object.keys(descriptor.outputSchema as object), ["path"]); + assert.equal((descriptor.annotations as { destructiveHint?: boolean }).destructiveHint, false); const fileSchema = (descriptor.inputSchema as z.ZodRawShape).file as z.ZodType; const valid = { @@ -78,7 +81,7 @@ function testOneToolContract(): void { assert.throws(() => fileSchema.parse({ file_id: "file_123" })); } -async function testSafeDownloadAndCollision(testRoot: string): Promise { +async function testSafeDownloadAndConflict(testRoot: string): Promise { const workspaceRoot = join(testRoot, "workspace"); await mkdir(workspaceRoot, { recursive: true }); const bytes = Buffer.from("native artifact bytes\u0000\xff", "latin1"); @@ -94,26 +97,46 @@ async function testSafeDownloadAndCollision(testRoot: string): Promise { workspaceRoot, maxFileBytes: 1024, file: { native: true }, + path: "public/images/generated.png", }); - assert.equal(first.path, ".devspace/incoming/generated.png"); + assert.equal(first.path, "public/images/generated.png"); assert.deepEqual(await readFile(join(workspaceRoot, first.path)), bytes); - const second = await downloadIncomingArtifact({ - registry: registryFor({ - name: "generated.png", - size: bytes.length, - stream: Readable.from([bytes]), + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ + name: "replacement.png", + stream: Readable.from(["replacement"]), + }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + path: "public/images/generated.png", }), - workspaceId: "ws_test", - workspaceRoot, - maxFileBytes: 1024, - file: { native: true }, - }); - assert.equal(second.path, ".devspace/incoming/generated (1).png"); - assert.deepEqual(await readFile(join(workspaceRoot, second.path)), bytes); + "artifact_destination_exists", + ); + assert.deepEqual(await readFile(join(workspaceRoot, first.path)), bytes); + assert.deepEqual(await readdir(join(workspaceRoot, ".devspace", "incoming")), []); +} - const entries = await readdir(join(workspaceRoot, ".devspace", "incoming")); - assert.deepEqual(entries.sort(), ["generated (1).png", "generated.png"]); +async function testDestinationValidation(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + + for (const path of ["../outside.txt", "nested/../outside.txt", "/absolute.txt", "folder/"]) { + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + path, + }), + "artifact_destination_invalid", + ); + } } async function testSizeLimitAndCleanup(testRoot: string): Promise { @@ -131,6 +154,7 @@ async function testSizeLimitAndCleanup(testRoot: string): Promise { workspaceRoot, maxFileBytes: 4, file: { native: true }, + path: "too-large.bin", }), "artifact_file_too_large", ); @@ -145,6 +169,7 @@ async function testSizeLimitAndCleanup(testRoot: string): Promise { workspaceRoot, maxFileBytes: 4, file: { native: true }, + path: "stream-too-large.bin", }), "artifact_file_too_large", ); @@ -162,6 +187,7 @@ async function testCrashLeftoverCleanup(testRoot: string): Promise { workspaceRoot, maxFileBytes: 1024, file: { native: true }, + path: "first.txt", }); const incoming = join(workspaceRoot, ".devspace", "incoming"); @@ -180,6 +206,7 @@ async function testCrashLeftoverCleanup(testRoot: string): Promise { workspaceRoot, maxFileBytes: 1024, file: { native: true }, + path: "second.txt", }); const entries = await readdir(incoming); @@ -203,6 +230,7 @@ async function testUnsafeDirectoryPermissions(testRoot: string): Promise { workspaceRoot, maxFileBytes: 1024, file: { native: true }, + path: "blocked.txt", }), "artifact_directory_permissions_unsafe", ); @@ -224,6 +252,7 @@ async function testSymlinkRejection(testRoot: string): Promise { workspaceRoot: linkedDevspaceRoot, maxFileBytes: 1024, file: { native: true }, + path: "blocked.txt", }), "artifact_directory_unsafe", ); @@ -241,9 +270,46 @@ async function testSymlinkRejection(testRoot: string): Promise { workspaceRoot: linkedIncomingRoot, maxFileBytes: 1024, file: { native: true }, + path: "blocked.txt", }), "artifact_directory_unsafe", ); + + const linkedDestinationRoot = join(testRoot, "linked-destination-workspace"); + await mkdir(linkedDestinationRoot, { recursive: true }); + await symlink(outside, join(linkedDestinationRoot, "assets"), "dir"); + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), + workspaceId: "ws_test", + workspaceRoot: linkedDestinationRoot, + maxFileBytes: 1024, + file: { native: true }, + path: "assets/blocked.txt", + }), + "artifact_destination_parent_unsafe", + ); +} + +async function testPrivateStagingPermissions(testRoot: string): Promise { + if (process.platform === "win32") return; + + const workspaceRoot = join(testRoot, "workspace"); + const incoming = join(workspaceRoot, ".devspace", "incoming"); + await mkdir(incoming, { recursive: true }); + await chmod(incoming, 0o755); + + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "private.txt", stream: Readable.from(["private"]) }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + path: "private.txt", + }), + "artifact_directory_permissions_unsafe", + ); } function testLogRedaction(): void { @@ -254,6 +320,7 @@ function testLogRedaction(): void { file_name: "generated.png", }, workspaceId: "ws_secret", + path: "private/generated.png", }); const serialized = JSON.stringify(fields); assert.equal(serialized.includes("super-secret"), false); diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index 702aafd6..57ed0712 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -9,7 +9,7 @@ import { unlink, type FileHandle, } from "node:fs/promises"; -import { basename, extname, join } from "node:path"; +import { isAbsolute, join, normalize, sep } from "node:path"; import { registerAppTool } from "@modelcontextprotocol/ext-apps/server"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import * as z from "zod/v4"; @@ -25,15 +25,12 @@ import type { WorkspaceRegistry } from "./workspaces.js"; const ARTIFACT_WRITE_ANNOTATIONS = { readOnlyHint: false, - destructiveHint: true, + destructiveHint: false, idempotentHint: false, openWorldHint: true, }; const NO_FOLLOW = fsConstants.O_NOFOLLOW ?? 0; const DIRECTORY_FLAGS = fsConstants.O_RDONLY | (fsConstants.O_DIRECTORY ?? 0) | NO_FOLLOW; -const MAX_RENAME_ATTEMPTS = 10_000; -const MAX_SAFE_FILENAME_BYTES = 180; -const MAX_SAFE_EXTENSION_BYTES = 32; const PARTIAL_PREFIX = ".devspace-download-"; const PARTIAL_SUFFIX = ".partial"; const STALE_PARTIAL_AGE_MS = 24 * 60 * 60 * 1_000; @@ -57,6 +54,7 @@ export interface ArtifactToolRegistrationOptions { export interface DownloadIncomingArtifactInput { file: unknown; workspaceId: string; + path: string; } export interface DownloadIncomingArtifactResult { @@ -73,6 +71,18 @@ interface SecureIncomingDirectory { close(): Promise; } +interface SecureDestinationDirectory { + handle: FileHandle; + anchorPath: string; + close(): Promise; +} + +interface ArtifactDestination { + path: string; + parentParts: string[]; + name: string; +} + export function registerArtifactTools( server: McpServer, { @@ -89,7 +99,7 @@ export function registerArtifactTools( { title: "Download attached or generated file", description: - "Stream one MCP-host-provided native file into the selected workspace's .devspace/incoming directory. DevSpace chooses a collision-free filename and returns its workspace-relative path. Arbitrary URLs, local paths, and malformed file objects are rejected.", + "Stream one MCP-host-provided native file to a requested relative path inside an already-open workspace. Existing destinations, arbitrary URLs, absolute paths, traversal, symlinked parents, local source paths, and malformed file objects are rejected.", inputSchema: { file: openAIFileReferenceInputSchema.describe( "Native file value authorized and supplied by the MCP host.", @@ -97,6 +107,9 @@ export function registerArtifactTools( workspaceId: z.string().min(1).describe( "Workspace identifier returned by open_workspace.", ), + path: z.string().min(1).describe( + "Relative destination path inside the selected workspace. The destination must not already exist.", + ), }, outputSchema: { path: z.string(), @@ -112,6 +125,7 @@ export function registerArtifactTools( workspaceRoot: workspace.root, maxFileBytes: config.artifactMaxFileBytes, file: input.file, + path: input.path, }); return { publicResult: { path: downloaded.path }, @@ -126,8 +140,8 @@ export function registerArtifactTools( * * Bytes are written to an exclusive private partial, hashed and size-checked, * chmodded through the still-open file descriptor, fsynced, and only then - * published with an atomic hard link. No path-based chmod/hash/verification is - * performed after publication. + * published without overwriting the requested workspace destination. No + * path-based chmod or hashing is performed after publication. */ export async function downloadIncomingArtifact({ registry, @@ -135,12 +149,14 @@ export async function downloadIncomingArtifact({ workspaceRoot, maxFileBytes, file, + path, }: { registry: IncomingArtifactAdapterRegistry; workspaceId: string; workspaceRoot: string; maxFileBytes: number; file: unknown; + path: string; }): Promise { if (!Number.isSafeInteger(maxFileBytes) || maxFileBytes < 1) { throw new ArtifactError( @@ -155,8 +171,10 @@ export async function downloadIncomingArtifact({ ); } + const destination = normalizeArtifactDestination(path); const opened = await registry.open(file); let secureDirectory: SecureIncomingDirectory | undefined; + let destinationDirectory: SecureDestinationDirectory | undefined; let partialPath: string | undefined; let handle: FileHandle | undefined; @@ -229,17 +247,21 @@ export async function downloadIncomingArtifact({ ); } - const safeName = normalizeArtifactFilename(opened.name); - const finalName = await publishCollisionFree( - secureDirectory, + destinationDirectory = await prepareDestinationDirectory( + secureDirectory.rootHandle, + destination.parentParts, + ); + await publishDestination( + destinationDirectory, partialPath, - safeName, + destination.name, + writtenEntry, ); - await unlink(partialPath); + await unlink(partialPath).catch(() => undefined); partialPath = undefined; return { - path: `.devspace/incoming/${finalName}`, + path: destination.path, size, sha256: `sha256:${hash.digest("hex")}`, }; @@ -249,6 +271,7 @@ export async function downloadIncomingArtifact({ } finally { await handle?.close().catch(() => undefined); if (partialPath) await unlink(partialPath).catch(() => undefined); + await destinationDirectory?.close().catch(() => undefined); await secureDirectory?.close().catch(() => undefined); } } @@ -261,6 +284,7 @@ export function artifactToolLogFields( fileReferenceShape: describeIncomingArtifactValue(input.file), downloadUrlHostname: incomingFileDownloadHostname(input.file), workspaceId: input.workspaceId, + path: input.path, }; } @@ -322,12 +346,18 @@ async function prepareIncomingDirectory( "Selected workspace root is not a real directory.", ); const rootAnchor = descriptorDirectoryPath(rootHandle); - devspaceHandle = await ensureChildDirectory(rootHandle, rootAnchor, ".devspace"); + devspaceHandle = await ensureChildDirectory( + rootHandle, + rootAnchor, + ".devspace", + false, + ); const devspaceAnchor = descriptorDirectoryPath(devspaceHandle); incomingHandle = await ensureChildDirectory( devspaceHandle, devspaceAnchor, "incoming", + true, ); const anchorPath = descriptorDirectoryPath(incomingHandle); @@ -354,6 +384,7 @@ async function ensureChildDirectory( parentHandle: FileHandle, parentAnchor: string, name: string, + privateAccess: boolean, ): Promise { await assertDirectoryHandle(parentHandle); const path = join(parentAnchor, name); @@ -369,7 +400,11 @@ async function ensureChildDirectory( "Artifact destination parent is not a real directory.", ); try { - await assertPrivateDirectoryHandle(child); + if (privateAccess) { + await assertPrivateDirectoryHandle(child); + } else { + await assertOwnedDirectoryHandle(child); + } return child; } catch (error) { await child.close().catch(() => undefined); @@ -377,6 +412,27 @@ async function ensureChildDirectory( } } +async function assertOwnedDirectoryHandle(handle: FileHandle): Promise { + const entry = await handle.stat(); + if (!entry.isDirectory()) { + throw new ArtifactError( + "artifact_directory_unsafe", + "Artifact staging parent is not a directory.", + ); + } + if (process.platform === "win32") return; + const currentUid = process.getuid?.(); + if ( + (currentUid !== undefined && entry.uid !== currentUid) + || (Number(entry.mode) & 0o022) !== 0 + ) { + throw new ArtifactError( + "artifact_directory_permissions_unsafe", + "Artifact staging parent must be owned by the current user and not group/world writable.", + ); + } +} + async function openDirectoryNoFollow( path: string, code: string, @@ -416,11 +472,11 @@ async function assertPrivateDirectoryHandle(handle: FileHandle): Promise { const currentUid = process.getuid?.(); if ( (currentUid !== undefined && entry.uid !== currentUid) - || (Number(entry.mode) & 0o022) !== 0 + || (Number(entry.mode) & 0o777) !== 0o700 ) { throw new ArtifactError( "artifact_directory_permissions_unsafe", - "Artifact destination directory must be owned by the current user and not group/world writable.", + "Artifact staging directory must be owned by and accessible only to the current user.", ); } } @@ -436,64 +492,147 @@ function descriptorDirectoryPath(handle: FileHandle): string { ); } -async function publishCollisionFree( - directory: SecureIncomingDirectory, - partialPath: string, - filename: string, -): Promise { - for (let attempt = 0; attempt < MAX_RENAME_ATTEMPTS; attempt += 1) { - await assertPrivateDirectoryHandle(directory.incomingHandle); - const candidateName = attempt === 0 - ? filename - : renamedFilename(filename, attempt); - const candidate = join(directory.anchorPath, candidateName); - try { - await link(partialPath, candidate); - return candidateName; - } catch (error) { - if (isNodeError(error) && error.code === "EEXIST") continue; - throw error; +function normalizeArtifactDestination(value: string): ArtifactDestination { + const rawParts = value.split(sep); + if ( + !value + || value.includes("\u0000") + || isAbsolute(value) + || value.endsWith(sep) + || rawParts.includes("..") + ) { + throw new ArtifactError( + "artifact_destination_invalid", + "Artifact destination must be a non-empty relative file path inside the workspace.", + ); + } + + const normalized = normalize(value); + if ( + normalized === "." + || normalized === ".." + || normalized.startsWith(`..${sep}`) + ) { + throw new ArtifactError( + "artifact_destination_invalid", + "Artifact destination must stay inside the selected workspace.", + ); + } + + const parts = normalized.split(sep); + const name = parts.at(-1); + if (!name || name === "." || name === "..") { + throw new ArtifactError( + "artifact_destination_invalid", + "Artifact destination must name a file inside the selected workspace.", + ); + } + + return { + path: normalized, + parentParts: parts.slice(0, -1), + name, + }; +} + +async function prepareDestinationDirectory( + rootHandle: FileHandle, + parentParts: readonly string[], +): Promise { + const openedHandles: FileHandle[] = []; + let parentHandle = rootHandle; + let parentAnchor = descriptorDirectoryPath(rootHandle); + + try { + for (const part of parentParts) { + const child = await ensureWorkspaceChildDirectory( + parentHandle, + parentAnchor, + part, + ); + openedHandles.push(child); + parentHandle = child; + parentAnchor = descriptorDirectoryPath(child); } + + return { + handle: parentHandle, + anchorPath: parentAnchor, + async close() { + for (const handle of openedHandles.reverse()) { + await handle.close().catch(() => undefined); + } + }, + }; + } catch (error) { + for (const handle of openedHandles.reverse()) { + await handle.close().catch(() => undefined); + } + throw error; } - throw new ArtifactError( - "artifact_filename_exhausted", - "Could not choose an available incoming filename.", - ); } -export function normalizeArtifactFilename(value: string): string { - const flattened = value.replaceAll("\\", "/"); - let candidate = basename(flattened) - .replace(/[\u0000-\u001F\u007F]/gu, "") - .trim(); - if (!candidate || candidate === "." || candidate === ".." || candidate.startsWith(".")) { - candidate = "download.bin"; +async function ensureWorkspaceChildDirectory( + parentHandle: FileHandle, + parentAnchor: string, + name: string, +): Promise { + await assertDirectoryHandle(parentHandle); + const path = join(parentAnchor, name); + try { + await mkdir(path, { mode: 0o755 }); + } catch (error) { + if (!isNodeError(error) || error.code !== "EEXIST") throw error; } - const extension = extname(candidate); - const safeExtension = Buffer.byteLength(extension, "utf8") <= MAX_SAFE_EXTENSION_BYTES - ? extension - : ""; - const stemCharacters = Array.from( - safeExtension ? candidate.slice(0, -safeExtension.length) : candidate, + return openDirectoryNoFollow( + path, + "artifact_destination_parent_unsafe", + "Artifact destination parent must be a real directory inside the workspace.", ); - while ( - stemCharacters.length > 1 - && Buffer.byteLength(`${stemCharacters.join("")}${safeExtension}`, "utf8") - > MAX_SAFE_FILENAME_BYTES - ) { - stemCharacters.pop(); - } - candidate = `${stemCharacters.join("")}${safeExtension}`; - return Buffer.byteLength(candidate, "utf8") <= MAX_SAFE_FILENAME_BYTES - ? candidate - : "download.bin"; } -function renamedFilename(filename: string, attempt: number): string { - const extension = extname(filename); - const stem = extension ? filename.slice(0, -extension.length) : filename; - return `${stem} (${attempt})${extension}`; +async function publishDestination( + directory: SecureDestinationDirectory, + partialPath: string, + filename: string, + writtenEntry: Awaited>, +): Promise { + await assertDirectoryHandle(directory.handle); + const candidate = join(directory.anchorPath, filename); + let published = false; + try { + await link(partialPath, candidate); + published = true; + const publishedEntry = await lstat(candidate); + if ( + publishedEntry.isSymbolicLink() + || !publishedEntry.isFile() + || publishedEntry.dev !== writtenEntry.dev + || publishedEntry.ino !== writtenEntry.ino + || publishedEntry.size !== writtenEntry.size + ) { + throw new ArtifactError( + "artifact_destination_publish_failed", + "Published artifact did not match the verified download.", + ); + } + } catch (error) { + if (published) await unlink(candidate).catch(() => undefined); + if (isNodeError(error) && error.code === "EEXIST") { + throw new ArtifactError( + "artifact_destination_exists", + "Artifact destination already exists.", + ); + } + if (isNodeError(error) && error.code === "EXDEV") { + throw new ArtifactError( + "artifact_destination_filesystem_unsupported", + "Artifact staging and destination must be on the same filesystem.", + ); + } + throw error; + } } async function cleanupStalePartials( diff --git a/src/server.ts b/src/server.ts index 1045ad2a..f13a79aa 100644 --- a/src/server.ts +++ b/src/server.ts @@ -185,7 +185,7 @@ interface ToolLogFields { function serverInstructions(config: ServerConfig): string { const artifactInstruction = config.artifactsEnabled - ? " When the user supplies or generates a file that is not present on the DevSpace host, use download_artifact with its native file value and the existing workspace ID. DevSpace chooses a collision-free path under .devspace/incoming and returns that workspace-relative path. Use the normal workspace tools for any later move, rename, or deletion. Do not recreate binary files with write/edit calls or place signed URLs, native file objects, base64 content, or invented host paths in shell commands or logs." + ? " When the user supplies or generates a file that is not present on the DevSpace host, use download_artifact with its native file value, the existing workspace ID, and a suitable relative destination path chosen from the user's request and project structure. The tool refuses to overwrite an existing destination and returns the normalized workspace-relative path. Use normal workspace tools when explicit inspection, replacement, movement, renaming, or deletion is needed. Do not recreate binary files with write/edit calls or place signed URLs, native file objects, base64 content, or invented host paths in shell commands or logs." : ""; const showChangesInstruction = config.widgets === "changes" From 54a75cbdcb3a91698b76e1e0e120de5defdfdac3 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 18:33:05 +0530 Subject: [PATCH 09/16] docs: describe artifact destination handoff --- .env.example | 4 ++-- docs/artifact-exchange.md | 36 ++++++++++++++++++++++++++---------- docs/configuration.md | 12 ++++++------ docs/security.md | 22 ++++++++++++---------- src/artifact-workspace.ts | 2 +- 5 files changed, 47 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index 7f4a48bf..e42f2130 100644 --- a/.env.example +++ b/.env.example @@ -27,8 +27,8 @@ DEVSPACE_LOG_SHELL_COMMANDS=0 # DEVSPACE_TRUST_PROXY=1 # DEVSPACE_STATE_DIR=/home/waishnav/.local/share/devspace # DEVSPACE_WORKTREE_ROOT=/home/waishnav/.devspace/worktrees -# Native-file download is opt-in. Files stream directly into the selected -# workspace's .devspace/incoming directory. +# Native-file download is opt-in. Files stream to a model-selected relative +# path inside an already-open workspace without overwriting existing files. # DEVSPACE_ARTIFACTS=1 # DEVSPACE_ARTIFACT_MAX_FILE_BYTES=104857600 # DEVSPACE_AUTO_LOAD_AGENTS_MD=1 diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index e0cd46d2..49640df1 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -6,14 +6,15 @@ DevSpace can stream a file attached or generated by an MCP host (such as ChatGPT ```text open_workspace - -> download_artifact({ file, workspaceId }) + -> download_artifact({ file, workspaceId, path }) -> { path } ``` -`download_artifact` is the only artifact-specific MCP tool. It accepts exactly two inputs: +`download_artifact` is the only artifact-specific MCP tool. It accepts exactly three inputs: - the native `file` value authorized by the MCP host; -- a `workspaceId` returned by `open_workspace`. +- a `workspaceId` returned by `open_workspace`; +- a relative destination `path` chosen for the selected workspace. ```json { @@ -23,19 +24,24 @@ open_workspace "mime_type": "image/png", "file_name": "generated-image.png" }, - "workspaceId": "ws_123" + "workspaceId": "ws_123", + "path": "public/images/generated-image.png" } ``` -DevSpace chooses a normalized, collision-free filename below the selected workspace's `.devspace/incoming/` directory. A successful call returns only the workspace-relative path: +DevSpace validates the requested path, safely creates missing parent directories, +and refuses to replace an existing destination. A successful call returns only +the normalized workspace-relative path: ```json { - "path": ".devspace/incoming/generated-image.png" + "path": "public/images/generated-image.png" } ``` -When that name already exists, DevSpace chooses `generated-image (1).png`, `generated-image (2).png`, and so on. Use the normal DevSpace filesystem tools for any later move, rename, replacement, or deletion. +If the destination already exists, the tool fails without modifying it. Use the +normal DevSpace filesystem tools when the user explicitly wants inspection, +replacement, movement, renaming, or deletion. ## Accepted file values @@ -59,14 +65,24 @@ The download is never buffered wholesale in memory. DevSpace: 3. streams bytes under `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` while computing SHA-256; 4. verifies any supplied size hint; 5. chmods and fsyncs through the still-open file descriptor; -6. publishes the verified inode with an atomic hard link to a collision-free final name; -7. removes the private partial. +6. opens or creates each requested destination parent without following symlinks; +7. publishes the verified inode with an atomic hard link to the exact requested path; +8. verifies that the published inode is the downloaded inode; +9. removes the private partial. There is no persistent artifact database, completed-object store, upload/chunk API, quota ledger, TTL, pinning, artifact ID, signed download route, or reusable artifact lifecycle. ## Directory safety -DevSpace opens the selected workspace without following symlinks, then creates or opens `.devspace` and `incoming` through already-open parent directory descriptors. All partial creation, cleanup, and final publication remain anchored to the open `incoming` descriptor, so replacing a pathname cannot redirect writes outside the selected workspace. Symlinked parents, non-directories, and group/world-writable incoming directories fail closed. Existing directories are inspected but never chmodded as a startup side effect. +DevSpace opens the selected workspace without following symlinks, then creates +or opens `.devspace` and its private `incoming` staging directory through +already-open parent directory descriptors. Destination parent components are +also created or opened one at a time through descriptor anchors. Absolute paths, +traversal, symlinked parents, non-directories, non-private staging directories, +and existing destination files fail closed. Existing directories are inspected +but never chmodded as a startup side effect. The staging directory and requested +destination must reside on the same filesystem so publication can remain +no-overwrite and atomic. Published files are not path-chmodded or path-hashed after publication. Temporary partial cleanup is bounded, only considers DevSpace partial names, and ignores symlinks and non-regular files. diff --git a/docs/configuration.md b/docs/configuration.md index 360de695..ad82a1ab 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,12 +56,12 @@ DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve The same settings may be persisted in `~/.devspace/config.json` as `artifactsEnabled` and `artifactMaxFileBytes`. -`download_artifact` accepts only the native file object supplied by the MCP -connector plus a `workspaceId` returned by `open_workspace`. DevSpace chooses a -collision-free path under `.devspace/incoming/` and returns only that -workspace-relative path. It does not accept destinations, conflict modes, -expected hashes, arbitrary URL strings, local paths, embedded credentials, or -extra object fields. +`download_artifact` accepts the native file object supplied by the MCP connector, +a `workspaceId` returned by `open_workspace`, and a relative workspace `path`. +DevSpace safely creates missing parent directories, refuses to overwrite an +existing destination, and returns only the normalized workspace-relative path. +It does not accept conflict modes, expected hashes, arbitrary URL strings, local +paths, embedded credentials, or extra object fields. There is no artifact root, total quota, TTL, pinning, persistent database record, or background artifact cleanup service. See [Native File Download](artifact-exchange.md) diff --git a/docs/security.md b/docs/security.md index e4d6ab41..3af44234 100644 --- a/docs/security.md +++ b/docs/security.md @@ -108,23 +108,25 @@ fragments, malformed IDs, extra fields, and redirects outside that boundary fail closed. Opaque IDs are bounded metadata and are never used as filenames or path components. -`download_artifact` accepts only the native file value plus a `workspaceId` -returned by `open_workspace`. DevSpace chooses a normalized, collision-free path -below `.devspace/incoming/`; callers cannot supply a destination, conflict mode, -expected hash, host path, or storage policy. +`download_artifact` accepts only the native file value, a `workspaceId` returned +by `open_workspace`, and a relative destination `path`. Absolute paths, +traversal, symlinked parents, and existing destinations fail closed. Callers +cannot supply a conflict mode, expected hash, host path, or storage policy. The selected workspace is opened without following symlinks. DevSpace then -creates or opens `.devspace` and `incoming` through already-open parent directory -descriptors, and keeps partial creation, cleanup, and final publication anchored -to the open `incoming` descriptor. Replacing a pathname therefore cannot redirect -writes outside the selected workspace. Symlinked components, non-directories, -and group/world-writable incoming directories fail closed. Existing directories +creates or opens `.devspace` and its private `incoming` staging directory through +already-open parent directory descriptors. Requested destination parents are +also created or opened component-by-component through descriptor anchors. +Replacing a pathname therefore cannot redirect writes outside the selected +workspace. Symlinked components, non-directories, non-private incoming +directories, and existing destination files fail closed. Existing directories are inspected but are never chmodded as a startup side effect. Bytes stream into an exclusive mode-`0600` partial under the configured per-file limit. DevSpace computes SHA-256 while writing, verifies any size hint, chmods and fsyncs through the still-open descriptor, then publishes the verified inode -with an atomic hard link. It does not path-chmod or path-hash the published file. +with a no-overwrite atomic hard link at the requested path and verifies the +published inode identity. It does not path-chmod or path-hash the published file. Partials are removed on success or failure; crash-leftover cleanup is bounded and only considers owned, regular DevSpace partial files. diff --git a/src/artifact-workspace.ts b/src/artifact-workspace.ts index d769053f..4f02f0e3 100644 --- a/src/artifact-workspace.ts +++ b/src/artifact-workspace.ts @@ -1,3 +1,3 @@ // The persistent-store workspace materializer was removed. Native files now -// stream directly into .devspace/incoming through download_artifact. +// stream directly to a requested workspace-relative path through download_artifact. export {}; From 335ad60aba7c8fbf3d7d7cec9d167ccf93d24e7b Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 20:18:59 +0530 Subject: [PATCH 10/16] refactor: stage artifacts beside destination --- docs/artifact-exchange.md | 36 +++--- docs/security.md | 27 ++--- src/artifact-download.test.ts | 93 +++------------ src/artifact-tools.ts | 208 +++++++--------------------------- 4 files changed, 88 insertions(+), 276 deletions(-) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 49640df1..53b45426 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -61,30 +61,28 @@ The tool retains the MCP metadata declaration `openai/fileParams: ["file"]` so c The download is never buffered wholesale in memory. DevSpace: 1. validates the trusted native reference and redirect chain; -2. opens a private exclusive partial below `.devspace/incoming/`; -3. streams bytes under `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` while computing SHA-256; -4. verifies any supplied size hint; -5. chmods and fsyncs through the still-open file descriptor; -6. opens or creates each requested destination parent without following symlinks; -7. publishes the verified inode with an atomic hard link to the exact requested path; -8. verifies that the published inode is the downloaded inode; -9. removes the private partial. +2. opens or creates each requested destination parent without following symlinks; +3. opens an exclusive mode-`0600` partial beside the requested destination; +4. streams bytes under `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` while computing SHA-256; +5. verifies any supplied size hint and fsyncs the partial; +6. publishes the verified inode with an atomic hard link to the exact requested path; +7. verifies that the published inode is the downloaded inode; +8. chmods and fsyncs through the still-open file descriptor; +9. removes the temporary partial. There is no persistent artifact database, completed-object store, upload/chunk API, quota ledger, TTL, pinning, artifact ID, signed download route, or reusable artifact lifecycle. ## Directory safety -DevSpace opens the selected workspace without following symlinks, then creates -or opens `.devspace` and its private `incoming` staging directory through -already-open parent directory descriptors. Destination parent components are -also created or opened one at a time through descriptor anchors. Absolute paths, -traversal, symlinked parents, non-directories, non-private staging directories, -and existing destination files fail closed. Existing directories are inspected -but never chmodded as a startup side effect. The staging directory and requested -destination must reside on the same filesystem so publication can remain -no-overwrite and atomic. - -Published files are not path-chmodded or path-hashed after publication. Temporary partial cleanup is bounded, only considers DevSpace partial names, and ignores symlinks and non-regular files. +DevSpace opens the selected workspace without following symlinks. Destination +parent components are then created or opened one at a time through descriptor +anchors. The temporary partial is created in that exact destination directory, +so DevSpace does not create a project-level staging area. Absolute paths, +traversal, symlinked parents, non-directories, and existing destination files +fail closed. Existing directories are inspected but never chmodded as a startup +side effect. + +Published files are not path-chmodded or path-hashed after publication. Temporary partial cleanup is bounded within the requested destination directory, only considers DevSpace partial names, and ignores symlinks and non-regular files. ## Logging diff --git a/docs/security.md b/docs/security.md index 3af44234..bb814483 100644 --- a/docs/security.md +++ b/docs/security.md @@ -113,21 +113,22 @@ by `open_workspace`, and a relative destination `path`. Absolute paths, traversal, symlinked parents, and existing destinations fail closed. Callers cannot supply a conflict mode, expected hash, host path, or storage policy. -The selected workspace is opened without following symlinks. DevSpace then -creates or opens `.devspace` and its private `incoming` staging directory through -already-open parent directory descriptors. Requested destination parents are -also created or opened component-by-component through descriptor anchors. -Replacing a pathname therefore cannot redirect writes outside the selected -workspace. Symlinked components, non-directories, non-private incoming -directories, and existing destination files fail closed. Existing directories -are inspected but are never chmodded as a startup side effect. +The selected workspace is opened without following symlinks. Requested +destination parents are created or opened component-by-component through +descriptor anchors, and the temporary partial is created beside the requested +destination. DevSpace does not create a project-level artifact or staging +directory. Replacing a pathname therefore cannot redirect writes outside the +selected workspace. Symlinked components, non-directories, and existing +destination files fail closed. Existing directories are inspected but are never +chmodded as a startup side effect. Bytes stream into an exclusive mode-`0600` partial under the configured per-file -limit. DevSpace computes SHA-256 while writing, verifies any size hint, chmods -and fsyncs through the still-open descriptor, then publishes the verified inode -with a no-overwrite atomic hard link at the requested path and verifies the -published inode identity. It does not path-chmod or path-hash the published file. -Partials are removed on success or failure; crash-leftover cleanup is bounded and +limit. DevSpace computes SHA-256 while writing, verifies any size hint, then +publishes the verified inode with a no-overwrite atomic hard link at the +requested path and verifies the published inode identity. Permissions and sync +are applied through the still-open descriptor. It does not path-chmod or +path-hash the published file. Partials are removed on success or failure; +crash-leftover cleanup is bounded within the requested destination directory and only considers owned, regular DevSpace partial files. The native download seam deliberately does not: diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index 36df6d5f..c7e09337 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -1,6 +1,5 @@ import assert from "node:assert/strict"; import { - chmod, mkdir, mkdtemp, readFile, @@ -33,9 +32,7 @@ try { await testDestinationValidation(join(root, "destinations")); await testSizeLimitAndCleanup(join(root, "size-limit")); await testCrashLeftoverCleanup(join(root, "stale-partials")); - await testUnsafeDirectoryPermissions(join(root, "unsafe-permissions")); await testSymlinkRejection(join(root, "symlinks")); - await testPrivateStagingPermissions(join(root, "permissions")); testLogRedaction(); } finally { await rm(root, { recursive: true, force: true }); @@ -117,7 +114,7 @@ async function testSafeDownloadAndConflict(testRoot: string): Promise { "artifact_destination_exists", ); assert.deepEqual(await readFile(join(workspaceRoot, first.path)), bytes); - assert.deepEqual(await readdir(join(workspaceRoot, ".devspace", "incoming")), []); + assert.deepEqual(await readdir(workspaceRoot), ["public"]); } async function testDestinationValidation(testRoot: string): Promise { @@ -174,8 +171,7 @@ async function testSizeLimitAndCleanup(testRoot: string): Promise { "artifact_file_too_large", ); - const incoming = join(workspaceRoot, ".devspace", "incoming"); - assert.deepEqual(await readdir(incoming), []); + assert.deepEqual(await readdir(workspaceRoot), []); } async function testCrashLeftoverCleanup(testRoot: string): Promise { @@ -187,13 +183,13 @@ async function testCrashLeftoverCleanup(testRoot: string): Promise { workspaceRoot, maxFileBytes: 1024, file: { native: true }, - path: "first.txt", + path: "downloads/first.txt", }); - const incoming = join(workspaceRoot, ".devspace", "incoming"); - const stalePartial = join(incoming, ".devspace-download-stale.partial"); - const recentPartial = join(incoming, ".devspace-download-recent.partial"); - const unrelated = join(incoming, "keep-me.partial"); + const destinationDirectory = join(workspaceRoot, "downloads"); + const stalePartial = join(destinationDirectory, ".devspace-download-stale.partial"); + const recentPartial = join(destinationDirectory, ".devspace-download-recent.partial"); + const unrelated = join(destinationDirectory, "keep-me.partial"); await writeFile(stalePartial, "stale"); await writeFile(recentPartial, "recent"); await writeFile(unrelated, "unrelated"); @@ -206,34 +202,15 @@ async function testCrashLeftoverCleanup(testRoot: string): Promise { workspaceRoot, maxFileBytes: 1024, file: { native: true }, - path: "second.txt", + path: "downloads/second.txt", }); - const entries = await readdir(incoming); + const entries = await readdir(destinationDirectory); assert.equal(entries.includes(".devspace-download-stale.partial"), false); assert.equal(entries.includes(".devspace-download-recent.partial"), true); assert.equal(entries.includes("keep-me.partial"), true); -} - -async function testUnsafeDirectoryPermissions(testRoot: string): Promise { - if (process.platform === "win32") return; - - const workspaceRoot = join(testRoot, "workspace"); - const devspaceDirectory = join(workspaceRoot, ".devspace"); - await mkdir(devspaceDirectory, { recursive: true, mode: 0o700 }); - await chmod(devspaceDirectory, 0o777); - - await expectArtifactError( - downloadIncomingArtifact({ - registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), - workspaceId: "ws_test", - workspaceRoot, - maxFileBytes: 1024, - file: { native: true }, - path: "blocked.txt", - }), - "artifact_directory_permissions_unsafe", - ); + assert.equal(entries.includes("first.txt"), true); + assert.equal(entries.includes("second.txt"), true); } async function testSymlinkRejection(testRoot: string): Promise { @@ -242,37 +219,18 @@ async function testSymlinkRejection(testRoot: string): Promise { const outside = join(testRoot, "outside"); await mkdir(outside, { recursive: true, mode: 0o700 }); - const linkedDevspaceRoot = join(testRoot, "linked-devspace-workspace"); - await mkdir(linkedDevspaceRoot, { recursive: true, mode: 0o700 }); - await symlink(outside, join(linkedDevspaceRoot, ".devspace"), "dir"); + const linkedWorkspaceRoot = join(testRoot, "linked-workspace"); + await symlink(outside, linkedWorkspaceRoot, "dir"); await expectArtifactError( downloadIncomingArtifact({ registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), workspaceId: "ws_test", - workspaceRoot: linkedDevspaceRoot, + workspaceRoot: linkedWorkspaceRoot, maxFileBytes: 1024, file: { native: true }, path: "blocked.txt", }), - "artifact_directory_unsafe", - ); - - const linkedIncomingRoot = join(testRoot, "linked-incoming-workspace"); - await mkdir(join(linkedIncomingRoot, ".devspace"), { - recursive: true, - mode: 0o700, - }); - await symlink(outside, join(linkedIncomingRoot, ".devspace", "incoming"), "dir"); - await expectArtifactError( - downloadIncomingArtifact({ - registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), - workspaceId: "ws_test", - workspaceRoot: linkedIncomingRoot, - maxFileBytes: 1024, - file: { native: true }, - path: "blocked.txt", - }), - "artifact_directory_unsafe", + "artifact_workspace_unsafe", ); const linkedDestinationRoot = join(testRoot, "linked-destination-workspace"); @@ -291,27 +249,6 @@ async function testSymlinkRejection(testRoot: string): Promise { ); } -async function testPrivateStagingPermissions(testRoot: string): Promise { - if (process.platform === "win32") return; - - const workspaceRoot = join(testRoot, "workspace"); - const incoming = join(workspaceRoot, ".devspace", "incoming"); - await mkdir(incoming, { recursive: true }); - await chmod(incoming, 0o755); - - await expectArtifactError( - downloadIncomingArtifact({ - registry: registryFor({ name: "private.txt", stream: Readable.from(["private"]) }), - workspaceId: "ws_test", - workspaceRoot, - maxFileBytes: 1024, - file: { native: true }, - path: "private.txt", - }), - "artifact_directory_permissions_unsafe", - ); -} - function testLogRedaction(): void { const fields = artifactToolLogFields({ file: { diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index 57ed0712..ae08fd96 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -63,14 +63,6 @@ export interface DownloadIncomingArtifactResult { sha256: string; } -interface SecureIncomingDirectory { - rootHandle: FileHandle; - devspaceHandle: FileHandle; - incomingHandle: FileHandle; - anchorPath: string; - close(): Promise; -} - interface SecureDestinationDirectory { handle: FileHandle; anchorPath: string; @@ -138,10 +130,9 @@ export function registerArtifactTools( /** * Stream a trusted native file directly into one already-open workspace. * - * Bytes are written to an exclusive private partial, hashed and size-checked, - * chmodded through the still-open file descriptor, fsynced, and only then - * published without overwriting the requested workspace destination. No - * path-based chmod or hashing is performed after publication. + * Bytes are written to an exclusive partial beside the requested destination, + * hashed and size-checked, fsynced, and only then published without overwriting + * the requested workspace path. No project-level staging directory is created. */ export async function downloadIncomingArtifact({ registry, @@ -173,7 +164,7 @@ export async function downloadIncomingArtifact({ const destination = normalizeArtifactDestination(path); const opened = await registry.open(file); - let secureDirectory: SecureIncomingDirectory | undefined; + let workspaceHandle: FileHandle | undefined; let destinationDirectory: SecureDestinationDirectory | undefined; let partialPath: string | undefined; let handle: FileHandle | undefined; @@ -186,12 +177,19 @@ export async function downloadIncomingArtifact({ ); } - secureDirectory = await prepareIncomingDirectory(workspaceRoot); - await cleanupStalePartials(secureDirectory); - await assertPrivateDirectoryHandle(secureDirectory.incomingHandle); + workspaceHandle = await openDirectoryNoFollow( + workspaceRoot, + "artifact_workspace_unsafe", + "Selected workspace root is not a real directory.", + ); + destinationDirectory = await prepareDestinationDirectory( + workspaceHandle, + destination.parentParts, + ); + await cleanupStalePartials(destinationDirectory); partialPath = join( - secureDirectory.anchorPath, + destinationDirectory.anchorPath, `${PARTIAL_PREFIX}${randomUUID()}${PARTIAL_SUFFIX}`, ); handle = await open( @@ -222,7 +220,6 @@ export async function downloadIncomingArtifact({ ); } - await handle.chmod(0o644); await handle.sync(); const writtenEntry = await handle.stat(); if (!writtenEntry.isFile() || writtenEntry.size !== size) { @@ -232,7 +229,6 @@ export async function downloadIncomingArtifact({ ); } - await assertPrivateDirectoryHandle(secureDirectory.incomingHandle); const partialEntry = await lstat(partialPath); if ( partialEntry.isSymbolicLink() @@ -247,15 +243,12 @@ export async function downloadIncomingArtifact({ ); } - destinationDirectory = await prepareDestinationDirectory( - secureDirectory.rootHandle, - destination.parentParts, - ); await publishDestination( destinationDirectory, partialPath, destination.name, writtenEntry, + handle, ); await unlink(partialPath).catch(() => undefined); partialPath = undefined; @@ -272,7 +265,7 @@ export async function downloadIncomingArtifact({ await handle?.close().catch(() => undefined); if (partialPath) await unlink(partialPath).catch(() => undefined); await destinationDirectory?.close().catch(() => undefined); - await secureDirectory?.close().catch(() => undefined); + await workspaceHandle?.close().catch(() => undefined); } } @@ -332,107 +325,6 @@ function artifactToolResponse(result: { path: string }) { }; } -async function prepareIncomingDirectory( - workspaceRoot: string, -): Promise { - let rootHandle: FileHandle | undefined; - let devspaceHandle: FileHandle | undefined; - let incomingHandle: FileHandle | undefined; - - try { - rootHandle = await openDirectoryNoFollow( - workspaceRoot, - "artifact_workspace_unsafe", - "Selected workspace root is not a real directory.", - ); - const rootAnchor = descriptorDirectoryPath(rootHandle); - devspaceHandle = await ensureChildDirectory( - rootHandle, - rootAnchor, - ".devspace", - false, - ); - const devspaceAnchor = descriptorDirectoryPath(devspaceHandle); - incomingHandle = await ensureChildDirectory( - devspaceHandle, - devspaceAnchor, - "incoming", - true, - ); - const anchorPath = descriptorDirectoryPath(incomingHandle); - - return { - rootHandle, - devspaceHandle, - incomingHandle, - anchorPath, - async close() { - await incomingHandle?.close().catch(() => undefined); - await devspaceHandle?.close().catch(() => undefined); - await rootHandle?.close().catch(() => undefined); - }, - }; - } catch (error) { - await incomingHandle?.close().catch(() => undefined); - await devspaceHandle?.close().catch(() => undefined); - await rootHandle?.close().catch(() => undefined); - throw error; - } -} - -async function ensureChildDirectory( - parentHandle: FileHandle, - parentAnchor: string, - name: string, - privateAccess: boolean, -): Promise { - await assertDirectoryHandle(parentHandle); - const path = join(parentAnchor, name); - try { - await mkdir(path, { mode: 0o700 }); - } catch (error) { - if (!isNodeError(error) || error.code !== "EEXIST") throw error; - } - - const child = await openDirectoryNoFollow( - path, - "artifact_directory_unsafe", - "Artifact destination parent is not a real directory.", - ); - try { - if (privateAccess) { - await assertPrivateDirectoryHandle(child); - } else { - await assertOwnedDirectoryHandle(child); - } - return child; - } catch (error) { - await child.close().catch(() => undefined); - throw error; - } -} - -async function assertOwnedDirectoryHandle(handle: FileHandle): Promise { - const entry = await handle.stat(); - if (!entry.isDirectory()) { - throw new ArtifactError( - "artifact_directory_unsafe", - "Artifact staging parent is not a directory.", - ); - } - if (process.platform === "win32") return; - const currentUid = process.getuid?.(); - if ( - (currentUid !== undefined && entry.uid !== currentUid) - || (Number(entry.mode) & 0o022) !== 0 - ) { - throw new ArtifactError( - "artifact_directory_permissions_unsafe", - "Artifact staging parent must be owned by the current user and not group/world writable.", - ); - } -} - async function openDirectoryNoFollow( path: string, code: string, @@ -460,27 +352,6 @@ async function assertDirectoryHandle(handle: FileHandle): Promise { } } -async function assertPrivateDirectoryHandle(handle: FileHandle): Promise { - const entry = await handle.stat(); - if (!entry.isDirectory()) { - throw new ArtifactError( - "artifact_directory_unsafe", - "Artifact destination parent is not a directory.", - ); - } - if (process.platform === "win32") return; - const currentUid = process.getuid?.(); - if ( - (currentUid !== undefined && entry.uid !== currentUid) - || (Number(entry.mode) & 0o777) !== 0o700 - ) { - throw new ArtifactError( - "artifact_directory_permissions_unsafe", - "Artifact staging directory must be owned by and accessible only to the current user.", - ); - } -} - function descriptorDirectoryPath(handle: FileHandle): string { if (process.platform === "linux") return `/proc/self/fd/${handle.fd}`; if (["darwin", "freebsd", "openbsd", "netbsd"].includes(process.platform)) { @@ -597,6 +468,7 @@ async function publishDestination( partialPath: string, filename: string, writtenEntry: Awaited>, + handle: FileHandle, ): Promise { await assertDirectoryHandle(directory.handle); const candidate = join(directory.anchorPath, filename); @@ -605,18 +477,10 @@ async function publishDestination( await link(partialPath, candidate); published = true; const publishedEntry = await lstat(candidate); - if ( - publishedEntry.isSymbolicLink() - || !publishedEntry.isFile() - || publishedEntry.dev !== writtenEntry.dev - || publishedEntry.ino !== writtenEntry.ino - || publishedEntry.size !== writtenEntry.size - ) { - throw new ArtifactError( - "artifact_destination_publish_failed", - "Published artifact did not match the verified download.", - ); - } + assertPublishedArtifactEntry(publishedEntry, writtenEntry); + await handle.chmod(0o644); + await handle.sync(); + assertPublishedArtifactEntry(await lstat(candidate), writtenEntry); } catch (error) { if (published) await unlink(candidate).catch(() => undefined); if (isNodeError(error) && error.code === "EEXIST") { @@ -625,20 +489,32 @@ async function publishDestination( "Artifact destination already exists.", ); } - if (isNodeError(error) && error.code === "EXDEV") { - throw new ArtifactError( - "artifact_destination_filesystem_unsupported", - "Artifact staging and destination must be on the same filesystem.", - ); - } throw error; } } +function assertPublishedArtifactEntry( + entry: Awaited>, + writtenEntry: Awaited>, +): void { + if ( + entry.isSymbolicLink() + || !entry.isFile() + || entry.dev !== writtenEntry.dev + || entry.ino !== writtenEntry.ino + || entry.size !== writtenEntry.size + ) { + throw new ArtifactError( + "artifact_destination_publish_failed", + "Published artifact did not match the verified download.", + ); + } +} + async function cleanupStalePartials( - directory: SecureIncomingDirectory, + directory: SecureDestinationDirectory, ): Promise { - await assertPrivateDirectoryHandle(directory.incomingHandle); + await assertDirectoryHandle(directory.handle); const entries = await readdir(directory.anchorPath, { withFileTypes: true }); let inspected = 0; const cutoff = Date.now() - STALE_PARTIAL_AGE_MS; From 28e1f1f6b94ac65d285159d1afa0ecf565456668 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 20:57:39 +0530 Subject: [PATCH 11/16] refactor: remove retired artifact migration --- src/db/migrations.ts | 8 -------- src/oauth-store.test.ts | 6 ------ 2 files changed, 14 deletions(-) diff --git a/src/db/migrations.ts b/src/db/migrations.ts index c122068e..2bda20c2 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -22,14 +22,6 @@ const migrations: Migration[] = [ name: "local-agent-sessions", up: migrateLocalAgentSessions, }, - { - version: 4, - name: "artifact-exchange-retired", - // Preserve the historical migration version without creating new - // persistent artifact tables. Existing user tables and bytes are left - // untouched; the one-shot download path does not read or mutate them. - up: () => undefined, - }, ]; export function migrateDatabase(sqlite: Database.Database): void { diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index 74e6ea8b..e1c00338 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -44,13 +44,7 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 1, name: "workspace-state" }, { version: 2, name: "oauth-state" }, { version: 3, name: "local-agent-sessions" }, - { version: 4, name: "artifact-exchange-retired" }, ]); - const artifactTables = database.sqlite - .prepare("select name from sqlite_master where type = 'table' and name in ('artifacts', 'artifact_uploads')") - .pluck() - .all(); - assert.deepEqual(artifactTables, []); } finally { database.close(); } From afd241ae0c247c3786605a559eab1bcc798535d4 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 21:01:01 +0530 Subject: [PATCH 12/16] fix: preserve raced artifact destinations --- src/artifact-download.test.ts | 32 ++++++++++++++++++++++++++++++++ src/artifact-tools.ts | 14 ++++++++------ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index c7e09337..a45c2611 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -1,11 +1,13 @@ import assert from "node:assert/strict"; import { + link, mkdir, mkdtemp, readFile, readdir, rm, symlink, + unlink, utimes, writeFile, } from "node:fs/promises"; @@ -33,6 +35,7 @@ try { await testSizeLimitAndCleanup(join(root, "size-limit")); await testCrashLeftoverCleanup(join(root, "stale-partials")); await testSymlinkRejection(join(root, "symlinks")); + await testPublicationFailurePreservesReplacement(join(root, "publication-race")); testLogRedaction(); } finally { await rm(root, { recursive: true, force: true }); @@ -249,6 +252,35 @@ async function testSymlinkRejection(testRoot: string): Promise { ); } +async function testPublicationFailurePreservesReplacement(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + const destinationPath = join(workspaceRoot, "generated.txt"); + + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ + name: "generated.txt", + stream: Readable.from(["downloaded"]), + }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + path: "generated.txt", + publishLink: async (partialPath, candidatePath) => { + await link(partialPath, candidatePath); + await unlink(candidatePath); + await writeFile(candidatePath, "replacement"); + }, + }), + "artifact_destination_publish_failed", + ); + + assert.equal(await readFile(destinationPath, "utf8"), "replacement"); + assert.deepEqual(await readdir(workspaceRoot), ["generated.txt"]); +} + function testLogRedaction(): void { const fields = artifactToolLogFields({ file: { diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index ae08fd96..5c9b0af7 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -141,6 +141,7 @@ export async function downloadIncomingArtifact({ maxFileBytes, file, path, + publishLink = link, }: { registry: IncomingArtifactAdapterRegistry; workspaceId: string; @@ -148,6 +149,7 @@ export async function downloadIncomingArtifact({ maxFileBytes: number; file: unknown; path: string; + publishLink?: typeof link; }): Promise { if (!Number.isSafeInteger(maxFileBytes) || maxFileBytes < 1) { throw new ArtifactError( @@ -249,6 +251,7 @@ export async function downloadIncomingArtifact({ destination.name, writtenEntry, handle, + publishLink, ); await unlink(partialPath).catch(() => undefined); partialPath = undefined; @@ -469,26 +472,25 @@ async function publishDestination( filename: string, writtenEntry: Awaited>, handle: FileHandle, + publishLink: typeof link, ): Promise { await assertDirectoryHandle(directory.handle); const candidate = join(directory.anchorPath, filename); - let published = false; try { - await link(partialPath, candidate); - published = true; - const publishedEntry = await lstat(candidate); - assertPublishedArtifactEntry(publishedEntry, writtenEntry); await handle.chmod(0o644); await handle.sync(); + await publishLink(partialPath, candidate); assertPublishedArtifactEntry(await lstat(candidate), writtenEntry); } catch (error) { - if (published) await unlink(candidate).catch(() => undefined); if (isNodeError(error) && error.code === "EEXIST") { throw new ArtifactError( "artifact_destination_exists", "Artifact destination already exists.", ); } + // Once the destination path exists, never unlink it during failure cleanup. + // Another process may have replaced that path after publication, and a + // path-based verification followed by unlink would introduce another race. throw error; } } From 6ba030d76997bc0e7e399371ff710b195297cec0 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 21:02:20 +0530 Subject: [PATCH 13/16] refactor: remove retired artifact store shims --- docs/artifact-exchange.md | 4 ---- docs/security.md | 1 - src/artifact-workspace.test.ts | 2 -- src/artifact-workspace.ts | 3 --- src/artifacts.test.ts | 2 -- src/artifacts.ts | 4 ---- 6 files changed, 16 deletions(-) delete mode 100644 src/artifact-workspace.test.ts delete mode 100644 src/artifact-workspace.ts delete mode 100644 src/artifacts.test.ts delete mode 100644 src/artifacts.ts diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 53b45426..096fc614 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -87,7 +87,3 @@ Published files are not path-chmodded or path-hashed after publication. Temporar ## Logging Tool logs contain bounded metadata such as workspace ID, validated download hostname, workspace-relative output path, byte count, SHA-256, duration, and stable error code. They never contain signed URLs, native file IDs, credentials, file contents, base64 data, host paths, or temporary paths. - -## Migration behavior - -Older DevSpace builds may have created artifact tables or private artifact bytes. The one-shot implementation does not read, mutate, or delete that legacy data. Existing users can remove it manually after confirming they no longer need it; DevSpace does not opportunistically delete user data during migration. diff --git a/docs/security.md b/docs/security.md index bb814483..b845f4d5 100644 --- a/docs/security.md +++ b/docs/security.md @@ -139,7 +139,6 @@ The native download seam deliberately does not: - extract archives or execute transferred content; - expand workspace allowlists; - preserve executable permissions; -- opportunistically delete legacy artifact tables or bytes. ## Logs diff --git a/src/artifact-workspace.test.ts b/src/artifact-workspace.test.ts deleted file mode 100644 index ae2e2ae0..00000000 --- a/src/artifact-workspace.test.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Workspace materializer coverage was replaced by artifact-download.test.ts. -export {}; diff --git a/src/artifact-workspace.ts b/src/artifact-workspace.ts deleted file mode 100644 index 4f02f0e3..00000000 --- a/src/artifact-workspace.ts +++ /dev/null @@ -1,3 +0,0 @@ -// The persistent-store workspace materializer was removed. Native files now -// stream directly to a requested workspace-relative path through download_artifact. -export {}; diff --git a/src/artifacts.test.ts b/src/artifacts.test.ts deleted file mode 100644 index adf921ef..00000000 --- a/src/artifacts.test.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Persistent artifact-store coverage was replaced by artifact-download.test.ts. -export {}; diff --git a/src/artifacts.ts b/src/artifacts.ts deleted file mode 100644 index fac7395a..00000000 --- a/src/artifacts.ts +++ /dev/null @@ -1,4 +0,0 @@ -// Persistent artifact storage was removed in favor of the one-shot -// download_artifact workspace handoff. Keep this re-export temporarily so -// downstream source imports of ArtifactError have a deliberate migration path. -export { ArtifactError } from "./artifact-error.js"; From e758e7860ecbb256118b166d3b82935b2120b25e Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 21:03:55 +0530 Subject: [PATCH 14/16] fix: gate artifact downloads by platform --- docs/artifact-exchange.md | 4 ++++ docs/configuration.md | 4 ++++ src/artifact-download.test.ts | 43 ++++++++++++++++++++++++++++++----- src/artifact-tools.ts | 21 ++++++++++++++++- src/server.ts | 16 +++++++++---- 5 files changed, 77 insertions(+), 11 deletions(-) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 096fc614..873fb72a 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -2,6 +2,10 @@ DevSpace can stream a file attached or generated by an MCP host (such as ChatGPT) into an already-open workspace. Enable this opt-in feature with `DEVSPACE_ARTIFACTS=1`. +Native-file download currently requires Linux, macOS, FreeBSD, OpenBSD, or +NetBSD because publication is anchored through open directory descriptors. The +tool is not registered on Windows. + ## Workflow ```text diff --git a/docs/configuration.md b/docs/configuration.md index ad82a1ab..1e2de63b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -48,6 +48,10 @@ an attached or generated file into an already-open workspace: DEVSPACE_ARTIFACTS=1 npx @waishnav/devspace serve ``` +This feature currently supports Linux, macOS, FreeBSD, OpenBSD, and NetBSD. It +is not registered on Windows because the secure publication path depends on +descriptor-anchored directory operations. + | Variable | Default | Purpose | | --- | --- | --- | | `DEVSPACE_ARTIFACTS` | `0` | Expose `download_artifact` for trusted native files. | diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index a45c2611..13932956 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -18,6 +18,7 @@ import * as z from "zod/v4"; import { artifactToolLogFields, downloadIncomingArtifact, + isArtifactDownloadSupportedPlatform, registerArtifactTools, } from "./artifact-tools.js"; import { ArtifactError } from "./artifact-error.js"; @@ -30,12 +31,17 @@ const root = await mkdtemp(join(tmpdir(), "devspace-artifact-download-test-")); try { testOneToolContract(); - await testSafeDownloadAndConflict(join(root, "downloads")); - await testDestinationValidation(join(root, "destinations")); - await testSizeLimitAndCleanup(join(root, "size-limit")); - await testCrashLeftoverCleanup(join(root, "stale-partials")); - await testSymlinkRejection(join(root, "symlinks")); - await testPublicationFailurePreservesReplacement(join(root, "publication-race")); + testPlatformSupportContract(); + if (isArtifactDownloadSupportedPlatform()) { + await testSafeDownloadAndConflict(join(root, "downloads")); + await testDestinationValidation(join(root, "destinations")); + await testSizeLimitAndCleanup(join(root, "size-limit")); + await testCrashLeftoverCleanup(join(root, "stale-partials")); + await testSymlinkRejection(join(root, "symlinks")); + await testPublicationFailurePreservesReplacement(join(root, "publication-race")); + } else { + await testUnsupportedPlatform(join(root, "unsupported-platform")); + } testLogRedaction(); } finally { await rm(root, { recursive: true, force: true }); @@ -81,6 +87,31 @@ function testOneToolContract(): void { assert.throws(() => fileSchema.parse({ file_id: "file_123" })); } +function testPlatformSupportContract(): void { + assert.equal(isArtifactDownloadSupportedPlatform("linux"), true); + assert.equal(isArtifactDownloadSupportedPlatform("darwin"), true); + assert.equal(isArtifactDownloadSupportedPlatform("freebsd"), true); + assert.equal(isArtifactDownloadSupportedPlatform("openbsd"), true); + assert.equal(isArtifactDownloadSupportedPlatform("netbsd"), true); + assert.equal(isArtifactDownloadSupportedPlatform("win32"), false); +} + +async function testUnsupportedPlatform(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + await expectArtifactError( + downloadIncomingArtifact({ + registry: registryFor({ name: "blocked.txt", stream: Readable.from(["blocked"]) }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + path: "blocked.txt", + }), + "artifact_platform_unsupported", + ); +} + async function testSafeDownloadAndConflict(testRoot: string): Promise { const workspaceRoot = join(testRoot, "workspace"); await mkdir(workspaceRoot, { recursive: true }); diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index 5c9b0af7..eb61e6df 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -35,6 +35,13 @@ const PARTIAL_PREFIX = ".devspace-download-"; const PARTIAL_SUFFIX = ".partial"; const STALE_PARTIAL_AGE_MS = 24 * 60 * 60 * 1_000; const MAX_STALE_PARTIAL_CLEANUP = 32; +const ARTIFACT_DOWNLOAD_PLATFORMS = new Set([ + "linux", + "darwin", + "freebsd", + "openbsd", + "netbsd", +]); const openAIFileReferenceInputSchema = z.object({ download_url: z.string(), @@ -63,6 +70,12 @@ export interface DownloadIncomingArtifactResult { sha256: string; } +export function isArtifactDownloadSupportedPlatform( + platform: NodeJS.Platform = process.platform, +): boolean { + return ARTIFACT_DOWNLOAD_PLATFORMS.has(platform); +} + interface SecureDestinationDirectory { handle: FileHandle; anchorPath: string; @@ -151,6 +164,12 @@ export async function downloadIncomingArtifact({ path: string; publishLink?: typeof link; }): Promise { + if (!isArtifactDownloadSupportedPlatform()) { + throw new ArtifactError( + "artifact_platform_unsupported", + "Native file download requires descriptor-anchored directory operations on this platform.", + ); + } if (!Number.isSafeInteger(maxFileBytes) || maxFileBytes < 1) { throw new ArtifactError( "artifact_limit_invalid", @@ -357,7 +376,7 @@ async function assertDirectoryHandle(handle: FileHandle): Promise { function descriptorDirectoryPath(handle: FileHandle): string { if (process.platform === "linux") return `/proc/self/fd/${handle.fd}`; - if (["darwin", "freebsd", "openbsd", "netbsd"].includes(process.platform)) { + if (isArtifactDownloadSupportedPlatform()) { return `/dev/fd/${handle.fd}`; } throw new ArtifactError( diff --git a/src/server.ts b/src/server.ts index f13a79aa..91f5df61 100644 --- a/src/server.ts +++ b/src/server.ts @@ -18,7 +18,10 @@ import express from "express"; import type { Request, Response } from "express"; import * as z from "zod/v4"; import { applyPatch } from "./apply-patch.js"; -import { registerArtifactTools } from "./artifact-tools.js"; +import { + isArtifactDownloadSupportedPlatform, + registerArtifactTools, +} from "./artifact-tools.js"; import { loadConfig, type ServerConfig, type WidgetMode } from "./config.js"; import { createOpenAIIncomingArtifactAdapter, @@ -184,7 +187,7 @@ interface ToolLogFields { } function serverInstructions(config: ServerConfig): string { - const artifactInstruction = config.artifactsEnabled + const artifactInstruction = config.artifactsEnabled && isArtifactDownloadSupportedPlatform() ? " When the user supplies or generates a file that is not present on the DevSpace host, use download_artifact with its native file value, the existing workspace ID, and a suitable relative destination path chosen from the user's request and project structure. The tool refuses to overwrite an existing destination and returns the normalized workspace-relative path. Use normal workspace tools when explicit inspection, replacement, movement, renaming, or deletion is needed. Do not recreate binary files with write/edit calls or place signed URLs, native file objects, base64 content, or invented host paths in shell commands or logs." : ""; const showChangesInstruction = @@ -1603,7 +1606,7 @@ function createMcpServer( registerCodexProcessTools(server, config, workspaces, processSessions); } - if (config.artifactsEnabled) { + if (config.artifactsEnabled && isArtifactDownloadSupportedPlatform()) { registerArtifactTools(server, { config, workspaces, @@ -1866,7 +1869,12 @@ if (await isMainModule()) { console.log(`request logging: ${config.logging.requests ? "enabled" : "disabled"}`); console.log(`asset logging: ${config.logging.assets ? "enabled" : "disabled"}`); console.log(`trust proxy: ${config.logging.trustProxy ? "enabled" : "disabled"}`); - console.log(`native artifact download: ${config.artifactsEnabled ? "enabled" : "disabled"}`); + const artifactDownloadStatus = !config.artifactsEnabled + ? "disabled" + : isArtifactDownloadSupportedPlatform() + ? "enabled" + : `unsupported on ${process.platform}`; + console.log(`native artifact download: ${artifactDownloadStatus}`); if (config.subagents) { console.log(`subagent providers: ${formatLocalAgentProviderAvailabilitySummary(localAgentProviders)}`); } From 45fb361ab9afb98aaf2d83e941832d6a040d47e4 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Tue, 21 Jul 2026 21:05:01 +0530 Subject: [PATCH 15/16] fix: keep downloaded artifacts private --- docs/artifact-exchange.md | 7 +++++-- docs/security.md | 5 +++-- src/artifact-download.test.ts | 25 +++++++++++++++++++++++++ src/artifact-tools.ts | 2 -- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 873fb72a..2a3fcb24 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -71,7 +71,7 @@ The download is never buffered wholesale in memory. DevSpace: 5. verifies any supplied size hint and fsyncs the partial; 6. publishes the verified inode with an atomic hard link to the exact requested path; 7. verifies that the published inode is the downloaded inode; -8. chmods and fsyncs through the still-open file descriptor; +8. retains owner-only mode `0600` on the published file; 9. removes the temporary partial. There is no persistent artifact database, completed-object store, upload/chunk API, quota ledger, TTL, pinning, artifact ID, signed download route, or reusable artifact lifecycle. @@ -86,7 +86,10 @@ traversal, symlinked parents, non-directories, and existing destination files fail closed. Existing directories are inspected but never chmodded as a startup side effect. -Published files are not path-chmodded or path-hashed after publication. Temporary partial cleanup is bounded within the requested destination directory, only considers DevSpace partial names, and ignores symlinks and non-regular files. +Published files retain owner-only mode `0600` and are not path-chmodded or +path-hashed after publication. Temporary partial cleanup is bounded within the +requested destination directory, only considers DevSpace partial names, and +ignores symlinks and non-regular files. ## Logging diff --git a/docs/security.md b/docs/security.md index b845f4d5..ed76037f 100644 --- a/docs/security.md +++ b/docs/security.md @@ -126,8 +126,9 @@ Bytes stream into an exclusive mode-`0600` partial under the configured per-file limit. DevSpace computes SHA-256 while writing, verifies any size hint, then publishes the verified inode with a no-overwrite atomic hard link at the requested path and verifies the published inode identity. Permissions and sync -are applied through the still-open descriptor. It does not path-chmod or -path-hash the published file. Partials are removed on success or failure; +are fixed before publication, and the published file retains owner-only mode +`0600`. It does not path-chmod or path-hash the published file. Partials are +removed on success or failure; crash-leftover cleanup is bounded within the requested destination directory and only considers owned, regular DevSpace partial files. diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index 13932956..3e27a547 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -6,6 +6,7 @@ import { readFile, readdir, rm, + stat, symlink, unlink, utimes, @@ -39,6 +40,7 @@ try { await testCrashLeftoverCleanup(join(root, "stale-partials")); await testSymlinkRejection(join(root, "symlinks")); await testPublicationFailurePreservesReplacement(join(root, "publication-race")); + await testPublishedPermissions(join(root, "permissions")); } else { await testUnsupportedPlatform(join(root, "unsupported-platform")); } @@ -312,6 +314,29 @@ async function testPublicationFailurePreservesReplacement(testRoot: string): Pro assert.deepEqual(await readdir(workspaceRoot), ["generated.txt"]); } +async function testPublishedPermissions(testRoot: string): Promise { + const workspaceRoot = join(testRoot, "workspace"); + await mkdir(workspaceRoot, { recursive: true }); + const previousUmask = process.umask(0o077); + try { + await downloadIncomingArtifact({ + registry: registryFor({ + name: "private.txt", + stream: Readable.from(["private"]), + }), + workspaceId: "ws_test", + workspaceRoot, + maxFileBytes: 1024, + file: { native: true }, + path: "private.txt", + }); + } finally { + process.umask(previousUmask); + } + + assert.equal((await stat(join(workspaceRoot, "private.txt"))).mode & 0o777, 0o600); +} + function testLogRedaction(): void { const fields = artifactToolLogFields({ file: { diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index eb61e6df..f108c3b2 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -496,8 +496,6 @@ async function publishDestination( await assertDirectoryHandle(directory.handle); const candidate = join(directory.anchorPath, filename); try { - await handle.chmod(0o644); - await handle.sync(); await publishLink(partialPath, candidate); assertPublishedArtifactEntry(await lstat(candidate), writtenEntry); } catch (error) { From ed2702b86599cb137e377c3b37cd21efcd22f9b5 Mon Sep 17 00:00:00 2001 From: JP Lew <462836+jplew@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:50:55 +0000 Subject: [PATCH 16/16] fix: address final artifact handoff review --- docs/artifact-exchange.md | 106 +++++++++------------------------- docs/security.md | 60 +++++-------------- src/artifact-download.test.ts | 10 ++++ src/artifact-tools.ts | 2 +- 4 files changed, 51 insertions(+), 127 deletions(-) diff --git a/docs/artifact-exchange.md b/docs/artifact-exchange.md index 2a3fcb24..c269ae96 100644 --- a/docs/artifact-exchange.md +++ b/docs/artifact-exchange.md @@ -1,10 +1,7 @@ -# Native file download +# Download a native file -DevSpace can stream a file attached or generated by an MCP host (such as ChatGPT) into an already-open workspace. Enable this opt-in feature with `DEVSPACE_ARTIFACTS=1`. - -Native-file download currently requires Linux, macOS, FreeBSD, OpenBSD, or -NetBSD because publication is anchored through open directory descriptors. The -tool is not registered on Windows. +DevSpace can save a file attached or generated by an MCP host, such as ChatGPT, +directly into an open workspace. Enable the tool with `DEVSPACE_ARTIFACTS=1`. ## Workflow @@ -14,83 +11,32 @@ open_workspace -> { path } ``` -`download_artifact` is the only artifact-specific MCP tool. It accepts exactly three inputs: - -- the native `file` value authorized by the MCP host; -- a `workspaceId` returned by `open_workspace`; -- a relative destination `path` chosen for the selected workspace. - -```json -{ - "file": { - "download_url": "", - "file_id": "", - "mime_type": "image/png", - "file_name": "generated-image.png" - }, - "workspaceId": "ws_123", - "path": "public/images/generated-image.png" -} -``` - -DevSpace validates the requested path, safely creates missing parent directories, -and refuses to replace an existing destination. A successful call returns only -the normalized workspace-relative path: +1. Open the project with `open_workspace`. +2. Pass the host-provided native `file`, the returned `workspaceId`, and an + unused workspace-relative `path` to `download_artifact`. +3. Use the returned path with the ordinary DevSpace filesystem tools. -```json -{ - "path": "public/images/generated-image.png" -} +```text +download_artifact({ + file: , + workspaceId: "ws_123", + path: "public/images/generated-image.png" +}) ``` -If the destination already exists, the tool fails without modifying it. Use the -normal DevSpace filesystem tools when the user explicitly wants inspection, -replacement, movement, renaming, or deletion. - -## Accepted file values - -The `file` value must be passed natively by the MCP host. DevSpace validates the complete object shape and rejects: - -- arbitrary URL strings; -- local filesystem paths; -- malformed or ambiguous objects; -- extra credential-bearing fields; -- untrusted hosts or redirect targets; -- unsupported adapter values. - -The tool retains the MCP metadata declaration `openai/fileParams: ["file"]` so compatible hosts treat `file` as a native file parameter. - -## Streaming and publication - -The download is never buffered wholesale in memory. DevSpace: - -1. validates the trusted native reference and redirect chain; -2. opens or creates each requested destination parent without following symlinks; -3. opens an exclusive mode-`0600` partial beside the requested destination; -4. streams bytes under `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` while computing SHA-256; -5. verifies any supplied size hint and fsyncs the partial; -6. publishes the verified inode with an atomic hard link to the exact requested path; -7. verifies that the published inode is the downloaded inode; -8. retains owner-only mode `0600` on the published file; -9. removes the temporary partial. - -There is no persistent artifact database, completed-object store, upload/chunk API, quota ledger, TTL, pinning, artifact ID, signed download route, or reusable artifact lifecycle. - -## Directory safety - -DevSpace opens the selected workspace without following symlinks. Destination -parent components are then created or opened one at a time through descriptor -anchors. The temporary partial is created in that exact destination directory, -so DevSpace does not create a project-level staging area. Absolute paths, -traversal, symlinked parents, non-directories, and existing destination files -fail closed. Existing directories are inspected but never chmodded as a startup -side effect. +DevSpace creates missing parent directories and refuses to overwrite an existing +file. After the download, normal tools can inspect, move, rename, replace, or +delete it. -Published files retain owner-only mode `0600` and are not path-chmodded or -path-hashed after publication. Temporary partial cleanup is bounded within the -requested destination directory, only considers DevSpace partial names, and -ignores symlinks and non-regular files. +## Safety and limits -## Logging +The `file` input must be the native value supplied by the MCP host. DevSpace does +not accept pasted download URLs or local source paths. It validates the complete +file-object shape, trusted OpenAI download hosts, and redirects before streaming. +Malformed references, unknown fields, absolute paths, traversal, and symlinked +parents are rejected. -Tool logs contain bounded metadata such as workspace ID, validated download hostname, workspace-relative output path, byte count, SHA-256, duration, and stable error code. They never contain signed URLs, native file IDs, credentials, file contents, base64 data, host paths, or temporary paths. +Downloads are streamed under `DEVSPACE_ARTIFACT_MAX_FILE_BYTES` and published as +owner-only files without overwriting an existing destination. The tool is +available on Linux, macOS, FreeBSD, OpenBSD, and NetBSD; it is not registered on +Windows. diff --git a/docs/security.md b/docs/security.md index ed76037f..d7ec0e1d 100644 --- a/docs/security.md +++ b/docs/security.md @@ -94,52 +94,20 @@ sessions. ## Native File Download -Native file download is an opt-in, one-shot byte-transfer seam. It has no -persistent artifact root, object database, upload lifecycle, reusable artifact -ID, public download route, TTL, pinning, or quota ledger. - -The production server declares ChatGPT's top-level `openai/fileParams` contract -and accepts only the documented `download_url`, `file_id`, optional -MIME/filename aliases, and optional size. Downloads use HTTPS on -`files.oaiusercontent.com` or the constrained regional OpenAI Azure account -family `oaisdmntpr.blob.core.windows.net`, where `` is lowercase -alphanumeric. Arbitrary Azure Blob accounts, alternate ports, credentials, -fragments, malformed IDs, extra fields, and redirects outside that boundary fail -closed. Opaque IDs are bounded metadata and are never used as filenames or path -components. - -`download_artifact` accepts only the native file value, a `workspaceId` returned -by `open_workspace`, and a relative destination `path`. Absolute paths, -traversal, symlinked parents, and existing destinations fail closed. Callers -cannot supply a conflict mode, expected hash, host path, or storage policy. - -The selected workspace is opened without following symlinks. Requested -destination parents are created or opened component-by-component through -descriptor anchors, and the temporary partial is created beside the requested -destination. DevSpace does not create a project-level artifact or staging -directory. Replacing a pathname therefore cannot redirect writes outside the -selected workspace. Symlinked components, non-directories, and existing -destination files fail closed. Existing directories are inspected but are never -chmodded as a startup side effect. - -Bytes stream into an exclusive mode-`0600` partial under the configured per-file -limit. DevSpace computes SHA-256 while writing, verifies any size hint, then -publishes the verified inode with a no-overwrite atomic hard link at the -requested path and verifies the published inode identity. Permissions and sync -are fixed before publication, and the published file retains owner-only mode -`0600`. It does not path-chmod or path-hash the published file. Partials are -removed on success or failure; -crash-leftover cleanup is bounded within the requested destination directory and -only considers owned, regular DevSpace partial files. - -The native download seam deliberately does not: - -- fetch arbitrary URLs or local paths; -- expose generic upload/chunk/stat/delete/copy tools; -- expose artifact IDs, signed URLs, host paths, temp paths, or raw content; -- extract archives or execute transferred content; -- expand workspace allowlists; -- preserve executable permissions; +Native file download is an opt-in, one-shot transfer into an already-open +workspace. `download_artifact` accepts the MCP host's native file value, the +`workspaceId` returned by `open_workspace`, and an unused relative destination +path. It returns only the workspace-relative path and does not create a +persistent artifact service or reusable artifact ID. + +DevSpace accepts only the documented native-file object and trusted OpenAI +download hosts and redirects. Arbitrary URL strings, local source paths, +credentials, malformed references, and unknown object fields are rejected. + +Absolute paths, traversal, symlinked parents, and existing destinations also +fail closed. Downloads stream under the configured per-file limit and are +published without overwrite as owner-only files. DevSpace does not extract or +execute transferred content. ## Logs diff --git a/src/artifact-download.test.ts b/src/artifact-download.test.ts index 3e27a547..028de112 100644 --- a/src/artifact-download.test.ts +++ b/src/artifact-download.test.ts @@ -87,6 +87,14 @@ function testOneToolContract(): void { }; assert.deepEqual(fileSchema.parse(valid), valid); assert.throws(() => fileSchema.parse({ file_id: "file_123" })); + + const sensitiveExtraValue = "Bearer should-not-leak"; + const rejected = fileSchema.safeParse({ + ...valid, + authorization: sensitiveExtraValue, + }); + assert.equal(rejected.success, false); + assert.equal(JSON.stringify(rejected).includes(sensitiveExtraValue), false); } function testPlatformSupportContract(): void { @@ -343,6 +351,7 @@ function testLogRedaction(): void { download_url: "https://files.oaiusercontent.com/file_123/download?sig=super-secret", file_id: "file_secret", file_name: "generated.png", + authorization: "Bearer log-secret", }, workspaceId: "ws_secret", path: "private/generated.png", @@ -350,6 +359,7 @@ function testLogRedaction(): void { const serialized = JSON.stringify(fields); assert.equal(serialized.includes("super-secret"), false); assert.equal(serialized.includes("file_secret"), false); + assert.equal(serialized.includes("log-secret"), false); assert.equal(serialized.includes("ws_secret"), true); assert.equal(serialized.includes("files.oaiusercontent.com"), true); } diff --git a/src/artifact-tools.ts b/src/artifact-tools.ts index f108c3b2..85806090 100644 --- a/src/artifact-tools.ts +++ b/src/artifact-tools.ts @@ -43,7 +43,7 @@ const ARTIFACT_DOWNLOAD_PLATFORMS = new Set([ "netbsd", ]); -const openAIFileReferenceInputSchema = z.object({ +const openAIFileReferenceInputSchema = z.strictObject({ download_url: z.string(), file_id: z.string(), mime_type: z.string().nullable().optional(),