From f7be8094bccd8be3b58922159ca6349bef8c34f9 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sun, 21 Jun 2026 10:51:05 +0800 Subject: [PATCH] fix: handle control characters in object keys --- eslint.config.js | 6 +- hooks/use-object.ts | 9 +- lib/delete-task.ts | 5 + lib/s3-object-encoding.ts | 73 ++++++++++ pnpm-lock.yaml | 199 --------------------------- tests/lib/s3-object-encoding.test.ts | 50 +++++++ 6 files changed, 139 insertions(+), 203 deletions(-) create mode 100644 lib/s3-object-encoding.ts create mode 100644 tests/lib/s3-object-encoding.test.ts diff --git a/eslint.config.js b/eslint.config.js index c226d89e..2a7ef4fe 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -17,9 +17,9 @@ const eslintConfig = defineConfig([ ]), { rules: { - "react-hooks/set-state-in-effect": "off" - } - } + "react-hooks/set-state-in-effect": "off", + }, + }, ]) export default eslintConfig diff --git a/hooks/use-object.ts b/hooks/use-object.ts index 0480a20b..c7ffa738 100644 --- a/hooks/use-object.ts +++ b/hooks/use-object.ts @@ -18,6 +18,7 @@ import { } from "@aws-sdk/client-s3" import { getSignedUrl } from "@aws-sdk/s3-request-presigner" import { useS3 } from "@/contexts/s3-context" +import { decodeS3UrlEncodedObjectList, decodeS3UrlEncodedObjectVersions } from "@/lib/s3-object-encoding" function attachIncludeDeletedHeader(command: ListObjectsV2Command) { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -101,11 +102,13 @@ export function useObject(bucket: string) { MaxKeys: pageSize, Delimiter: "/", ContinuationToken: continuationToken, + EncodingType: "url", }) if (options?.includeDeleted) { attachIncludeDeletedHeader(command) } - return client.send(command) + const response = await client.send(command) + return decodeS3UrlEncodedObjectList(response) }, [client], ) @@ -121,8 +124,10 @@ export function useObject(bucket: string) { Bucket: bucketName, Prefix: prefix, ContinuationToken: continuationToken, + EncodingType: "url", }), ) + decodeS3UrlEncodedObjectList(data) data.Contents?.forEach((item) => { if (item.Key) callback(item.Key) @@ -256,8 +261,10 @@ export function useObject(bucket: string) { Bucket: bucket, Prefix: key, Delimiter: "/", + EncodingType: "url", }), ) + decodeS3UrlEncodedObjectVersions(res) const Versions = (res.Versions ?? []).filter((v) => v.Key === key) const DeleteMarkers = (res.DeleteMarkers ?? []).filter((m) => m.Key === key) Versions.sort((a, b) => new Date(b.LastModified!).getTime() - new Date(a.LastModified!).getTime()) diff --git a/lib/delete-task.ts b/lib/delete-task.ts index 08155552..710351db 100644 --- a/lib/delete-task.ts +++ b/lib/delete-task.ts @@ -5,6 +5,7 @@ import { ListObjectVersionsCommand, S3Client, } from "@aws-sdk/client-s3" +import { decodeS3UrlEncodedObjectList, decodeS3UrlEncodedObjectVersions } from "./s3-object-encoding" import type { ManagedTask, TaskHandler, TaskLifecycleStatus } from "./task-manager" import { createTaskId } from "./task-id" @@ -116,9 +117,11 @@ export function createDeleteTaskHelpers(s3Client: S3Client, config: DeleteTaskCo Prefix: prefix, KeyMarker: keyMarker, VersionIdMarker: versionIdMarker, + EncodingType: "url", }), { abortSignal: abortController.signal }, ) + decodeS3UrlEncodedObjectVersions(data) const objectsToDelete: { Key: string; VersionId?: string }[] = [] data.Versions?.forEach((v) => { @@ -151,9 +154,11 @@ export function createDeleteTaskHelpers(s3Client: S3Client, config: DeleteTaskCo Bucket: bucketName, Prefix: prefix, ContinuationToken: continuationToken, + EncodingType: "url", }), { abortSignal: abortController.signal }, ) + decodeS3UrlEncodedObjectList(data) const objectsToDelete = (data.Contents ?? []).filter((item) => item.Key).map((item) => ({ Key: item.Key! })) diff --git a/lib/s3-object-encoding.ts b/lib/s3-object-encoding.ts new file mode 100644 index 00000000..d174fbae --- /dev/null +++ b/lib/s3-object-encoding.ts @@ -0,0 +1,73 @@ +type S3KeyedItem = { + Key?: string +} + +type S3PrefixedItem = { + Prefix?: string +} + +type S3ObjectListResponse = { + EncodingType?: string + Prefix?: string + Delimiter?: string + StartAfter?: string + Contents?: S3KeyedItem[] + CommonPrefixes?: S3PrefixedItem[] +} + +type S3ObjectVersionsResponse = { + EncodingType?: string + Prefix?: string + Delimiter?: string + KeyMarker?: string + NextKeyMarker?: string + Versions?: S3KeyedItem[] + DeleteMarkers?: S3KeyedItem[] + CommonPrefixes?: S3PrefixedItem[] +} + +function decodeS3UrlValue(value: string | undefined): string | undefined { + if (value == null) return value + + try { + return decodeURIComponent(value) + } catch { + return value + } +} + +export function decodeS3UrlEncodedObjectList(response: T): T { + if (response.EncodingType !== "url") return response + + response.Prefix = decodeS3UrlValue(response.Prefix) + response.Delimiter = decodeS3UrlValue(response.Delimiter) + response.StartAfter = decodeS3UrlValue(response.StartAfter) + response.Contents?.forEach((item) => { + item.Key = decodeS3UrlValue(item.Key) + }) + response.CommonPrefixes?.forEach((item) => { + item.Prefix = decodeS3UrlValue(item.Prefix) + }) + + return response +} + +export function decodeS3UrlEncodedObjectVersions(response: T): T { + if (response.EncodingType !== "url") return response + + response.Prefix = decodeS3UrlValue(response.Prefix) + response.Delimiter = decodeS3UrlValue(response.Delimiter) + response.KeyMarker = decodeS3UrlValue(response.KeyMarker) + response.NextKeyMarker = decodeS3UrlValue(response.NextKeyMarker) + response.Versions?.forEach((item) => { + item.Key = decodeS3UrlValue(item.Key) + }) + response.DeleteMarkers?.forEach((item) => { + item.Key = decodeS3UrlValue(item.Key) + }) + response.CommonPrefixes?.forEach((item) => { + item.Prefix = decodeS3UrlValue(item.Prefix) + }) + + return response +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d000be88..78b0a462 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,202 +1,3 @@ ---- -lockfileVersion: '9.0' - -importers: - - .: - configDependencies: {} - packageManagerDependencies: - '@pnpm/exe': - specifier: 11.5.0 - version: 11.5.0 - pnpm: - specifier: 11.5.0 - version: 11.5.0 - -packages: - - '@pnpm/exe@11.5.0': - resolution: {integrity: sha512-4hzOXq1HHrNPjwI8k1rt7Ot/Yrdx1JX3pn/L/M95ii1gid1Q6ZK6dVg4+gbSgUdPsYmYDZ4/Yfc0A7vd5C0ndg==} - hasBin: true - - '@pnpm/linux-arm64@11.5.0': - resolution: {integrity: sha512-NV9HdzzCB0epuI9LqZZeTaqjH3OweNQSQCS76GzEkFxJHS9e5Gvu7tgex91gxVL7bCZ+R4yr/3d3yexBFtr2ug==} - cpu: [arm64] - os: [linux] - - '@pnpm/linux-x64@11.5.0': - resolution: {integrity: sha512-vH83rRx4iPk/bwm9pBVCn+5hXbcQI66I/4zk6Vc09SusJgTqOdbN4U6VhMcGIqSEdr901ksYGCyIbMv7f6Guew==} - cpu: [x64] - os: [linux] - - '@pnpm/linuxstatic-arm64@11.5.0': - resolution: {integrity: sha512-2nOnMW1rSwGv22q2yZz1HlGT3ly/Ij8wUlX0NB4n+Krx7nETRHA3MgWsbkVejxHknDcTulRVudAghuX9rgrXcw==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@pnpm/linuxstatic-x64@11.5.0': - resolution: {integrity: sha512-ONOC1Mg0JusHtjzkRlre9di1QO+GAjy4HP7jMjDx21yGhrSheNdUweTXbekMH1EflRd19kTU6d8M3zewJFPtVg==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@pnpm/macos-arm64@11.5.0': - resolution: {integrity: sha512-od0ALdTxs4A7s5vAH5q2l2phzCJb98+PVOW1rq7BGpWGeYxQ+EwvL+vq0KaO6iLsn/eVVoncCkgZ/k6QNYuTgw==} - cpu: [arm64] - os: [darwin] - - '@pnpm/win-arm64@11.5.0': - resolution: {integrity: sha512-9HqbI80FjVVqFx4+EPxYYNfeP9Sx69W6kYqUDvOJn9G7RJ/2NNNQ898cVHTMpXlW1/PrMEcijmdpa/NjZIrWiQ==} - cpu: [arm64] - os: [win32] - - '@pnpm/win-x64@11.5.0': - resolution: {integrity: sha512-Q89CQqFGAsWmfvHZs5Kbbar45q3GBYtfAdPUCiVMVNJoLi3dsBS2LCvUq8ak3AufkFDaJBpvhaFcDP2M1NXr3A==} - cpu: [x64] - os: [win32] - - '@reflink/reflink-darwin-arm64@0.1.19': - resolution: {integrity: sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [darwin] - - '@reflink/reflink-darwin-x64@0.1.19': - resolution: {integrity: sha512-By85MSWrMZa+c26TcnAy8SDk0sTUkYlNnwknSchkhHpGXOtjNDUOxJE9oByBnGbeuIE1PiQsxDG3Ud+IVV9yuA==} - engines: {node: '>= 10'} - cpu: [x64] - os: [darwin] - - '@reflink/reflink-linux-arm64-gnu@0.1.19': - resolution: {integrity: sha512-7P+er8+rP9iNeN+bfmccM4hTAaLP6PQJPKWSA4iSk2bNvo6KU6RyPgYeHxXmzNKzPVRcypZQTpFgstHam6maVg==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@reflink/reflink-linux-arm64-musl@0.1.19': - resolution: {integrity: sha512-37iO/Dp6m5DDaC2sf3zPtx/hl9FV3Xze4xoYidrxxS9bgP3S8ALroxRK6xBG/1TtfXKTvolvp+IjrUU6ujIGmA==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@reflink/reflink-linux-x64-gnu@0.1.19': - resolution: {integrity: sha512-jbI8jvuYCaA3MVUdu8vLoLAFqC+iNMpiSuLbxlAgg7x3K5bsS8nOpTRnkLF7vISJ+rVR8W+7ThXlXlUQ93ulkw==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@reflink/reflink-linux-x64-musl@0.1.19': - resolution: {integrity: sha512-e9FBWDe+lv7QKAwtKOt6A2W/fyy/aEEfr0g6j/hWzvQcrzHCsz07BNQYlNOjTfeytrtLU7k449H1PI95jA4OjQ==} - engines: {node: '>= 10'} - cpu: [x64] - os: [linux] - libc: [musl] - - '@reflink/reflink-win32-arm64-msvc@0.1.19': - resolution: {integrity: sha512-09PxnVIQcd+UOn4WAW73WU6PXL7DwGS6wPlkMhMg2zlHHG65F3vHepOw06HFCq+N42qkaNAc8AKIabWvtk6cIQ==} - engines: {node: '>= 10'} - cpu: [arm64] - os: [win32] - - '@reflink/reflink-win32-x64-msvc@0.1.19': - resolution: {integrity: sha512-E//yT4ni2SyhwP8JRjVGWr3cbnhWDiPLgnQ66qqaanjjnMiu3O/2tjCPQXlcGc/DEYofpDc9fvhv6tALQsMV9w==} - engines: {node: '>= 10'} - cpu: [x64] - os: [win32] - - '@reflink/reflink@0.1.19': - resolution: {integrity: sha512-DmCG8GzysnCZ15bres3N5AHCmwBwYgp0As6xjhQ47rAUTUXxJiK+lLUxaGsX3hd/30qUpVElh05PbGuxRPgJwA==} - engines: {node: '>= 10'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - pnpm@11.5.0: - resolution: {integrity: sha512-2/zE+Bz0hZev1Lw5H/3xLBHxqfuDo5W/prCi2cwv2P/rr9scy9UpYyFT95OQTCYVt/Cf4aNFRz/Rw1hFFyqOsQ==} - engines: {node: '>=22.13'} - hasBin: true - -snapshots: - - '@pnpm/exe@11.5.0': - dependencies: - '@reflink/reflink': 0.1.19 - detect-libc: 2.1.2 - optionalDependencies: - '@pnpm/linux-arm64': 11.5.0 - '@pnpm/linux-x64': 11.5.0 - '@pnpm/linuxstatic-arm64': 11.5.0 - '@pnpm/linuxstatic-x64': 11.5.0 - '@pnpm/macos-arm64': 11.5.0 - '@pnpm/win-arm64': 11.5.0 - '@pnpm/win-x64': 11.5.0 - - '@pnpm/linux-arm64@11.5.0': - optional: true - - '@pnpm/linux-x64@11.5.0': - optional: true - - '@pnpm/linuxstatic-arm64@11.5.0': - optional: true - - '@pnpm/linuxstatic-x64@11.5.0': - optional: true - - '@pnpm/macos-arm64@11.5.0': - optional: true - - '@pnpm/win-arm64@11.5.0': - optional: true - - '@pnpm/win-x64@11.5.0': - optional: true - - '@reflink/reflink-darwin-arm64@0.1.19': - optional: true - - '@reflink/reflink-darwin-x64@0.1.19': - optional: true - - '@reflink/reflink-linux-arm64-gnu@0.1.19': - optional: true - - '@reflink/reflink-linux-arm64-musl@0.1.19': - optional: true - - '@reflink/reflink-linux-x64-gnu@0.1.19': - optional: true - - '@reflink/reflink-linux-x64-musl@0.1.19': - optional: true - - '@reflink/reflink-win32-arm64-msvc@0.1.19': - optional: true - - '@reflink/reflink-win32-x64-msvc@0.1.19': - optional: true - - '@reflink/reflink@0.1.19': - optionalDependencies: - '@reflink/reflink-darwin-arm64': 0.1.19 - '@reflink/reflink-darwin-x64': 0.1.19 - '@reflink/reflink-linux-arm64-gnu': 0.1.19 - '@reflink/reflink-linux-arm64-musl': 0.1.19 - '@reflink/reflink-linux-x64-gnu': 0.1.19 - '@reflink/reflink-linux-x64-musl': 0.1.19 - '@reflink/reflink-win32-arm64-msvc': 0.1.19 - '@reflink/reflink-win32-x64-msvc': 0.1.19 - - detect-libc@2.1.2: {} - - pnpm@11.5.0: {} - ---- lockfileVersion: '9.0' settings: diff --git a/tests/lib/s3-object-encoding.test.ts b/tests/lib/s3-object-encoding.test.ts new file mode 100644 index 00000000..13fcd1e9 --- /dev/null +++ b/tests/lib/s3-object-encoding.test.ts @@ -0,0 +1,50 @@ +import test from "node:test" +import assert from "node:assert/strict" + +const loadS3ObjectEncoding = () => import(new URL("../../lib/s3-object-encoding.ts", import.meta.url).href) + +test("decodeS3UrlEncodedObjectList restores control-character object keys", async () => { + const { decodeS3UrlEncodedObjectList } = await loadS3ObjectEncoding() + const response = decodeS3UrlEncodedObjectList({ + EncodingType: "url", + Prefix: "bad%2F", + Delimiter: "%2F", + StartAfter: "%04bad-prefix%2Fbefore.txt", + Contents: [{ Key: "%04bad-prefix%2Fobject.txt" }], + CommonPrefixes: [{ Prefix: "normal%2F" }], + }) + + assert.equal(response.StartAfter, "\x04bad-prefix/before.txt") + assert.equal(response.Contents?.[0]?.Key, "\x04bad-prefix/object.txt") + assert.equal(response.CommonPrefixes?.[0]?.Prefix, "normal/") + assert.equal(response.Prefix, "bad/") + assert.equal(response.Delimiter, "/") +}) + +test("decodeS3UrlEncodedObjectVersions restores encoded key markers", async () => { + const { decodeS3UrlEncodedObjectVersions } = await loadS3ObjectEncoding() + const response = decodeS3UrlEncodedObjectVersions({ + EncodingType: "url", + Prefix: "%04bad-prefix%2F", + KeyMarker: "%04bad-prefix%2Ffrom.txt", + NextKeyMarker: "%04bad-prefix%2Fnext.txt", + Versions: [{ Key: "%04bad-prefix%2Fobject.txt" }], + DeleteMarkers: [{ Key: "%04bad-prefix%2Fdeleted.txt" }], + }) + + assert.equal(response.Prefix, "\x04bad-prefix/") + assert.equal(response.KeyMarker, "\x04bad-prefix/from.txt") + assert.equal(response.NextKeyMarker, "\x04bad-prefix/next.txt") + assert.equal(response.Versions?.[0]?.Key, "\x04bad-prefix/object.txt") + assert.equal(response.DeleteMarkers?.[0]?.Key, "\x04bad-prefix/deleted.txt") +}) + +test("S3 URL decoding leaves malformed values unchanged", async () => { + const { decodeS3UrlEncodedObjectList } = await loadS3ObjectEncoding() + const response = decodeS3UrlEncodedObjectList({ + EncodingType: "url", + Contents: [{ Key: "%E0%A4%A" }], + }) + + assert.equal(response.Contents?.[0]?.Key, "%E0%A4%A") +})