feat(upload): dedup encrypted binaries on the plaintext hash (#101)

`--encrypt` uses a fresh random DEK per upload, so identical plaintext produces
different ciphertext every time. Since the dedup key was the hash of whatever
gets uploaded, encryption silently disabled dedup entirely: every run re-uploaded
and re-stored the full binary. Verified on dev — two `--encrypt` runs of a
byte-identical APK produced two distinct binaries, while the plaintext run
deduped.

Hash the plaintext first, dedup on that, and only encrypt on a miss. Because the
check now happens before encryption, a hit skips the encryption work too, not
just the upload — an encrypted re-run is now faster than the old plaintext path
rather than merely matching it.

`binaries.sha` is untouched and remains the ciphertext hash: the hosts that
verify it (B2, Supabase Storage, storage-cache, Mac LRU) hold ciphertext and no
DEK, so they can only hash the bytes they actually have. The plaintext hash rides
alongside as `shaPlain` and is sent only when encrypting.

A dedup hit is honoured only if the server confirms the matched binary is itself
encrypted. This matters: a previously-uploaded *plaintext* copy of the same app
has the same plaintext hash, so a lookup blind to encryption state would hand
back an unencrypted binary to someone who asked for encryption. The API applies
the same predicate (devicecloud-dev/dcd#1168); this is the client refusing to
depend on that, so an older or misbehaving deployment degrades into a redundant
upload instead of a silent loss of encryption.

Requires the API side of dcd#1168 for the encrypted path to dedup; against an
older deployment the check finds nothing and behaviour is exactly as it is today.
The unencrypted path is unchanged on the wire.

One cost worth naming: an encrypted upload that misses now hashes twice (once
plaintext, once ciphertext). Hashing is cheap next to encrypt + upload, and only
the miss path pays it.

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: riglar

src/config/flags/binary.flags.ts

@@ -27,6 +27,6 @@ export const binaryFlags = {
   encrypt: {
     type: 'boolean',
     description:
-      'Encrypt the app binary, flow zip, and env vars before upload (client-side envelope encryption; each gets its own key). Can also be enabled with DCD_ENCRYPT=1.',
+      'Encrypt the app binary, flow zip, and env vars before upload (client-side envelope encryption; each gets its own key). The binary is still deduplicated across runs, so an unchanged app is not re-uploaded. Can also be enabled with DCD_ENCRYPT=1.',
   },
 } as const satisfies ArgsDef;

src/gateways/api-gateway.ts

@@ -257,14 +257,32 @@ export const ApiGateway = {
     return true;
   },
 
+  /**
+   * Look for an already-uploaded binary to skip re-uploading.
+   *
+   * Two lookup keys, because encryption changes what is stable (dcd#1168).
+   * Unencrypted uploads pass `sha` (the hash of exactly what gets stored).
+   * Encrypted uploads pass `shaPlain` plus `encrypted: true`: their ciphertext is
+   * freshly keyed on every upload, so its hash never matches, and the plaintext
+   * hash is the only stable key. `sha` is deliberately not sent in that case —
+   * see the caller in methods.ts for why sending it would be unsafe.
+   *
+   * `encrypted` in the response reports whether the *matched* binary is stored
+   * encrypted, so the caller can verify the invariant it asked for instead of
+   * trusting the server to have applied the right predicate.
+   */
   async checkForExistingUpload(
     baseUrl: string,
     auth: AuthContext,
-    sha: string,
+    lookup: { encrypted?: boolean; sha?: string; shaPlain?: string } | string,
   ) {
+    // Historically this took a bare sha string; keep that shape working.
+    const body =
+      typeof lookup === 'string' ? { sha: lookup } : { ...lookup };
+
     try {
       const res = await fetch(`${baseUrl}/uploads/checkForExistingUpload`, {
-        body: JSON.stringify({ sha }),
+        body: JSON.stringify(body),
         headers: {
           'content-type': 'application/json',
           ...auth.headers,
@@ -277,7 +295,9 @@ export const ApiGateway = {
       }
 
       return await parseJsonResponse<
-        paths['/uploads/checkForExistingUpload']['post']['responses']['201']['content']['application/json']
+        paths['/uploads/checkForExistingUpload']['post']['responses']['201']['content']['application/json'] & {
+          encrypted?: boolean;
+        }
       >(res, 'Failed to check for existing upload');
     } catch (error) {
       // Handle network-level errors (DNS, connection refused, timeout, etc.)
@@ -342,9 +362,15 @@ export const ApiGateway = {
     metadata: TAppMetadata;
     path: string;
     sha?: string;
+    /**
+     * Hash of the PLAINTEXT, sent only for encrypted uploads (dcd#1168). Stored
+     * as `binaries.sha_plain` so later encrypted uploads of the same input can
+     * dedup; `sha` remains the ciphertext hash.
+     */
+    shaPlain?: string;
     supabaseSuccess: boolean;
   }) {
-    const { baseUrl, auth, id, metadata, path, sha, supabaseSuccess, backblazeSuccess, bytes } = config;
+    const { baseUrl, auth, id, metadata, path, sha, shaPlain, supabaseSuccess, backblazeSuccess, bytes } = config;
     try {
       const res = await fetch(`${baseUrl}/uploads/finaliseUpload`, {
         body: JSON.stringify({
@@ -354,6 +380,7 @@ export const ApiGateway = {
           metadata,
           path, // This is tempPath for TUS uploads
           ...(sha ? { sha } : {}),
+          ...(shaPlain ? { shaPlain } : {}),
           supabaseSuccess,
         }),
         headers: {

src/methods.ts

@@ -173,23 +173,21 @@ export const uploadBinary = async (config: UploadBinaryConfig) => {
     // Prepare file for upload
     source = await prepareFileForUpload(filePath, debug, startTime);
 
-    // Encrypt before hashing/upload so the SHA, dedup check, and both uploaders
-    // all operate on ciphertext (binaries.sha = ciphertext hash, per #1138).
-    if (encrypt) {
-      const encrypted = await encryptUploadSource(source, apiUrl, debug);
-      enc = encrypted.enc;
-      encCleanupDir = encrypted.cleanupDir;
-      if (log) {
-        ux.info(colors.dim(`Encrypting binary before upload (KEK v${enc.kek})`));
-      }
-    }
-
-    // Calculate SHA hash
-    const sha = await calculateFileHash(source, debug, log);
-
-    // Check for existing upload with same SHA
-    if (!ignoreShaCheck && sha) {
-      const { exists, binaryId } = await checkExistingUpload(apiUrl, auth, sha, debug);
+    // Hash the PLAINTEXT first, before any encryption (dcd#1168). Encryption
+    // uses a fresh random DEK per upload, so the ciphertext hash differs every
+    // time and cannot dedup — the plaintext hash is the only stable key. Doing it
+    // in this order also means a dedup hit skips the encryption work entirely,
+    // not just the upload.
+    const shaPlain = await calculateFileHash(source, debug, log);
+
+    // Check for an existing upload before spending anything on encryption.
+    if (!ignoreShaCheck && shaPlain) {
+      const { exists, binaryId } = await checkExistingUpload(
+        apiUrl,
+        auth,
+        encrypt ? { encrypted: true, shaPlain } : { sha: shaPlain },
+        debug,
+      );
 
       if (exists && binaryId) {
         if (log) {
@@ -203,8 +201,35 @@ export const uploadBinary = async (config: UploadBinaryConfig) => {
       }
     }
 
+    // Encrypt after the dedup check, so the SHA sent at finalise, and both
+    // uploaders, operate on ciphertext (binaries.sha = ciphertext hash, #1138).
+    if (encrypt) {
+      const encrypted = await encryptUploadSource(source, apiUrl, debug);
+      enc = encrypted.enc;
+      encCleanupDir = encrypted.cleanupDir;
+      if (log) {
+        ux.info(colors.dim(`Encrypting binary before upload (KEK v${enc.kek})`));
+      }
+    }
+
+    // Re-hash once encrypted: what lands in storage is the ciphertext, and every
+    // downstream verifySha hashes the bytes it actually holds (no DEK required).
+    const sha = encrypt ? await calculateFileHash(source, debug, false) : shaPlain;
+
     // Perform the upload
-    const uploadId = await performUpload({ auth, apiUrl, debug, enc, filePath, sha, source, startTime });
+    const uploadId = await performUpload({
+      auth,
+      apiUrl,
+      debug,
+      enc,
+      filePath,
+      sha,
+      // Only encrypted uploads record a plaintext hash; it is what makes the
+      // next encrypted run of this binary dedupable.
+      shaPlain: encrypt ? shaPlain : undefined,
+      source,
+      startTime,
+    });
 
     if (log) {
       ux.action.stop(colors.success('\n✓ Binary uploaded with ID: ') + formatId(uploadId));
@@ -425,30 +450,43 @@ async function calculateFileHash(
 }
 
 /**
- * Checks if an upload with the same SHA already exists
+ * Checks whether a matching binary has already been uploaded.
+ *
+ * `lookup` is `{ sha }` for a plaintext upload, or `{ shaPlain, encrypted: true }`
+ * for an encrypted one (dcd#1168) — see {@link ApiGateway.checkForExistingUpload}.
+ *
+ * When asking as an encrypting client, a hit is only honoured if the server
+ * confirms the matched binary is itself encrypted. Any binary is a *plausible*
+ * match on plaintext hash, including a previously-uploaded plaintext copy of the
+ * same app, and reusing that would hand back an unencrypted binary while the user
+ * had asked for encryption. The server applies the same predicate; this is the
+ * client refusing to depend on that, so an older or misbehaving deployment
+ * degrades into a redundant upload rather than a silent loss of encryption.
+ *
  * @param apiUrl API base URL
  * @param auth AuthContext carrying request headers
- * @param sha SHA-256 hash to check
+ * @param lookup Dedup key — plaintext hash for encrypted uploads, else the sha
  * @param debug Whether debug logging is enabled
  * @returns Promise resolving to object with exists flag and optional binaryId
  */
 async function checkExistingUpload(
   apiUrl: string,
   auth: AuthContext,
-  sha: string,
+  lookup: { encrypted?: boolean; sha?: string; shaPlain?: string },
   debug: boolean,
 ): Promise<{ binaryId?: string; exists: boolean }> {
   try {
     if (debug) {
       console.log('[DEBUG] Checking for existing upload with matching SHA...');
+      console.log(`[DEBUG] Lookup: ${JSON.stringify(lookup)}`);
       console.log(`[DEBUG] Target endpoint: ${apiUrl}/uploads/checkForExistingUpload`);
     }
 
     const shaCheckStartTime = Date.now();
-    const { appBinaryId, exists } = await ApiGateway.checkForExistingUpload(
+    const { appBinaryId, encrypted, exists } = await ApiGateway.checkForExistingUpload(
       apiUrl,
       auth,
-      sha as string,
+      lookup,
     );
 
     if (debug) {
@@ -459,6 +497,16 @@ async function checkExistingUpload(
       }
     }
 
+    if (exists && lookup.encrypted && encrypted !== true) {
+      if (debug) {
+        console.log(
+          '[DEBUG] Ignoring dedup hit: encryption was requested but the matched binary is not encrypted',
+        );
+      }
+
+      return { exists: false };
+    }
+
     return { binaryId: appBinaryId, exists };
   } catch (error) {
     // Invalid credentials will fail every subsequent request — surface now
@@ -493,6 +541,11 @@ interface PerformUploadConfig {
   enc?: BinaryEnvelope;
   filePath: string;
   sha: string | undefined;
+  /**
+   * Hash of the plaintext, set only for encrypted uploads (#1168). Persisted as
+   * `binaries.sha_plain` so the next encrypted upload of this binary can dedup.
+   */
+  shaPlain?: string;
   source: UploadSource;
   startTime: number;
 }
@@ -769,7 +822,7 @@ function validateUploadResults(
  * @returns Promise resolving to upload ID
  */
 async function performUpload(config: PerformUploadConfig): Promise<string> {
-  const { filePath, apiUrl, auth, enc, source, sha, debug, startTime } = config;
+  const { filePath, apiUrl, auth, enc, source, sha, shaPlain, debug, startTime } = config;
 
   // Request upload URL and paths
   const { id, tempPath, finalPath, b2 } = await requestUploadPaths(apiUrl, auth, filePath, source.size, debug);
@@ -832,6 +885,7 @@ async function performUpload(config: PerformUploadConfig): Promise<string> {
     path: tempPath,
     // sha is undefined when hash calculation failed — omit it explicitly
     ...(sha ? { sha } : {}),
+    ...(shaPlain ? { shaPlain } : {}),
     supabaseSuccess: supabaseResult.success,
   });