feat: add bounded LZMA2 multithreading

Author: sunnylqm

README.md

@@ -31,10 +31,16 @@ bun run test:bun   # run the same tests under the Bun runtime
 
 ## Usage
 
-### diff(originBuf, newBuf)
+### diff(originBuf, newBuf[, options])
 
 Compare two buffers and return a new hdiffpatch patch as return value.
 
+All diff entry points accept an optional `options` object with
+`compressionThreads: 1 | 2`. Two threads enable LZMA's internal parallel
+match finder while preserving compression level 9 and the 8 MiB dictionary.
+The default remains one thread. `diffWindow()` also accepts `windowSize` in
+the options object; the legacy positional `windowSize` remains supported.
+
 ### diffSingleStream(oldPath, newPath, outDiffPath[, cb])
 
 Create a **single-format** (same wire format as `diff()`) patch by streaming
@@ -67,6 +73,7 @@ additional memory.
 `capabilities.diffWindowVerifiesOutput` are `true`. Each native diff function
 applies and compares the generated patch before it returns, so orchestration
 layers can avoid running a redundant second round-trip check.
+`capabilities.maxCompressionThreads` is `2`.
 
 ### patchSingleStream(oldPath, diffPath, outNewPath[, cb])
 

binding.gyp

@@ -24,15 +24,17 @@
         "HDiffPatch/libHDiffPatch/HDiff/private_diff/match_inplace.cpp",
         "lzma/C/CpuArch.c",
         "lzma/C/LzFind.c",
+        "lzma/C/LzFindMt.c",
+        "lzma/C/LzFindOpt.c",
         "lzma/C/LzmaDec.c",
         "lzma/C/LzmaEnc.c",
         "lzma/C/Lzma2Dec.c",
-        "lzma/C/Lzma2Enc.c"
+        "lzma/C/Lzma2Enc.c",
+        "lzma/C/MtCoder.c",
+        "lzma/C/Threads.c"
       ],
       "defines": [
         "_IS_NEED_DIR_DIFF_PATCH=0",
-        "_7ZIP_ST",
-        "Z7_ST",
         "_IS_USED_MULTITHREAD=0",
         "_IS_OUT_DIFF_INFO=0",
         "NAPI_VERSION=8"

index.d.ts

@@ -5,18 +5,48 @@ export type BinaryLike = Buffer | ArrayBufferView;
 export type DiffCallback = (err: Error | null, result?: Buffer) => void;
 export type StreamCallback = (err: Error | null, outPath?: string) => void;
 
+export interface CompressionOptions {
+  /** LZMA2 compression workers. Level 9 and the 8 MiB dictionary are unchanged. */
+  compressionThreads?: 1 | 2;
+}
+
+export interface DiffWindowOptions extends CompressionOptions {
+  /** Old-data sliding window bytes; 0 uses the native 2 MiB default. */
+  windowSize?: number;
+}
+
 export interface NativeAddon {
   diff(oldBuf: BinaryLike, newBuf: BinaryLike): Buffer;
+  diff(oldBuf: BinaryLike, newBuf: BinaryLike, options: CompressionOptions): Buffer;
   diff(oldBuf: BinaryLike, newBuf: BinaryLike, cb: DiffCallback): void;
+  diff(
+    oldBuf: BinaryLike,
+    newBuf: BinaryLike,
+    options: CompressionOptions,
+    cb: DiffCallback
+  ): void;
   patch(oldBuf: BinaryLike, diffBuf: BinaryLike): Buffer;
   patch(oldBuf: BinaryLike, diffBuf: BinaryLike, cb: DiffCallback): void;
   diffStream(oldPath: string, newPath: string, outDiffPath: string): string;
+  diffStream(
+    oldPath: string,
+    newPath: string,
+    outDiffPath: string,
+    options: CompressionOptions
+  ): string;
   diffStream(
     oldPath: string,
     newPath: string,
     outDiffPath: string,
     cb: StreamCallback
   ): void;
+  diffStream(
+    oldPath: string,
+    newPath: string,
+    outDiffPath: string,
+    options: CompressionOptions,
+    cb: StreamCallback
+  ): void;
   patchStream(oldPath: string, diffPath: string, outNewPath: string): string;
   patchStream(
     oldPath: string,
@@ -29,6 +59,19 @@ export interface NativeAddon {
     oldPath: string,
     newPath: string,
     outDiffPath: string,
+    options: CompressionOptions
+  ): string;
+  diffSingleStream(
+    oldPath: string,
+    newPath: string,
+    outDiffPath: string,
+    cb: StreamCallback,
+  ): void;
+  diffSingleStream(
+    oldPath: string,
+    newPath: string,
+    outDiffPath: string,
+    options: CompressionOptions,
     cb: StreamCallback,
   ): void;
   patchSingleStream(oldPath: string, diffPath: string, outNewPath: string): string;
@@ -48,6 +91,19 @@ export interface NativeAddon {
     oldPath: string,
     newPath: string,
     outDiffPath: string,
+    options: DiffWindowOptions
+  ): string;
+  diffWindow(
+    oldPath: string,
+    newPath: string,
+    outDiffPath: string,
+    cb: StreamCallback
+  ): void;
+  diffWindow(
+    oldPath: string,
+    newPath: string,
+    outDiffPath: string,
+    options: DiffWindowOptions,
     cb: StreamCallback
   ): void;
   diffWindow(
@@ -65,6 +121,7 @@ export interface HdiffpatchCapabilities {
   readonly diffStreamVerifiesOutput: true;
   readonly diffSingleStreamVerifiesOutput: true;
   readonly diffWindowVerifiesOutput: true;
+  readonly maxCompressionThreads: 2;
 }
 
 /** Native diff functions apply and compare their output before returning. */
@@ -74,6 +131,17 @@ export function diff(oldBuf: BinaryLike, newBuf: BinaryLike): Buffer;
 export function diff(
   oldBuf: BinaryLike,
   newBuf: BinaryLike,
+  options: CompressionOptions
+): Buffer;
+export function diff(
+  oldBuf: BinaryLike,
+  newBuf: BinaryLike,
+  cb: DiffCallback
+): void;
+export function diff(
+  oldBuf: BinaryLike,
+  newBuf: BinaryLike,
+  options: CompressionOptions,
   cb: DiffCallback
 ): void;
 
@@ -93,6 +161,19 @@ export function diffStream(
   oldPath: string,
   newPath: string,
   outDiffPath: string,
+  options: CompressionOptions
+): string;
+export function diffStream(
+  oldPath: string,
+  newPath: string,
+  outDiffPath: string,
+  cb: StreamCallback
+): void;
+export function diffStream(
+  oldPath: string,
+  newPath: string,
+  outDiffPath: string,
+  options: CompressionOptions,
   cb: StreamCallback
 ): void;
 
@@ -116,6 +197,19 @@ export function diffSingleStream(
   oldPath: string,
   newPath: string,
   outDiffPath: string,
+  options: CompressionOptions,
+): string;
+export function diffSingleStream(
+  oldPath: string,
+  newPath: string,
+  outDiffPath: string,
+  cb: StreamCallback,
+): void;
+export function diffSingleStream(
+  oldPath: string,
+  newPath: string,
+  outDiffPath: string,
+  options: CompressionOptions,
   cb: StreamCallback,
 ): void;
 export function patchSingleStream(
@@ -144,6 +238,19 @@ export function diffWindow(
   oldPath: string,
   newPath: string,
   outDiffPath: string,
+  options: DiffWindowOptions
+): string;
+export function diffWindow(
+  oldPath: string,
+  newPath: string,
+  outDiffPath: string,
+  cb: StreamCallback
+): void;
+export function diffWindow(
+  oldPath: string,
+  newPath: string,
+  outDiffPath: string,
+  options: DiffWindowOptions,
   cb: StreamCallback
 ): void;
 export function diffWindow(

index.js

@@ -46,4 +46,5 @@ exports.capabilities = Object.freeze({
   diffStreamVerifiesOutput: true,
   diffSingleStreamVerifiesOutput: true,
   diffWindowVerifiesOutput: true,
+  maxCompressionThreads: 2,
 });

package.json

@@ -18,6 +18,7 @@
     "test:bun": "bun ./test/test.js",
     "prepublishOnly": "bun scripts/prepublish.ts",
     "benchmark": "node --expose-gc test/benchmark.js",
+    "benchmark:threads": "node test/benchmark-threads.js",
     "prebuild": "prebuildify --napi --strip"
   },
   "gypfile": true,

src/hdiff.cpp

@@ -10,19 +10,28 @@
 #define IS_NOTICE_compress_canceled 0
 #include "../lzma/C/Lzma2Dec.h"
 #include "../lzma/C/Lzma2Enc.h"
-#include "../lzma/C/MtCoder.h"
+// The public API deliberately caps LZMA's internal match-finder parallelism
+// at two workers, so the C++ wrapper only needs the compressor's bound here.
+// MtCoder itself is compiled as C from binding.gyp.
+#define MTCODER_THREADS_MAX 2
 #include "../HDiffPatch/compress_plugin_demo.h"
 #include "../HDiffPatch/decompress_plugin_demo.h"
 
 namespace {
     // 与 v1.0.6 起的历史产物保持一致的压缩参数(patch 兼容性由 single
     // 格式规范保证,这里的一致性只为产物尺寸/确定性稳定)
-    void configure_lzma2(TCompressPlugin_lzma2& compressPlugin) {
+    void configure_lzma2(TCompressPlugin_lzma2& compressPlugin,
+                         size_t compressionThreads) {
+        if (compressionThreads < 1 || compressionThreads > 2) {
+            throw std::runtime_error("compressionThreads must be 1 or 2.");
+        }
         const size_t myBestDictSize = (1 << 20) * 8;  // 固定 8MB
         compressPlugin = lzma2CompressPlugin;
         compressPlugin.compress_level = 9;
         compressPlugin.dict_size = myBestDictSize;
-        compressPlugin.thread_num = 1;
+        // 2 线程只启用 LZMA 内部并行匹配,不切 LZMA2 block,避免额外的
+        // block 级内存放大。压缩级别和字典保持历史值不变。
+        compressPlugin.thread_num = static_cast<int>(compressionThreads);
     }
 
     // 升级到 HDiffPatch v5 前的历史参数:匹配分 3、步进内存 256KB。
@@ -255,10 +264,10 @@ namespace {
 }
 
 void hdiff(const uint8_t* old, size_t oldsize, const uint8_t* _new, size_t newsize,
-           std::vector<uint8_t>& out_codeBuf) {
+           std::vector<uint8_t>& out_codeBuf, size_t compressionThreads) {
     hpatch_TDecompress* decompressPlugin = &lzma2DecompressPlugin;
     TCompressPlugin_lzma2 compressPlugin;
-    configure_lzma2(compressPlugin);
+    configure_lzma2(compressPlugin, compressionThreads);
 
     create_single_compressed_diff(_new, _new + newsize, old, old + oldsize, out_codeBuf,
                                   &compressPlugin.base, kPatchStepMemSize,
@@ -273,14 +282,15 @@ void hdiff(const uint8_t* old, size_t oldsize, const uint8_t* _new, size_t newsi
     }
 }
 
-void hdiff_stream(const char* oldPath,const char* newPath,const char* outDiffPath){
+void hdiff_stream(const char* oldPath,const char* newPath,const char* outDiffPath,
+                  size_t compressionThreads){
     if (!oldPath || !newPath || !outDiffPath) {
         throw std::runtime_error("Invalid file path.");
     }
 
     hpatch_TDecompress* decompressPlugin = &lzma2DecompressPlugin;
     TCompressPlugin_lzma2 compressPlugin;
-    configure_lzma2(compressPlugin);
+    configure_lzma2(compressPlugin, compressionThreads);
 
     FileStreamGuard streams;
     streams.openInputs(oldPath, newPath);
@@ -300,14 +310,14 @@ void hdiff_stream(const char* oldPath,const char* newPath,const char* outDiffPat
 }
 
 void hdiff_window(const char* oldPath,const char* newPath,const char* outDiffPath,
-                  size_t windowSize){
+                  size_t windowSize,size_t compressionThreads){
     if (!oldPath || !newPath || !outDiffPath) {
         throw std::runtime_error("Invalid file path.");
     }
 
     hpatch_TDecompress* decompressPlugin = &lzma2DecompressPlugin;
     TCompressPlugin_lzma2 compressPlugin;
-    configure_lzma2(compressPlugin);
+    configure_lzma2(compressPlugin, compressionThreads);
 
     FileStreamGuard streams;
     streams.openInputs(oldPath, newPath);
@@ -336,14 +346,15 @@ void hdiff_window(const char* oldPath,const char* newPath,const char* outDiffPat
     streams.closeAllOrThrow();
 }
 
-void hdiff_single_stream(const char* oldPath,const char* newPath,const char* outDiffPath){
+void hdiff_single_stream(const char* oldPath,const char* newPath,const char* outDiffPath,
+                         size_t compressionThreads){
     if (!oldPath || !newPath || !outDiffPath) {
         throw std::runtime_error("Invalid file path.");
     }
 
     hpatch_TDecompress* decompressPlugin = &lzma2DecompressPlugin;
     TCompressPlugin_lzma2 compressPlugin;
-    configure_lzma2(compressPlugin);
+    configure_lzma2(compressPlugin, compressionThreads);
 
     FileStreamGuard streams;
     streams.openInputs(oldPath, newPath);

src/hdiff.h

@@ -9,17 +9,19 @@
 #include <vector>
 
 void hdiff(const uint8_t* old,size_t oldsize,const uint8_t* _new,size_t newsize,
-		   std::vector<uint8_t>& out_codeBuf);
+		   std::vector<uint8_t>& out_codeBuf,size_t compressionThreads=1);
 // HDIFF13 流式(生成端低内存,产物需 patchStream 应用)
-void hdiff_stream(const char* oldPath,const char* newPath,const char* outDiffPath);
+void hdiff_stream(const char* oldPath,const char* newPath,const char* outDiffPath,
+                  size_t compressionThreads=1);
 // HDIFFSF20 single 格式的流式生成(生成端低内存,产物与 diff() 同格式,
 // 任何既有 single 应用端可直接使用)
-void hdiff_single_stream(const char* oldPath,const char* newPath,const char* outDiffPath);
+void hdiff_single_stream(const char* oldPath,const char* newPath,const char* outDiffPath,
+                         size_t compressionThreads=1);
 // HDIFFSF20 single 格式的 window 模式生成:大块流式匹配 + 窗口内后缀串
 // 精修,匹配质量接近内存版而内存占用保持流式档;产物与 diff() 同格式。
 // windowSize 为 old 数据滑动窗口字节数,0 表示用默认值(2MB);窗口越大
 // 能捕获越长距离的内容移动,内存占用近似随之线性增长。
 void hdiff_window(const char* oldPath,const char* newPath,const char* outDiffPath,
-                  size_t windowSize=0);
+                  size_t windowSize=0,size_t compressionThreads=1);
 
 #endif