diff --git a/.github/workflows/full-validation.yml b/.github/workflows/full-validation.yml index 816246d8..9bee934c 100644 --- a/.github/workflows/full-validation.yml +++ b/.github/workflows/full-validation.yml @@ -296,6 +296,47 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' }} - run: cargo xtask release-cpu + t803-cpu: + name: T.803 CPU (${{ matrix.lane }}) + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, lane: linux-x86_64 } + - { os: macos-latest, lane: macos-aarch64 } + - { os: windows-latest, lane: windows-x86_64 } + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 0 + - uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c + with: + toolchain: stable + - name: Generate exact-reference CPU evidence + shell: bash + run: | + status=0 + cargo xtask t803 fetch || status=$? + if [ "${status}" -eq 0 ]; then + cargo xtask t803 run --iut cpu --out-dir target/t803/reports || status=$? + fi + echo "T803_STATUS=${status}" >> "${GITHUB_ENV}" + - name: Upload CPU T.803 reports only + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: j2k-t803-cpu-${{ matrix.lane }}-${{ github.sha }} + path: | + target/t803/reports/cpu.json + target/t803/reports/cpu.md + if-no-files-found: error + - name: Enforce CPU T.803 result after evidence upload + if: ${{ always() }} + shell: bash + run: exit "${T803_STATUS:-1}" + metal-compile: name: Metal compile and pure tests runs-on: macos-latest @@ -459,6 +500,7 @@ jobs: - typos - test - release-cpu + - t803-cpu - metal-compile - no-std - miri diff --git a/.github/workflows/gpu-benchmarks.yml b/.github/workflows/gpu-benchmarks.yml index 3d36a8d3..02370318 100644 --- a/.github/workflows/gpu-benchmarks.yml +++ b/.github/workflows/gpu-benchmarks.yml @@ -13,7 +13,7 @@ on: type: choice required: true default: smoke - options: [smoke, criterion, profile, adoption] + options: [smoke, criterion, profile, adoption, routing] baseline-ref: description: "Immutable baseline revision for Criterion comparisons" type: string @@ -134,6 +134,28 @@ jobs: --encode-fixtures "${J2K_ADOPTION_ENCODE_FIXTURES}" \ --encode-manifest "${J2K_ADOPTION_ENCODE_MANIFEST}" \ --require-cuda --out-dir target/gpu-benchmark/adoption + - name: Measure and verify CUDA Auto routing + if: ${{ inputs.suite == 'routing' }} + env: + J2K_AUTO_ROUTING_MANIFEST: ${{ vars.J2K_AUTO_ROUTING_MANIFEST }} + J2K_AUTO_ROUTING_ROOT: ${{ vars.J2K_AUTO_ROUTING_ROOT }} + PROFILE_MODE: ${{ inputs.profile-mode }} + run: | + : "${J2K_AUTO_ROUTING_MANIFEST:?Set J2K_AUTO_ROUTING_MANIFEST}" + : "${J2K_AUTO_ROUTING_ROOT:?Set J2K_AUTO_ROUTING_ROOT}" + mkdir -p target/gpu-benchmark/auto-routing + export J2K_AUTO_ROUTING_CANDIDATE_SHA="${GITHUB_SHA}" + export J2K_AUTO_ROUTING_EVIDENCE="${GITHUB_WORKSPACE}/target/gpu-benchmark/auto-routing/evidence.json" + export J2K_AUTO_ROUTING_HARDWARE="$(nvidia-smi --query-gpu=name --format=csv,noheader | paste -sd ';' -)" + export J2K_AUTO_ROUTING_DRIVER="$(nvidia-smi --query-gpu=driver_version --format=csv,noheader | sort -u | paste -sd ';' -)" + args=() + if [ "${PROFILE_MODE}" = quick ]; then args+=(--quick); fi + cargo bench --profile release-bench -p j2k-cuda --bench auto_routing --features cuda-runtime -- "${args[@]}" + cargo xtask auto-routing verify \ + --evidence "${J2K_AUTO_ROUTING_EVIDENCE}" \ + --external-manifest "${J2K_AUTO_ROUTING_MANIFEST}" \ + --criterion-root target/criterion \ + --out target/gpu-benchmark/auto-routing/verified.json - name: Require unchanged CUDA device identity if: always() run: | @@ -231,6 +253,28 @@ jobs: --encode-fixtures "${J2K_ADOPTION_ENCODE_FIXTURES}" \ --encode-manifest "${J2K_ADOPTION_ENCODE_MANIFEST}" \ --require-metal --out-dir target/gpu-benchmark/adoption + - name: Measure and verify Metal Auto routing + if: ${{ inputs.suite == 'routing' }} + env: + J2K_AUTO_ROUTING_MANIFEST: ${{ vars.J2K_AUTO_ROUTING_MANIFEST }} + J2K_AUTO_ROUTING_ROOT: ${{ vars.J2K_AUTO_ROUTING_ROOT }} + PROFILE_MODE: ${{ inputs.profile-mode }} + run: | + : "${J2K_AUTO_ROUTING_MANIFEST:?Set J2K_AUTO_ROUTING_MANIFEST}" + : "${J2K_AUTO_ROUTING_ROOT:?Set J2K_AUTO_ROUTING_ROOT}" + mkdir -p target/gpu-benchmark/auto-routing + export J2K_AUTO_ROUTING_CANDIDATE_SHA="${GITHUB_SHA}" + export J2K_AUTO_ROUTING_EVIDENCE="${GITHUB_WORKSPACE}/target/gpu-benchmark/auto-routing/evidence.json" + export J2K_AUTO_ROUTING_HARDWARE="$(sysctl -n machdep.cpu.brand_string)" + export J2K_AUTO_ROUTING_DRIVER="macOS $(sw_vers -productVersion); $(xcrun metal --version | head -n 1)" + args=() + if [ "${PROFILE_MODE}" = quick ]; then args+=(--quick); fi + cargo bench --profile release-bench -p j2k-metal --bench auto_routing -- "${args[@]}" + cargo xtask auto-routing verify \ + --evidence "${J2K_AUTO_ROUTING_EVIDENCE}" \ + --external-manifest "${J2K_AUTO_ROUTING_MANIFEST}" \ + --criterion-root target/criterion \ + --out target/gpu-benchmark/auto-routing/verified.json - name: Require unchanged Metal device identity if: always() run: | diff --git a/.github/workflows/gpu-validation.yml b/.github/workflows/gpu-validation.yml index d30ecc32..222d5315 100644 --- a/.github/workflows/gpu-validation.yml +++ b/.github/workflows/gpu-validation.yml @@ -199,6 +199,26 @@ jobs: elapsed="$(( $(date +%s) - started ))" echo "CUDA full wall time: ${elapsed}s" | tee -a "${GITHUB_STEP_SUMMARY}" exit "${status}" + - name: Compile packaged CUDA adapter and dependencies as an external consumer + run: cargo xtask package-consumer-smoke --target cuda --cuda-runtime + - name: Generate exact-reference CUDA adapter evidence + if: ${{ always() }} + run: | + status=0 + cargo xtask t803 fetch || status=$? + if [ "${status}" -eq 0 ]; then + cargo xtask t803 run --iut cuda --out-dir target/t803/reports || status=$? + fi + echo "T803_STATUS=${status}" >> "${GITHUB_ENV}" + - name: Upload CUDA T.803 reports only + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: j2k-t803-cuda-linux-x86_64-${{ github.sha }} + path: | + target/t803/reports/cuda.json + target/t803/reports/cuda.md + if-no-files-found: error - name: Ensure pinned coverage tool run: scripts/ensure-cargo-llvm-cov.sh - name: Collect changed-line CUDA host coverage @@ -215,6 +235,9 @@ jobs: coverage-cuda-summary.json coverage-cuda-regions.json if-no-files-found: error + - name: Enforce CUDA T.803 result after evidence upload + if: ${{ always() }} + run: exit "${T803_STATUS:-1}" metal-full: name: Metal full release validation @@ -261,6 +284,26 @@ jobs: elapsed="$(( $(date +%s) - started ))" echo "Metal full wall time: ${elapsed}s" | tee -a "${GITHUB_STEP_SUMMARY}" exit "${status}" + - name: Compile packaged Metal adapter and dependencies as an external consumer + run: cargo xtask package-consumer-smoke --target metal + - name: Generate exact-reference Metal adapter evidence + if: ${{ always() }} + run: | + status=0 + cargo xtask t803 fetch || status=$? + if [ "${status}" -eq 0 ]; then + cargo xtask t803 run --iut metal --out-dir target/t803/reports || status=$? + fi + echo "T803_STATUS=${status}" >> "${GITHUB_ENV}" + - name: Upload Metal T.803 reports only + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: j2k-t803-metal-macos-aarch64-${{ github.sha }} + path: | + target/t803/reports/metal.json + target/t803/reports/metal.md + if-no-files-found: error - name: Ensure pinned coverage tool run: scripts/ensure-cargo-llvm-cov.sh - name: Collect changed-line Metal host coverage @@ -277,3 +320,6 @@ jobs: coverage-metal-summary.json coverage-metal-regions.json if-no-files-found: error + - name: Enforce Metal T.803 result after evidence upload + if: ${{ always() }} + run: exit "${T803_STATUS:-1}" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 971333b9..f052646c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -56,7 +56,19 @@ jobs: --aggregate-job "Release candidate aggregate" \ --gpu-workflow gpu-validation.yml \ --cuda-job "CUDA full release validation" \ - --metal-job "Metal full release validation" + --metal-job "Metal full release validation" \ + --t803-scope all \ + --t803-out-dir target/t803/release-evidence + - name: Verify downloaded T.803 report contents + if: ${{ github.event_name == 'push' }} + run: | + candidate_sha="$(git rev-parse HEAD)" + cargo xtask t803 verify --scope all --candidate-sha "${candidate_sha}" \ + --report "target/t803/release-evidence/j2k-t803-cpu-linux-x86_64-${candidate_sha}/cpu.json" \ + --report "target/t803/release-evidence/j2k-t803-cpu-macos-aarch64-${candidate_sha}/cpu.json" \ + --report "target/t803/release-evidence/j2k-t803-cpu-windows-x86_64-${candidate_sha}/cpu.json" \ + --report "target/t803/release-evidence/j2k-t803-cuda-linux-x86_64-${candidate_sha}/cuda.json" \ + --report "target/t803/release-evidence/j2k-t803-metal-macos-aarch64-${candidate_sha}/metal.json" - name: Verify release metadata integrity run: cargo xtask release-integrity - name: Verify final publish metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5b6473..48edf90e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,38 @@ This changelog tracks the current release line. Historical phase notes and stale roadmap entries have been removed from the public documentation set. +## [0.8.1] - 2026-08-06 + +- Adds release-scoped ISO/IEC 15444-4:2024 / ITU-T T.803 v3 decoder evidence. + The CPU IUT is Profile-1 Cclass-1 compliant, Profile-1 Cclass-1HF compliant, + and Annex G JP2 reader compliant across all 90 selected cases with zero skips + on macOS arm64, Linux x86-64, and Windows x86-64. +- Publishes separate CUDA and Metal adapter-IUT results for the same selected + codestream classes. Each headline is 0/90 device-native, 48/90 hybrid, and + 42/90 CPU-routed; reports disclose parsing, Tier-1, dequantization, IDWT, + MCT, color/output, and transfer execution per case. +- Adds exact codestream-resolution decoding through + `decode_native_components_at_reduction` and explicit Annex G normalization + through `decode_srgb8`, including Gray/RGB/RGBA layouts, component mapping, + palettes, CDEF ordering, subsampling, enumerated color spaces, and restricted + ICC conversion. +- Fixes Part 1 decoder conformance defects in irreversible midpoint + reconstruction, ROI handling, packet/header parsing, progression and tile-part + handling, component transforms, and JP2 validation without vector-specific + exceptions or relaxed tolerances. +- Adds deterministic Annex D/F encoder ICS matrices. The CPU matrix passes + 28/28 cases and the CUDA and Metal matrices pass 25/25 through the pinned + T.804 OpenJPEG decoder; this is informative encoder evidence, not the formal + decoder claim. +- Independently verifies `p0_13.j2k` before harness normalization: the + production decoder and OpenJPEG match all 257 native components exactly. +- Promotes only benchmark-qualified fixed `Auto` hybrid cells after identical + output, at least 10% median improvement, and non-overlapping Criterion 95% + confidence intervals. Explicit CUDA and Metal requests remain strict. +- Makes tag publication verify all three CPU reports plus the CUDA and Metal + reports for the exact release SHA, and rotates the public API compatibility + baseline to the published `v0.8.0` release. + ## [0.8.0] - 2026-07-29 - Breaking: decoding is strict by default in `j2k` and `j2k-native`. Callers diff --git a/Cargo.lock b/Cargo.lock index 3d49a31b..de452509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2554,7 +2554,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k-codec-math", @@ -2562,13 +2562,14 @@ dependencies = [ "j2k-native", "j2k-test-support", "j2k-types", + "moxcms", "proptest", "thiserror 2.0.18", ] [[package]] name = "j2k-alloc-probe" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-native", "j2k-profile", @@ -2578,7 +2579,7 @@ dependencies = [ [[package]] name = "j2k-cli" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k", "j2k-jpeg", @@ -2588,7 +2589,7 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.8.0" +version = "0.8.1" [[package]] name = "j2k-compare" @@ -2605,14 +2606,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.8.0" +version = "0.8.1" dependencies = [ "thiserror 2.0.18", ] [[package]] name = "j2k-cuda" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k", @@ -2626,7 +2627,7 @@ dependencies = [ [[package]] name = "j2k-cuda-runtime" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -2637,7 +2638,7 @@ dependencies = [ [[package]] name = "j2k-jpeg" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k-codec-math", @@ -2655,7 +2656,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-cuda" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k-core", @@ -2668,7 +2669,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-metal" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k-core", @@ -2684,9 +2685,10 @@ dependencies = [ [[package]] name = "j2k-metal" -version = "0.8.0" +version = "0.8.1" dependencies = [ "cc", + "criterion", "j2k", "j2k-codec-math", "j2k-core", @@ -2703,7 +2705,7 @@ dependencies = [ [[package]] name = "j2k-metal-support" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-core", "j2k-test-support", @@ -2713,7 +2715,7 @@ dependencies = [ [[package]] name = "j2k-ml" -version = "0.8.0" +version = "0.8.1" dependencies = [ "burn-core", "burn-cuda", @@ -2733,7 +2735,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "fearless_simd", @@ -2747,18 +2749,43 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.0" +version = "0.8.1" + +[[package]] +name = "j2k-t803" +version = "0.8.1" +dependencies = [ + "image", + "j2k", + "j2k-codec-math", + "j2k-compare", + "j2k-core", + "j2k-cuda", + "j2k-cuda-runtime", + "j2k-metal", + "j2k-native", + "j2k-test-support", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.18", + "toml", + "zip", +] [[package]] name = "j2k-test-support" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-native", + "serde", + "serde_json", + "sha2", ] [[package]] name = "j2k-tilecodec" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "flate2", @@ -2770,7 +2797,7 @@ dependencies = [ [[package]] name = "j2k-transcode" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k", @@ -2787,7 +2814,7 @@ dependencies = [ [[package]] name = "j2k-transcode-cuda" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-core", "j2k-cuda-runtime", @@ -2799,7 +2826,7 @@ dependencies = [ [[package]] name = "j2k-transcode-metal" -version = "0.8.0" +version = "0.8.1" dependencies = [ "criterion", "j2k-codec-math", @@ -2818,7 +2845,7 @@ dependencies = [ [[package]] name = "j2k-transcode-test-support" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-native", "j2k-transcode", @@ -2827,7 +2854,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.8.0" +version = "0.8.1" [[package]] name = "jni-sys" @@ -4839,6 +4866,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" @@ -5684,6 +5717,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", + "sha2", "syn", "tar", "toml", @@ -5798,6 +5832,19 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 95a80d92..d86e2c81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/j2k-native", "crates/j2k-profile", "crates/j2k-test-support", + "crates/j2k-t803", "crates/j2k-tilecodec", "crates/j2k-transcode", "crates/j2k-transcode-cuda", @@ -31,7 +32,7 @@ exclude = [ ] [workspace.package] -version = "0.8.0" +version = "0.8.1" edition = "2021" rust-version = "1.96" license = "MIT OR Apache-2.0" @@ -56,6 +57,7 @@ libc = "0.2" log = "0.4" memchr = { version = "2.8", default-features = false } image = { version = "0.25.8", default-features = false, features = ["bmp", "jpeg", "png", "pnm", "tiff"] } +moxcms = { version = "=0.8.1", default-features = false } jpeg-encoder = "0.7.0" jpeg-decoder = { version = "0.3.2", default-features = false } zune-core = "0.5.1" @@ -67,8 +69,10 @@ proc-macro2 = "1.0.106" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml_ng = "0.10" +sha2 = "0.10.9" syn = "2.0.117" toml = "1.1.2" +zip = { version = "8.6.0", default-features = false, features = ["deflate-flate2-zlib"] } burn-core = { version = "0.21.0", default-features = false, features = ["dataset", "std"] } burn-autodiff = { version = "0.21.0", default-features = false, features = ["std"] } burn-flex = { version = "0.21.0", default-features = false, features = ["std"] } @@ -101,6 +105,9 @@ must_use_candidate = "allow" missing_errors_doc = "allow" missing_panics_doc = "allow" +[profile.dev] +debug = "line-tables-only" + [profile.release] lto = "fat" codegen-units = 1 diff --git a/README.md b/README.md index 588fa449..e35f0daa 100644 --- a/README.md +++ b/README.md @@ -8,18 +8,38 @@ **Docs & guides:** [Pure-Rust JPEG 2000 codec documentation](https://frames-sg.github.io/j2k/rust-jpeg2000-codec/) -**Release status:** `0.8.0` is published and security-supported. It replaces -the defective `j2k-ml 0.7.5` accelerator packages with CUDA and Metal adapters -that use released dependency APIs. See the [release notes](CHANGELOG.md), -[release policy](docs/release.md), and [security policy](SECURITY.md). - -**Safe public Rust APIs, audited unsafe boundaries, and vendor-independent JPEG 2000 / HTJ2K.** - -J2K is a Rust image-codec workspace for JPEG 2000 / HTJ2K decode, encode, -recode, and JPEG-to-HTJ2K coefficient-domain transcoding. It is built for teams -that need safe Rust integration for untrusted still-image inputs, permissive -MIT/Apache-2.0 licensing, and optional acceleration across both CUDA and Apple -Metal without making a GPU vendor SDK the public API. +**Release status:** `0.8.0` is published and security-supported. See the +[release notes](CHANGELOG.md), [release policy](docs/release.md), and +[security policy](SECURITY.md). + +**A general-purpose JPEG 2000 and HTJ2K codec with safe Rust APIs, a portable +CPU baseline, and optional CUDA and Metal acceleration.** + +J2K provides JPEG 2000 / HTJ2K decode, encode, recode, and +JPEG-to-HTJ2K coefficient-domain transcoding. Its public APIs cover whole-image, +region, reduced-resolution, tile, batch, host-output, and resident-device +workflows without coupling the codec to a particular application domain. The +workspace is dual-licensed under MIT/Apache-2.0. + +Region and reduced-resolution decoding plus retained tiled batch plans avoid +whole-image work for slide-scale and other large-image readers; those are codec +capabilities, not domain-specific APIs. + +Formal decoder claim for `0.8.1`: the `j2k` CPU IUT is ISO/IEC 15444-4:2024 / +ITU-T T.803 v3 **Profile-1 Cclass-1 compliant**, +**Profile-1 Cclass-1HF compliant**, and **Annex G JP2 reader compliant**. +Exact-SHA macOS arm64, +Linux x86-64, and Windows x86-64 reports each contain all 90 selected cases +with zero skips and outputs within the applicable peak-error and MSE bounds. + +The real-hardware adapter-IUT headlines are **CUDA: 0/90 device-native, 48/90 hybrid, 42/90 CPU-routed** +and **Metal: 0/90 device-native, 48/90 hybrid, 42/90 CPU-routed**. Both pass +the selected Profile-1 Cclass-1 and Cclass-1HF +cases, but neither is a device-native conformance result. Every parser, +Tier-1, transform, color/output, and transfer stage is disclosed per case. The +exact scope, evidence rules, report inventory, and informative encoder results +are in [docs/t803-conformance.md](docs/t803-conformance.md). T.803 does not +establish robustness, security, adoption, or performance. Speed matters, but it is not the reason this project exists. The strategic gap is a memory-safety-oriented Rust codec with a portable CPU baseline, @@ -28,20 +48,18 @@ benchmark gates. The public crate release centers on `j2k`, with lower-level crates for native codec internals, device adapters, JPEG input, and transcode pipelines. -The codec support claim is intentionally scoped and explicit: full JPEG 2000 -Part 1 codestream support for still-image workflows, JP2 wrapping, HTJ2K -Part 15 codestream support, and JPH wrapping. JPX / JPEG 2000 Part 2 -extensions are outside this claim unless a feature is required for standard -JP2/JPH still-image correctness. The living support boundary is +The codec support boundary is intentionally scoped and explicit: JPEG 2000 +Part 1 still-image codestream features, JP2 wrapping, HTJ2K Part 15 +codestreams, and JPH wrapping. JPX / JPEG 2000 Part 2 extensions are outside +this boundary unless a feature is required for standard JP2/JPH still-image +correctness. The implementation matrix is [docs/public-support.md](docs/public-support.md). -The APIs are general codec APIs. Whole-slide imaging and DICOM tile workloads -are the main public examples and benchmark fixtures because they stress -large tiled images, strict color handling, and high-throughput GPU paths, but -the decoder, encoder, and transcode crates are not WSI-only. The -[digital-pathology workflow audit](docs/digital-pathology-workflow-audit.md) -defines the container, indexing, color, memory, and validation responsibilities -that remain outside the codec layer. +The APIs expose codec operations rather than application-specific workflow +abstractions. Medical imaging, geospatial systems, digital preservation, +servers, desktop applications, and large tiled-image readers can use the same +decoder, encoder, and transcode surfaces. Domain containers, indexing, +application metadata, and workflow validation remain outside the codec layer. ## Why J2K exists @@ -132,6 +150,13 @@ device requests are strict. Unsupported device shapes return errors instead of silently changing the requested backend. `Auto` is an optimization policy, not a promise to use a device whenever one is available. +A new fixed hybrid threshold is eligible for `Auto` only when identical-output +external-corpus Criterion evidence shows a median at least 10% faster than CPU +and any supported strict-device route, with non-overlapping 95% confidence +intervals. The policy never calibrates at runtime. Explicit `Cuda` and `Metal` +requests remain strict, and an accelerator failure after `Auto` selects a device +is an error rather than a silent CPU retry. + CUDA paths use J2K-owned CUDA Oxide device kernels through `cuda-runtime`. NVIDIA performance claims require self-hosted benchmark evidence; hosted CI is not treated as NVIDIA performance evidence. @@ -193,8 +218,6 @@ Use lower-level crates only when you need a specific integration point: | Tile compression codecs | `j2k-tilecodec` | | Command-line inspection and JPEG-to-HTJ2K smoke transcode | `j2k-cli` | -The names `statumen` and `wsi-dicom` are not current package names. - ## Support and evidence The living codec support matrix is @@ -202,19 +225,14 @@ The living codec support matrix is adapter has a narrower, explicit boundary in [docs/j2k-ml.md](docs/j2k-ml.md). Hardware measurements and their publication qualifications are recorded separately in -[docs/benchmark-evidence.md](docs/benchmark-evidence.md). - -## Fast Path For LLM-Assisted Use - -For normal JPEG 2000 / HTJ2K work, start with the public codec crate: - -```bash -cargo add j2k -``` +[docs/benchmark-evidence.md](docs/benchmark-evidence.md). Candidate Part 1 +decoder conformance evidence is tracked separately in +[docs/t803-conformance.md](docs/t803-conformance.md). -The shared decode traits live in `j2k-core` and are implemented by codec -crates: `ImageDecode`, `ImageDecodeRows`, `TileBatchDecode`, and -device-surface traits. +The previous `j2k-ml 0.7.5` accelerator features were defective. That release +history is retained in the [release policy](docs/release.md); current CUDA and +Metal adapters use released dependency APIs and are validated as clean +packaged consumers before publication. ## Current backend posture @@ -275,6 +293,8 @@ Reference files: environment variables - [docs/public-support.md](docs/public-support.md) - exact J2K Part 1, HTJ2K Part 15, JP2/JPH, and out-of-scope support boundary +- [docs/t803-conformance.md](docs/t803-conformance.md) - candidate T.803 v3 + decoder claims, encoder procedure, blockers, and release evidence rules - [docs/j2k-ml.md](docs/j2k-ml.md) - Burn native integer batch groups, prepared reuse, and explicit accelerator decode/upload adapters - [docs/release.md](docs/release.md) - release and package validation policy diff --git a/corpus/j2k-conformance/README.md b/corpus/j2k-conformance/README.md index 6721a5b6..9485cf18 100644 --- a/corpus/j2k-conformance/README.md +++ b/corpus/j2k-conformance/README.md @@ -1,11 +1,27 @@ -# J2K Conformance Corpus +# J2K Conformance Metadata -This directory holds metadata for JPEG 2000 conformance vectors. +This directory contains authored metadata only. The official ISO/IEC 15444-4 / +ITU-T T.803 v3 copyrighted electronic attachment remains external. It must not be committed, +copied into a package, or uploaded as a CI artifact. -The manifest is the source of truth: +The blocking Part 1 decoder and Annex G inventory is `t803-v3.toml`. It pins the +official attachment URL, archive size and SHA-256, every selected codestream and +reference hash, all entries in the five selected decoder tables, and all nine +Annex G JP2 files. `cargo xtask t803 fetch` materializes the verified corpus +only under `target/t803/`; `run` fails when that corpus is absent or altered. -- `manifest.tsv` +The deterministic encoder procedure is described by: -Release signoff should classify vectors by shipped feature coverage, known -limitations, and investigation status. Narrative summaries must not override the -manifest. +- `encoder-ics-cpu.toml` +- `encoder-ics-cuda.toml` +- `encoder-ics-metal.toml` +- `encoder-matrix-v1.toml` + +Those Annex D/F results are informative under T.803 and are not decoder +conformance claims. `support-inventory.tsv` is the feature-support ledger used +by `cargo xtask public-support`; it contains no corpus paths, is not consumed by +the T.803 runner, and must not be cited as exact-reference evidence. + +Generated JSON and Markdown reports, not narrative summaries, are the release +evidence. The claim policy and current blocker are documented in +[`docs/t803-conformance.md`](../../docs/t803-conformance.md). diff --git a/corpus/j2k-conformance/encoder-ics-cpu.toml b/corpus/j2k-conformance/encoder-ics-cpu.toml new file mode 100644 index 00000000..c7b2af06 --- /dev/null +++ b/corpus/j2k-conformance/encoder-ics-cpu.toml @@ -0,0 +1,97 @@ +# T.803 Annex F encoder implementation compliance statement. +# Encoder testing is informative and is not a decoder conformance claim. +schema_version = 1 +standard = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3" +iut = "cpu" +scope = "Informative Annex D/F evidence for the listed public JPEG 2000 Part 1 encoder surfaces" +surfaces = [ + "j2k::encode_j2k_lossless", + "j2k::encode_j2k_lossless_components", + "j2k::encode_j2k_lossless_typed_components", + "j2k::encode_j2k_lossless_with_roi_regions", + "j2k::encode_j2k_lossy", +] +matrix_path = "corpus/j2k-conformance/encoder-matrix-v1.toml" +matrix_case_count = 28 +matrix_case_sha256 = "6168ada6be73822b06b8b1b8564af009be17592b5b6d8b2e6bd0d578a69af4d5" +reference_decoder_standard = "ISO/IEC 15444-5 / ITU-T T.804" +reference_decoder_implementation = "OpenJPEG" +reference_decoder_version = "2.5.3" +public_max_bit_depth = 38 +reference_validated_max_bit_depth = 31 +public_max_components = 16384 +component_sampling = true +reference_limitations = [ + "OpenJPEG stores decoded component samples in 32-bit signed integers, so 32-38 bit public encoder inputs are outside this matrix", + "The matrix samples the declared API range; Annex D does not define an exhaustive encoder corpus", +] + +[[markers]] +marker = "SOC" +usage = "always" +[[markers]] +marker = "CAP" +usage = "outside-part1" +[[markers]] +marker = "PRF" +usage = "not-produced" +[[markers]] +marker = "CPF" +usage = "outside-part1" +[[markers]] +marker = "SOT" +usage = "always" +[[markers]] +marker = "SOD" +usage = "always" +[[markers]] +marker = "EOC" +usage = "always" +[[markers]] +marker = "SIZ" +usage = "always" +[[markers]] +marker = "COD" +usage = "always" +[[markers]] +marker = "COC" +usage = "conditional" +[[markers]] +marker = "RGN" +usage = "caller-controlled" +[[markers]] +marker = "QCD" +usage = "always" +[[markers]] +marker = "QCC" +usage = "conditional" +[[markers]] +marker = "POC" +usage = "not-produced" +[[markers]] +marker = "TLM" +usage = "caller-controlled" +[[markers]] +marker = "PLM" +usage = "caller-controlled" +[[markers]] +marker = "PLT" +usage = "caller-controlled" +[[markers]] +marker = "PPM" +usage = "caller-controlled" +[[markers]] +marker = "PPT" +usage = "caller-controlled" +[[markers]] +marker = "SOP" +usage = "caller-controlled" +[[markers]] +marker = "EPH" +usage = "caller-controlled" +[[markers]] +marker = "CRG" +usage = "not-produced" +[[markers]] +marker = "COM" +usage = "not-produced" diff --git a/corpus/j2k-conformance/encoder-ics-cuda.toml b/corpus/j2k-conformance/encoder-ics-cuda.toml new file mode 100644 index 00000000..310dd056 --- /dev/null +++ b/corpus/j2k-conformance/encoder-ics-cuda.toml @@ -0,0 +1,94 @@ +# T.803 Annex F encoder implementation compliance statement. +# This is an adapter IUT: CPU-assisted cases remain labelled CPU or hybrid. +schema_version = 1 +standard = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3" +iut = "cuda" +scope = "Informative Annex D/F evidence for public JPEG 2000 Part 1 CUDA adapter encoder surfaces with per-stage route disclosure" +surfaces = [ + "j2k_cuda::CudaLosslessEncoder", + "j2k::encode_j2k_lossy_with_accelerator(j2k_cuda::CudaEncodeStageAccelerator)", +] +matrix_path = "corpus/j2k-conformance/encoder-matrix-v1.toml" +matrix_case_count = 25 +matrix_case_sha256 = "0ed86e8b051afd9b05a8364f4daca421b87add4576e49af543dccca81d4e2138" +reference_decoder_standard = "ISO/IEC 15444-5 / ITU-T T.804" +reference_decoder_implementation = "OpenJPEG" +reference_decoder_version = "2.5.3" +public_max_bit_depth = 38 +reference_validated_max_bit_depth = 31 +public_max_components = 16384 +component_sampling = false +reference_limitations = [ + "OpenJPEG stores decoded component samples in 32-bit signed integers, so 32-38 bit public encoder inputs are outside this matrix", + "CUDA adapter evidence covers interleaved host-input surfaces and records every CPU fallback stage", +] + +[[markers]] +marker = "SOC" +usage = "always" +[[markers]] +marker = "CAP" +usage = "outside-part1" +[[markers]] +marker = "PRF" +usage = "not-produced" +[[markers]] +marker = "CPF" +usage = "outside-part1" +[[markers]] +marker = "SOT" +usage = "always" +[[markers]] +marker = "SOD" +usage = "always" +[[markers]] +marker = "EOC" +usage = "always" +[[markers]] +marker = "SIZ" +usage = "always" +[[markers]] +marker = "COD" +usage = "always" +[[markers]] +marker = "COC" +usage = "conditional" +[[markers]] +marker = "RGN" +usage = "not-produced" +[[markers]] +marker = "QCD" +usage = "always" +[[markers]] +marker = "QCC" +usage = "conditional" +[[markers]] +marker = "POC" +usage = "not-produced" +[[markers]] +marker = "TLM" +usage = "caller-controlled" +[[markers]] +marker = "PLM" +usage = "caller-controlled" +[[markers]] +marker = "PLT" +usage = "caller-controlled" +[[markers]] +marker = "PPM" +usage = "caller-controlled" +[[markers]] +marker = "PPT" +usage = "caller-controlled" +[[markers]] +marker = "SOP" +usage = "caller-controlled" +[[markers]] +marker = "EPH" +usage = "caller-controlled" +[[markers]] +marker = "CRG" +usage = "not-produced" +[[markers]] +marker = "COM" +usage = "not-produced" diff --git a/corpus/j2k-conformance/encoder-ics-metal.toml b/corpus/j2k-conformance/encoder-ics-metal.toml new file mode 100644 index 00000000..82c504a0 --- /dev/null +++ b/corpus/j2k-conformance/encoder-ics-metal.toml @@ -0,0 +1,94 @@ +# T.803 Annex F encoder implementation compliance statement. +# This is an adapter IUT: CPU-assisted cases remain labelled CPU or hybrid. +schema_version = 1 +standard = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3" +iut = "metal" +scope = "Informative Annex D/F evidence for public JPEG 2000 Part 1 Metal adapter encoder surfaces with per-stage route disclosure" +surfaces = [ + "j2k::encode_j2k_lossless_with_accelerator(j2k_metal::MetalEncodeStageAccelerator)", + "j2k::encode_j2k_lossy_with_accelerator(j2k_metal::MetalEncodeStageAccelerator)", +] +matrix_path = "corpus/j2k-conformance/encoder-matrix-v1.toml" +matrix_case_count = 25 +matrix_case_sha256 = "0ed86e8b051afd9b05a8364f4daca421b87add4576e49af543dccca81d4e2138" +reference_decoder_standard = "ISO/IEC 15444-5 / ITU-T T.804" +reference_decoder_implementation = "OpenJPEG" +reference_decoder_version = "2.5.3" +public_max_bit_depth = 38 +reference_validated_max_bit_depth = 31 +public_max_components = 16384 +component_sampling = false +reference_limitations = [ + "OpenJPEG stores decoded component samples in 32-bit signed integers, so 32-38 bit public encoder inputs are outside this matrix", + "Metal adapter evidence covers interleaved host-input surfaces and records every CPU fallback stage", +] + +[[markers]] +marker = "SOC" +usage = "always" +[[markers]] +marker = "CAP" +usage = "outside-part1" +[[markers]] +marker = "PRF" +usage = "not-produced" +[[markers]] +marker = "CPF" +usage = "outside-part1" +[[markers]] +marker = "SOT" +usage = "always" +[[markers]] +marker = "SOD" +usage = "always" +[[markers]] +marker = "EOC" +usage = "always" +[[markers]] +marker = "SIZ" +usage = "always" +[[markers]] +marker = "COD" +usage = "always" +[[markers]] +marker = "COC" +usage = "conditional" +[[markers]] +marker = "RGN" +usage = "not-produced" +[[markers]] +marker = "QCD" +usage = "always" +[[markers]] +marker = "QCC" +usage = "conditional" +[[markers]] +marker = "POC" +usage = "not-produced" +[[markers]] +marker = "TLM" +usage = "caller-controlled" +[[markers]] +marker = "PLM" +usage = "caller-controlled" +[[markers]] +marker = "PLT" +usage = "caller-controlled" +[[markers]] +marker = "PPM" +usage = "caller-controlled" +[[markers]] +marker = "PPT" +usage = "caller-controlled" +[[markers]] +marker = "SOP" +usage = "caller-controlled" +[[markers]] +marker = "EPH" +usage = "caller-controlled" +[[markers]] +marker = "CRG" +usage = "not-produced" +[[markers]] +marker = "COM" +usage = "not-produced" diff --git a/corpus/j2k-conformance/encoder-matrix-v1.toml b/corpus/j2k-conformance/encoder-matrix-v1.toml new file mode 100644 index 00000000..219981a9 --- /dev/null +++ b/corpus/j2k-conformance/encoder-matrix-v1.toml @@ -0,0 +1,446 @@ +# Informative T.803 Annex D encoder procedure for the public Part 1 surfaces. +# Cases are synthetic and distributable; no T.803 attachment content is embedded here. +schema_version = 1 +standard = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3" + +[pairwise] +modes = ["lossless", "lossy"] +dimensions = [[32, 32], [63, 47]] +signedness = [false, true] +bit_depths = [8, 12] +component_counts = [1, 3] +progressions = ["lrcp", "rlcp", "rpcl", "pcrl", "cprl"] + +[[inventories]] +iut = "cpu" +case_count = 28 +case_sha256 = "6168ada6be73822b06b8b1b8564af009be17592b5b6d8b2e6bd0d578a69af4d5" + +[[inventories]] +iut = "cuda" +case_count = 25 +case_sha256 = "0ed86e8b051afd9b05a8364f4daca421b87add4576e49af543dccca81d4e2138" + +[[inventories]] +iut = "metal" +case_count = 25 +case_sha256 = "0ed86e8b051afd9b05a8364f4daca421b87add4576e49af543dccca81d4e2138" + +[[cases]] +id = "boundary-components-2" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 37 +height = 29 +components = 2 +bit_depth = 8 +signed = false +pattern = "checkerboard" +progression = "lrcp" +decomposition_levels = 2 + +[[cases]] +id = "boundary-components-4" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 37 +height = 29 +components = 4 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "rlcp" +decomposition_levels = 2 + +[[cases]] +id = "boundary-components-5" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 37 +height = 29 +components = 5 +bit_depth = 8 +signed = true +pattern = "deterministic-noise" +progression = "cprl" +decomposition_levels = 2 + +[[cases]] +id = "boundary-gray1-singleton" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 1 +height = 1 +components = 1 +bit_depth = 1 +signed = false +pattern = "impulse" +progression = "lrcp" +decomposition_levels = 0 + +[[cases]] +id = "boundary-gray16-large" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 513 +height = 513 +components = 1 +bit_depth = 16 +signed = false +pattern = "gradient" +progression = "rpcl" +decomposition_levels = 5 + +[[cases]] +id = "boundary-gray31-signed" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 17 +height = 19 +components = 1 +bit_depth = 31 +signed = true +pattern = "deterministic-noise" +progression = "pcrl" +decomposition_levels = 2 + +[[cases]] +id = "layers-lossless-3" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 65 +height = 49 +components = 3 +bit_depth = 8 +signed = false +pattern = "checkerboard" +progression = "lrcp" +decomposition_levels = 3 +lossless_quality_layers = 3 + +[[cases]] +id = "layers-lossy-3" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 96 +height = 80 +components = 3 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "lrcp" +decomposition_levels = 3 +lossy_quality_layers = [ + { kind = "bits-per-pixel", value = 1.0 }, + { kind = "bits-per-pixel", value = 2.0 }, + { kind = "bits-per-pixel", value = 4.0 }, +] +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 + +[[cases]] +id = "marker-ppm" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 73 +height = 57 +components = 3 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "lrcp" +decomposition_levels = 3 +markers = ["PPM"] + +[[cases]] +id = "marker-ppt" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 73 +height = 57 +components = 3 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "rlcp" +decomposition_levels = 3 +markers = ["PPT"] + +[[cases]] +id = "marker-table-f1-optionals" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 80 +height = 64 +components = 3 +bit_depth = 8 +signed = false +pattern = "checkerboard" +progression = "lrcp" +decomposition_levels = 3 +markers = ["TLM", "PLM", "PLT", "SOP", "EPH"] + +[[cases]] +id = "pairwise-01" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 32 +height = 32 +components = 1 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "lrcp" +decomposition_levels = 1 +pairwise = true + +[[cases]] +id = "pairwise-02" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 63 +height = 47 +components = 3 +bit_depth = 12 +signed = true +pattern = "checkerboard" +progression = "rlcp" +decomposition_levels = 3 +pairwise = true + +[[cases]] +id = "pairwise-03" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 32 +height = 32 +components = 3 +bit_depth = 12 +signed = false +pattern = "deterministic-noise" +progression = "rpcl" +decomposition_levels = 1 +lossy_rate_target = { kind = "bits-per-pixel", value = 12.0 } +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 +pairwise = true + +[[cases]] +id = "pairwise-04" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 63 +height = 47 +components = 1 +bit_depth = 8 +signed = true +pattern = "impulse" +progression = "pcrl" +decomposition_levels = 3 +lossy_rate_target = { kind = "bits-per-pixel", value = 8.0 } +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 +pairwise = true + +[[cases]] +id = "pairwise-05" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 32 +height = 32 +components = 3 +bit_depth = 8 +signed = true +pattern = "deterministic-noise" +progression = "cprl" +decomposition_levels = 1 +pairwise = true + +[[cases]] +id = "pairwise-06" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 63 +height = 47 +components = 1 +bit_depth = 12 +signed = false +pattern = "gradient" +progression = "cprl" +decomposition_levels = 3 +lossy_rate_target = { kind = "bits-per-pixel", value = 8.0 } +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 +pairwise = true + +[[cases]] +id = "pairwise-07" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 32 +height = 32 +components = 3 +bit_depth = 12 +signed = false +pattern = "checkerboard" +progression = "pcrl" +decomposition_levels = 1 +pairwise = true + +[[cases]] +id = "pairwise-08" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 63 +height = 47 +components = 1 +bit_depth = 8 +signed = true +pattern = "impulse" +progression = "rpcl" +decomposition_levels = 3 +pairwise = true + +[[cases]] +id = "pairwise-09" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 32 +height = 32 +components = 1 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "rlcp" +decomposition_levels = 1 +lossy_rate_target = { kind = "bits-per-pixel", value = 8.0 } +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 +pairwise = true + +[[cases]] +id = "pairwise-10" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 63 +height = 47 +components = 3 +bit_depth = 12 +signed = true +pattern = "deterministic-noise" +progression = "lrcp" +decomposition_levels = 3 +lossy_rate_target = { kind = "bits-per-pixel", value = 8.0 } +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 +pairwise = true + +[[cases]] +id = "planar-sampled" +iuts = ["cpu"] +mode = "lossless" +input = "component-planes" +width = 65 +height = 49 +components = 3 +bit_depth = 12 +signed = false +pattern = "gradient" +sampling = [[1, 1], [2, 2], [2, 2]] +progression = "lrcp" +decomposition_levels = 3 + +[[cases]] +id = "planar-typed" +iuts = ["cpu"] +mode = "lossless" +input = "typed-component-planes" +width = 41 +height = 35 +components = 3 +bit_depth = 16 +signed = false +pattern = "deterministic-noise" +sampling = [[1, 1], [2, 1], [1, 2]] +component_bit_depths = [8, 12, 16] +component_signedness = [false, true, false] +progression = "rlcp" +decomposition_levels = 2 + +[[cases]] +id = "precinct-lossy" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 129 +height = 97 +components = 3 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "rpcl" +decomposition_levels = 1 +lossy_rate_target = { kind = "bits-per-pixel", value = 4.0 } +precinct_exponents = [[6, 6], [7, 7]] +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 25.0 + +[[cases]] +id = "rate-bytes" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 96 +height = 80 +components = 3 +bit_depth = 8 +signed = false +pattern = "checkerboard" +progression = "pcrl" +decomposition_levels = 3 +lossy_rate_target = { kind = "bytes", value = 3072 } +minimum_psnr_db = 20.0 +maximum_rate_overshoot_percent = 10.0 + +[[cases]] +id = "rate-psnr" +iuts = ["cpu", "cuda", "metal"] +mode = "lossy" +width = 96 +height = 80 +components = 3 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "cprl" +decomposition_levels = 3 +lossy_rate_target = { kind = "psnr-db", value = 36.0 } +minimum_psnr_db = 35.75 + +[[cases]] +id = "roi-lossless" +iuts = ["cpu"] +mode = "lossless" +width = 81 +height = 67 +components = 3 +bit_depth = 8 +signed = false +pattern = "gradient" +progression = "lrcp" +decomposition_levels = 3 +roi = { component = 1, x = 13, y = 11, width = 31, height = 27, shift = 12 } +markers = ["RGN"] + +[[cases]] +id = "tile-multitile" +iuts = ["cpu", "cuda", "metal"] +mode = "lossless" +width = 96 +height = 73 +components = 3 +bit_depth = 8 +signed = false +pattern = "deterministic-noise" +progression = "rpcl" +decomposition_levels = 2 +tile_size = [40, 32] +tile_part_packet_limit = 2 diff --git a/corpus/j2k-conformance/manifest.tsv b/corpus/j2k-conformance/manifest.tsv deleted file mode 100644 index da07e421..00000000 --- a/corpus/j2k-conformance/manifest.tsv +++ /dev/null @@ -1,11 +0,0 @@ -# id path classification features reason -part1_core_lossless_53 codestreams_profile0/p0_01.j2k blocking part1-core;lossless-5-3 Part 1 reversible core codestream support is shipped by the release scope. -part1_core_lossy_97_layers_precincts codestreams_profile0/p0_04.j2k blocking part1-core;lossy-9-7;quality-layers;precincts;progression-orders Part 1 irreversible lossy multi-layer codestream support, precincts, and progression-order support are shipped by the release scope. -part1_poc_tlm_sop codestreams_profile0/p0_03.j2k blocking part1-core;poc;progression-orders;tlm;sop POC packet iteration, TLM metadata, SOP markers, and progression-order support are shipped by the release scope. -part1_plt_sop_eph codestreams_profile0/p0_07.j2k blocking part1-core;poc;progression-orders;plt;sop;eph PLT packet-length metadata, chunked PPM/PPT separated packet headers, SOP/EPH packet markers, and POC progression changes are shipped by the release scope. -openhtj2k_ds0_ht_12_b11 htj2k_bsets_profile0/p0_12_bset/ds0_ht_12_b11.j2k blocking part15-core;ht-refinement HTJ2K core refinement-pass conformance vector shipped by the release scope. -openhtj2k_ds0_ht_09_b11 htj2k_bsets_profile0/p0_09_bset/ds0_ht_09_b11.j2k blocking part15-core;ht-refinement HTJ2K core refinement-pass conformance vector shipped by the release scope. -plm_iso_vector_absent known-limitations/no-plm-vector-in-t803v3.txt blocking plm;conformance-coverage-gap The available T.803v3 extraction has no described PLM-bearing vector; PLM behavior and oversized separated packet-header chunking remain covered by repo-local self-checks until external vectors are pinned. -jpx_part2_deferred known-limitations/jpx-part2-placeholder.jp2 out-of-scope jpx;part2 JPX and JPEG 2000 Part 2 extensions are outside the Part 1 plus Part 15 support claim unless required for standard JP2/JPH still-image correctness. -icc_roundtrip_deferred known-limitations/icc-roundtrip-placeholder.jp2 known-limitation jp2;icc-roundtrip;external-parity JP2/JPH ICC write, inspect, rewrap, and coefficient-preserving JPH recode are covered by repo-local tests; external ICC parity fixtures remain pending. -encode_gt16_deferred known-limitations/gt16-placeholder.j2k known-limitation encode-25-38-bit;external-parity High-bit publication-evidence row: signed/unsigned 24-bit raw-pixel encode plus exact single-tile and multi-tile classic reversible 29-bit DWT-safe, 32/35-bit classic DWT, 31/37-bit classic no-DWT, full-resolution 29/35-bit, sampled 29-bit, sampling-aligned sampled multi-tile 29-bit component-plane, unaligned sampled multi-tile 29-bit component-plane, sampled high-bit native region decode, classic irreversible 29/38-bit lossy encode/decode, mixed full-resolution 29/12-bit and 35/12-bit single-tile typed component-plane plus 29/12-bit multi-tile typed component-plane, and 29/31-bit HTJ2K no-DWT encode/decode are covered locally; HTJ2K low-bit ROI is covered at the 31 coded-bitplane edge, classic high-bit ROI is covered at 50 coded bitplanes, and classic 56/HT >31 coded-bitplane ROI overflows are reported explicitly; classic reversible requests beyond the Part 1 no-quantization bitplane field and HTJ2K requests beyond the Part 15 31-bitplane HT block limit reject explicitly; external parity remains publication evidence. diff --git a/corpus/j2k-conformance/support-inventory.tsv b/corpus/j2k-conformance/support-inventory.tsv new file mode 100644 index 00000000..3c135786 --- /dev/null +++ b/corpus/j2k-conformance/support-inventory.tsv @@ -0,0 +1,11 @@ +# id status features reason +part1_core_lossless_53 implemented part1-core;lossless-5-3 Part 1 reversible core support is covered by repo-local tests and the exact T.803 matrix. +part1_core_lossy_97_layers_precincts implemented part1-core;lossy-9-7;quality-layers;precincts;progression-orders Part 1 irreversible multi-layer support is covered by repo-local tests and the exact T.803 matrix. +part1_poc_tlm_sop implemented part1-core;poc;progression-orders;tlm;sop POC packet iteration, TLM metadata, SOP markers, and progression orders are covered locally and by selected T.803 cases. +part1_plt_sop_eph implemented part1-core;poc;progression-orders;plt;sop;eph PLT lengths, separated packet headers, SOP/EPH, and POC changes are covered locally and by selected T.803 cases. +openhtj2k_ds0_ht_12_b11 implemented part15-core;ht-refinement The committed OpenHTJ2K-derived fixture covers HT refinement decoding; Part 15 is outside the Part 1 T.803 claim. +openhtj2k_ds0_ht_09_b11 implemented part15-core;ht-refinement The committed OpenHTJ2K-derived fixture covers HT refinement decoding; Part 15 is outside the Part 1 T.803 claim. +plm_iso_vector_absent coverage-gap plm;conformance-coverage-gap The selected T.803 v3 Part 1 tables contain no described PLM-bearing case; PLM remains covered by repo-local tests. +jpx_part2_deferred out-of-scope jpx;part2 JPX and JPEG 2000 Part 2 extensions are outside the scope except where Annex G requires JP2-compatible input handling. +icc_roundtrip_deferred coverage-gap jp2;icc-roundtrip;external-parity JP2/JPH ICC write, inspect, rewrap, and decode are covered locally; broader external ICC parity remains pending. +encode_gt16_deferred coverage-gap encode-25-38-bit;external-parity High-bit-depth paths have repo-local boundary coverage; broader external encoder parity remains publication evidence. diff --git a/corpus/j2k-conformance/t803-v3.toml b/corpus/j2k-conformance/t803-v3.toml new file mode 100644 index 00000000..c86ec558 --- /dev/null +++ b/corpus/j2k-conformance/t803-v3.toml @@ -0,0 +1,1732 @@ +# T.803 v3 / ISO/IEC 15444-4:2024 Part 1 decoder and Annex G JP2 selection. +# The copyrighted electronic attachment remains external and must not be committed. +schema_version = 1 +standard = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3" + +[source] +url = "https://www.itu.int/wftp3/public/t/testsignal/SpeImage/T803/v2024_02/T.803v3_15444-4ed4-ElecAtt-codestreams.zip" +archive_sha256 = "ac04b52e1fe38404912036c14f215099ea9a785f38644fbe76ae8f3d1523c86d" +archive_bytes = 131660076 + +[[files]] +path = "files/codestreams_hifi/hifi_p1_02.j2k" +sha256 = "2870eaf3a8f7ed91b8db07663b0c0ffa3de11b6565dfafd62e3571232a766085" + +[[files]] +path = "files/codestreams_profile0/p0_01.j2k" +sha256 = "a61ea8d21ad0f7f9b76796e0f841c3e125240f83e308e3b8d626f4356b9a1113" + +[[files]] +path = "files/codestreams_profile0/p0_02.j2k" +sha256 = "150c8554b827c25b48029f3455b43ce3c029a26bea06eea2c60076c868bb07be" + +[[files]] +path = "files/codestreams_profile0/p0_03.j2k" +sha256 = "0aa26db75e8414d554e1fa93a0c7e101f16bb203f1fe06983fdd0c01fa61ddee" + +[[files]] +path = "files/codestreams_profile0/p0_04.j2k" +sha256 = "3d35a4eeffb00e150bb69611afec80fba9570d5fca18a04c4e644badfd052c6f" + +[[files]] +path = "files/codestreams_profile0/p0_05.j2k" +sha256 = "35f3840030802898d5ff2fc6ff39eea3b8080cad65db859830b869d48579cf9f" + +[[files]] +path = "files/codestreams_profile0/p0_06.j2k" +sha256 = "8a98292f29c4f04ef57c9a2974a1c254cca8297ce270c8baeb911bfe975c013d" + +[[files]] +path = "files/codestreams_profile0/p0_07.j2k" +sha256 = "46c4f5d64880a2df20d3faa22f6697384a6599dedb1b283e41a2efb1009cd9d1" + +[[files]] +path = "files/codestreams_profile0/p0_08.j2k" +sha256 = "4eb4eb2b356e16640ee94a24327fb66385c387657863072d774d633cf4379ef8" + +[[files]] +path = "files/codestreams_profile0/p0_09.j2k" +sha256 = "409c62a227497e7f2fc7e49055c02530cce742b6b385800b8a5aa4ae5bfab1a4" + +[[files]] +path = "files/codestreams_profile0/p0_10.j2k" +sha256 = "0146480434740e7580ec5a413fd19654a2f1273dd7db25dc5d0b75ab98a9d706" + +[[files]] +path = "files/codestreams_profile0/p0_11.j2k" +sha256 = "d20d8a13b570ef4adb0c8d4166568b043d00124c2db652ce7f78d85ac1d77b62" + +[[files]] +path = "files/codestreams_profile0/p0_12.j2k" +sha256 = "741885b801b7b8a901b61792f52e0d67d05b7a87a7fc2a718c92cb6c969416fc" + +[[files]] +path = "files/codestreams_profile0/p0_13.j2k" +sha256 = "12463d0c67e803fac6637d384263abbb6b546826419ecb725aed18fd45bcabb0" + +[[files]] +path = "files/codestreams_profile0/p0_14.j2k" +sha256 = "3e8df4a1dc7e5edcd9478d2e5d350e16f802ef8da3161d63a979f94906440e54" + +[[files]] +path = "files/codestreams_profile0/p0_15.j2k" +sha256 = "0aa26db75e8414d554e1fa93a0c7e101f16bb203f1fe06983fdd0c01fa61ddee" + +[[files]] +path = "files/codestreams_profile0/p0_16.j2k" +sha256 = "853d78ff805eff1aa9d6b97cea42ea4975d334b0d4d6b36f217baa0bc1314d27" + +[[files]] +path = "files/codestreams_profile1/p1_01.j2k" +sha256 = "26919bf56968ab57137bb478badb0bde8d2d53dfa26233935eecfbd40f235b81" + +[[files]] +path = "files/codestreams_profile1/p1_02.j2k" +sha256 = "9fb9a26fbd769ea4fce1ca450ed605764af9931783eba49b3551b3a37b7a6f87" + +[[files]] +path = "files/codestreams_profile1/p1_03.j2k" +sha256 = "0ff5572be1d0411550ceaf3b935f2c9f100bbd0399efbb8d48143add8d56ffb7" + +[[files]] +path = "files/codestreams_profile1/p1_04.j2k" +sha256 = "fbbe812a6ce6abfc5d1eef02c2e3838c023c24f0ff0b1dcd79c4e1d018b3351f" + +[[files]] +path = "files/codestreams_profile1/p1_05.j2k" +sha256 = "6b1eb070cb40585a9c0ce8569a957dcdbb1948e6129c0af50ec38ee9c0cef994" + +[[files]] +path = "files/codestreams_profile1/p1_06.j2k" +sha256 = "08a7fb96ba93bb3cd57183eee56ee92e91cb58193a378c6a4208d8cf1877459b" + +[[files]] +path = "files/codestreams_profile1/p1_07.j2k" +sha256 = "59576c42568098e4e5d9967eefb0a59b0002fedd7776c201c2c9f9aeb3cf1949" + +[[files]] +path = "files/reference_class0_profile0/c0p0_01.pgx" +sha256 = "268a50614b74bb375092d11aa21594069666568a73fe0b83f3ca9cfbda389139" + +[[files]] +path = "files/reference_class0_profile0/c0p0_02.pgx" +sha256 = "7d7924248477c4368eadd508234545acca9f790e5cd2f888651de44b0ae9e9f9" + +[[files]] +path = "files/reference_class0_profile0/c0p0_03r0.pgx" +sha256 = "29aa1042c71b3868bf607e4e3b29436604f512c392131925523a060554aea9f5" + +[[files]] +path = "files/reference_class0_profile0/c0p0_03r1.pgx" +sha256 = "3e3e2055b0b7783d16a339447a0dcc50d6ddb2381ac5d2769d3bd4206f30a68d" + +[[files]] +path = "files/reference_class0_profile0/c0p0_04.pgx" +sha256 = "3bd0ba5ab481a5415e7272fd03a5c8463ffc223c09b3954ead086c7556502cea" + +[[files]] +path = "files/reference_class0_profile0/c0p0_05.pgx" +sha256 = "008e9c31c4409706048d59d3dd9f79c87db0ea8c45eb49d53449c56c8919a2df" + +[[files]] +path = "files/reference_class0_profile0/c0p0_06.pgx" +sha256 = "2a2c37396620bc14f37c578c74b2894ecfeb5e20bd30dc67fc0e8db4bbfb592b" + +[[files]] +path = "files/reference_class0_profile0/c0p0_07.pgx" +sha256 = "55ee08494d259e5879aa4695d8c9d1afca4f4bb0be65bde61b55c7120d425d38" + +[[files]] +path = "files/reference_class0_profile0/c0p0_08.pgx" +sha256 = "ad58ffadc846fc3ce72d1280ef3f2300c3697bfa754c6eb292167f9f835716a6" + +[[files]] +path = "files/reference_class0_profile0/c0p0_09.pgx" +sha256 = "ce4dcf09fbd8076527f8d3ea83d9a6c23e24c8aa50f74ac7cd943695f59c875a" + +[[files]] +path = "files/reference_class0_profile0/c0p0_10.pgx" +sha256 = "c58b9eb2901a78519af6eec0946a38a9ebb01bc9bb429fb3ea7cc4fc7e9a0bfb" + +[[files]] +path = "files/reference_class0_profile0/c0p0_11.pgx" +sha256 = "4a4457e31466d9a78d332fea16237da57cd4b2f3d4fa6dee0074fa568b282765" + +[[files]] +path = "files/reference_class0_profile0/c0p0_12.pgx" +sha256 = "69a27310574c9849c6af457de29ba5eec34fd2b2037cd96e6ac7eca0a4fa40e9" + +[[files]] +path = "files/reference_class0_profile0/c0p0_13.pgx" +sha256 = "7f2bc12d19df4b2dd7b651c48d87c419ba8caa0cb7ec76424f672ba0ab53261c" + +[[files]] +path = "files/reference_class0_profile0/c0p0_14.pgx" +sha256 = "c556ab741f9b7d0a519830d9d609fd68366d7af200a9c2dd740e18218ced8fba" + +[[files]] +path = "files/reference_class0_profile0/c0p0_15r0.pgx" +sha256 = "12e00c98f9ad94527542d99c0763955614352ece12f22afa8e418c77fdcb9d35" + +[[files]] +path = "files/reference_class0_profile0/c0p0_15r1.pgx" +sha256 = "b95be5fbc4272595678af5955805ea073decb7bbb4e60708eaf41853c5c1311a" + +[[files]] +path = "files/reference_class0_profile0/c0p0_16.pgx" +sha256 = "48876e9905a149b70299072d8f148bf697589c6c5aa15b328af8566bb4daecf5" + +[[files]] +path = "files/reference_class0_profile1/c0p1_01.pgx" +sha256 = "00bdf8d02daa6eafb36a4563dd6c0f6125e2df75b2eb66f38b122a1c105849db" + +[[files]] +path = "files/reference_class0_profile1/c0p1_02.pgx" +sha256 = "23b2f1215dc6581c3f7014224f02f208ae7b05bc977300bc75a516160e4c7f27" + +[[files]] +path = "files/reference_class0_profile1/c0p1_03.pgx" +sha256 = "d3836ed021026ec9b5b13ad0d0cbc00446318134143a8dcba32ba6b5f6d78d6f" + +[[files]] +path = "files/reference_class0_profile1/c0p1_04r0.pgx" +sha256 = "63cc22c9b256116fae5976fdce1f7254f9743cba571dc2b4f5ddba1b11883a6a" + +[[files]] +path = "files/reference_class0_profile1/c0p1_04r3.pgx" +sha256 = "caddf11fa88f7ca8bc80ad11188203e69496d1437f7edb3930370b862cfd5489" + +[[files]] +path = "files/reference_class0_profile1/c0p1_05.pgx" +sha256 = "06595f8b3d171dbb5538bba8c9ba701e4e8e6c50d2208a28623d01a02d65a279" + +[[files]] +path = "files/reference_class0_profile1/c0p1_06.pgx" +sha256 = "25053fa746c9c552ab67b7da59b60384b6e1b8aba688d0314550beb5ba561f80" + +[[files]] +path = "files/reference_class0_profile1/c0p1_07.pgx" +sha256 = "ee0f0dd90be110241c60dbed84e03c94ff4f83d5cbdae603db9182f24cf5f8a4" + +[[files]] +path = "files/reference_class1HF_profile1/hifi_02-0.pgx" +sha256 = "d043e2685517ac444fd2b5e5b9785bdc8e0ec767dcea8fece3ff450f599ab549" + +[[files]] +path = "files/reference_class1HF_profile1/hifi_02-1.pgx" +sha256 = "ad44bd5c5fba4e5cc46a332b1f2543ef92207b37ef6ee5c5fb4db7e842d15c5f" + +[[files]] +path = "files/reference_class1HF_profile1/hifi_02-2.pgx" +sha256 = "a8624f77a99fd4264abe8dea5ff266da36f55ad4d35ffb261c9721ec01293d27" + +[[files]] +path = "files/reference_class1_profile0/c1p0_01-0.pgx" +sha256 = "d1c24494c873c6c6bf84021bcbbf4606ade7c1755f9bf4ee0c2fd999ce9ae211" + +[[files]] +path = "files/reference_class1_profile0/c1p0_02-0.pgx" +sha256 = "f8928b6d8d200c7dbc56e754c1826caf773d99e52c2c1bc4bbfdea7bc1e0d236" + +[[files]] +path = "files/reference_class1_profile0/c1p0_03-0.pgx" +sha256 = "edbdfc559b314d6ec3b3f75d94c50ba2b65040cf02ef0017a97ba92e9fedaa86" + +[[files]] +path = "files/reference_class1_profile0/c1p0_04-0.pgx" +sha256 = "51705f9c5e9a0abf2890a6d05d8e5cc509bc0dedc46a590acf04255318e25001" + +[[files]] +path = "files/reference_class1_profile0/c1p0_04-1.pgx" +sha256 = "9949ce4c13203132f6200926fec8c24ccb2381b8888ade607d9b73f403ee6c42" + +[[files]] +path = "files/reference_class1_profile0/c1p0_04-2.pgx" +sha256 = "81c8dda3f54ac601b0e9a4e18d7556ec69fddbb7997ae8aed75e3b512e6cbd60" + +[[files]] +path = "files/reference_class1_profile0/c1p0_05-0.pgx" +sha256 = "87122bc8f4a15c2b507d3d766c91b3fee54142c1935c0cdef43444a319f0a77f" + +[[files]] +path = "files/reference_class1_profile0/c1p0_05-1.pgx" +sha256 = "92876bda577d5ea7b2ba7c4c2c2a646251043f681f9e3cbf4f92260fac3ee3b7" + +[[files]] +path = "files/reference_class1_profile0/c1p0_05-2.pgx" +sha256 = "8183c2a7682d34b62cb09a747ab803207282f882590f5a533e61a9d6b1922511" + +[[files]] +path = "files/reference_class1_profile0/c1p0_05-3.pgx" +sha256 = "1d52fc3949c1110036d7f478f378a11bc7d91a2d02743d6c5eccde72e345cc6f" + +[[files]] +path = "files/reference_class1_profile0/c1p0_06-0.pgx" +sha256 = "8e4fe489e03fd5579f28c527bb966526a4cbc8bd2957277f9ef71f4d366ad286" + +[[files]] +path = "files/reference_class1_profile0/c1p0_06-1.pgx" +sha256 = "37cd7cbb59d8a8a6868d90b458fe7f1c1ca9cc8c3d1aaeb3279401cdac0a007f" + +[[files]] +path = "files/reference_class1_profile0/c1p0_06-2.pgx" +sha256 = "37da1bfb1f6fc90e65ecb62efab284daaf8828235e808bb3f1dcc819e242a87f" + +[[files]] +path = "files/reference_class1_profile0/c1p0_06-3.pgx" +sha256 = "e3a2ddfc7a37a16dde4c51d4bca43bc07fe65505d1d03986f9516713e6527f57" + +[[files]] +path = "files/reference_class1_profile0/c1p0_07-0.pgx" +sha256 = "2f59fff782b20c1648b68019ce6069960aac5efb2bf98a4df509ad4ba5a72b3b" + +[[files]] +path = "files/reference_class1_profile0/c1p0_07-1.pgx" +sha256 = "967181e82d33e3442160bf0ae3e5ffa6db99a21584e6736acb2719030ad1e412" + +[[files]] +path = "files/reference_class1_profile0/c1p0_07-2.pgx" +sha256 = "2d8d2ae1d2e33a486c3187206599944856b8a75525a165069042f4cfc295dc74" + +[[files]] +path = "files/reference_class1_profile0/c1p0_08-0.pgx" +sha256 = "69a2af90074593b007d50c96b1c5e4ecb1e591e265dc4f2ab5a230359ab2c87d" + +[[files]] +path = "files/reference_class1_profile0/c1p0_08-1.pgx" +sha256 = "fa2598b68dde7613b50caa1e5692dcecec934b33e2eb8327f44a274869bc3025" + +[[files]] +path = "files/reference_class1_profile0/c1p0_08-2.pgx" +sha256 = "7c3c05a5c5d4fe8efa55975e1b6d24869b3e1bf140db2b32b5dd14e42d65f77e" + +[[files]] +path = "files/reference_class1_profile0/c1p0_09-0.pgx" +sha256 = "174557fca48a83f7a474d2307984051326350266fec1acf623466e8f92bc8a29" + +[[files]] +path = "files/reference_class1_profile0/c1p0_10-0.pgx" +sha256 = "feee51c4c4519bd4ddbd53fb9e8186ddc95c4e64b42ffc9681d65d26172f76c4" + +[[files]] +path = "files/reference_class1_profile0/c1p0_10-1.pgx" +sha256 = "4857f0f3247bac8bef2354dc88727cece95bb5713cbe1e83c346601c8cd395d1" + +[[files]] +path = "files/reference_class1_profile0/c1p0_10-2.pgx" +sha256 = "460dede2e04d2be3f2c74334784b58bac59bd640c2f2fbad64489e788d2c511a" + +[[files]] +path = "files/reference_class1_profile0/c1p0_11-0.pgx" +sha256 = "2255812746edf02b988b0efefedf99ad2a9c1e01f2260a223490480d53e69c99" + +[[files]] +path = "files/reference_class1_profile0/c1p0_12-0.pgx" +sha256 = "2336f73b2596e62ce5fd46a09813c4e88f2b57eabdfa5178008f8ef6982de8bd" + +[[files]] +path = "files/reference_class1_profile0/c1p0_13-0.pgx" +sha256 = "a3250795aa7c31d0caf0a9ad706e21652060768b404210624e5dd695787dbc07" + +[[files]] +path = "files/reference_class1_profile0/c1p0_13-1.pgx" +sha256 = "ff079e7fe2ca5fd542f6b85bda838fa9f11428a59d7584f6523cb649cb8b79c7" + +[[files]] +path = "files/reference_class1_profile0/c1p0_13-2.pgx" +sha256 = "83d2079dcfaee85fcd75e8a23d6dd69686b966ba334d2c8ca9a25b2bbc7079b0" + +[[files]] +path = "files/reference_class1_profile0/c1p0_13-3.pgx" +sha256 = "d1bb447c46cf81b0673ff570eb5d5f940aeebd8e6fee68f21e061c6d5cb064ae" + +[[files]] +path = "files/reference_class1_profile0/c1p0_14-0.pgx" +sha256 = "c4e96524ccf8f0520e92f335dc4bf968c74187ab491660992fff4740e6a1cd63" + +[[files]] +path = "files/reference_class1_profile0/c1p0_14-1.pgx" +sha256 = "b1e1b60cda396c16803da500f8b3006a2bf7375f39f1c79cf79f827881bbd714" + +[[files]] +path = "files/reference_class1_profile0/c1p0_14-2.pgx" +sha256 = "c4e96524ccf8f0520e92f335dc4bf968c74187ab491660992fff4740e6a1cd63" + +[[files]] +path = "files/reference_class1_profile0/c1p0_15-0.pgx" +sha256 = "edbdfc559b314d6ec3b3f75d94c50ba2b65040cf02ef0017a97ba92e9fedaa86" + +[[files]] +path = "files/reference_class1_profile0/c1p0_16-0.pgx" +sha256 = "48876e9905a149b70299072d8f148bf697589c6c5aa15b328af8566bb4daecf5" + +[[files]] +path = "files/reference_class1_profile1/c1p1_01-0.pgx" +sha256 = "fd1730adccecdfa90c31e895ea58083670ccedbe6fe053b20c3bb7c7cf8fb96b" + +[[files]] +path = "files/reference_class1_profile1/c1p1_02-0.pgx" +sha256 = "0ed11a110b40f1f7174546decf075a12ba201e1d5bbba18dcfa638d6b320bc3f" + +[[files]] +path = "files/reference_class1_profile1/c1p1_02-1.pgx" +sha256 = "23551bc04b229de63344f7a7f4b0cc19d8512681ad3fbd5d7b3fbd55d23228dc" + +[[files]] +path = "files/reference_class1_profile1/c1p1_02-2.pgx" +sha256 = "46349e1ccb1df738b58ce7d0f706bd6c2937eb6457596e9ec9f858ed0d9b47a4" + +[[files]] +path = "files/reference_class1_profile1/c1p1_03-0.pgx" +sha256 = "416ff6c9b3d917ff20d3bd6405e224049137d04ed80dec9f8f42dfa070deb794" + +[[files]] +path = "files/reference_class1_profile1/c1p1_03-1.pgx" +sha256 = "9cfbc8ebb61d6447cc4fc98d8cca63016a288037a4609b7424685fd72a984ca1" + +[[files]] +path = "files/reference_class1_profile1/c1p1_03-2.pgx" +sha256 = "71fd301fb0da107998a8805a46faff754c14316d1544e4159778b43055b210a5" + +[[files]] +path = "files/reference_class1_profile1/c1p1_03-3.pgx" +sha256 = "1d52fc3949c1110036d7f478f378a11bc7d91a2d02743d6c5eccde72e345cc6f" + +[[files]] +path = "files/reference_class1_profile1/c1p1_04-0.pgx" +sha256 = "46208571732cee8d274ec11629da785c6358599a040ddeb3d5e8f0b76e38594a" + +[[files]] +path = "files/reference_class1_profile1/c1p1_05-0.pgx" +sha256 = "e60798e11ca7af7fba5af5805f0cdb0682d903f698b4f4fc3f2bceab56331ad0" + +[[files]] +path = "files/reference_class1_profile1/c1p1_05-1.pgx" +sha256 = "c9858b293f37c4c831cec4c7fc21cae83493b6f60995ddd8cfe67e5f7c85c1c1" + +[[files]] +path = "files/reference_class1_profile1/c1p1_05-2.pgx" +sha256 = "c9279428c9c879a590fe70e991ddc76ab41a9441b43c349020a1b8582fb2671a" + +[[files]] +path = "files/reference_class1_profile1/c1p1_06-0.pgx" +sha256 = "5f1fea232444c1afa6f1e06738a64910436d122f2fe69b0bf5ea59a55ac7f66e" + +[[files]] +path = "files/reference_class1_profile1/c1p1_06-1.pgx" +sha256 = "52acc2a2c4374dcdcdf4b795c430fef1e632e5fc35590401c7c0a618935f6749" + +[[files]] +path = "files/reference_class1_profile1/c1p1_06-2.pgx" +sha256 = "01cb2817388ee05f1f4524b0420f86580e62e098ac17680f18ec53cf7f470fc0" + +[[files]] +path = "files/reference_class1_profile1/c1p1_07-0.pgx" +sha256 = "ee0f0dd90be110241c60dbed84e03c94ff4f83d5cbdae603db9182f24cf5f8a4" + +[[files]] +path = "files/reference_class1_profile1/c1p1_07-1.pgx" +sha256 = "b72f60a12ec7dcb1360cc9adfd86bfb2e81d929262df5e0167d00c02e61b4df4" + +[[files]] +path = "files/reference_jp2/jp2_1.tif" +sha256 = "1d88f16d47e304affde8fdb3a593a18b328dfa28c04884ac6c889cdefa154d19" + +[[files]] +path = "files/reference_jp2/jp2_2.tif" +sha256 = "9850a8d54b227dcb931f43c96e3522fcf106629c60e9126381f3b39009c40c91" + +[[files]] +path = "files/reference_jp2/jp2_3.tif" +sha256 = "512a8827b98d71051c3cba52b96a323e879870ba90dc254016befaa1aa90dbd5" + +[[files]] +path = "files/reference_jp2/jp2_4.tif" +sha256 = "97395ed8d03798ef25ab7e53d904c288a37c8a07d1510ec1e897cdd923cf8eb1" + +[[files]] +path = "files/reference_jp2/jp2_5.tif" +sha256 = "434b199ca176bb87de22d3e7413bf791602a7d0c21056745fb052d395dd52751" + +[[files]] +path = "files/reference_jp2/jp2_6.tif" +sha256 = "927541305c4f46e1eb43e5e551260bfee8b0c395b5ffba95a77483b0216bd2c6" + +[[files]] +path = "files/reference_jp2/jp2_7.tif" +sha256 = "65972c102d9277fbc901fbc60ee2253542f665b558c4548f986834a5488d6ed8" + +[[files]] +path = "files/reference_jp2/jp2_8.tif" +sha256 = "4f0187ac2cf3ab95739f642cafcf96fcb1f42d0de9a0ce92464d7b10f80d4b31" + +[[files]] +path = "files/reference_jp2/jp2_9.tif" +sha256 = "f092ba8dcffe34f289dbb258de2c3c22eb1eab7bd511096eacf812c29c2883fb" + +[[files]] +path = "files/testfiles_jp2/file1.jp2" +sha256 = "4e3d51df7bc66cf367162acfff88b0889d2b2c79ea8d99d93b2d2bd165398deb" + +[[files]] +path = "files/testfiles_jp2/file2.jp2" +sha256 = "c87b78e7c3298ddb5e2c4b9a27485ae52ed0906184ae4eecf13b7525ad160960" + +[[files]] +path = "files/testfiles_jp2/file3.jp2" +sha256 = "fe922461d6928f9b9c86c222a133c42c19d119351400d5e8dd6a1e60db437e66" + +[[files]] +path = "files/testfiles_jp2/file4.jp2" +sha256 = "b3474f23bcf622f4f82c8ede5c473a4a1bae1c287b713fbd8447442e37edcee0" + +[[files]] +path = "files/testfiles_jp2/file5.jp2" +sha256 = "dbaeb5aaadf2f911c38f2585fa7d87a490c5abcf6742d6c91ecdebe42077ecd7" + +[[files]] +path = "files/testfiles_jp2/file6.jp2" +sha256 = "78ae553dc97b22352a2417aba739160e3b6c5b187b2efb867fb50f9814f1bc67" + +[[files]] +path = "files/testfiles_jp2/file7.jp2" +sha256 = "c9e6c845fde3db494035b9be8728239da45791b3269687a043a1fcedd8d11dbf" + +[[files]] +path = "files/testfiles_jp2/file8.jp2" +sha256 = "81e8a8bae1e3d7e632091a120a8a6650a04636b2276a550889b153290348eb01" + +[[files]] +path = "files/testfiles_jp2/file9.jp2" +sha256 = "6dfea011b013b62353fedcce9520e7c061dbbe8bf0c06d9edad4ac8889274b91" + +[[decoder_cases]] +id = "c1-c0p0-01" +table = "C.1" +codestream = "files/codestreams_profile0/p0_01.j2k" +reference = "files/reference_class0_profile0/c0p0_01.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-02" +table = "C.1" +codestream = "files/codestreams_profile0/p0_02.j2k" +reference = "files/reference_class0_profile0/c0p0_02.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 64 +height = 126 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-03r0" +table = "C.1" +codestream = "files/codestreams_profile0/p0_03.j2k" +reference = "files/reference_class0_profile0/c0p0_03r0.pgx" +component = 0 +reduction_levels = 0 +signed = true +bit_depth = 4 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-03r1" +table = "C.1" +codestream = "files/codestreams_profile0/p0_03.j2k" +reference = "files/reference_class0_profile0/c0p0_03r1.pgx" +component = 0 +reduction_levels = 1 +signed = true +bit_depth = 4 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-04" +table = "C.1" +codestream = "files/codestreams_profile0/p0_04.j2k" +reference = "files/reference_class0_profile0/c0p0_04.pgx" +component = 0 +reduction_levels = 3 +signed = false +bit_depth = 8 +width = 80 +height = 60 +peak = 33 +mse = 55.8 + +[[decoder_cases]] +id = "c1-c0p0-05" +table = "C.1" +codestream = "files/codestreams_profile0/p0_05.j2k" +reference = "files/reference_class0_profile0/c0p0_05.pgx" +component = 0 +reduction_levels = 3 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 54 +mse = 68.0 + +[[decoder_cases]] +id = "c1-c0p0-06" +table = "C.1" +codestream = "files/codestreams_profile0/p0_06.j2k" +reference = "files/reference_class0_profile0/c0p0_06.pgx" +component = 0 +reduction_levels = 3 +signed = false +bit_depth = 8 +width = 65 +height = 17 +peak = 109 +mse = 743.0 + +[[decoder_cases]] +id = "c1-c0p0-07" +table = "C.1" +codestream = "files/codestreams_profile0/p0_07.j2k" +reference = "files/reference_class0_profile0/c0p0_07.pgx" +component = 0 +reduction_levels = 0 +signed = true +bit_depth = 8 +width = 128 +height = 128 +peak = 10 +mse = 0.34 + +[[decoder_cases]] +id = "c1-c0p0-08" +table = "C.1" +codestream = "files/codestreams_profile0/p0_08.j2k" +reference = "files/reference_class0_profile0/c0p0_08.pgx" +component = 0 +reduction_levels = 5 +signed = true +bit_depth = 8 +width = 17 +height = 96 +peak = 7 +mse = 6.72 + +[[decoder_cases]] +id = "c1-c0p0-09" +table = "C.1" +codestream = "files/codestreams_profile0/p0_09.j2k" +reference = "files/reference_class0_profile0/c0p0_09.pgx" +component = 0 +reduction_levels = 2 +signed = false +bit_depth = 8 +width = 5 +height = 10 +peak = 4 +mse = 1.47 + +[[decoder_cases]] +id = "c1-c0p0-10" +table = "C.1" +codestream = "files/codestreams_profile0/p0_10.j2k" +reference = "files/reference_class0_profile0/c0p0_10.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 64 +height = 64 +peak = 10 +mse = 2.84 + +[[decoder_cases]] +id = "c1-c0p0-11" +table = "C.1" +codestream = "files/codestreams_profile0/p0_11.j2k" +reference = "files/reference_class0_profile0/c0p0_11.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-12" +table = "C.1" +codestream = "files/codestreams_profile0/p0_12.j2k" +reference = "files/reference_class0_profile0/c0p0_12.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 3 +height = 5 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +# This 257-component codestream enables RCT. T.803 B.2.5 compares Cclass-0 +# component 0 before inverse MCT, while Cclass-1 compares decoded components. +# The runner therefore reads COD transform metadata instead of inferring MCT +# from the display colorspace, which is intentionally unknown for this shape. +id = "c1-c0p0-13" +table = "C.1" +codestream = "files/codestreams_profile0/p0_13.j2k" +reference = "files/reference_class0_profile0/c0p0_13.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-14" +table = "C.1" +codestream = "files/codestreams_profile0/p0_14.j2k" +reference = "files/reference_class0_profile0/c0p0_14.pgx" +component = 0 +reduction_levels = 2 +signed = false +bit_depth = 8 +width = 13 +height = 13 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-15r0" +table = "C.1" +codestream = "files/codestreams_profile0/p0_15.j2k" +reference = "files/reference_class0_profile0/c0p0_15r0.pgx" +component = 0 +reduction_levels = 0 +signed = true +bit_depth = 4 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-15r1" +table = "C.1" +codestream = "files/codestreams_profile0/p0_15.j2k" +reference = "files/reference_class0_profile0/c0p0_15r1.pgx" +component = 0 +reduction_levels = 1 +signed = true +bit_depth = 4 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c1-c0p0-16" +table = "C.1" +codestream = "files/codestreams_profile0/p0_16.j2k" +reference = "files/reference_class0_profile0/c0p0_16.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c4-c0p1-01" +table = "C.4" +codestream = "files/codestreams_profile1/p1_01.j2k" +reference = "files/reference_class0_profile1/c0p1_01.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 61 +height = 99 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c4-c0p1-02" +table = "C.4" +codestream = "files/codestreams_profile1/p1_02.j2k" +reference = "files/reference_class0_profile1/c0p1_02.pgx" +component = 0 +reduction_levels = 3 +signed = false +bit_depth = 8 +width = 80 +height = 60 +peak = 35 +mse = 74.0 + +[[decoder_cases]] +id = "c4-c0p1-03" +table = "C.4" +codestream = "files/codestreams_profile1/p1_03.j2k" +reference = "files/reference_class0_profile1/c0p1_03.pgx" +component = 0 +reduction_levels = 3 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 28 +mse = 18.8 + +[[decoder_cases]] +id = "c4-c0p1-04r0" +table = "C.4" +codestream = "files/codestreams_profile1/p1_04.j2k" +reference = "files/reference_class0_profile1/c0p1_04r0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 2 +mse = 0.55 + +[[decoder_cases]] +id = "c4-c0p1-04r3" +table = "C.4" +codestream = "files/codestreams_profile1/p1_04.j2k" +reference = "files/reference_class0_profile1/c0p1_04r3.pgx" +component = 0 +reduction_levels = 3 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 128 +mse = 2042.0 + +[[decoder_cases]] +id = "c4-c0p1-05" +table = "C.4" +codestream = "files/codestreams_profile1/p1_05.j2k" +reference = "files/reference_class0_profile1/c0p1_05.pgx" +component = 0 +reduction_levels = 4 +signed = false +bit_depth = 8 +width = 32 +height = 32 +peak = 128 +mse = 16384.0 + +[[decoder_cases]] +id = "c4-c0p1-06" +table = "C.4" +codestream = "files/codestreams_profile1/p1_06.j2k" +reference = "files/reference_class0_profile1/c0p1_06.pgx" +component = 0 +reduction_levels = 1 +signed = false +bit_depth = 8 +width = 6 +height = 6 +peak = 128 +mse = 16384.0 + +[[decoder_cases]] +id = "c4-c0p1-07" +table = "C.4" +codestream = "files/codestreams_profile1/p1_07.j2k" +reference = "files/reference_class0_profile1/c0p1_07.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 2 +height = 12 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-01-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_01.j2k" +reference = "files/reference_class1_profile0/c1p0_01-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-02-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_02.j2k" +reference = "files/reference_class1_profile0/c1p0_02-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 64 +height = 126 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-03-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_03.j2k" +reference = "files/reference_class1_profile0/c1p0_03-0.pgx" +component = 0 +reduction_levels = 0 +signed = true +bit_depth = 4 +width = 256 +height = 256 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-04-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_04.j2k" +reference = "files/reference_class1_profile0/c1p0_04-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 640 +height = 480 +peak = 5 +mse = 0.776 + +[[decoder_cases]] +id = "c6-c1p0-04-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_04.j2k" +reference = "files/reference_class1_profile0/c1p0_04-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 640 +height = 480 +peak = 4 +mse = 0.626 + +[[decoder_cases]] +id = "c6-c1p0-04-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_04.j2k" +reference = "files/reference_class1_profile0/c1p0_04-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 640 +height = 480 +peak = 6 +mse = 1.07 + +[[decoder_cases]] +id = "c6-c1p0-05-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_05.j2k" +reference = "files/reference_class1_profile0/c1p0_05-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1024 +height = 1024 +peak = 2 +mse = 0.319 + +[[decoder_cases]] +id = "c6-c1p0-05-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_05.j2k" +reference = "files/reference_class1_profile0/c1p0_05-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1024 +height = 1024 +peak = 2 +mse = 0.323 + +[[decoder_cases]] +id = "c6-c1p0-05-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_05.j2k" +reference = "files/reference_class1_profile0/c1p0_05-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 2 +mse = 0.317 + +[[decoder_cases]] +id = "c6-c1p0-05-3" +table = "C.6" +codestream = "files/codestreams_profile0/p0_05.j2k" +reference = "files/reference_class1_profile0/c1p0_05-3.pgx" +component = 3 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-06-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_06.j2k" +reference = "files/reference_class1_profile0/c1p0_06-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 513 +height = 129 +peak = 635 +mse = 11287.0 + +[[decoder_cases]] +id = "c6-c1p0-06-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_06.j2k" +reference = "files/reference_class1_profile0/c1p0_06-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 257 +height = 129 +peak = 403 +mse = 6124.0 + +[[decoder_cases]] +id = "c6-c1p0-06-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_06.j2k" +reference = "files/reference_class1_profile0/c1p0_06-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 513 +height = 65 +peak = 378 +mse = 3968.0 + +[[decoder_cases]] +id = "c6-c1p0-06-3" +table = "C.6" +codestream = "files/codestreams_profile0/p0_06.j2k" +reference = "files/reference_class1_profile0/c1p0_06-3.pgx" +component = 3 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 257 +height = 65 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-07-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_07.j2k" +reference = "files/reference_class1_profile0/c1p0_07-0.pgx" +component = 0 +reduction_levels = 0 +signed = true +bit_depth = 12 +width = 2048 +height = 2048 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-07-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_07.j2k" +reference = "files/reference_class1_profile0/c1p0_07-1.pgx" +component = 1 +reduction_levels = 0 +signed = true +bit_depth = 12 +width = 2048 +height = 2048 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-07-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_07.j2k" +reference = "files/reference_class1_profile0/c1p0_07-2.pgx" +component = 2 +reduction_levels = 0 +signed = true +bit_depth = 12 +width = 2048 +height = 2048 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-08-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_08.j2k" +reference = "files/reference_class1_profile0/c1p0_08-0.pgx" +component = 0 +reduction_levels = 1 +signed = true +bit_depth = 12 +width = 257 +height = 1536 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-08-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_08.j2k" +reference = "files/reference_class1_profile0/c1p0_08-1.pgx" +component = 1 +reduction_levels = 1 +signed = true +bit_depth = 12 +width = 257 +height = 1536 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-08-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_08.j2k" +reference = "files/reference_class1_profile0/c1p0_08-2.pgx" +component = 2 +reduction_levels = 1 +signed = true +bit_depth = 12 +width = 257 +height = 1536 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-09-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_09.j2k" +reference = "files/reference_class1_profile0/c1p0_09-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 17 +height = 37 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-10-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_10.j2k" +reference = "files/reference_class1_profile0/c1p0_10-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 64 +height = 64 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-10-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_10.j2k" +reference = "files/reference_class1_profile0/c1p0_10-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 64 +height = 64 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-10-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_10.j2k" +reference = "files/reference_class1_profile0/c1p0_10-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 64 +height = 64 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-11-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_11.j2k" +reference = "files/reference_class1_profile0/c1p0_11-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-12-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_12.j2k" +reference = "files/reference_class1_profile0/c1p0_12-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 3 +height = 5 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-13-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_13.j2k" +reference = "files/reference_class1_profile0/c1p0_13-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-13-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_13.j2k" +reference = "files/reference_class1_profile0/c1p0_13-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-13-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_13.j2k" +reference = "files/reference_class1_profile0/c1p0_13-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-13-3" +table = "C.6" +codestream = "files/codestreams_profile0/p0_13.j2k" +reference = "files/reference_class1_profile0/c1p0_13-3.pgx" +component = 3 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1 +height = 1 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +# Table C.6 prints 49x149, while both the pinned v3 PGX header and p0_14 +# codestream declare this component as 49x49. Comparison follows the official +# electronic attachment and records its hashes above. +id = "c6-c1p0-14-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_14.j2k" +reference = "files/reference_class1_profile0/c1p0_14-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 49 +height = 49 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-14-1" +table = "C.6" +codestream = "files/codestreams_profile0/p0_14.j2k" +reference = "files/reference_class1_profile0/c1p0_14-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 49 +height = 49 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-14-2" +table = "C.6" +codestream = "files/codestreams_profile0/p0_14.j2k" +reference = "files/reference_class1_profile0/c1p0_14-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 49 +height = 49 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-15-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_15.j2k" +reference = "files/reference_class1_profile0/c1p0_15-0.pgx" +component = 0 +reduction_levels = 0 +signed = true +bit_depth = 4 +width = 256 +height = 256 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c6-c1p0-16-0" +table = "C.6" +codestream = "files/codestreams_profile0/p0_16.j2k" +reference = "files/reference_class1_profile0/c1p0_16-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 128 +height = 128 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c7-c1p1-01-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_01.j2k" +reference = "files/reference_class1_profile1/c1p1_01-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 61 +height = 99 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c7-c1p1-02-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_02.j2k" +reference = "files/reference_class1_profile1/c1p1_02-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 640 +height = 480 +peak = 5 +mse = 0.765 + +[[decoder_cases]] +id = "c7-c1p1-02-1" +table = "C.7" +codestream = "files/codestreams_profile1/p1_02.j2k" +reference = "files/reference_class1_profile1/c1p1_02-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 640 +height = 480 +peak = 4 +mse = 0.616 + +[[decoder_cases]] +id = "c7-c1p1-02-2" +table = "C.7" +codestream = "files/codestreams_profile1/p1_02.j2k" +reference = "files/reference_class1_profile1/c1p1_02-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 640 +height = 480 +peak = 6 +mse = 1.051 + +[[decoder_cases]] +id = "c7-c1p1-03-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_03.j2k" +reference = "files/reference_class1_profile1/c1p1_03-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1024 +height = 1024 +peak = 2 +mse = 0.311 + +[[decoder_cases]] +id = "c7-c1p1-03-1" +table = "C.7" +codestream = "files/codestreams_profile1/p1_03.j2k" +reference = "files/reference_class1_profile1/c1p1_03-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1024 +height = 1024 +peak = 2 +mse = 0.28 + +[[decoder_cases]] +id = "c7-c1p1-03-2" +table = "C.7" +codestream = "files/codestreams_profile1/p1_03.j2k" +reference = "files/reference_class1_profile1/c1p1_03-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 1 +mse = 0.267 + +[[decoder_cases]] +id = "c7-c1p1-03-3" +table = "C.7" +codestream = "files/codestreams_profile1/p1_03.j2k" +reference = "files/reference_class1_profile1/c1p1_03-3.pgx" +component = 3 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c7-c1p1-04-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_04.j2k" +reference = "files/reference_class1_profile1/c1p1_04-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 1024 +height = 1024 +peak = 624 +mse = 3080.0 + +[[decoder_cases]] +id = "c7-c1p1-05-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_05.j2k" +reference = "files/reference_class1_profile1/c1p1_05-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 40 +mse = 8.458 + +[[decoder_cases]] +id = "c7-c1p1-05-1" +table = "C.7" +codestream = "files/codestreams_profile1/p1_05.j2k" +reference = "files/reference_class1_profile1/c1p1_05-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 40 +mse = 9.716 + +[[decoder_cases]] +id = "c7-c1p1-05-2" +table = "C.7" +codestream = "files/codestreams_profile1/p1_05.j2k" +reference = "files/reference_class1_profile1/c1p1_05-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 512 +height = 512 +peak = 40 +mse = 10.154 + +[[decoder_cases]] +id = "c7-c1p1-06-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_06.j2k" +reference = "files/reference_class1_profile1/c1p1_06-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 12 +height = 12 +peak = 2 +mse = 0.6 + +[[decoder_cases]] +id = "c7-c1p1-06-1" +table = "C.7" +codestream = "files/codestreams_profile1/p1_06.j2k" +reference = "files/reference_class1_profile1/c1p1_06-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 12 +height = 12 +peak = 2 +mse = 0.6 + +[[decoder_cases]] +id = "c7-c1p1-06-2" +table = "C.7" +codestream = "files/codestreams_profile1/p1_06.j2k" +reference = "files/reference_class1_profile1/c1p1_06-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 12 +height = 12 +peak = 2 +mse = 0.6 + +[[decoder_cases]] +id = "c7-c1p1-07-0" +table = "C.7" +codestream = "files/codestreams_profile1/p1_07.j2k" +reference = "files/reference_class1_profile1/c1p1_07-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 2 +height = 12 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c7-c1p1-07-1" +table = "C.7" +codestream = "files/codestreams_profile1/p1_07.j2k" +reference = "files/reference_class1_profile1/c1p1_07-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 8 +height = 12 +peak = 0 +mse = 0.0 + +[[decoder_cases]] +id = "c8-hifi-02-0" +table = "C.8" +codestream = "files/codestreams_hifi/hifi_p1_02.j2k" +reference = "files/reference_class1HF_profile1/hifi_02-0.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 128 +height = 128 +peak = 43 +mse = 80.0 + +[[decoder_cases]] +id = "c8-hifi-02-1" +table = "C.8" +codestream = "files/codestreams_hifi/hifi_p1_02.j2k" +reference = "files/reference_class1HF_profile1/hifi_02-1.pgx" +component = 1 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 128 +height = 128 +peak = 33 +mse = 62.0 + +[[decoder_cases]] +id = "c8-hifi-02-2" +table = "C.8" +codestream = "files/codestreams_hifi/hifi_p1_02.j2k" +reference = "files/reference_class1HF_profile1/hifi_02-2.pgx" +component = 2 +reduction_levels = 0 +signed = false +bit_depth = 12 +width = 128 +height = 128 +peak = 38 +mse = 72.0 + +[[jp2_cases]] +id = "jp2-1" +input = "files/testfiles_jp2/file1.jp2" +reference = "files/reference_jp2/jp2_1.tif" +components = 3 +bit_depth = 8 +width = 768 +height = 512 +peak = 4 + +[[jp2_cases]] +id = "jp2-2" +input = "files/testfiles_jp2/file2.jp2" +reference = "files/reference_jp2/jp2_2.tif" +components = 3 +bit_depth = 8 +width = 480 +height = 640 +peak = 4 + +[[jp2_cases]] +id = "jp2-3" +input = "files/testfiles_jp2/file3.jp2" +reference = "files/reference_jp2/jp2_3.tif" +components = 3 +bit_depth = 8 +width = 480 +height = 640 +peak = 4 + +[[jp2_cases]] +id = "jp2-4" +input = "files/testfiles_jp2/file4.jp2" +reference = "files/reference_jp2/jp2_4.tif" +components = 1 +bit_depth = 8 +width = 768 +height = 512 +peak = 4 + +[[jp2_cases]] +id = "jp2-5" +input = "files/testfiles_jp2/file5.jp2" +reference = "files/reference_jp2/jp2_5.tif" +components = 3 +bit_depth = 8 +width = 768 +height = 512 +peak = 4 + +[[jp2_cases]] +id = "jp2-6" +input = "files/testfiles_jp2/file6.jp2" +reference = "files/reference_jp2/jp2_6.tif" +components = 1 +bit_depth = 12 +width = 768 +height = 512 +peak = 4 + +[[jp2_cases]] +id = "jp2-7" +input = "files/testfiles_jp2/file7.jp2" +reference = "files/reference_jp2/jp2_7.tif" +components = 3 +bit_depth = 16 +width = 480 +height = 640 +peak = 4 + +[[jp2_cases]] +id = "jp2-8" +input = "files/testfiles_jp2/file8.jp2" +reference = "files/reference_jp2/jp2_8.tif" +components = 1 +bit_depth = 8 +width = 700 +height = 400 +peak = 4 + +[[jp2_cases]] +id = "jp2-9" +input = "files/testfiles_jp2/file9.jp2" +reference = "files/reference_jp2/jp2_9.tif" +components = 1 +bit_depth = 8 +width = 768 +height = 512 +peak = 4 diff --git a/crates/j2k-cli/Cargo.toml b/crates/j2k-cli/Cargo.toml index a330ee12..53c438f9 100644 --- a/crates/j2k-cli/Cargo.toml +++ b/crates/j2k-cli/Cargo.toml @@ -18,9 +18,9 @@ path = "src/main.rs" doc = false [dependencies] -j2k = { path = "../j2k", version = "=0.8.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0" } +j2k = { path = "../j2k", version = "=0.8.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.1" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-codec-math/src/classic.rs b/crates/j2k-codec-math/src/classic.rs index bd575228..b8bcbb63 100644 --- a/crates/j2k-codec-math/src/classic.rs +++ b/crates/j2k-codec-math/src/classic.rs @@ -100,6 +100,43 @@ pub const MQ_QE_VALUES: [u32; 47] = mq_qe_values(); /// Packed MQ MPS/LPS transitions and switch flag for device lookup tables. pub const PACKED_MQ_TRANSITION_VALUES: [u32; 47] = packed_mq_transitions(); +/// Returns the bit position of the half-bin term used to reconstruct an +/// irreversible coefficient after the final decoded coding pass. +/// +/// `None` means there is no non-zero coefficient to reconstruct or the pass +/// count is inconsistent with the number of decoded bitplanes. +pub const fn irreversible_midpoint_bit( + magnitude: u64, + decoded_bitplanes: u32, + number_of_coding_passes: u32, +) -> Option { + if magnitude == 0 || decoded_bitplanes == 0 || number_of_coding_passes == 0 { + return None; + } + + let final_pass = number_of_coding_passes - 1; + let decoded_plane = final_pass.div_ceil(3); + let Some(mut lowest_decoded_bit) = decoded_bitplanes.checked_sub(decoded_plane + 1) else { + return None; + }; + if lowest_decoded_bit >= u64::BITS { + return None; + } + + // A final significance-propagation pass does not refine coefficients that + // were already significant. A newly significant coefficient has the + // current bit set, distinguishing the two cases without decoder state. + if final_pass % 3 == 1 && magnitude & (1_u64 << lowest_decoded_bit) == 0 { + lowest_decoded_bit += 1; + } + + if lowest_decoded_bit < u64::BITS { + Some(lowest_decoded_bit) + } else { + None + } +} + /// Sign-coding context and XOR bit indexed by packed cardinal-neighbor state. #[rustfmt::skip] pub const SIGN_CONTEXT_LOOKUP: [(u8, u8); 256] = [ @@ -198,6 +235,22 @@ mod tests { use super::*; + #[test] + fn irreversible_midpoint_bit_tracks_the_last_decoded_pass() { + assert_eq!(irreversible_midpoint_bit(4, 3, 1), Some(2)); + assert_eq!(irreversible_midpoint_bit(2, 3, 2), Some(1)); + assert_eq!(irreversible_midpoint_bit(4, 3, 2), Some(2)); + assert_eq!(irreversible_midpoint_bit(4, 3, 3), Some(1)); + } + + #[test] + fn irreversible_midpoint_bit_rejects_empty_or_inconsistent_state() { + assert_eq!(irreversible_midpoint_bit(0, 3, 1), None); + assert_eq!(irreversible_midpoint_bit(4, 0, 1), None); + assert_eq!(irreversible_midpoint_bit(4, 3, 0), None); + assert_eq!(irreversible_midpoint_bit(1, 1, 5), None); + } + #[test] fn device_tables_match_their_structured_sources_at_runtime() { let build_qe: fn() -> [u32; 47] = mq_qe_values; diff --git a/crates/j2k-compare/Cargo.toml b/crates/j2k-compare/Cargo.toml index ad6d0368..04b32628 100644 --- a/crates/j2k-compare/Cargo.toml +++ b/crates/j2k-compare/Cargo.toml @@ -13,10 +13,10 @@ publish = false ignored = ["cc"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } -j2k = { path = "../j2k", version = "=0.8.0" } -j2k-test-support = { path = "../j2k-test-support", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } +j2k = { path = "../j2k", version = "=0.8.1" } +j2k-test-support = { path = "../j2k-test-support", version = "=0.8.1" } image = { workspace = true } openjpeg-sys = { workspace = true } diff --git a/crates/j2k-compare/src/openjpeg.rs b/crates/j2k-compare/src/openjpeg.rs index 43639c5c..db3f97f4 100644 --- a/crates/j2k-compare/src/openjpeg.rs +++ b/crates/j2k-compare/src/openjpeg.rs @@ -42,8 +42,37 @@ pub fn library_path() -> &'static str { "openjpeg-sys vendored openjp2" } +/// Native component image decoded by the vendored `OpenJPEG` reference implementation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpenJpegDecodedImage { + /// Reference-grid image dimensions. + pub dimensions: (u32, u32), + /// Decoded components in codestream order. + pub components: Vec, +} + +/// One native component decoded by the vendored `OpenJPEG` reference implementation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct OpenJpegDecodedComponent { + /// Component-grid dimensions. + pub dimensions: (u32, u32), + /// Horizontal and vertical SIZ sampling factors. + pub sampling: (u32, u32), + /// Significant component precision. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Row-major decoded component samples. + pub samples: Vec, +} + crate::external_decode_wrappers!(decode); +/// Decode native component samples without display scaling or interleaving. +pub fn decode_components(bytes: &[u8]) -> Result { + decode_with_image(bytes, ExternalDecodeRequest::gray(), pack_components) +} + /// Owns an `OpenJPEG` stream; never null. struct StreamGuard(*mut opj_stream_t); @@ -91,14 +120,22 @@ impl Drop for ImageGuard { } } +fn decode(bytes: &[u8], request: ExternalDecodeRequest) -> Result, String> { + let channels = usize::try_from(request.color.channels()) + .map_err(|_| "openjpeg: channel count exceeds platform usize".to_string())?; + decode_with_image(bytes, request, |image| pack_image(image, channels)) +} + #[expect( unsafe_code, reason = "decode coordinates checked RAII-owned OpenJPEG stream, codec, and image handles" )] -fn decode(bytes: &[u8], request: ExternalDecodeRequest) -> Result, String> { +fn decode_with_image( + bytes: &[u8], + request: ExternalDecodeRequest, + consume: impl FnOnce(*mut opj_image_t) -> Result, +) -> Result { let codec_format = codec_format(bytes)?; - let channels = usize::try_from(request.color.channels()) - .map_err(|_| "openjpeg: channel count exceeds platform usize".to_string())?; let decode_area = request.region.map(checked_decode_area).transpose()?; // Declaration order matters: drops run in reverse (image, codec, stream), // and the RAII guards free the FFI resources on every exit path, @@ -125,11 +162,11 @@ fn decode(bytes: &[u8], request: ExternalDecodeRequest) -> Result, Strin if opj_decode(codec.0, stream.0, image.0) == bool_false() { return Err("openjpeg: decode failed".to_string()); } - let packed = pack_image(image.0, channels)?; + let decoded = consume(image.0)?; if opj_end_decompress(codec.0, stream.0) == bool_false() { return Err("openjpeg: end_decompress failed".to_string()); } - Ok(packed) + Ok(decoded) } } @@ -266,6 +303,85 @@ fn pack_image(image: *mut opj_image_t, channels: usize) -> Result, Strin Ok(out) } +#[expect( + unsafe_code, + reason = "component extraction validates OpenJPEG image arrays and copies their bounded sample buffers" +)] +fn pack_components(image: *mut opj_image_t) -> Result { + if image.is_null() { + return Err("openjpeg: null image".to_string()); + } + // SAFETY: `image` was checked non-null and remains owned by the caller's `ImageGuard`. + let image = unsafe { &*image }; + if image.numcomps == 0 || image.comps.is_null() { + return Err("openjpeg: image has no components".to_string()); + } + let width = image + .x1 + .checked_sub(image.x0) + .ok_or_else(|| "openjpeg: invalid reference-grid width".to_string())?; + let height = image + .y1 + .checked_sub(image.y0) + .ok_or_else(|| "openjpeg: invalid reference-grid height".to_string())?; + if width == 0 || height == 0 { + return Err("openjpeg: image has zero-sized output".to_string()); + } + let component_count = usize::try_from(image.numcomps) + .map_err(|_| "openjpeg: component count exceeds platform usize".to_string())?; + let mut components = Vec::new(); + components + .try_reserve_exact(component_count) + .map_err(|_| "openjpeg: cannot allocate component owners".to_string())?; + let mut total_samples = 0usize; + for index in 0..component_count { + // SAFETY: `index < numcomps`; OpenJPEG owns a contiguous component array. + let component = unsafe { &*image.comps.add(index) }; + if component.w == 0 || component.h == 0 || component.dx == 0 || component.dy == 0 { + return Err(format!("openjpeg: component {index} has invalid geometry")); + } + if component.data.is_null() { + return Err(format!("openjpeg: component {index} data missing")); + } + let component_width = usize::try_from(component.w) + .map_err(|_| "openjpeg: component width exceeds platform usize".to_string())?; + let component_height = usize::try_from(component.h) + .map_err(|_| "openjpeg: component height exceeds platform usize".to_string())?; + let sample_len = checked_component_sample_len(component_width, component_height)?; + total_samples = total_samples + .checked_add(sample_len) + .ok_or_else(|| "openjpeg: total component sample count overflow".to_string())?; + let total_bytes = total_samples + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| "openjpeg: total component byte count overflow".to_string())?; + if total_bytes > MAX_EXTERNAL_OUTPUT_BYTES { + return Err(format!( + "openjpeg: native component output exceeds {MAX_EXTERNAL_OUTPUT_BYTES} byte cap" + )); + } + let bit_depth = u8::try_from(component_precision(component.prec)?.0) + .map_err(|_| "openjpeg: component precision exceeds u8".to_string())?; + // SAFETY: `data` is non-null and OpenJPEG allocated `w * h` i32 samples. + let source = unsafe { slice::from_raw_parts(component.data, sample_len) }; + let mut samples = Vec::new(); + samples + .try_reserve_exact(sample_len) + .map_err(|_| "openjpeg: cannot allocate component samples".to_string())?; + samples.extend_from_slice(source); + components.push(OpenJpegDecodedComponent { + dimensions: (component.w, component.h), + sampling: (component.dx, component.dy), + bit_depth, + signed: component.sgnd != 0, + samples, + }); + } + Ok(OpenJpegDecodedImage { + dimensions: (width, height), + components, + }) +} + #[expect( unsafe_code, reason = "component lookup validates OpenJPEG's component array and sample buffer bounds" @@ -379,11 +495,11 @@ fn scale_to_u8(value: i32, precision: ComponentPrecision, signed: bool) -> u8 { value }; if precision <= 8 { - u8::try_from(adjusted.clamp(0, 255)).expect("sample was clamped to the u8 range") + u8::try_from(adjusted.clamp(0, 255)).unwrap_or(0) } else { let max = i64::from((1_u32 << precision.min(31)) - 1); let scaled = (i64::from(adjusted.max(0)) * 255 + max / 2) / max.max(1); - u8::try_from(scaled.clamp(0, 255)).expect("sample was clamped to the u8 range") + u8::try_from(scaled.clamp(0, 255)).unwrap_or(0) } } @@ -652,4 +768,35 @@ mod tests { [1, 2, 4, 6] ); } + + #[test] + fn native_component_decode_preserves_signed_samples_and_metadata() { + let samples = [-2048_i16, -1, 0, 2047]; + let bytes = samples + .into_iter() + .flat_map(i16::to_le_bytes) + .collect::>(); + let encoded = j2k_native::encode( + &bytes, + 2, + 2, + 1, + 12, + true, + &j2k_native::EncodeOptions { + num_decomposition_levels: 0, + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode signed OpenJPEG oracle fixture"); + + let decoded = super::decode_components(&encoded).expect("OpenJPEG component decode"); + assert_eq!(decoded.dimensions, (2, 2)); + assert_eq!(decoded.components.len(), 1); + assert_eq!(decoded.components[0].dimensions, (2, 2)); + assert_eq!(decoded.components[0].sampling, (1, 1)); + assert_eq!(decoded.components[0].bit_depth, 12); + assert!(decoded.components[0].signed); + assert_eq!(decoded.components[0].samples, [-2048, -1, 0, 2047]); + } } diff --git a/crates/j2k-cuda-runtime/Cargo.toml b/crates/j2k-cuda-runtime/Cargo.toml index fbc7c492..d82c73eb 100644 --- a/crates/j2k-cuda-runtime/Cargo.toml +++ b/crates/j2k-cuda-runtime/Cargo.toml @@ -48,13 +48,13 @@ cuda-oxide-jpeg-decode = [] cuda-oxide-jpeg-encode = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } libloading = { workspace = true } thiserror = { workspace = true } [build-dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-cuda-runtime/src/bytes/abi.rs b/crates/j2k-cuda-runtime/src/bytes/abi.rs index c735b4ed..da588205 100644 --- a/crates/j2k-cuda-runtime/src/bytes/abi.rs +++ b/crates/j2k-cuda-runtime/src/bytes/abi.rs @@ -309,7 +309,9 @@ impl_cuda_gpu_abi! { sub_band_type: u32, style_flags: u32, strict: u32, + irreversible_midpoint: u32, dequantization_step: f32, + roi_shift: u32, }, CudaClassicKernelSegment { data_offset: u32, diff --git a/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs b/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs index d05ace87..c9347bd1 100644 --- a/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs +++ b/crates/j2k-cuda-runtime/src/bytes/abi/tests.rs @@ -17,8 +17,9 @@ fn explicit_tail_fields_preserve_cuda_host_abi_sizes_and_offsets() { ); assert_eq!(size_of::(), 40); assert_eq!(offset_of!(CudaHtj2kDequantizeKernelJob, reserved_tail), 36); - assert_eq!(size_of::(), 72); - assert_eq!(offset_of!(CudaClassicKernelJob, dequantization_step), 68); + assert_eq!(size_of::(), 80); + assert_eq!(offset_of!(CudaClassicKernelJob, dequantization_step), 72); + assert_eq!(offset_of!(CudaClassicKernelJob, roi_shift), 76); assert_eq!(size_of::(), 20); assert_eq!(size_of::(), 1_656); assert_eq!(offset_of!(CudaClassicKernelTables, mq_transitions), 188); diff --git a/crates/j2k-cuda-runtime/src/classic_decode/abi.rs b/crates/j2k-cuda-runtime/src/classic_decode/abi.rs index 08c4e800..daf8948a 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode/abi.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode/abi.rs @@ -40,6 +40,10 @@ pub struct CudaClassicCodeBlockJob { pub style_flags: u32, /// Whether malformed entropy data is rejected. pub strict: bool, + /// Whether coefficients use irreversible midpoint reconstruction. + pub irreversible_midpoint: bool, + /// JPEG 2000 Part 1 ROI maxshift applied to this code-block. + pub roi_shift: u32, /// Fused coefficient dequantization multiplier. pub dequantization_step: f32, } @@ -93,7 +97,9 @@ pub(crate) struct CudaClassicKernelJob { pub(crate) sub_band_type: u32, pub(crate) style_flags: u32, pub(crate) strict: u32, + pub(crate) irreversible_midpoint: u32, pub(crate) dequantization_step: f32, + pub(crate) roi_shift: u32, } #[repr(C)] diff --git a/crates/j2k-cuda-runtime/src/classic_decode/prepare.rs b/crates/j2k-cuda-runtime/src/classic_decode/prepare.rs index 659a4cb7..b4a0f445 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode/prepare.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode/prepare.rs @@ -117,7 +117,9 @@ pub(super) fn prepare_classic_decode( sub_band_type: job.sub_band_type, style_flags: job.style_flags, strict: u32::from(job.strict), + irreversible_midpoint: u32::from(job.irreversible_midpoint), dequantization_step: job.dequantization_step, + roi_shift: job.roi_shift, }); let segment_end = job.segment_start.checked_add(job.segment_count).ok_or( CudaError::LengthTooLarge { @@ -159,10 +161,16 @@ pub(super) fn validate_classic_job( output_words: usize, job: &CudaClassicCodeBlockJob, ) -> Result<(), CudaError> { + let Some(coded_bitplanes) = job.total_bitplanes.checked_add(job.roi_shift) else { + return Err(invalid( + "classic code-block dimensions, bitplanes, or sub-band are invalid", + )); + }; if !(1..=MAX_CODEBLOCK_DIMENSION).contains(&job.width) || !(1..=MAX_CODEBLOCK_DIMENSION).contains(&job.height) || !(1..=MAX_BITPLANES).contains(&job.total_bitplanes) - || job.missing_bitplanes >= job.total_bitplanes + || coded_bitplanes > MAX_BITPLANES + || job.missing_bitplanes >= coded_bitplanes || job.sub_band_type > 3 || job.style_flags & !KNOWN_STYLE_FLAGS != 0 { @@ -170,8 +178,8 @@ pub(super) fn validate_classic_job( "classic code-block dimensions, bitplanes, or sub-band are invalid", )); } - let coded_bitplanes = job.total_bitplanes - job.missing_bitplanes; - if job.number_of_coding_passes > 1 + 3 * (coded_bitplanes - 1) { + let decoded_bitplanes = coded_bitplanes - job.missing_bitplanes; + if job.number_of_coding_passes > 1 + 3 * (decoded_bitplanes - 1) { return Err(invalid( "classic code-block pass count exceeds its coded bitplanes", )); diff --git a/crates/j2k-cuda-runtime/src/classic_decode/tests.rs b/crates/j2k-cuda-runtime/src/classic_decode/tests.rs index add1a5e0..24470868 100644 --- a/crates/j2k-cuda-runtime/src/classic_decode/tests.rs +++ b/crates/j2k-cuda-runtime/src/classic_decode/tests.rs @@ -18,10 +18,30 @@ fn max_job() -> CudaClassicCodeBlockJob { sub_band_type: 3, style_flags: 0, strict: true, + irreversible_midpoint: false, + roi_shift: 0, dequantization_step: 1.0, } } +#[test] +fn classic_preflight_accepts_roi_maxshift_and_rejects_bitplane_overflow() { + let segments = [CudaClassicSegment { + data_offset: 0, + data_length: 7, + start_coding_pass: 0, + end_coding_pass: 91, + use_arithmetic: true, + }]; + let mut job = max_job(); + job.total_bitplanes = 24; + job.roi_shift = 7; + validate_classic_job(7, &segments, 64 * 64, &job).expect("valid ROI maxshift"); + + job.roi_shift = 8; + assert!(validate_classic_job(7, &segments, 64 * 64, &job).is_err()); +} + #[test] fn classic_preflight_accepts_maximum_contract() { let segments = [CudaClassicSegment { diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_classic_decode/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_classic_decode/simt/src/main.rs index 18320652..e7312da5 100644 --- a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_classic_decode/simt/src/main.rs +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_classic_decode/simt/src/main.rs @@ -5,6 +5,7 @@ use cuda_device::{kernel, thread, SharedArray}; use cuda_host::cuda_module; +use j2k_codec_math::classic::irreversible_midpoint_bit; include!("../../../cuda_oxide_simt_prelude.rs"); const MAX_PADDED_COEFFICIENTS: usize = 66 * 66; @@ -47,7 +48,9 @@ struct ClassicJob { sub_band_type: u32, style_flags: u32, strict: u32, + irreversible_midpoint: u32, dequantization_step: f32, + roi_shift: u32, } #[repr(C)] @@ -165,6 +168,33 @@ fn set_coefficient_sign(coefficients: *mut u32, index: u32, negative: u32) { ); } +#[inline(always)] +fn reconstructed_classic_sample(coefficient: u32, job: ClassicJob) -> f32 { + let magnitude = coefficient & 0x7fff_ffff; + let mut reconstructed = magnitude as f32; + let decoded_bitplanes = job.total_bitplanes + job.roi_shift - job.missing_msbs; + if job.irreversible_midpoint != 0 { + if let Some(lowest_decoded_bit) = irreversible_midpoint_bit( + u64::from(magnitude), + decoded_bitplanes, + job.number_of_coding_passes, + ) { + let mut fixed_magnitude = (magnitude << 1) | (1 << lowest_decoded_bit); + if job.roi_shift != 0 && fixed_magnitude >= 1 << job.roi_shift { + fixed_magnitude >>= job.roi_shift; + } + reconstructed = fixed_magnitude as f32 * 0.5; + } + } else if job.roi_shift != 0 && magnitude >= 1 << job.roi_shift { + reconstructed = (magnitude >> job.roi_shift) as f32; + } + if coefficient & 0x8000_0000 != 0 { + -reconstructed + } else { + reconstructed + } +} + #[inline(always)] fn neighbor_in_next_stripe(y: u32, height: u32) -> bool { let real_y = y - 1; @@ -395,9 +425,8 @@ fn arithmetic_decode_bit( fn raw_read_bit(decoder: &mut BypassDecoder) -> u32 { let byte_position = decoder.bit_pos / 8; if byte_position >= decoder.data_len { - if decoder.strict { - return RAW_READ_FAILED; - } + // T.800 D.4.1 extends a cleanly exhausted terminated segment with + // 0xFF bytes. Strict mode still rejects a malformed stuffed bit below. decoder.bit_pos += 1; return 1; } @@ -512,13 +541,17 @@ fn validate_job_header( } if job.total_bitplanes == 0 || job.total_bitplanes > 31 - || job.missing_msbs >= job.total_bitplanes + || job.roi_shift > 31 - job.total_bitplanes || job.sub_band_type > 3 || job.style_flags & !KNOWN_STYLE_FLAGS != 0 { return fail(statuses, job_index, STATUS_UNSUPPORTED, 2); } - let bitplanes = job.total_bitplanes - job.missing_msbs; + let coded_bitplanes = job.total_bitplanes + job.roi_shift; + if job.missing_msbs >= coded_bitplanes { + return fail(statuses, job_index, STATUS_UNSUPPORTED, 2); + } + let bitplanes = coded_bitplanes - job.missing_msbs; let max_passes = 1 + 3 * (bitplanes - 1); if job.number_of_coding_passes > max_passes { return fail(statuses, job_index, STATUS_UNSUPPORTED, 3); @@ -541,7 +574,7 @@ fn decode_job( statuses: *mut ClassicStatus, job_index: u32, ) -> bool { - let bitplanes = job.total_bitplanes - job.missing_msbs; + let bitplanes = job.total_bitplanes + job.roi_shift - job.missing_msbs; if job.number_of_coding_passes == 0 { return true; } @@ -888,18 +921,12 @@ mod kernels { coefficients.cast_const(), coefficient_index(padded_width, x + 1, y + 1) as usize, ); - let magnitude = (packed & 0x7fff_ffff) as i32; - let signed = if packed & 0x8000_0000 != 0 { - -magnitude - } else { - magnitude - }; simt_store( output, job.output_offset as usize + y as usize * job.output_stride as usize + x as usize, - signed as f32 * job.dequantization_step, + reconstructed_classic_sample(packed, job) * job.dequantization_step, ); sample += CLASSIC_DECODE_THREADS; } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs index 34cda77d..43e8bc5e 100644 --- a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/exports.rs @@ -20,7 +20,7 @@ mod kernels { }, helpers::{ floor_f32, load_f32, load_f32_u64, load_job, load_u8, load_u32, store_f32, - store_f32_u64, store_i32, store_u8, store_u32, + sign_extend_u32, store_f32_u64, store_i32, store_u8, store_u32, }, packetization::{ j2k_packet_build_header_serial, j2k_packet_copy_body_cooperative, j2k_packet_status, @@ -53,21 +53,16 @@ mod kernels { let mut component = 0_u32; while component < num_components { let sample_base = pixel_base + component as u64 * bytes_per_sample as u64; - let sample = if bit_depth <= 8 { - let raw = load_u8(pixels, sample_base); - if is_signed != 0 { - (raw as i8) as f32 - } else { - raw as f32 - unsigned_offset - } + let raw = if bit_depth <= 8 { + load_u8(pixels, sample_base) as u32 } else { - let raw = load_u8(pixels, sample_base) as u16 - | ((load_u8(pixels, sample_base + 1) as u16) << 8); - if is_signed != 0 { - (raw as i16) as f32 - } else { - raw as f32 - unsigned_offset - } + load_u8(pixels, sample_base) as u32 + | ((load_u8(pixels, sample_base + 1) as u32) << 8) + }; + let sample = if is_signed != 0 { + sign_extend_u32(raw, bit_depth) as f32 + } else { + raw as f32 - unsigned_offset }; store_f32_u64(components, component as u64 * num_pixels + idx, sample); component += 1; @@ -105,21 +100,16 @@ mod kernels { let mut component = 0_u32; while component < num_components { let sample_base = pixel_base + component as u64 * bytes_per_sample as u64; - let sample = if bit_depth <= 8 { - let raw = load_u8(pixels, sample_base); - if is_signed != 0 { - (raw as i8) as f32 - } else { - raw as f32 - unsigned_offset - } + let raw = if bit_depth <= 8 { + load_u8(pixels, sample_base) as u32 } else { - let raw = load_u8(pixels, sample_base) as u16 - | ((load_u8(pixels, sample_base + 1) as u16) << 8); - if is_signed != 0 { - (raw as i16) as f32 - } else { - raw as f32 - unsigned_offset - } + load_u8(pixels, sample_base) as u32 + | ((load_u8(pixels, sample_base + 1) as u32) << 8) + }; + let sample = if is_signed != 0 { + sign_extend_u32(raw, bit_depth) as f32 + } else { + raw as f32 - unsigned_offset }; store_f32_u64(components, component as u64 * num_pixels + idx, sample); component += 1; diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/helpers.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/helpers.rs index 856e54f5..0f377353 100644 --- a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/helpers.rs +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/helpers.rs @@ -52,6 +52,12 @@ pub(crate) fn load_job(ptr: *const T, index: u32) -> T { simt_load(ptr, index as usize) } +#[inline(always)] +pub(crate) fn sign_extend_u32(raw: u32, bit_depth: u32) -> i32 { + let shift = 32 - bit_depth; + ((raw << shift) as i32) >> shift +} + #[inline(always)] pub(crate) fn floor_f32(value: f32) -> f32 { // f32::floor routes through libdevice in cuda-oxide, which emits NVVM IR diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/idwt.rs b/crates/j2k-cuda-runtime/src/j2k_decode/idwt.rs index 84dc3669..a323eedc 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/idwt.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/idwt.rs @@ -146,6 +146,7 @@ impl CudaContext { CudaJ2kIdwtBatchKernelMode::Cooperative53 => self .launch_j2k_idwt_interleave_horizontal_53_multi( jobs_device, + max_width as usize, max_height as usize, kernel_jobs.len(), false, @@ -173,6 +174,7 @@ impl CudaContext { CudaJ2kIdwtBatchKernelMode::Cooperative53 => self.launch_j2k_idwt_vertical_53_multi( jobs_device, max_width as usize, + max_height as usize, kernel_jobs.len(), false, ), @@ -227,6 +229,7 @@ impl CudaContext { CudaJ2kIdwtBatchKernelMode::Cooperative53 => self .launch_j2k_idwt_interleave_horizontal_53_multi( jobs_device, + max_width as usize, max_height as usize, kernel_jobs.len(), true, @@ -252,6 +255,7 @@ impl CudaContext { CudaJ2kIdwtBatchKernelMode::Cooperative53 => self.launch_j2k_idwt_vertical_53_multi( jobs_device, max_width as usize, + max_height as usize, kernel_jobs.len(), true, ), diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/idwt/launch_validation.rs b/crates/j2k-cuda-runtime/src/j2k_decode/idwt/launch_validation.rs index f5510444..c6a2b45d 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/idwt/launch_validation.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/idwt/launch_validation.rs @@ -5,7 +5,7 @@ use crate::{ kernels::{ j2k_dwt53_launch_geometry, j2k_idwt_multi_1d_launch_geometry, j2k_idwt_multi_coop_axis_launch_geometry, j2k_idwt_multi_coop_columns_launch_geometry, - j2k_idwt_multi_coop_launch_geometry, CudaKernel, CudaLaunchGeometry, + CudaKernel, CudaLaunchGeometry, }, }; @@ -67,22 +67,23 @@ pub(super) fn validate_idwt_batch_launch( CudaJ2kIdwtBatchKernelMode::Generic => { j2k_idwt_multi_1d_launch_geometry(max_height as usize, job_count) } - CudaJ2kIdwtBatchKernelMode::Cooperative53 => { - j2k_idwt_multi_coop_launch_geometry(max_height as usize, job_count) + CudaJ2kIdwtBatchKernelMode::Cooperative53 | CudaJ2kIdwtBatchKernelMode::Cooperative97 => { + j2k_idwt_multi_coop_axis_launch_geometry( + max_height as usize, + max_width as usize, + job_count, + ) } - CudaJ2kIdwtBatchKernelMode::Cooperative97 => j2k_idwt_multi_coop_axis_launch_geometry( - max_height as usize, - max_width as usize, - job_count, - ), }; let vertical = match kernel_mode { CudaJ2kIdwtBatchKernelMode::Generic => { j2k_idwt_multi_1d_launch_geometry(max_width as usize, job_count) } - CudaJ2kIdwtBatchKernelMode::Cooperative53 => { - j2k_idwt_multi_coop_launch_geometry(max_width as usize, job_count) - } + CudaJ2kIdwtBatchKernelMode::Cooperative53 => j2k_idwt_multi_coop_axis_launch_geometry( + max_width as usize, + max_height as usize, + job_count, + ), CudaJ2kIdwtBatchKernelMode::Cooperative97 => idwt_vertical_97_multi_launch_geometry( max_width as usize, max_height as usize, diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/idwt/sequence.rs b/crates/j2k-cuda-runtime/src/j2k_decode/idwt/sequence.rs index 5066e26e..e45dd26f 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/idwt/sequence.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/idwt/sequence.rs @@ -129,6 +129,7 @@ impl CudaContext { CudaJ2kIdwtBatchKernelMode::Cooperative53 => self .launch_j2k_idwt_interleave_horizontal_53_multi_ptr( jobs_ptr, + max_width as usize, max_height as usize, count, false, @@ -156,6 +157,7 @@ impl CudaContext { .launch_j2k_idwt_vertical_53_multi_ptr( jobs_ptr, max_width as usize, + max_height as usize, count, false, ), diff --git a/crates/j2k-cuda-runtime/src/j2k_decode/idwt_launch.rs b/crates/j2k-cuda-runtime/src/j2k_decode/idwt_launch.rs index b65fd5cb..07959cb7 100644 --- a/crates/j2k-cuda-runtime/src/j2k_decode/idwt_launch.rs +++ b/crates/j2k-cuda-runtime/src/j2k_decode/idwt_launch.rs @@ -7,8 +7,8 @@ use crate::{ execution::{cuda_kernel_param, CudaLaunchMode}, kernels::{ j2k_dwt53_launch_geometry, j2k_forward_rct_launch_geometry, - j2k_idwt_multi_1d_launch_geometry, j2k_idwt_multi_coop_axis_launch_geometry, - j2k_idwt_multi_coop_launch_geometry, CudaKernel, CudaLaunchGeometry, + j2k_idwt_multi_1d_launch_geometry, j2k_idwt_multi_coop_axis_launch_geometry, CudaKernel, + CudaLaunchGeometry, }, memory::CudaDeviceBuffer, }; @@ -91,12 +91,14 @@ impl CudaContext { pub(in crate::j2k_decode) fn launch_j2k_idwt_interleave_horizontal_53_multi( &self, jobs: &CudaDeviceBuffer, + max_width: usize, max_rows: usize, job_count: usize, synchronize: bool, ) -> Result<(), CudaError> { self.launch_j2k_idwt_interleave_horizontal_53_multi_ptr( jobs.device_ptr(), + max_width, max_rows, job_count, synchronize, @@ -106,13 +108,14 @@ impl CudaContext { pub(in crate::j2k_decode) fn launch_j2k_idwt_interleave_horizontal_53_multi_ptr( &self, jobs_ptr: CuDevicePtr, + max_width: usize, max_rows: usize, job_count: usize, synchronize: bool, ) -> Result<(), CudaError> { let mut jobs_ptr = jobs_ptr; let mut params = cuda_kernel_params!(jobs_ptr); - let geometry = j2k_idwt_multi_coop_launch_geometry(max_rows, job_count) + let geometry = j2k_idwt_multi_coop_axis_launch_geometry(max_rows, max_width, job_count) .ok_or(CudaError::LengthTooLarge { len: job_count })?; self.launch_j2k_idwt_named_kernel( CudaKernel::J2kIdwtInterleaveHorizontal53Multi, @@ -220,12 +223,14 @@ impl CudaContext { &self, jobs: &CudaDeviceBuffer, max_columns: usize, + max_height: usize, job_count: usize, synchronize: bool, ) -> Result<(), CudaError> { self.launch_j2k_idwt_vertical_53_multi_ptr( jobs.device_ptr(), max_columns, + max_height, job_count, synchronize, ) @@ -235,12 +240,13 @@ impl CudaContext { &self, jobs_ptr: CuDevicePtr, max_columns: usize, + max_height: usize, job_count: usize, synchronize: bool, ) -> Result<(), CudaError> { let mut jobs_ptr = jobs_ptr; let mut params = cuda_kernel_params!(jobs_ptr); - let geometry = j2k_idwt_multi_coop_launch_geometry(max_columns, job_count) + let geometry = j2k_idwt_multi_coop_axis_launch_geometry(max_columns, max_height, job_count) .ok_or(CudaError::LengthTooLarge { len: job_count })?; self.launch_j2k_idwt_named_kernel( CudaKernel::J2kIdwtVertical53Multi, diff --git a/crates/j2k-cuda-runtime/src/kernels.rs b/crates/j2k-cuda-runtime/src/kernels.rs index 3a4827f0..433fc8b7 100644 --- a/crates/j2k-cuda-runtime/src/kernels.rs +++ b/crates/j2k-cuda-runtime/src/kernels.rs @@ -35,7 +35,7 @@ pub(crate) use j2k::{ j2k_classic_codeblock_launch_geometry, j2k_dwt53_launch_geometry, j2k_forward_rct_launch_geometry, j2k_idwt_multi_1d_launch_geometry, j2k_idwt_multi_coop_axis_launch_geometry, j2k_idwt_multi_coop_columns_launch_geometry, - j2k_idwt_multi_coop_launch_geometry, j2k_store_batch_launch_geometry, + j2k_store_batch_launch_geometry, }; #[cfg(feature = "cuda-oxide-jpeg-decode")] pub(crate) use jpeg::cuda_oxide_jpeg_decode_ptx; diff --git a/crates/j2k-cuda-runtime/src/kernels/j2k.rs b/crates/j2k-cuda-runtime/src/kernels/j2k.rs index e0e8b379..59aa2591 100644 --- a/crates/j2k-cuda-runtime/src/kernels/j2k.rs +++ b/crates/j2k-cuda-runtime/src/kernels/j2k.rs @@ -205,20 +205,6 @@ pub(crate) fn j2k_idwt_multi_1d_launch_geometry( x_blocks_launch_geometry(max_len, job_count, COPY_U8_THREADS) } -pub(crate) fn j2k_idwt_multi_coop_launch_geometry( - max_len: usize, - job_count: usize, -) -> Option { - let lanes = c_uint::try_from(max_len).ok()?; - let jobs = c_uint::try_from(job_count).ok()?; - let threads = if max_len > COPY_U8_THREADS { - J2K_IDWT_COOP_THREADS_LARGE_CUDA - } else { - J2K_IDWT_COOP_THREADS_SMALL_CUDA - }; - CudaLaunchGeometry::new((lanes, jobs, 1), (threads, 1, 1)) -} - pub(crate) fn j2k_idwt_multi_coop_axis_launch_geometry( work_items: usize, lane_count: usize, diff --git a/crates/j2k-cuda-runtime/src/tests/pipeline.rs b/crates/j2k-cuda-runtime/src/tests/pipeline.rs index 4ad2694b..e72a5943 100644 --- a/crates/j2k-cuda-runtime/src/tests/pipeline.rs +++ b/crates/j2k-cuda-runtime/src/tests/pipeline.rs @@ -708,6 +708,119 @@ fn j2k_inverse_dwt_batch_512_reversible_matches_single_when_runtime_required() { assert_eq!(batch_actual, single_actual); } +#[test] +#[expect( + clippy::cast_precision_loss, + clippy::similar_names, + reason = "fixture coordinates and parallel plane names mirror the CUDA API" +)] +fn j2k_inverse_dwt_batch_640x480_reversible_matches_single_when_runtime_required() { + const WIDTH: usize = 640; + const HEIGHT: usize = 480; + const BAND_WIDTH: usize = WIDTH / 2; + const BAND_HEIGHT: usize = HEIGHT / 2; + + if !cuda_runtime_gate() { + return; + } + + let width_u32 = u32::try_from(WIDTH).expect("fixture width fits u32"); + let height_u32 = u32::try_from(HEIGHT).expect("fixture height fits u32"); + let band_width_u32 = u32::try_from(BAND_WIDTH).expect("fixture band width fits u32"); + let band_height_u32 = u32::try_from(BAND_HEIGHT).expect("fixture band height fits u32"); + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let band_len = BAND_WIDTH * BAND_HEIGHT; + let ll_values: Vec = (0..band_len).map(|idx| (idx % 43) as f32).collect(); + let hl_values: Vec = (0..band_len).map(|idx| ((idx * 3) % 47) as f32).collect(); + let lh_values: Vec = (0..band_len).map(|idx| ((idx * 5) % 53) as f32).collect(); + let hh_values: Vec = (0..band_len).map(|idx| ((idx * 7) % 59) as f32).collect(); + let ll = context + .upload(super::super::f32_slice_as_bytes(&ll_values)) + .expect("upload 640x480 LL"); + let hl = context + .upload(super::super::f32_slice_as_bytes(&hl_values)) + .expect("upload 640x480 HL"); + let lh = context + .upload(super::super::f32_slice_as_bytes(&lh_values)) + .expect("upload 640x480 LH"); + let hh = context + .upload(super::super::f32_slice_as_bytes(&hh_values)) + .expect("upload 640x480 HH"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: width_u32, + y1: height_u32, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: band_width_u32, + y1: band_height_u32, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: band_width_u32, + y1: band_height_u32, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: band_width_u32, + y1: band_height_u32, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: band_width_u32, + y1: band_height_u32, + }, + irreversible97: 0, + }; + + let single = context + .j2k_inverse_dwt_single_device_with_pool(&ll, &hl, &lh, &hh, job, &pool) + .expect("640x480 single CUDA inverse DWT"); + let batch_output = pool + .take(WIDTH * HEIGHT * std::mem::size_of::()) + .expect("640x480 batched IDWT output"); + let execution = context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: batch_output + .as_device_buffer() + .expect("640x480 batch output device buffer"), + job, + }], + &pool, + ) + .expect("640x480 batched CUDA inverse DWT"); + assert_eq!(execution.kernel_dispatches(), 2); + + let mut single_actual = vec![0.0f32; WIDTH * HEIGHT]; + single + .buffer() + .expect("640x480 single output device buffer") + .copy_to_host(super::super::f32_slice_as_bytes_mut(&mut single_actual)) + .expect("download 640x480 single IDWT"); + let mut batch_actual = vec![0.0f32; WIDTH * HEIGHT]; + batch_output + .copy_to_host(super::super::f32_slice_as_bytes_mut(&mut batch_actual)) + .expect("download 640x480 batch IDWT"); + let first_mismatch = batch_actual + .iter() + .zip(&single_actual) + .position(|(batch, single)| batch.to_bits() != single.to_bits()); + assert_eq!(first_mismatch, None); +} + #[test] fn j2k_inverse_dwt_batch_enqueue_matches_expected_outputs_when_runtime_required() { if !cuda_runtime_gate() { @@ -985,6 +1098,199 @@ fn j2k_inverse_dwt_batch_sequence_enqueue_matches_two_stage_path_when_runtime_re assert_eq!(sequence_actual, legacy_actual); } +#[test] +#[expect( + clippy::cast_precision_loss, + clippy::similar_names, + clippy::too_many_lines, + reason = "two-stage reversible pipeline fixture keeps stage buffers explicit" +)] +fn j2k_inverse_dwt_batch_sequence_large_mixed_modes_matches_single_path_when_runtime_required() { + const STAGE1_WIDTH: usize = 320; + const STAGE1_HEIGHT: usize = 240; + const STAGE2_WIDTH: usize = 640; + const STAGE2_HEIGHT: usize = 480; + + if !cuda_runtime_gate() { + return; + } + + let stage1_width_u32 = u32::try_from(STAGE1_WIDTH).expect("stage 1 width fits u32"); + let stage1_height_u32 = u32::try_from(STAGE1_HEIGHT).expect("stage 1 height fits u32"); + let stage2_width_u32 = u32::try_from(STAGE2_WIDTH).expect("stage 2 width fits u32"); + let stage2_height_u32 = u32::try_from(STAGE2_HEIGHT).expect("stage 2 height fits u32"); + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let stage1_band_len = STAGE1_WIDTH / 2 * (STAGE1_HEIGHT / 2); + let stage2_band_len = STAGE2_WIDTH / 2 * (STAGE2_HEIGHT / 2); + let stage1_ll_values: Vec = (0..stage1_band_len).map(|idx| (idx % 43) as f32).collect(); + let stage1_hl_values: Vec = (0..stage1_band_len) + .map(|idx| ((idx * 3) % 47) as f32) + .collect(); + let stage1_lh_values: Vec = (0..stage1_band_len) + .map(|idx| ((idx * 5) % 53) as f32) + .collect(); + let stage1_hh_values: Vec = (0..stage1_band_len) + .map(|idx| ((idx * 7) % 59) as f32) + .collect(); + let stage2_hl_values: Vec = (0..stage2_band_len) + .map(|idx| ((idx * 11) % 61) as f32) + .collect(); + let stage2_lh_values: Vec = (0..stage2_band_len) + .map(|idx| ((idx * 13) % 67) as f32) + .collect(); + let stage2_hh_values: Vec = (0..stage2_band_len) + .map(|idx| ((idx * 17) % 71) as f32) + .collect(); + let stage1_ll = context + .upload(super::super::f32_slice_as_bytes(&stage1_ll_values)) + .expect("upload stage1 LL"); + let stage1_hl = context + .upload(super::super::f32_slice_as_bytes(&stage1_hl_values)) + .expect("upload stage1 HL"); + let stage1_lh = context + .upload(super::super::f32_slice_as_bytes(&stage1_lh_values)) + .expect("upload stage1 LH"); + let stage1_hh = context + .upload(super::super::f32_slice_as_bytes(&stage1_hh_values)) + .expect("upload stage1 HH"); + let stage2_hl = context + .upload(super::super::f32_slice_as_bytes(&stage2_hl_values)) + .expect("upload stage2 HL"); + let stage2_lh = context + .upload(super::super::f32_slice_as_bytes(&stage2_lh_values)) + .expect("upload stage2 LH"); + let stage2_hh = context + .upload(super::super::f32_slice_as_bytes(&stage2_hh_values)) + .expect("upload stage2 HH"); + let job = |width: u32, height: u32| CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: width, + y1: height, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: width / 2, + y1: height / 2, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: width / 2, + y1: height / 2, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: width / 2, + y1: height / 2, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: width / 2, + y1: height / 2, + }, + irreversible97: 0, + }; + let stage1_job = job(stage1_width_u32, stage1_height_u32); + let stage2_job = job(stage2_width_u32, stage2_height_u32); + + let single_stage1 = context + .j2k_inverse_dwt_single_device_with_pool( + &stage1_ll, &stage1_hl, &stage1_lh, &stage1_hh, stage1_job, &pool, + ) + .expect("single stage1 IDWT"); + let single_stage2 = context + .j2k_inverse_dwt_single_device_with_pool( + single_stage1.buffer().expect("single stage1 output"), + &stage2_hl, + &stage2_lh, + &stage2_hh, + stage2_job, + &pool, + ) + .expect("single stage2 IDWT"); + let sequence_stage1 = pool + .take(STAGE1_WIDTH * STAGE1_HEIGHT * std::mem::size_of::()) + .expect("sequence stage1 output"); + let sequence_stage2 = pool + .take(STAGE2_WIDTH * STAGE2_HEIGHT * std::mem::size_of::()) + .expect("sequence stage2 output"); + let stage1_targets = [CudaJ2kIdwtTarget { + ll: &stage1_ll, + hl: &stage1_hl, + lh: &stage1_lh, + hh: &stage1_hh, + output: sequence_stage1 + .as_device_buffer() + .expect("sequence stage1 device buffer"), + job: stage1_job, + }]; + let stage2_targets = [CudaJ2kIdwtTarget { + ll: sequence_stage1 + .as_device_buffer() + .expect("sequence stage1 device buffer"), + hl: &stage2_hl, + lh: &stage2_lh, + hh: &stage2_hh, + output: sequence_stage2 + .as_device_buffer() + .expect("sequence stage2 device buffer"), + job: stage2_job, + }]; + // SAFETY: all inputs and outputs remain live and untouched until the + // returned execution is explicitly finished below. + let queued = unsafe { + context.j2k_inverse_dwt_batch_sequence_enqueue_with_pool( + &[&stage1_targets, &stage2_targets], + &pool, + ) + } + .expect("large queued IDWT sequence"); + assert_eq!(queued.execution().kernel_dispatches(), 4); + queued.finish().expect("finish large queued IDWT sequence"); + + let mut single_stage1_actual = vec![0.0f32; STAGE1_WIDTH * STAGE1_HEIGHT]; + single_stage1 + .buffer() + .expect("single stage1 output") + .copy_to_host(super::super::f32_slice_as_bytes_mut( + &mut single_stage1_actual, + )) + .expect("download single stage1 IDWT"); + let mut sequence_stage1_actual = vec![0.0f32; STAGE1_WIDTH * STAGE1_HEIGHT]; + sequence_stage1 + .copy_to_host(super::super::f32_slice_as_bytes_mut( + &mut sequence_stage1_actual, + )) + .expect("download sequence stage1 IDWT"); + let stage1_first_mismatch = sequence_stage1_actual + .iter() + .zip(&single_stage1_actual) + .position(|(sequence, single)| sequence.to_bits() != single.to_bits()); + assert_eq!(stage1_first_mismatch, None, "stage1 mismatch"); + + let mut single_actual = vec![0.0f32; STAGE2_WIDTH * STAGE2_HEIGHT]; + single_stage2 + .buffer() + .expect("single stage2 output") + .copy_to_host(super::super::f32_slice_as_bytes_mut(&mut single_actual)) + .expect("download single two-stage IDWT"); + let mut sequence_actual = vec![0.0f32; STAGE2_WIDTH * STAGE2_HEIGHT]; + sequence_stage2 + .copy_to_host(super::super::f32_slice_as_bytes_mut(&mut sequence_actual)) + .expect("download sequence two-stage IDWT"); + let first_mismatch = sequence_actual + .iter() + .zip(&single_actual) + .position(|(sequence, single)| sequence.to_bits() != single.to_bits()); + assert_eq!(first_mismatch, None); +} + #[test] fn j2k_store_rgb8_mct_matches_inverse_mct_plus_store_when_runtime_required() { if !cuda_runtime_gate() { diff --git a/crates/j2k-cuda/Cargo.toml b/crates/j2k-cuda/Cargo.toml index 54cc204b..271c98b3 100644 --- a/crates/j2k-cuda/Cargo.toml +++ b/crates/j2k-cuda/Cargo.toml @@ -34,11 +34,11 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.0", optional = true } -j2k = { path = "../j2k", version = "=0.8.0" } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.1", optional = true } +j2k = { path = "../j2k", version = "=0.8.1" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1" } thiserror = { workspace = true } [dev-dependencies] @@ -50,6 +50,12 @@ name = "encode_stages" harness = false test = false +[[bench]] +name = "auto_routing" +harness = false +test = false +required-features = ["cuda-runtime"] + [[bench]] name = "htj2k_decode" harness = false diff --git a/crates/j2k-cuda/benches/auto_routing.rs b/crates/j2k-cuda/benches/auto_routing.rs new file mode 100644 index 00000000..2736083a --- /dev/null +++ b/crates/j2k-cuda/benches/auto_routing.rs @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{path::PathBuf, time::Duration}; + +use criterion::Criterion; +use j2k::{ + encode_j2k_lossless, encode_j2k_lossless_with_accelerator, encode_j2k_lossy, + encode_j2k_lossy_with_accelerator, EncodeBackendPreference, J2kBlockCodingMode, + J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, J2kLossyEncodeOptions, + J2kLossySamples, J2kRateTarget, +}; +use j2k_core::{ + BackendKind, BackendRequest, DeviceSubmission, DeviceSurface, Downscale, ImageDecodeSubmit, + PixelFormat, Rect, TileBatchDecodeManyDevice, +}; +use j2k_cuda::{Codec, CudaEncodeStageAccelerator, CudaSession, J2kDecoder, SurfaceResidency}; +use j2k_test_support::{ + append_auto_routing_output as append_output, auto_routing_operation_label as operation_label, + auto_routing_route_cell as route_cell, auto_routing_sha256, load_auto_routing_manifest, + load_auto_routing_pnm, write_auto_routing_evidence, AutoRoutingBackend, AutoRoutingCell, + AutoRoutingEvidence, AutoRoutingOperation, AutoRoutingPixelFormat, AutoRoutingPlatform, + AutoRoutingPnm, AutoRoutingWorkload, AutoRoutingWorkloadKind, +}; + +const SAMPLE_SIZE: usize = 10; +const WARM_UP: Duration = Duration::from_secs(1); +const MEASUREMENT: Duration = Duration::from_secs(3); +const BATCH_SIZE: usize = 16; + +fn main() { + let manifest_path = required_path("J2K_AUTO_ROUTING_MANIFEST"); + let corpus_root = required_path("J2K_AUTO_ROUTING_ROOT"); + let evidence_path = required_path("J2K_AUTO_ROUTING_EVIDENCE"); + let workloads = load_auto_routing_manifest(&manifest_path, &corpus_root) + .unwrap_or_else(|error| panic!("load CUDA Auto-routing workloads: {error}")); + let mut criterion = Criterion::default() + .sample_size(SAMPLE_SIZE) + .warm_up_time(WARM_UP) + .measurement_time(MEASUREMENT) + .configure_from_args(); + let mut cells = Vec::new(); + + for workload in &workloads.workloads { + match workload.kind { + AutoRoutingWorkloadKind::Decode => { + let decode = DecodeCase::new(workload); + for operation in [ + AutoRoutingOperation::FullDecode, + AutoRoutingOperation::RoiDecode, + AutoRoutingOperation::ScaledDecode, + AutoRoutingOperation::BatchDecode, + ] { + cells.push(bench_decode_cell(&mut criterion, &decode, operation)); + } + } + AutoRoutingWorkloadKind::Encode => { + let encode = load_auto_routing_pnm(workload).unwrap_or_else(|error| { + panic!("load CUDA encode workload {}: {error}", workload.id) + }); + for operation in [ + AutoRoutingOperation::LosslessEncode, + AutoRoutingOperation::LossyEncode, + ] { + cells.push(bench_encode_cell(&mut criterion, &encode, operation)); + } + } + } + } + criterion.final_summary(); + + let evidence = AutoRoutingEvidence { + schema_version: 1, + candidate_sha: required_env("J2K_AUTO_ROUTING_CANDIDATE_SHA"), + backend: AutoRoutingBackend::Cuda, + platform: AutoRoutingPlatform { + os: "linux".to_string(), + arch: "x86_64".to_string(), + hardware: required_env("J2K_AUTO_ROUTING_HARDWARE"), + driver: required_env("J2K_AUTO_ROUTING_DRIVER"), + }, + external_manifest_sha256: workloads.manifest_sha256, + external_case_count: workloads.workloads.len(), + cells, + }; + write_auto_routing_evidence(&evidence_path, &evidence) + .unwrap_or_else(|error| panic!("write CUDA Auto-routing evidence: {error}")); +} + +struct DecodeCase<'a> { + id: &'a str, + bytes: &'a [u8], + fmt: PixelFormat, + dimensions: (u32, u32), +} + +impl<'a> DecodeCase<'a> { + fn new(workload: &'a AutoRoutingWorkload) -> Self { + let info = j2k::J2kDecoder::inspect(&workload.bytes) + .unwrap_or_else(|error| panic!("inspect decode workload {}: {error}", workload.id)); + Self { + id: &workload.id, + bytes: &workload.bytes, + fmt: pixel_format(workload.pixel_format), + dimensions: info.dimensions, + } + } +} + +type EncodeCase = AutoRoutingPnm; + +fn bench_decode_cell( + criterion: &mut Criterion, + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, +) -> AutoRoutingCell { + let mut cpu_probe_session = CudaSession::default(); + let cpu = decode_once(case, operation, BackendRequest::Cpu, &mut cpu_probe_session) + .unwrap_or_else(|error| panic!("CPU {} {}: {error}", case.id, operation_label(operation))); + let mut hybrid_probe_session = CudaSession::default(); + let hybrid = decode_once( + case, + operation, + BackendRequest::Cuda, + &mut hybrid_probe_session, + ) + .unwrap_or_else(|error| { + panic!( + "hybrid CUDA {} {}: {error}", + case.id, + operation_label(operation) + ) + }); + if operation == AutoRoutingOperation::BatchDecode { + let mut reference_session = CudaSession::default(); + let single = decode_once( + case, + AutoRoutingOperation::FullDecode, + BackendRequest::Cpu, + &mut reference_session, + ) + .unwrap_or_else(|error| panic!("CPU {} batch reference: {error}", case.id)); + let reference = repeated_batch_output(&single) + .unwrap_or_else(|error| panic!("CPU {} batch reference: {error}", case.id)); + assert_output_parity( + &format!("{} CPU batch versus repeated full decode", case.id), + operation, + &reference, + &cpu, + ); + assert_output_parity( + &format!("{} CUDA batch versus repeated CPU full decode", case.id), + operation, + &reference, + &hybrid, + ); + } + assert_output_parity(case.id, operation, &cpu, &hybrid); + + let group_id = format!( + "auto-routing_{}_{id}", + operation_label(operation), + id = case.id + ); + let mut cpu_session = CudaSession::default(); + let mut hybrid_session = hybrid_probe_session; + let mut group = criterion.benchmark_group(&group_id); + group.bench_function("cpu", |bencher| { + bencher.iter(|| { + std::hint::black_box( + decode_once(case, operation, BackendRequest::Cpu, &mut cpu_session) + .expect("measured CPU CUDA-adapter decode"), + ) + }); + }); + group.bench_function("hybrid", |bencher| { + bencher.iter(|| { + std::hint::black_box( + decode_once(case, operation, BackendRequest::Cuda, &mut hybrid_session) + .expect("measured hybrid CUDA decode"), + ) + }); + }); + group.finish(); + route_cell(case.id, operation, &group_id, auto_routing_sha256(&cpu)) +} + +fn decode_once( + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, + backend: BackendRequest, + session: &mut CudaSession, +) -> Result, String> { + if operation == AutoRoutingOperation::BatchDecode { + return decode_batch_once(case, backend, session); + } + let mut decoder = J2kDecoder::new(case.bytes).map_err(|error| error.to_string())?; + let submission = match operation { + AutoRoutingOperation::FullDecode => decoder + .submit_to_device(session, case.fmt, backend) + .map_err(|error| error.to_string())?, + AutoRoutingOperation::RoiDecode => decoder + .submit_region_to_device(session, case.fmt, benchmark_roi(case.dimensions), backend) + .map_err(|error| error.to_string())?, + AutoRoutingOperation::ScaledDecode => decoder + .submit_scaled_to_device(session, case.fmt, Downscale::Half, backend) + .map_err(|error| error.to_string())?, + AutoRoutingOperation::BatchDecode + | AutoRoutingOperation::LosslessEncode + | AutoRoutingOperation::LossyEncode => { + return Err("invalid single-image decode operation".to_string()) + } + }; + let surface = submission.wait().map_err(|error| error.to_string())?; + assert_surface_route(&surface, backend)?; + surface_bytes(&surface) +} + +fn decode_batch_once( + case: &DecodeCase<'_>, + backend: BackendRequest, + session: &mut CudaSession, +) -> Result, String> { + let mut inputs = Vec::new(); + inputs + .try_reserve_exact(BATCH_SIZE) + .map_err(|_| "allocate CUDA Auto-routing batch inputs".to_string())?; + inputs.resize(BATCH_SIZE, case.bytes); + let surfaces = if backend == BackendRequest::Cuda { + J2kDecoder::decode_batch_to_device_with_session(&inputs, case.fmt, session) + .map_err(|error| error.to_string())? + } else { + let mut context = j2k_cuda::J2kContext::default(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + Codec::decode_tiles_to_device(&mut context, &mut pool, &inputs, case.fmt, backend) + .map_err(|error| error.to_string())? + }; + if surfaces.len() != BATCH_SIZE { + return Err(format!( + "CUDA batch returned {} surfaces for {BATCH_SIZE} inputs", + surfaces.len() + )); + } + let mut output = Vec::new(); + for surface in &surfaces { + assert_surface_route(surface, backend)?; + let bytes = surface_bytes(surface)?; + append_output(&mut output, &bytes)?; + } + Ok(output) +} + +fn repeated_batch_output(single: &[u8]) -> Result, String> { + let mut output = Vec::new(); + for _ in 0..BATCH_SIZE { + append_output(&mut output, single)?; + } + Ok(output) +} + +fn surface_bytes(surface: &j2k_cuda::Surface) -> Result, String> { + let (width, height) = surface.dimensions(); + let stride = usize::try_from(width) + .ok() + .and_then(|width| width.checked_mul(surface.pixel_format().bytes_per_pixel())) + .ok_or_else(|| "CUDA surface stride overflow".to_string())?; + let len = stride + .checked_mul(usize::try_from(height).map_err(|_| "CUDA surface height overflow")?) + .ok_or_else(|| "CUDA surface byte length overflow".to_string())?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(len) + .map_err(|_| "allocate CUDA Auto-routing surface readback".to_string())?; + bytes.resize(len, 0); + surface + .download_into(&mut bytes, stride) + .map_err(|error| error.to_string())?; + Ok(bytes) +} + +fn assert_surface_route( + surface: &j2k_cuda::Surface, + backend: BackendRequest, +) -> Result<(), String> { + match backend { + BackendRequest::Cpu if surface.backend_kind() == BackendKind::Cpu => Ok(()), + BackendRequest::Cuda + if surface.backend_kind() == BackendKind::Cuda + && surface.residency() == SurfaceResidency::CudaResidentDecode => + { + Ok(()) + } + _ => Err(format!( + "requested {backend:?} but received {:?}/{:?}", + surface.backend_kind(), + surface.residency() + )), + } +} + +fn bench_encode_cell( + criterion: &mut Criterion, + case: &EncodeCase, + operation: AutoRoutingOperation, +) -> AutoRoutingCell { + let cpu = encode_cpu(case, operation) + .unwrap_or_else(|error| panic!("CPU {} {}: {error}", case.id, operation_label(operation))); + let mut probe_accelerator = CudaEncodeStageAccelerator::for_auto_host_output(); + let hybrid = encode_hybrid(case, operation, &mut probe_accelerator).unwrap_or_else(|error| { + panic!( + "hybrid CUDA {} {}: {error}", + case.id, + operation_label(operation) + ) + }); + assert_output_parity(&case.id, operation, &cpu, &hybrid); + + let group_id = format!( + "auto-routing_{}_{id}", + operation_label(operation), + id = case.id + ); + let mut accelerator = probe_accelerator; + let mut group = criterion.benchmark_group(&group_id); + group.bench_function("cpu", |bencher| { + bencher.iter(|| { + std::hint::black_box( + encode_cpu(case, operation).expect("measured CPU JPEG 2000 encode"), + ) + }); + }); + group.bench_function("hybrid", |bencher| { + bencher.iter(|| { + std::hint::black_box( + encode_hybrid(case, operation, &mut accelerator) + .expect("measured hybrid CUDA JPEG 2000 encode"), + ) + }); + }); + group.finish(); + route_cell(&case.id, operation, &group_id, auto_routing_sha256(&cpu)) +} + +fn encode_cpu(case: &EncodeCase, operation: AutoRoutingOperation) -> Result, String> { + match operation { + AutoRoutingOperation::LosslessEncode => { + let encoded = encode_j2k_lossless( + lossless_samples(case)?, + &lossless_options(EncodeBackendPreference::CpuOnly), + ) + .map_err(|error| error.to_string())?; + Ok(encoded.codestream) + } + AutoRoutingOperation::LossyEncode => { + let encoded = encode_j2k_lossy( + lossy_samples(case)?, + &lossy_options(EncodeBackendPreference::CpuOnly), + ) + .map_err(|error| error.to_string())?; + Ok(encoded.codestream) + } + _ => Err("invalid encode operation".to_string()), + } +} + +fn encode_hybrid( + case: &EncodeCase, + operation: AutoRoutingOperation, + accelerator: &mut CudaEncodeStageAccelerator, +) -> Result, String> { + let (codestream, dispatches) = match operation { + AutoRoutingOperation::LosslessEncode => { + let encoded = encode_j2k_lossless_with_accelerator( + lossless_samples(case)?, + &lossless_options(EncodeBackendPreference::Auto), + BackendKind::Cuda, + accelerator, + ) + .map_err(|error| error.to_string())?; + (encoded.codestream, encoded.dispatch_report.total()) + } + AutoRoutingOperation::LossyEncode => { + let encoded = encode_j2k_lossy_with_accelerator( + lossy_samples(case)?, + &lossy_options(EncodeBackendPreference::Auto), + BackendKind::Cuda, + accelerator, + ) + .map_err(|error| error.to_string())?; + (encoded.codestream, encoded.dispatch_report.total()) + } + _ => return Err("invalid encode operation".to_string()), + }; + if dispatches == 0 { + return Err("CUDA hybrid encode did not dispatch any device stage".to_string()); + } + Ok(codestream) +} + +fn lossless_samples(case: &EncodeCase) -> Result, String> { + J2kLosslessSamples::new( + &case.pixels, + case.width, + case.height, + case.components, + 8, + false, + ) + .map_err(|error| error.to_string()) +} + +fn lossy_samples(case: &EncodeCase) -> Result, String> { + J2kLossySamples::new( + &case.pixels, + case.width, + case.height, + case.components, + 8, + false, + ) + .map_err(|error| error.to_string()) +} + +fn lossless_options(backend: EncodeBackendPreference) -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(3)) + .with_validation(J2kEncodeValidation::External) +} + +fn lossy_options(backend: EncodeBackendPreference) -> J2kLossyEncodeOptions { + let mut options = J2kLossyEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(3)) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel(4.0))) + .with_validation(J2kEncodeValidation::External); + options.psnr_iteration_budget = 1; + options +} + +fn benchmark_roi(dimensions: (u32, u32)) -> Rect { + let width = (dimensions.0 / 2).max(1); + let height = (dimensions.1 / 2).max(1); + Rect { + x: dimensions.0.saturating_sub(width) / 2, + y: dimensions.1.saturating_sub(height) / 2, + w: width, + h: height, + } +} + +fn assert_output_parity(case_id: &str, operation: AutoRoutingOperation, cpu: &[u8], hybrid: &[u8]) { + if cpu == hybrid { + return; + } + let first_difference = cpu + .iter() + .zip(hybrid) + .position(|(cpu, hybrid)| cpu != hybrid) + .map(|index| (index, cpu[index], hybrid[index])); + let (mismatch_count, max_delta) = cpu.iter().zip(hybrid).fold( + (0_usize, 0_u8), + |(mismatch_count, max_delta), (&cpu, &hybrid)| { + ( + mismatch_count + usize::from(cpu != hybrid), + max_delta.max(cpu.abs_diff(hybrid)), + ) + }, + ); + panic!( + "CUDA {} output differs for {case_id}: cpu_len={}, hybrid_len={}, mismatch_count={mismatch_count}, max_delta={max_delta}, first_difference={first_difference:?}", + operation_label(operation), + cpu.len(), + hybrid.len(), + ); +} + +const fn pixel_format(format: AutoRoutingPixelFormat) -> PixelFormat { + match format { + AutoRoutingPixelFormat::Gray8 => PixelFormat::Gray8, + AutoRoutingPixelFormat::Rgb8 => PixelFormat::Rgb8, + } +} + +fn required_path(name: &str) -> PathBuf { + PathBuf::from(required_env(name)) +} + +fn required_env(name: &str) -> String { + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set for Auto-routing benchmarks")) +} diff --git a/crates/j2k-cuda/src/allocation.rs b/crates/j2k-cuda/src/allocation.rs index 986d9916..22a199da 100644 --- a/crates/j2k-cuda/src/allocation.rs +++ b/crates/j2k-cuda/src/allocation.rs @@ -90,12 +90,14 @@ impl HostPhaseBudget { Ok(values) } + #[cfg(feature = "cuda-runtime")] pub(crate) fn try_clone_slice(&mut self, source: &[T]) -> Result, Error> { let mut values = self.try_vec_with_capacity(source.len())?; values.extend_from_slice(source); Ok(values) } + #[cfg(feature = "cuda-runtime")] pub(crate) fn try_collect_exact(&mut self, iter: I) -> Result, Error> where I: ExactSizeIterator, diff --git a/crates/j2k-cuda/src/batch/decoder.rs b/crates/j2k-cuda/src/batch/decoder.rs index 28864d82..d9b87a54 100644 --- a/crates/j2k-cuda/src/batch/decoder.rs +++ b/crates/j2k-cuda/src/batch/decoder.rs @@ -2,6 +2,8 @@ //! Persistent CUDA batch decoder facade. +#[cfg(not(feature = "cuda-runtime"))] +use super::IndexedBatchError; #[cfg(feature = "cuda-runtime")] use super::{ decode_warnings, group_pixel_format, native_color_inputs, native_decode_settings, @@ -142,9 +144,20 @@ impl CudaBatchDecoder { if let Some(group) = prepared.groups().first() { return Err(CudaBatchError::group(group, Error::CudaUnavailable)); } + let mut errors = Vec::new(); + errors + .try_reserve_exact(prepared.errors().len()) + .map_err(|_| BatchInfrastructureError::HostAllocationFailed { + what: "CUDA stub indexed errors", + bytes: prepared + .errors() + .len() + .saturating_mul(core::mem::size_of::()), + })?; + errors.extend_from_slice(prepared.errors()); Ok(CudaBatchDecodeResult { groups: Vec::new(), - errors: prepared.errors().to_vec(), + errors, group_errors: Vec::new(), }) } diff --git a/crates/j2k-cuda/src/codec.rs b/crates/j2k-cuda/src/codec.rs index 72afeb30..04285c0a 100644 --- a/crates/j2k-cuda/src/codec.rs +++ b/crates/j2k-cuda/src/codec.rs @@ -13,6 +13,7 @@ use j2k_core::{ use crate::{ allocation::{try_collect_results_exact, try_vec_filled}, + routing::{auto_cuda_available, auto_repeated_decode_uses_cuda, inputs_repeat_one_slice}, runtime::{validate_surface_request, wrap_surface}, }; use crate::{CudaSession, Error, J2kDecoder, Surface}; @@ -298,6 +299,23 @@ impl TileBatchDecodeManyDevice for Codec { if matches!(backend, BackendRequest::Cuda) && Self::supports_cuda_batch_format(fmt) { return Self::decode_tiles_to_cuda_batch(inputs, fmt, &mut session); } + if backend == BackendRequest::Auto + && Self::supports_cuda_batch_format(fmt) + && inputs_repeat_one_slice(inputs) + { + let support = CpuDecoder::inspect_support(inputs[0])?; + if auto_repeated_decode_uses_cuda( + support.info.dimensions, + support.info.components, + fmt, + support.transfer_syntax, + support.payload_kind, + inputs.len(), + ) && auto_cuda_available(&mut session)? + { + return Self::decode_tiles_to_cuda_batch(inputs, fmt, &mut session); + } + } try_collect_results_exact( inputs.iter().map(|input| { diff --git a/crates/j2k-cuda/src/decoder.rs b/crates/j2k-cuda/src/decoder.rs index 070f5b45..18a401de 100644 --- a/crates/j2k-cuda/src/decoder.rs +++ b/crates/j2k-cuda/src/decoder.rs @@ -107,6 +107,8 @@ pub struct J2kDecoder<'a> { )] bytes: &'a [u8], inner: CpuDecoder<'a>, + transfer_syntax: Option, + payload_kind: Option, pool: CpuJ2kScratchPool, } diff --git a/crates/j2k-cuda/src/decoder/api.rs b/crates/j2k-cuda/src/decoder/api.rs index 074cb74d..b7da42f4 100644 --- a/crates/j2k-cuda/src/decoder/api.rs +++ b/crates/j2k-cuda/src/decoder/api.rs @@ -17,18 +17,46 @@ use super::{ J2kDecodeWarning, J2kDecoder, J2kView, PixelFormat, ReadySubmission, Rect, Surface, DEFAULT_MAX_HOST_ALLOCATION_BYTES, }; -use crate::allocation::try_vec_filled; +use crate::{ + allocation::try_vec_filled, + routing::{auto_cuda_available, auto_decode_uses_cuda, AutoDecodeOperation}, +}; impl<'a> J2kDecoder<'a> { /// Create a CUDA-facing decoder from compressed bytes. pub fn new(input: &'a [u8]) -> Result { + let view = J2kView::parse(input)?; + let (transfer_syntax, payload_kind) = view.support_info().map_or((None, None), |support| { + (Some(support.transfer_syntax), Some(support.payload_kind)) + }); Ok(Self { bytes: input, - inner: CpuDecoder::new(input)?, + inner: CpuDecoder::from_view(view)?, + transfer_syntax, + payload_kind, pool: CpuJ2kScratchPool::new(), }) } + fn auto_decode_uses_cuda( + &self, + work_dimensions: (u32, u32), + fmt: PixelFormat, + operation: AutoDecodeOperation, + ) -> bool { + match (self.transfer_syntax, self.payload_kind) { + (Some(transfer_syntax), Some(payload_kind)) => auto_decode_uses_cuda( + work_dimensions, + self.inner.info().components, + fmt, + transfer_syntax, + payload_kind, + operation, + ), + _ => false, + } + } + fn decode_to_surface_impl( &mut self, session: &mut CudaSession, @@ -36,7 +64,15 @@ impl<'a> J2kDecoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - if matches!(backend, BackendRequest::Cuda) { + if matches!(backend, BackendRequest::Cuda) + || (backend == BackendRequest::Auto + && self.auto_decode_uses_cuda( + self.inner.info().dimensions, + fmt, + AutoDecodeOperation::Full, + ) + && auto_cuda_available(session)?) + { return decode_to_cuda_resident_surface_impl(self, session, fmt); } let dims = self.inner.info().dimensions; @@ -73,6 +109,12 @@ impl<'a> J2kDecoder<'a> { DeviceDecodeRequest::Region { roi }, )?; let dims = plan.output_dims(); + if backend == BackendRequest::Auto + && self.auto_decode_uses_cuda(dims, fmt, AutoDecodeOperation::Region) + && auto_cuda_available(session)? + { + return decode_region_to_cuda_resident_surface_impl(self, session, fmt, roi); + } let (mut out, stride) = allocate_cpu_surface(dims, fmt)?; self.inner .decode_region_into(&mut self.pool, &mut out, stride, fmt, plan.source_rect())?; @@ -90,11 +132,18 @@ impl<'a> J2kDecoder<'a> { if matches!(backend, BackendRequest::Cuda) { return decode_scaled_to_cuda_resident_surface_impl(self, session, fmt, scale); } - let dims = DeviceDecodePlan::for_image( + let plan = DeviceDecodePlan::for_image( self.inner.info().dimensions, DeviceDecodeRequest::Scaled { scale }, - )? - .output_dims(); + )?; + let dims = plan.output_dims(); + if backend == BackendRequest::Auto + && scale == Downscale::Half + && self.auto_decode_uses_cuda(dims, fmt, AutoDecodeOperation::ScaledHalf) + && auto_cuda_available(session)? + { + return decode_scaled_to_cuda_resident_surface_impl(self, session, fmt, scale); + } let (mut out, stride) = allocate_cpu_surface(dims, fmt)?; self.inner .decode_scaled_into(&mut self.pool, &mut out, stride, fmt, scale)?; @@ -435,9 +484,14 @@ impl<'a> CpuBackedImageDecode<'a> for J2kDecoder<'a> { fn from_cpu_view(view: Self::View) -> Result { let bytes = view.bytes(); + let (transfer_syntax, payload_kind) = view.support_info().map_or((None, None), |support| { + (Some(support.transfer_syntax), Some(support.payload_kind)) + }); Ok(Self { bytes, inner: CpuDecoder::from_view(view)?, + transfer_syntax, + payload_kind, pool: CpuJ2kScratchPool::new(), }) } diff --git a/crates/j2k-cuda/src/decoder/resident/component/classic.rs b/crates/j2k-cuda/src/decoder/resident/component/classic.rs index c77f0a4d..3fa7568e 100644 --- a/crates/j2k-cuda/src/decoder/resident/component/classic.rs +++ b/crates/j2k-cuda/src/decoder/resident/component/classic.rs @@ -51,11 +51,14 @@ pub(super) fn append_classic_subbands( .iter() .map(|segment| Ok::<_, Error>(cuda_classic_segment_from_plan(segment))), )?; - let jobs = host_budget.try_collect_results_exact( - code_blocks - .iter() - .map(|block| cuda_classic_job_from_plan(block, subband.width, segment_base)), - )?; + let jobs = host_budget.try_collect_results_exact(code_blocks.iter().map(|block| { + cuda_classic_job_from_plan( + block, + subband.width, + subband.irreversible_midpoint, + segment_base, + ) + }))?; let output_words = checked_cuda_element_count(subband.width, subband.height).ok_or( Error::UnsupportedCudaRequest { reason: CUDA_HTJ2K_KERNELS_NOT_READY, @@ -98,6 +101,7 @@ fn cuda_classic_segment_from_plan( fn cuda_classic_job_from_plan( block: &crate::direct_plan::CudaClassicCodeBlock, subband_width: u32, + irreversible_midpoint: bool, segment_base: u32, ) -> Result { let output_offset = block @@ -129,6 +133,8 @@ fn cuda_classic_job_from_plan( sub_band_type: u32::from(block.sub_band_type), style_flags: block.style_flags, strict: block.strict, + irreversible_midpoint, + roi_shift: u32::from(block.roi_shift), dequantization_step: block.dequantization_step, }) } diff --git a/crates/j2k-cuda/src/direct_plan.rs b/crates/j2k-cuda/src/direct_plan.rs index 1ce791b1..97180e16 100644 --- a/crates/j2k-cuda/src/direct_plan.rs +++ b/crates/j2k-cuda/src/direct_plan.rs @@ -138,6 +138,7 @@ pub(crate) struct CudaClassicCodeBlock { pub(crate) missing_bit_planes: u8, pub(crate) number_of_coding_passes: u8, pub(crate) total_bitplanes: u8, + pub(crate) roi_shift: u8, pub(crate) sub_band_type: u8, pub(crate) style_flags: u32, pub(crate) strict: bool, @@ -162,6 +163,7 @@ pub(crate) struct CudaClassicSubband { pub(crate) band_id: CudaHtj2kBandId, pub(crate) width: u32, pub(crate) height: u32, + pub(crate) irreversible_midpoint: bool, pub(crate) code_block_start: u32, pub(crate) code_block_count: u32, } @@ -287,7 +289,7 @@ pub(crate) struct CudaHtj2kDecodePlan { classic_code_blocks: Vec, classic_segments: Vec, #[cfg_attr( - not(feature = "cuda-runtime"), + all(not(feature = "cuda-runtime"), not(test)), expect( dead_code, reason = "classic subband metadata is consumed only by CUDA decode routes" diff --git a/crates/j2k-cuda/src/direct_plan/classic.rs b/crates/j2k-cuda/src/direct_plan/classic.rs index b0c8b5a2..866e34ef 100644 --- a/crates/j2k-cuda/src/direct_plan/classic.rs +++ b/crates/j2k-cuda/src/direct_plan/classic.rs @@ -41,6 +41,7 @@ pub(super) fn append_classic_subband( band_id: subband.band_id, width: subband.width, height: subband.height, + irreversible_midpoint: subband.irreversible_midpoint, code_block_start, code_block_count: checked_u32( owners.classic_code_blocks.len() - code_block_start as usize, @@ -88,6 +89,7 @@ fn append_classic_job_metadata( missing_bit_planes: job.missing_bit_planes, number_of_coding_passes: job.number_of_coding_passes, total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, sub_band_type: classic_subband_type(job.sub_band_type), style_flags: classic_style_flags(job.style), strict: job.strict, @@ -97,16 +99,19 @@ fn append_classic_job_metadata( } fn validate_classic_job(job: &J2kOwnedCodeBlockBatchJob, payload_len: usize) -> Result<(), Error> { - if job.roi_shift != 0 - || !(1..=64).contains(&job.width) + let Some(coded_bitplanes) = job.total_bitplanes.checked_add(job.roi_shift) else { + return invalid_classic_plan(); + }; + if !(1..=64).contains(&job.width) || !(1..=64).contains(&job.height) || !(1..=31).contains(&job.total_bitplanes) - || job.missing_bit_planes >= job.total_bitplanes + || coded_bitplanes > 31 + || job.missing_bit_planes >= coded_bitplanes { return invalid_classic_plan(); } - let coded_bitplanes = job.total_bitplanes - job.missing_bit_planes; - let max_passes = 1 + 3 * (coded_bitplanes - 1); + let decoded_bitplanes = coded_bitplanes - job.missing_bit_planes; + let max_passes = 1 + 3 * (decoded_bitplanes - 1); if job.number_of_coding_passes > max_passes { return invalid_classic_plan(); } diff --git a/crates/j2k-cuda/src/direct_plan/classic/referenced.rs b/crates/j2k-cuda/src/direct_plan/classic/referenced.rs index a6d78353..398c32d5 100644 --- a/crates/j2k-cuda/src/direct_plan/classic/referenced.rs +++ b/crates/j2k-cuda/src/direct_plan/classic/referenced.rs @@ -62,6 +62,7 @@ pub(in crate::direct_plan) fn append_referenced_classic_subband<'a>( band_id: subband.band_id, width: subband.width, height: subband.height, + irreversible_midpoint: subband.irreversible_midpoint, code_block_start, code_block_count: checked_u32( owners.classic_code_blocks.len() - code_block_start as usize, diff --git a/crates/j2k-cuda/src/direct_plan/tests.rs b/crates/j2k-cuda/src/direct_plan/tests.rs index 27b8d34c..baee7238 100644 --- a/crates/j2k-cuda/src/direct_plan/tests.rs +++ b/crates/j2k-cuda/src/direct_plan/tests.rs @@ -3,7 +3,8 @@ use super::*; use j2k_core::CodecError; use j2k_native::{ - HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, J2kDirectIdwtStep, J2kDirectStoreStep, J2kRect, + encode, DecodeSettings, DecoderContext, EncodeOptions, HtOwnedCodeBlockBatchJob, + HtOwnedSubBandPlan, Image, J2kDirectIdwtStep, J2kDirectStoreStep, J2kRect, }; fn one_block_direct_plan( @@ -147,6 +148,74 @@ fn two_block_direct_plan() -> J2kDirectGrayscalePlan { } } +#[test] +fn classic_cuda_plan_retains_irreversible_midpoint_reconstruction() { + let pixels = j2k_test_support::gradient_u8(16, 16, 1); + let bytes = encode( + &pixels, + 16, + 16, + 1, + 8, + false, + &EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode irreversible grayscale"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct plan"); + + let cuda = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect("CUDA plan"); + + assert!( + cuda.classic_subbands + .iter() + .all(|subband| subband.irreversible_midpoint), + "every classic CUDA sub-band must retain the 9/7 reconstruction rule" + ); +} + +#[test] +fn classic_cuda_plan_accepts_roi_maxshift() { + let pixels = j2k_test_support::gradient_u8(16, 16, 1); + let bytes = encode( + &pixels, + 16, + 16, + 1, + 8, + false, + &EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + roi_component_shifts: vec![7], + ..EncodeOptions::default() + }, + ) + .expect("encode ROI maxshift grayscale"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let direct = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct ROI plan"); + + let cuda = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect("CUDA classic ROI plan"); + assert!( + cuda.classic_code_blocks + .iter() + .all(|block| block.roi_shift == 7), + "every classic CUDA code-block must retain the component ROI maxshift" + ); +} + #[test] fn append_payload_to_shared_offsets_blocks_and_drains_local_payload() { let mut first = one_block_plan(vec![1, 2]); @@ -225,7 +294,7 @@ fn rejects_block_length_mismatch() { } #[test] -fn rejects_roi_maxshift_jobs() { +fn rejects_ht_roi_maxshift_jobs() { let mut direct = one_block_direct_plan(1, 0, vec![0xAA], 1); let J2kDirectGrayscaleStep::HtSubBand(subband) = &mut direct.steps[0] else { panic!("fixture starts with one HT sub-band"); diff --git a/crates/j2k-cuda/src/encode/stage.rs b/crates/j2k-cuda/src/encode/stage.rs index f69b7f66..f4609e4e 100644 --- a/crates/j2k-cuda/src/encode/stage.rs +++ b/crates/j2k-cuda/src/encode/stage.rs @@ -40,9 +40,7 @@ use super::stage_error::{adapter_error, arithmetic_overflow, CudaStageResult}; #[cfg(feature = "cuda-runtime")] mod dwt_output; #[cfg(feature = "cuda-runtime")] -pub(super) use self::dwt_output::cuda_dwt53_output_to_j2k; -#[cfg(feature = "cuda-runtime")] -use self::dwt_output::cuda_dwt97_output_to_j2k; +pub(super) use self::dwt_output::{cuda_dwt53_output_to_j2k, cuda_dwt97_output_to_j2k}; macro_rules! emit_cuda_encode_route { ($(($key:expr, $value:expr)),+ $(,)?) => {{ @@ -137,6 +135,10 @@ impl CudaEncodeStageAccelerator { { self.device_unavailable_observed = false; } + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = self; + } } pub(super) const fn device_unavailable_observed(&self) -> bool { @@ -146,6 +148,7 @@ impl CudaEncodeStageAccelerator { } #[cfg(not(feature = "cuda-runtime"))] { + let _ = self; true } } @@ -503,6 +506,15 @@ impl J2kEncodeStageAccelerator for CudaEncodeStageAccelerator { job: J2kDeinterleaveToF32Job<'_>, ) -> CudaStageResult>>> { self.deinterleave_attempts = self.deinterleave_attempts.saturating_add(1); + if job.num_components > 4 { + emit_cuda_encode_route!( + ("op", "encode_deinterleave"), + ("decision", "cpu_fallback"), + ("reason", "component_count_unsupported"), + ("components", job.num_components), + ); + return Ok(None); + } #[cfg(feature = "cuda-runtime")] if let Some(context) = self.cuda_context()? { let num_components = cuda_component_count_u8( diff --git a/crates/j2k-cuda/src/encode/stage/dwt_output.rs b/crates/j2k-cuda/src/encode/stage/dwt_output.rs index 81c6b3a3..c45b5638 100644 --- a/crates/j2k-cuda/src/encode/stage/dwt_output.rs +++ b/crates/j2k-cuda/src/encode/stage/dwt_output.rs @@ -43,7 +43,7 @@ pub(in crate::encode) fn cuda_dwt53_output_to_j2k( }) } -pub(super) fn cuda_dwt97_output_to_j2k( +pub(in crate::encode) fn cuda_dwt97_output_to_j2k( output: &CudaDwt97Output, ) -> CudaStageResult { let (ll_width, ll_height) = output.ll_dimensions(); diff --git a/crates/j2k-cuda/src/encode/tests/mod.rs b/crates/j2k-cuda/src/encode/tests/mod.rs index 77d96765..779fff2b 100644 --- a/crates/j2k-cuda/src/encode/tests/mod.rs +++ b/crates/j2k-cuda/src/encode/tests/mod.rs @@ -15,9 +15,9 @@ use super::packetization::{ cuda_ht_segment_lengths, flatten_cuda_htj2k_packetization_job, CudaHtj2kPacketizationPlanError, CudaHtj2kPacketizationPlanTagNodeState, }; -#[cfg(feature = "cuda-runtime")] -use super::stage::cuda_dwt53_output_to_j2k; use super::stage::cuda_packetization_plan_fallback_reason; +#[cfg(feature = "cuda-runtime")] +use super::stage::{cuda_dwt53_output_to_j2k, cuda_dwt97_output_to_j2k}; #[cfg(not(feature = "cuda-runtime"))] use super::CudaEncodeFallbackReason; #[cfg(feature = "cuda-runtime")] @@ -52,12 +52,15 @@ use j2k_core::{BackendKind, CodecError}; use j2k_cuda_runtime::{ CudaContext, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, CudaJ2kQuantizeJob, }; -#[cfg(feature = "cuda-runtime")] -use j2k_native::forward_dwt53_reference; use j2k_native::{ encode_with_accelerator as encode_with_native_accelerator, DecodeSettings, EncodeOptions, EncodeResult, Image, }; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{ + forward_dwt53_reference, forward_dwt97_reference, forward_ict_reference, + try_deinterleave_reference, +}; fn assert_strict_cuda_classic_tier1_error(err: &E, context: &str) { assert!(err.is_unsupported()); diff --git a/crates/j2k-cuda/src/encode/tests/routing.rs b/crates/j2k-cuda/src/encode/tests/routing.rs index 8c60791f..332d159a 100644 --- a/crates/j2k-cuda/src/encode/tests/routing.rs +++ b/crates/j2k-cuda/src/encode/tests/routing.rs @@ -14,6 +14,8 @@ use super::{ cuda_resident_input_error, encode_j2k_lossy_with_accelerator, BackendKind, J2kLossyEncodeOptions, J2kLossySamples, J2kResidentEncodeInputError, }; +#[cfg(feature = "cuda-runtime")] +use j2k::{encode_j2k_lossy, J2kRateTarget}; #[cfg(feature = "cuda-runtime")] #[test] @@ -183,6 +185,89 @@ fn cuda_lossy_htj2k_facade_require_device_dispatches_supported_stages_when_runti assert_eq!(accelerator.packetization_dispatches(), 1); } +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_auto_host_output_lossy_classic_matches_cpu_for_rgb_fixture_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + + let (pixels, width, height, components) = + if let Some(path) = std::env::var_os("J2K_CUDA_LOSSY_PARITY_PNM") { + let image = j2k_test_support::read_pnm_image(path).expect("read parity PNM fixture"); + ( + image.pixels, + image.width, + image.height, + u16::try_from(image.channels).expect("PNM component count fits u16"), + ) + } else { + const WIDTH: u32 = 640; + const HEIGHT: u32 = 480; + let pixels = (0u32..WIDTH * HEIGHT) + .flat_map(|index| { + let x = index % WIDTH; + let y = index / WIDTH; + [ + u8::try_from((x * 17 + y * 31 + index / 7) & 0xFF) + .expect("masked red sample fits u8"), + u8::try_from((x * 11 + y * 47 + index / 13) & 0xFF) + .expect("masked green sample fits u8"), + u8::try_from((x * 43 + y * 5 + index / 29) & 0xFF) + .expect("masked blue sample fits u8"), + ] + }) + .collect(); + (pixels, WIDTH, HEIGHT, 3) + }; + let samples = || { + J2kLossySamples::new(&pixels, width, height, components, 8, false) + .expect("valid RGB8 samples") + }; + let options = |backend| { + let mut options = J2kLossyEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(3)) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel(4.0))) + .with_validation(J2kEncodeValidation::External); + options.psnr_iteration_budget = 1; + options + }; + + let cpu = encode_j2k_lossy(samples(), &options(EncodeBackendPreference::CpuOnly)) + .expect("CPU lossy encode"); + let mut accelerator = CudaEncodeStageAccelerator::for_auto_host_output(); + let hybrid = encode_j2k_lossy_with_accelerator( + samples(), + &options(EncodeBackendPreference::Auto), + BackendKind::Cuda, + &mut accelerator, + ) + .expect("hybrid CUDA lossy encode"); + + assert!(hybrid.dispatch_report.total() > 0); + if hybrid.codestream != cpu.codestream { + let first_difference = hybrid + .codestream + .iter() + .zip(&cpu.codestream) + .position(|(hybrid, cpu)| hybrid != cpu) + .map(|index| (index, cpu.codestream[index], hybrid.codestream[index])); + let mismatch_count = hybrid + .codestream + .iter() + .zip(&cpu.codestream) + .filter(|(hybrid, cpu)| hybrid != cpu) + .count(); + panic!( + "hybrid lossy codestream differs from CPU: cpu_len={}, hybrid_len={}, mismatch_count={mismatch_count}, first_difference={first_difference:?}", + cpu.codestream.len(), + hybrid.codestream.len(), + ); + } +} + #[test] fn cuda_encode_stage_accelerator_preserves_cpu_codestream_validity() { let pixels: Vec = (0u8..192).collect(); diff --git a/crates/j2k-cuda/src/encode/tests/transforms.rs b/crates/j2k-cuda/src/encode/tests/transforms.rs index 42120da0..afa8b9cb 100644 --- a/crates/j2k-cuda/src/encode/tests/transforms.rs +++ b/crates/j2k-cuda/src/encode/tests/transforms.rs @@ -2,7 +2,9 @@ #[cfg(feature = "cuda-runtime")] use super::{ - cuda_dwt53_output_to_j2k, forward_dwt53_reference, CudaContext, J2kDeinterleaveToF32Job, + cuda_dwt53_output_to_j2k, cuda_dwt97_output_to_j2k, forward_dwt53_reference, + forward_dwt97_reference, forward_ict_reference, try_deinterleave_reference, CudaContext, + J2kDeinterleaveToF32Job, }; #[cfg(feature = "cuda-runtime")] use super::{ @@ -37,6 +39,28 @@ fn cuda_deinterleave_stage_dispatches_when_runtime_required() { ); } +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_deinterleave_declines_more_than_four_components_for_cpu_fallback_when_runtime_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + + let mut accelerator = CudaEncodeStageAccelerator::default(); + let components = accelerator + .encode_deinterleave(J2kDeinterleaveToF32Job { + pixels: &[0; 5], + num_pixels: 1, + num_components: 5, + bit_depth: 8, + signed: false, + }) + .expect("unsupported CUDA component count should decline to the CPU stage"); + + assert!(components.is_none()); + assert_eq!(accelerator.deinterleave_dispatches(), 0); +} + #[cfg(feature = "cuda-runtime")] #[test] fn cuda_forward_rct_dispatches_when_runtime_required() { @@ -114,6 +138,131 @@ fn cuda_forward_ict_dispatches_when_runtime_required() { assert_eq!(accelerator.forward_ict_dispatches(), 1); } +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_forward_ict_matches_native_for_external_parity_fixture_when_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let Some(path) = std::env::var_os("J2K_CUDA_LOSSY_PARITY_PNM") else { + return; + }; + let image = j2k_test_support::read_pnm_image(path).expect("read parity PNM fixture"); + assert_eq!(image.channels, 3, "ICT parity fixture must be RGB"); + let num_pixels = usize::try_from(image.width) + .expect("fixture width fits usize") + .checked_mul(usize::try_from(image.height).expect("fixture height fits usize")) + .expect("fixture sample count"); + let native_planes = try_deinterleave_reference(&image.pixels, num_pixels, 3, 8, false) + .expect("native deinterleave"); + let expected = forward_ict_reference(native_planes.clone()); + let mut actual = native_planes; + let context = CudaContext::system_default().expect("CUDA context"); + let (plane0, rest) = actual.split_at_mut(1); + let (plane1, plane2) = rest.split_at_mut(1); + context + .j2k_forward_ict(&mut plane0[0], &mut plane1[0], &mut plane2[0]) + .expect("CUDA forward ICT"); + + for (component, (actual_plane, expected_plane)) in actual.iter().zip(&expected).enumerate() { + if let Some(index) = actual_plane + .iter() + .zip(expected_plane) + .position(|(actual, expected)| actual.to_bits() != expected.to_bits()) + { + let pixel = &image.pixels[index * 3..index * 3 + 3]; + panic!( + "forward ICT differs at component {component}, sample {index}, RGB={pixel:?}: CPU={:?} ({:x?}), CUDA={:?} ({:x?})", + expected.iter().map(|plane| plane[index]).collect::>(), + expected + .iter() + .map(|plane| plane[index].to_bits()) + .collect::>(), + actual.iter().map(|plane| plane[index]).collect::>(), + actual + .iter() + .map(|plane| plane[index].to_bits()) + .collect::>(), + ); + } + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_forward_dwt97_matches_native_for_external_parity_fixture_when_required() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let Some(path) = std::env::var_os("J2K_CUDA_LOSSY_PARITY_PNM") else { + return; + }; + let image = j2k_test_support::read_pnm_image(path).expect("read parity PNM fixture"); + assert_eq!(image.channels, 3, "DWT parity fixture must be RGB"); + let num_pixels = usize::try_from(image.width) + .expect("fixture width fits usize") + .checked_mul(usize::try_from(image.height).expect("fixture height fits usize")) + .expect("fixture sample count"); + let native_planes = try_deinterleave_reference(&image.pixels, num_pixels, 3, 8, false) + .expect("native deinterleave"); + let transformed = forward_ict_reference(native_planes); + let context = CudaContext::system_default().expect("CUDA context"); + + for (component, plane) in transformed.iter().enumerate() { + let expected = forward_dwt97_reference(plane, image.width, image.height, 3) + .expect("native forward DWT 9/7 reference"); + let cuda = context + .j2k_forward_dwt97(plane, image.width, image.height, 3) + .expect("CUDA forward DWT 9/7"); + let actual = cuda_dwt97_output_to_j2k(&cuda).expect("reshape CUDA forward DWT 9/7"); + + assert_f32_bits_equal(component, "LL", &actual.ll, &expected.ll); + assert_eq!(actual.levels.len(), expected.levels.len()); + for (level, (actual, expected)) in actual.levels.iter().zip(&expected.levels).enumerate() { + assert_f32_bits_equal( + component, + &format!("level {level} HL"), + &actual.hl, + &expected.hl, + ); + assert_f32_bits_equal( + component, + &format!("level {level} LH"), + &actual.lh, + &expected.lh, + ); + assert_f32_bits_equal( + component, + &format!("level {level} HH"), + &actual.hh, + &expected.hh, + ); + } + } +} + +#[cfg(feature = "cuda-runtime")] +fn assert_f32_bits_equal(component: usize, band: &str, actual: &[f32], expected: &[f32]) { + assert_eq!( + actual.len(), + expected.len(), + "component {component} {band} length" + ); + if let Some(index) = actual + .iter() + .zip(expected) + .position(|(actual, expected)| actual.to_bits() != expected.to_bits()) + { + panic!( + "forward DWT 9/7 differs at component {component} {band} sample {index}: CPU={} ({:#010x}), CUDA={} ({:#010x})", + expected[index], + expected[index].to_bits(), + actual[index], + actual[index].to_bits(), + ); + } +} + #[cfg(feature = "cuda-runtime")] #[test] fn cuda_forward_dwt53_dispatches_when_runtime_required() { diff --git a/crates/j2k-cuda/src/lib.rs b/crates/j2k-cuda/src/lib.rs index ada5da85..30ba5427 100644 --- a/crates/j2k-cuda/src/lib.rs +++ b/crates/j2k-cuda/src/lib.rs @@ -19,6 +19,7 @@ mod direct_plan; mod encode; mod error; mod profile; +mod routing; mod runtime; mod session; mod surface; diff --git a/crates/j2k-cuda/src/routing.rs b/crates/j2k-cuda/src/routing.rs new file mode 100644 index 00000000..1bfb8aee --- /dev/null +++ b/crates/j2k-cuda/src/routing.rs @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_core::{CompressedPayloadKind, CompressedTransferSyntax, PixelFormat}; + +use crate::{CudaSession, Error}; + +// Minimum qualified cells from verified CUDA Auto-routing artifact +// ded1eb045f9673e5bbe64dc873be3ba227ecb61ec11b6c9ad53653dbcc993f44. +// These thresholds apply only to measured raw Part 1 Gray8/Rgb8 surfaces, +// including host readback. No CUDA encode cell qualified. +const RGB_SMALL_FULL: (u32, u32) = (256, 149); +const RGB_SMALL_ROI: (u32, u32) = (128, 74); +const MEDIUM_FULL: (u32, u32) = (640, 480); +const MEDIUM_HALF: (u32, u32) = (320, 240); +const GRAY_LARGE_FULL: (u32, u32) = (3323, 891); +const GRAY_LARGE_ROI: (u32, u32) = (1661, 445); +const GRAY_LARGE_HALF: (u32, u32) = (1662, 446); +const RGB_LARGE_HALF: (u32, u32) = (1296, 972); +const REPEATED_DECODE_MIN_COUNT: usize = 16; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum AutoDecodeOperation { + Full, + Region, + ScaledHalf, +} + +pub(crate) fn auto_decode_uses_cuda( + work_dimensions: (u32, u32), + source_components: u16, + fmt: PixelFormat, + transfer_syntax: CompressedTransferSyntax, + payload_kind: CompressedPayloadKind, + operation: AutoDecodeOperation, +) -> bool { + if payload_kind != CompressedPayloadKind::Jpeg2000Codestream { + return false; + } + let minimum = match (operation, source_components, fmt, transfer_syntax) { + ( + AutoDecodeOperation::Full, + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossless, + ) => RGB_SMALL_FULL, + ( + AutoDecodeOperation::Region, + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossless, + ) => RGB_SMALL_ROI, + ( + AutoDecodeOperation::Full + | AutoDecodeOperation::Region + | AutoDecodeOperation::ScaledHalf, + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossy, + ) => match operation { + AutoDecodeOperation::Full => MEDIUM_FULL, + AutoDecodeOperation::Region | AutoDecodeOperation::ScaledHalf => MEDIUM_HALF, + }, + ( + AutoDecodeOperation::Full, + 1, + PixelFormat::Gray8, + CompressedTransferSyntax::Jpeg2000Lossless, + ) => MEDIUM_FULL, + ( + AutoDecodeOperation::Full + | AutoDecodeOperation::Region + | AutoDecodeOperation::ScaledHalf, + 1, + PixelFormat::Gray8, + CompressedTransferSyntax::Jpeg2000Lossy, + ) => match operation { + AutoDecodeOperation::Full => GRAY_LARGE_FULL, + AutoDecodeOperation::Region => GRAY_LARGE_ROI, + AutoDecodeOperation::ScaledHalf => GRAY_LARGE_HALF, + }, + ( + AutoDecodeOperation::ScaledHalf, + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossless, + ) => RGB_LARGE_HALF, + _ => return false, + }; + dimensions_at_least(work_dimensions, minimum) +} + +pub(crate) fn auto_repeated_decode_uses_cuda( + dimensions: (u32, u32), + source_components: u16, + fmt: PixelFormat, + transfer_syntax: CompressedTransferSyntax, + payload_kind: CompressedPayloadKind, + count: usize, +) -> bool { + if count < REPEATED_DECODE_MIN_COUNT + || payload_kind != CompressedPayloadKind::Jpeg2000Codestream + { + return false; + } + let minimum = match (source_components, fmt, transfer_syntax) { + ( + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossless | CompressedTransferSyntax::Jpeg2000Lossy, + ) => RGB_SMALL_FULL, + (1, PixelFormat::Gray8, CompressedTransferSyntax::Jpeg2000Lossless) => MEDIUM_FULL, + (1, PixelFormat::Gray8, CompressedTransferSyntax::Jpeg2000Lossy) => GRAY_LARGE_FULL, + _ => return false, + }; + dimensions_at_least(dimensions, minimum) +} + +pub(crate) fn inputs_repeat_one_slice(inputs: &[&[u8]]) -> bool { + let Some(first) = inputs.first().copied() else { + return false; + }; + inputs + .iter() + .copied() + .all(|input| core::ptr::eq(input, first)) +} + +pub(crate) fn auto_cuda_available(session: &mut CudaSession) -> Result { + #[cfg(feature = "cuda-runtime")] + { + match session.cuda_context() { + Ok(_) => Ok(true), + Err(Error::CudaUnavailable) => Ok(false), + Err(error) => Err(error), + } + } + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = session; + Ok(false) + } +} + +fn dimensions_at_least(actual: (u32, u32), minimum: (u32, u32)) -> bool { + actual.0 >= minimum.0 && actual.1 >= minimum.1 +} + +#[cfg(test)] +mod tests { + use j2k_core::{CompressedPayloadKind, CompressedTransferSyntax, PixelFormat}; + + use super::{ + auto_decode_uses_cuda, auto_repeated_decode_uses_cuda, inputs_repeat_one_slice, + AutoDecodeOperation, + }; + + #[test] + fn single_image_thresholds_match_verified_external_cells() { + use AutoDecodeOperation::{Full, Region, ScaledHalf}; + use CompressedTransferSyntax::{Jpeg2000Lossless as Lossless, Jpeg2000Lossy as Lossy}; + const RAW: CompressedPayloadKind = CompressedPayloadKind::Jpeg2000Codestream; + + let cases = [ + ((256, 149), 3, PixelFormat::Rgb8, Lossless, Full, true), + ((128, 74), 3, PixelFormat::Rgb8, Lossless, Region, true), + ((128, 75), 3, PixelFormat::Rgb8, Lossless, ScaledHalf, false), + ((256, 149), 3, PixelFormat::Rgb8, Lossy, Full, false), + ((640, 480), 3, PixelFormat::Rgb8, Lossy, Full, true), + ((320, 240), 3, PixelFormat::Rgb8, Lossy, Region, true), + ((320, 240), 3, PixelFormat::Rgb8, Lossy, ScaledHalf, true), + ((640, 480), 1, PixelFormat::Gray8, Lossless, Full, true), + ((320, 240), 1, PixelFormat::Gray8, Lossless, Region, false), + ( + (320, 240), + 1, + PixelFormat::Gray8, + Lossless, + ScaledHalf, + false, + ), + ((3323, 891), 1, PixelFormat::Gray8, Lossy, Full, true), + ((1661, 445), 1, PixelFormat::Gray8, Lossy, Region, true), + ((1662, 446), 1, PixelFormat::Gray8, Lossy, ScaledHalf, true), + ( + (1296, 972), + 3, + PixelFormat::Rgb8, + Lossless, + ScaledHalf, + true, + ), + ]; + for (dimensions, components, fmt, transfer_syntax, operation, expected) in cases { + assert_eq!( + auto_decode_uses_cuda(dimensions, components, fmt, transfer_syntax, RAW, operation,), + expected, + "{dimensions:?} {fmt:?} {transfer_syntax:?} {operation:?}", + ); + } + } + + #[test] + fn auto_decode_keeps_unmeasured_surfaces_on_cpu() { + for transfer_syntax in [ + CompressedTransferSyntax::HtJpeg2000Lossless, + CompressedTransferSyntax::HtJpeg2000Lossy, + ] { + assert!(!auto_decode_uses_cuda( + (4096, 4096), + 3, + PixelFormat::Rgb8, + transfer_syntax, + CompressedPayloadKind::Jpeg2000Codestream, + AutoDecodeOperation::Full, + )); + } + for fmt in [ + PixelFormat::Gray16, + PixelFormat::Rgb16, + PixelFormat::Rgba8, + PixelFormat::Rgba16, + ] { + assert!(!auto_decode_uses_cuda( + (4096, 4096), + match fmt { + PixelFormat::Gray16 => 1, + PixelFormat::Rgb16 => 3, + PixelFormat::Rgba8 | PixelFormat::Rgba16 => 4, + _ => unreachable!("test enumerates only higher-depth and alpha formats"), + }, + fmt, + CompressedTransferSyntax::Jpeg2000Lossy, + CompressedPayloadKind::Jpeg2000Codestream, + AutoDecodeOperation::Full, + )); + } + } + + #[test] + fn auto_decode_requires_the_measured_source_component_count() { + assert!(!auto_decode_uses_cuda( + (2592, 1944), + 1, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossless, + CompressedPayloadKind::Jpeg2000Codestream, + AutoDecodeOperation::Full, + )); + assert!(!auto_decode_uses_cuda( + (3323, 891), + 3, + PixelFormat::Gray8, + CompressedTransferSyntax::Jpeg2000Lossy, + CompressedPayloadKind::Jpeg2000Codestream, + AutoDecodeOperation::Full, + )); + } + + #[test] + fn auto_decode_keeps_wrapped_and_below_threshold_work_on_cpu() { + assert!(!auto_decode_uses_cuda( + (640, 480), + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossy, + CompressedPayloadKind::Jp2File, + AutoDecodeOperation::Full, + )); + assert!(!auto_decode_uses_cuda( + (319, 240), + 3, + PixelFormat::Rgb8, + CompressedTransferSyntax::Jpeg2000Lossy, + CompressedPayloadKind::Jpeg2000Codestream, + AutoDecodeOperation::Region, + )); + } + + #[test] + fn repeated_batch_thresholds_match_verified_external_cells() { + use CompressedTransferSyntax::{Jpeg2000Lossless as Lossless, Jpeg2000Lossy as Lossy}; + const RAW: CompressedPayloadKind = CompressedPayloadKind::Jpeg2000Codestream; + + let cases = [ + ((256, 149), 3, PixelFormat::Rgb8, Lossy, 16, true), + ((640, 480), 1, PixelFormat::Gray8, Lossless, 16, true), + ((256, 149), 1, PixelFormat::Gray8, Lossless, 16, false), + ((2592, 1944), 3, PixelFormat::Rgb8, Lossless, 15, false), + ]; + for (dimensions, components, fmt, transfer_syntax, count, expected) in cases { + assert_eq!( + auto_repeated_decode_uses_cuda( + dimensions, + components, + fmt, + transfer_syntax, + RAW, + count, + ), + expected, + "{dimensions:?} {fmt:?} {transfer_syntax:?} count={count}", + ); + } + } + + #[test] + fn repeated_batch_requires_one_shared_input_slice() { + let bytes = [1, 2, 3, 4]; + let copied = bytes; + + assert!(inputs_repeat_one_slice(&[&bytes, &bytes])); + assert!(!inputs_repeat_one_slice(&[&bytes, &copied])); + assert!(!inputs_repeat_one_slice(&[])); + } +} diff --git a/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs b/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs index 279eba77..8b54f8f3 100644 --- a/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs +++ b/crates/j2k-cuda/tests/batch_decoder_api/classic_native.rs @@ -92,7 +92,7 @@ fn prepared_classic_multitile_gray_and_rgb_are_resident_and_external_bit_exact() } #[test] -fn classic_irreversible_gray_and_rgb_match_cpu_within_one_lsb_for_all_requests_and_layouts() { +fn classic_irreversible_gray_and_rgb_match_cpu_exactly_for_all_requests_and_layouts() { if !j2k_test_support::cuda_runtime_gate(module_path!()) { return; } @@ -130,6 +130,34 @@ fn classic_irreversible_gray_and_rgb_match_cpu_within_one_lsb_for_all_requests_a } } +#[test] +fn classic_roi_maxshift_gray_and_rgb_match_cpu_exactly() { + if !j2k_test_support::cuda_runtime_gate(module_path!()) { + return; + } + let context = CudaContext::system_default().expect("CUDA context"); + let requests = [ + DecodeRequest::Full, + DecodeRequest::RegionReduced { + roi: Rect { + x: 2, + y: 4, + w: 10, + h: 8, + }, + scale: Downscale::Half, + }, + ]; + for channels in [1, 3] { + let encoded = classic_roi_fixture(channels); + for layout in [BatchLayout::Nhwc, BatchLayout::Nchw] { + for request in requests { + assert_classic_case(&context, &encoded, layout, request); + } + } + } +} + fn classic_fixtures(channels: u16) -> [Arc<[u8]>; 3] { let sample_count = 16 * 16 * channels as usize; let u8_samples = (0..sample_count) @@ -227,6 +255,31 @@ fn classic_irreversible_fixture(channels: u16) -> Arc<[u8]> { ) } +fn classic_roi_fixture(channels: u16) -> Arc<[u8]> { + let sample_count = 16 * 16 * channels as usize; + let samples = (0..sample_count) + .map(|index| u8::try_from((index * 53 + 11) & 0xff).expect("masked sample fits u8")) + .collect::>(); + Arc::from( + j2k_native::encode( + &samples, + 16, + 16, + channels, + 8, + false, + &j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + use_mct: channels == 3, + roi_component_shifts: vec![7; channels as usize], + ..j2k_native::EncodeOptions::default() + }, + ) + .expect("encode classic ROI fixture"), + ) +} + fn native_bytes(samples: &[T]) -> Vec { // SAFETY: integers have no invalid bit patterns and the returned bytes are copied. unsafe { @@ -361,13 +414,21 @@ fn assert_classic_irreversible_case( allocation .copy_to_host(&mut external) .expect("download classic irreversible external output"); - assert_within_one_lsb(&external, expected, "external", layout, request); + assert_eq!( + external.as_slice(), + expected.as_slice(), + "external {layout:?} {request:?}" + ); let resident = decoder .decode_prepared(&prepared) .expect("decode classic irreversible resident batch"); let actual = download_resident_bytes(&resident.groups()[0], expected.len()); - assert_within_one_lsb(&actual, expected, "resident", layout, request); + assert_eq!( + actual.as_slice(), + expected.as_slice(), + "resident {layout:?} {request:?}" + ); } fn download_resident_bytes(group: &CudaBatchGroup, expected_len: usize) -> Vec { @@ -384,22 +445,6 @@ fn download_resident_bytes(group: &CudaBatchGroup, expected_len: usize) -> Vec Vec { match samples { CpuBatchSamples::U8(samples) => samples.clone(), diff --git a/crates/j2k-cuda/tests/classic_tier1_parity.rs b/crates/j2k-cuda/tests/classic_tier1_parity.rs index 6d9ef42a..9748bff9 100644 --- a/crates/j2k-cuda/tests/classic_tier1_parity.rs +++ b/crates/j2k-cuda/tests/classic_tier1_parity.rs @@ -241,6 +241,8 @@ fn cuda_job( sub_band_type: subband_tag(case.subband), style_flags: style_flags(case.style), strict, + irreversible_midpoint: false, + roi_shift: 0, dequantization_step: 1.0, } } @@ -355,7 +357,9 @@ fn check_truncated_bypass( } let expected = native_decode(case, encoded, data, &segments, false) .expect("native lenient truncated bypass decode"); - assert!(native_decode(case, encoded, data, &segments, true).is_err()); + let expected_strict = native_decode(case, encoded, data, &segments, true) + .expect("native strict truncated bypass decode extends a clean segment end"); + assert_eq!(expected_strict, expected, "native strict truncated parity"); let actual = cuda_decode( context, pool, @@ -366,16 +370,14 @@ fn check_truncated_bypass( ) .expect("CUDA lenient truncated bypass decode"); assert_eq!(actual, expected, "lenient truncated parity"); - assert!( - cuda_decode( - context, - pool, - data, - cuda_job(case, encoded, truncated_len, true), - &cuda_segments(&segments), - output_words, - ) - .is_err(), - "CUDA strict truncated bypass decode must fail" - ); + let actual_strict = cuda_decode( + context, + pool, + data, + cuda_job(case, encoded, truncated_len, true), + &cuda_segments(&segments), + output_words, + ) + .expect("CUDA strict truncated bypass decode extends a clean segment end"); + assert_eq!(actual_strict, expected_strict, "strict truncated parity"); } diff --git a/crates/j2k-cuda/tests/host_surface.rs b/crates/j2k-cuda/tests/host_surface.rs index 9961a68c..50473884 100644 --- a/crates/j2k-cuda/tests/host_surface.rs +++ b/crates/j2k-cuda/tests/host_surface.rs @@ -1,7 +1,7 @@ use j2k_core::{ - BackendRequest, CodecError, DeviceSubmission, DeviceSurface, Downscale, ImageDecode, - ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, - TileBatchDecodeManyDevice, + BackendRequest, CodecError, CompressedTransferSyntax, DeviceSubmission, DeviceSurface, + Downscale, ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, + TileBatchDecodeDevice, TileBatchDecodeManyDevice, }; use j2k_cuda::{Codec, CudaSession, Error, J2kDecoder, SurfaceResidency}; use j2k_native::{encode, EncodeOptions}; @@ -184,6 +184,111 @@ fn auto_falls_back_to_cpu_surface() { assert!(surface.as_host_bytes().is_some()); } +#[test] +fn auto_routes_only_benchmark_qualified_rgb_lossy_cells_when_runtime_required() { + if !cuda_runtime_and_strict_oxide_gate(module_path!()) { + return; + } + let bytes = fixture_classic(640, 480, 3, false); + let support = j2k::J2kDecoder::inspect_support(&bytes).expect("inspect promoted workload"); + assert_eq!(support.info.dimensions, (640, 480)); + assert_eq!( + support.transfer_syntax, + CompressedTransferSyntax::Jpeg2000Lossy + ); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) + .expect("promoted Auto surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + + let roi = Rect { + x: 160, + y: 120, + w: 320, + h: 240, + }; + let mut decoder = J2kDecoder::new(&bytes).expect("ROI decoder"); + let surface = decoder + .decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Auto) + .expect("promoted Auto ROI surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + + let mut decoder = J2kDecoder::new(&bytes).expect("scaled decoder"); + let surface = decoder + .decode_scaled_to_device(PixelFormat::Rgb8, Downscale::Half, BackendRequest::Auto) + .expect("promoted Auto scaled surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + + let mut decoder = J2kDecoder::new(&bytes).expect("quarter-scale decoder"); + let surface = decoder + .decode_scaled_to_device(PixelFormat::Rgb8, Downscale::Quarter, BackendRequest::Auto) + .expect("unmeasured Auto quarter-scale surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + + let mut decoder = J2kDecoder::new(&bytes).expect("tiny ROI decoder"); + let surface = decoder + .decode_region_to_device( + PixelFormat::Rgb8, + Rect { + x: 0, + y: 0, + w: 1, + h: 1, + }, + BackendRequest::Auto, + ) + .expect("unmeasured tiny Auto ROI surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + + let jp2 = j2k::wrap_j2k_codestream(&bytes, j2k::J2kFileWrapOptions::jp2()) + .expect("wrap promoted workload as JP2"); + let mut decoder = J2kDecoder::new(&jp2).expect("JP2 decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) + .expect("unmeasured JP2 Auto surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + + let inputs = vec![bytes.as_slice(); 16]; + let mut context = j2k_cuda::J2kContext::default(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let surfaces = Codec::decode_tiles_to_device( + &mut context, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("promoted repeated Auto batch"); + assert_eq!(surfaces.len(), inputs.len()); + assert!(surfaces.iter().all(|surface| { + surface.backend_kind() == j2k_core::BackendKind::Cuda + && surface.residency() == SurfaceResidency::CudaResidentDecode + })); + + let retained = fixture_classic(256, 149, 3, false); + let retained_support = + j2k::J2kDecoder::inspect_support(&retained).expect("inspect retained workload"); + assert_eq!(retained_support.info.dimensions, (256, 149)); + assert_eq!( + retained_support.transfer_syntax, + CompressedTransferSyntax::Jpeg2000Lossy + ); + let mut decoder = J2kDecoder::new(&retained).expect("retained decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) + .expect("retained Auto surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); +} + #[test] fn explicit_cuda_classic_j2k_request_matches_native() { if !cuda_runtime_and_strict_oxide_gate(module_path!()) { @@ -1642,6 +1747,60 @@ fn decode_tiles_to_device_explicit_cuda_mixed_grayscale_batch_matches_host_bytes assert_eq!(actual, expected); } +#[test] +fn decode_tiles_to_device_explicit_cuda_repeated_large_classic_gray_matches_host_bytes() { + const WIDTH: usize = 640; + const HEIGHT: usize = 480; + + if !cuda_runtime_and_strict_oxide_gate(module_path!()) { + return; + } + let width_u32 = u32::try_from(WIDTH).expect("fixture width fits u32"); + let height_u32 = u32::try_from(HEIGHT).expect("fixture height fits u32"); + let classic = fixture_classic(width_u32, height_u32, 1, true); + let mut single = vec![0; WIDTH * HEIGHT]; + J2kDecoder::new(&classic) + .expect("host decoder") + .decode_into(&mut single, WIDTH, PixelFormat::Gray8) + .expect("host repeated grayscale reference"); + + for (profiled, batch_size) in [(true, 1), (false, 1), (false, 2), (false, 16)] { + let inputs = vec![classic.as_slice(); batch_size]; + let mut session = CudaSession::default(); + let surfaces = if profiled { + J2kDecoder::decode_batch_to_device_with_session_and_profile( + &inputs, + PixelFormat::Gray8, + &mut session, + ) + .map(|(surfaces, _report)| surfaces) + } else { + J2kDecoder::decode_batch_to_device_with_session( + &inputs, + PixelFormat::Gray8, + &mut session, + ) + } + .expect("strict CUDA repeated classic grayscale batch"); + assert_eq!(surfaces.len(), batch_size); + for surface in &surfaces { + assert_resident_cuda_surface(surface); + } + + let actual = j2k_cuda::Surface::download_batch_tight(&surfaces) + .expect("download repeated gray batch"); + let expected = single.repeat(batch_size); + let first_mismatch = actual + .iter() + .zip(&expected) + .position(|(lhs, rhs)| lhs != rhs); + assert_eq!( + first_mismatch, None, + "profiled={profiled} batch size {batch_size} first mismatch: {first_mismatch:?}" + ); + } +} + #[test] fn decode_tiles_to_device_explicit_cuda_rgba8_batch_matches_host_bytes() { let first = fixture_ht_rgb8_pattern(32, 32, 31); diff --git a/crates/j2k-cuda/tests/htj2k_encode_parity.rs b/crates/j2k-cuda/tests/htj2k_encode_parity.rs index 9e3a0796..a44921de 100644 --- a/crates/j2k-cuda/tests/htj2k_encode_parity.rs +++ b/crates/j2k-cuda/tests/htj2k_encode_parity.rs @@ -114,7 +114,7 @@ fn cuda_quantize_reversible_matches_native_reference_when_required() { } // --------------------------------------------------------------------------- -// Test 4: pixel deinterleave (covers 8-bit unsigned, 8-bit signed, 16-bit unsigned) +// Test 4: pixel deinterleave across supported precision and signedness boundaries // --------------------------------------------------------------------------- #[cfg(feature = "cuda-runtime")] @@ -261,6 +261,37 @@ fn cuda_deinterleave_matches_native_reference_when_required() { "16-bit signed deinterleave mismatch" ); } + + // --- 4e: 12-bit signed single-component, including both sign boundaries --- + { + let values: &[u16] = &[0x0800, 0x0fff, 0x0000, 0x07ff]; + let mut pixels: Vec = Vec::with_capacity(values.len() * 2); + for value in values { + pixels.extend_from_slice(&value.to_le_bytes()); + } + let num_pixels = values.len(); + let num_components = 1u8; + let bit_depth = 12u8; + let signed = true; + + let native = try_deinterleave_reference( + &pixels, + num_pixels, + u16::from(num_components), + bit_depth, + signed, + ) + .expect("valid native 12-bit signed deinterleave input"); + let cuda_out = context + .j2k_deinterleave_to_f32(&pixels, num_pixels, num_components, bit_depth, signed) + .expect("CUDA deinterleave 12-bit signed gray"); + + assert_eq!( + cuda_out.components(), + native.as_slice(), + "12-bit signed deinterleave mismatch" + ); + } } // --------------------------------------------------------------------------- diff --git a/crates/j2k-jpeg-cuda/Cargo.toml b/crates/j2k-jpeg-cuda/Cargo.toml index 768ccc5b..b2afc09c 100644 --- a/crates/j2k-jpeg-cuda/Cargo.toml +++ b/crates/j2k-jpeg-cuda/Cargo.toml @@ -27,10 +27,10 @@ cuda-runtime = [ ] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.0", optional = true } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.1", optional = true } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/j2k-jpeg-metal/Cargo.toml b/crates/j2k-jpeg-metal/Cargo.toml index 605caa97..1d49abcc 100644 --- a/crates/j2k-jpeg-metal/Cargo.toml +++ b/crates/j2k-jpeg-metal/Cargo.toml @@ -19,10 +19,10 @@ name = "j2k_jpeg_metal" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.1" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-jpeg/Cargo.toml b/crates/j2k-jpeg/Cargo.toml index 3f389277..398ceacf 100644 --- a/crates/j2k-jpeg/Cargo.toml +++ b/crates/j2k-jpeg/Cargo.toml @@ -27,9 +27,9 @@ bench-libjpeg-turbo = [] bench-internals = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1" } thiserror = { workspace = true } memchr = { workspace = true } rayon = { workspace = true } diff --git a/crates/j2k-jpeg/fuzz/Cargo.lock b/crates/j2k-jpeg/fuzz/Cargo.lock index a97aeec2..d516b86b 100644 --- a/crates/j2k-jpeg/fuzz/Cargo.lock +++ b/crates/j2k-jpeg/fuzz/Cargo.lock @@ -77,18 +77,18 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.8.0" +version = "0.8.1" [[package]] name = "j2k-core" -version = "0.8.0" +version = "0.8.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-jpeg" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -108,7 +108,7 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.0" +version = "0.8.1" [[package]] name = "jobserver" diff --git a/crates/j2k-metal-support/Cargo.toml b/crates/j2k-metal-support/Cargo.toml index ef27d9a1..b8d58f7f 100644 --- a/crates/j2k-metal-support/Cargo.toml +++ b/crates/j2k-metal-support/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_metal_support" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } log = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-metal/Cargo.toml b/crates/j2k-metal/Cargo.toml index 157c4db7..f32ec82c 100644 --- a/crates/j2k-metal/Cargo.toml +++ b/crates/j2k-metal/Cargo.toml @@ -24,12 +24,12 @@ name = "j2k_metal" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k = { path = "../j2k", version = "=0.8.0" } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k = { path = "../j2k", version = "=0.8.1" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] @@ -38,6 +38,7 @@ metal = { workspace = true } rayon = { workspace = true } [dev-dependencies] +criterion = { workspace = true } rayon = { workspace = true } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } syn = { workspace = true, features = ["full"] } @@ -57,3 +58,8 @@ undocumented_unsafe_blocks = "warn" # implementation-quality lint gate to avoid fingerprint-only release churn. must_use_candidate = "allow" missing_errors_doc = "allow" + +[[bench]] +name = "auto_routing" +harness = false +test = false diff --git a/crates/j2k-metal/benches/auto_routing.rs b/crates/j2k-metal/benches/auto_routing.rs new file mode 100644 index 00000000..fd243b85 --- /dev/null +++ b/crates/j2k-metal/benches/auto_routing.rs @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[cfg(not(target_os = "macos"))] +fn main() { + assert!( + std::env::var_os("J2K_REQUIRE_METAL_BENCH").is_none(), + "J2K Metal Auto-routing benchmark requires macOS" + ); + eprintln!("J2K Metal Auto-routing benchmark skipped outside macOS"); +} + +#[cfg(target_os = "macos")] +fn main() { + macos::run(); +} + +#[cfg(target_os = "macos")] +mod macos { + use std::{path::PathBuf, sync::Arc, time::Duration}; + + use criterion::Criterion; + use j2k::{ + encode_j2k_lossless, encode_j2k_lossless_with_accelerator, encode_j2k_lossy, + encode_j2k_lossy_with_accelerator, EncodeBackendPreference, J2kBlockCodingMode, + J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, J2kLossyEncodeOptions, + J2kLossySamples, J2kRateTarget, + }; + use j2k_core::{ + BackendKind, BackendRequest, CompressedTransferSyntax, DeviceSurface, Downscale, + PixelFormat, Rect, + }; + use j2k_metal::{ + J2kDecoder, MetalBackendSession, MetalDecodeRequest, MetalEncodeStageAccelerator, + MetalTileBatch, SurfaceResidency, + }; + use j2k_test_support::{ + append_auto_routing_output as append_output, + auto_routing_operation_label as operation_label, auto_routing_route_cell as route_cell, + auto_routing_sha256, load_auto_routing_manifest, load_auto_routing_pnm, + write_auto_routing_evidence, AutoRoutingBackend, AutoRoutingCell, AutoRoutingEvidence, + AutoRoutingOperation, AutoRoutingPixelFormat, AutoRoutingPlatform, AutoRoutingPnm, + AutoRoutingWorkload, AutoRoutingWorkloadKind, + }; + + const SAMPLE_SIZE: usize = 10; + const WARM_UP: Duration = Duration::from_secs(1); + const MEASUREMENT: Duration = Duration::from_secs(3); + const BATCH_SIZE: usize = 16; + const AUTO_GRAY8_MIN_PIXELS: u64 = 2_960_793; + const AUTO_RGB8_BATCH_MIN_PIXELS: u64 = 307_200; + const AUTO_RGB8_LARGE_MIN_PIXELS: u64 = 5_038_848; + + pub(super) fn run() { + let manifest_path = required_path("J2K_AUTO_ROUTING_MANIFEST"); + let corpus_root = required_path("J2K_AUTO_ROUTING_ROOT"); + let evidence_path = required_path("J2K_AUTO_ROUTING_EVIDENCE"); + let workloads = load_auto_routing_manifest(&manifest_path, &corpus_root) + .unwrap_or_else(|error| panic!("load Metal Auto-routing workloads: {error}")); + let session = MetalBackendSession::system_default() + .unwrap_or_else(|error| panic!("Metal Auto-routing benchmark needs a device: {error}")); + let mut criterion = Criterion::default() + .sample_size(SAMPLE_SIZE) + .warm_up_time(WARM_UP) + .measurement_time(MEASUREMENT) + .configure_from_args(); + let mut cells = Vec::new(); + + for workload in &workloads.workloads { + match workload.kind { + AutoRoutingWorkloadKind::Decode => { + let decode = DecodeCase::new(workload); + for operation in [ + AutoRoutingOperation::FullDecode, + AutoRoutingOperation::RoiDecode, + AutoRoutingOperation::ScaledDecode, + AutoRoutingOperation::BatchDecode, + ] { + cells.push(bench_decode_cell( + &mut criterion, + &decode, + operation, + &session, + )); + } + } + AutoRoutingWorkloadKind::Encode => { + let encode = load_auto_routing_pnm(workload).unwrap_or_else(|error| { + panic!("load Metal encode workload {}: {error}", workload.id) + }); + for operation in [ + AutoRoutingOperation::LosslessEncode, + AutoRoutingOperation::LossyEncode, + ] { + cells.push(bench_encode_cell(&mut criterion, &encode, operation)); + } + } + } + } + criterion.final_summary(); + + let evidence = AutoRoutingEvidence { + schema_version: 1, + candidate_sha: required_env("J2K_AUTO_ROUTING_CANDIDATE_SHA"), + backend: AutoRoutingBackend::Metal, + platform: AutoRoutingPlatform { + os: "macos".to_string(), + arch: "aarch64".to_string(), + hardware: required_env("J2K_AUTO_ROUTING_HARDWARE"), + driver: required_env("J2K_AUTO_ROUTING_DRIVER"), + }, + external_manifest_sha256: workloads.manifest_sha256, + external_case_count: workloads.workloads.len(), + cells, + }; + write_auto_routing_evidence(&evidence_path, &evidence) + .unwrap_or_else(|error| panic!("write Metal Auto-routing evidence: {error}")); + } + + struct DecodeCase<'a> { + id: &'a str, + bytes: &'a [u8], + fmt: PixelFormat, + dimensions: (u32, u32), + transfer_syntax: CompressedTransferSyntax, + shared: Arc<[u8]>, + } + + impl<'a> DecodeCase<'a> { + fn new(workload: &'a AutoRoutingWorkload) -> Self { + let fmt = pixel_format(workload.pixel_format); + let info = j2k::J2kDecoder::inspect(&workload.bytes) + .unwrap_or_else(|error| panic!("inspect decode workload {}: {error}", workload.id)); + let support = j2k::J2kDecoder::inspect_support(&workload.bytes) + .unwrap_or_else(|error| panic!("inspect decode support {}: {error}", workload.id)); + Self { + id: &workload.id, + bytes: &workload.bytes, + fmt, + dimensions: info.dimensions, + transfer_syntax: support.transfer_syntax, + shared: Arc::from(workload.bytes.clone()), + } + } + } + + type EncodeCase = AutoRoutingPnm; + + fn bench_decode_cell( + criterion: &mut Criterion, + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, + session: &MetalBackendSession, + ) -> AutoRoutingCell { + let cpu = + decode_once(case, operation, BackendRequest::Cpu, session).unwrap_or_else(|error| { + panic!("CPU {} {}: {error}", case.id, operation_label(operation)) + }); + let hybrid = + decode_once(case, operation, BackendRequest::Metal, session).unwrap_or_else(|error| { + panic!( + "hybrid Metal {} {}: {error}", + case.id, + operation_label(operation) + ) + }); + assert_output_parity(case.id, operation, &cpu, &hybrid); + let auto = + decode_once(case, operation, BackendRequest::Auto, session).unwrap_or_else(|error| { + panic!("Auto {} {}: {error}", case.id, operation_label(operation)) + }); + assert_output_parity(case.id, operation, &cpu, &auto); + + let group_id = format!( + "auto-routing_{}_{id}", + operation_label(operation), + id = case.id + ); + let mut group = criterion.benchmark_group(&group_id); + group.bench_function("cpu", |bencher| { + bencher.iter(|| { + std::hint::black_box( + decode_once(case, operation, BackendRequest::Cpu, session) + .expect("measured CPU Metal-adapter decode"), + ) + }); + }); + group.bench_function("hybrid", |bencher| { + bencher.iter(|| { + std::hint::black_box( + decode_once(case, operation, BackendRequest::Metal, session) + .expect("measured hybrid Metal decode"), + ) + }); + }); + group.finish(); + route_cell(case.id, operation, &group_id, auto_routing_sha256(&cpu)) + } + + fn decode_once( + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, + backend: BackendRequest, + session: &MetalBackendSession, + ) -> Result, String> { + if operation == AutoRoutingOperation::BatchDecode { + return decode_batch_once(case, backend); + } + let request = decode_request(case, operation, backend)?; + let mut decoder = J2kDecoder::new(case.bytes).map_err(|error| error.to_string())?; + let surface = decoder + .decode_request_to_device_with_session(request, session) + .map_err(|error| error.to_string())?; + assert_surface_route(&surface, backend, case, operation)?; + surface + .as_bytes() + .map(std::borrow::Cow::into_owned) + .map_err(|error| error.to_string()) + } + + fn decode_batch_once( + case: &DecodeCase<'_>, + backend: BackendRequest, + ) -> Result, String> { + let mut batch = MetalTileBatch::with_capacity(BATCH_SIZE); + for _ in 0..BATCH_SIZE { + batch + .push_shared_tile_request( + Arc::clone(&case.shared), + MetalDecodeRequest::full(case.fmt, backend), + ) + .map_err(|error| error.to_string())?; + } + let surfaces = batch.decode_all().map_err(|error| error.to_string())?; + if surfaces.len() != BATCH_SIZE { + return Err(format!( + "Metal batch returned {} surfaces for {BATCH_SIZE} inputs", + surfaces.len() + )); + } + let mut output = Vec::new(); + for surface in &surfaces { + assert_surface_route(surface, backend, case, AutoRoutingOperation::BatchDecode)?; + let bytes = surface.as_bytes().map_err(|error| error.to_string())?; + append_output(&mut output, &bytes)?; + } + Ok(output) + } + + fn decode_request( + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, + backend: BackendRequest, + ) -> Result { + match operation { + AutoRoutingOperation::FullDecode => Ok(MetalDecodeRequest::full(case.fmt, backend)), + AutoRoutingOperation::RoiDecode => Ok(MetalDecodeRequest::region( + case.fmt, + benchmark_roi(case.dimensions), + backend, + )), + AutoRoutingOperation::ScaledDecode => Ok(MetalDecodeRequest::scaled( + case.fmt, + Downscale::Half, + backend, + )), + AutoRoutingOperation::BatchDecode + | AutoRoutingOperation::LosslessEncode + | AutoRoutingOperation::LossyEncode => { + Err("invalid single-image decode operation".to_string()) + } + } + } + + fn assert_surface_route( + surface: &j2k_metal::Surface, + backend: BackendRequest, + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, + ) -> Result<(), String> { + match backend { + BackendRequest::Cpu if surface.backend_kind() == BackendKind::Cpu => Ok(()), + BackendRequest::Metal + if surface.backend_kind() == BackendKind::Metal + && surface.residency() == SurfaceResidency::MetalResidentDecode => + { + Ok(()) + } + BackendRequest::Auto + if surface.backend_kind() == expected_auto_decode_backend(case, operation) + && (surface.backend_kind() != BackendKind::Metal + || surface.residency() == SurfaceResidency::MetalResidentDecode) => + { + Ok(()) + } + _ => Err(format!( + "requested {backend:?} but received {:?}/{:?}", + surface.backend_kind(), + surface.residency() + )), + } + } + + fn expected_auto_decode_backend( + case: &DecodeCase<'_>, + operation: AutoRoutingOperation, + ) -> BackendKind { + if operation != AutoRoutingOperation::BatchDecode { + return BackendKind::Cpu; + } + let pixels = u64::from(case.dimensions.0) * u64::from(case.dimensions.1); + let promoted = match (case.fmt, case.transfer_syntax) { + (PixelFormat::Gray8, CompressedTransferSyntax::Jpeg2000Lossy) => { + pixels >= AUTO_GRAY8_MIN_PIXELS + } + (PixelFormat::Rgb8, CompressedTransferSyntax::Jpeg2000Lossy) => { + pixels >= AUTO_RGB8_BATCH_MIN_PIXELS + } + (PixelFormat::Rgb8, CompressedTransferSyntax::Jpeg2000Lossless) => { + pixels >= AUTO_RGB8_LARGE_MIN_PIXELS + } + _ => false, + }; + if promoted { + BackendKind::Metal + } else { + BackendKind::Cpu + } + } + + fn bench_encode_cell( + criterion: &mut Criterion, + case: &EncodeCase, + operation: AutoRoutingOperation, + ) -> AutoRoutingCell { + let cpu = encode_cpu(case, operation).unwrap_or_else(|error| { + panic!("CPU {} {}: {error}", case.id, operation_label(operation)) + }); + let mut probe_accelerator = MetalEncodeStageAccelerator::for_host_output_benchmark(); + let hybrid = + encode_hybrid(case, operation, &mut probe_accelerator).unwrap_or_else(|error| { + panic!( + "hybrid Metal {} {}: {error}", + case.id, + operation_label(operation) + ) + }); + assert_output_parity(&case.id, operation, &cpu, &hybrid); + let (auto, auto_dispatches) = encode_auto(case, operation).unwrap_or_else(|error| { + panic!("Auto {} {}: {error}", case.id, operation_label(operation)) + }); + assert_output_parity(&case.id, operation, &cpu, &auto); + let expected_dispatch = expected_auto_encode_dispatch(case, operation); + assert_eq!( + auto_dispatches > 0, + expected_dispatch, + "Auto dispatch decision for {} {}", + case.id, + operation_label(operation) + ); + + let group_id = format!( + "auto-routing_{}_{id}", + operation_label(operation), + id = case.id + ); + let mut accelerator = MetalEncodeStageAccelerator::for_host_output_benchmark(); + let mut group = criterion.benchmark_group(&group_id); + group.bench_function("cpu", |bencher| { + bencher.iter(|| { + std::hint::black_box( + encode_cpu(case, operation).expect("measured CPU JPEG 2000 encode"), + ) + }); + }); + group.bench_function("hybrid", |bencher| { + bencher.iter(|| { + std::hint::black_box( + encode_hybrid(case, operation, &mut accelerator) + .expect("measured hybrid Metal JPEG 2000 encode"), + ) + }); + }); + group.finish(); + route_cell(&case.id, operation, &group_id, auto_routing_sha256(&cpu)) + } + + fn encode_cpu(case: &EncodeCase, operation: AutoRoutingOperation) -> Result, String> { + match operation { + AutoRoutingOperation::LosslessEncode => { + let encoded = encode_j2k_lossless( + lossless_samples(case)?, + &lossless_options(EncodeBackendPreference::CpuOnly), + ) + .map_err(|error| error.to_string())?; + Ok(encoded.codestream) + } + AutoRoutingOperation::LossyEncode => { + let encoded = encode_j2k_lossy( + lossy_samples(case)?, + &lossy_options(EncodeBackendPreference::CpuOnly), + ) + .map_err(|error| error.to_string())?; + Ok(encoded.codestream) + } + _ => Err("invalid encode operation".to_string()), + } + } + + fn encode_hybrid( + case: &EncodeCase, + operation: AutoRoutingOperation, + accelerator: &mut MetalEncodeStageAccelerator, + ) -> Result, String> { + let (codestream, dispatches) = match operation { + AutoRoutingOperation::LosslessEncode => { + let encoded = encode_j2k_lossless_with_accelerator( + lossless_samples(case)?, + &lossless_options(EncodeBackendPreference::Auto), + BackendKind::Metal, + accelerator, + ) + .map_err(|error| error.to_string())?; + (encoded.codestream, encoded.dispatch_report.total()) + } + AutoRoutingOperation::LossyEncode => { + let encoded = encode_j2k_lossy_with_accelerator( + lossy_samples(case)?, + &lossy_options(EncodeBackendPreference::Auto), + BackendKind::Metal, + accelerator, + ) + .map_err(|error| error.to_string())?; + (encoded.codestream, encoded.dispatch_report.total()) + } + _ => return Err("invalid encode operation".to_string()), + }; + if dispatches == 0 { + return Err("Metal hybrid encode did not dispatch any device stage".to_string()); + } + Ok(codestream) + } + + fn encode_auto( + case: &EncodeCase, + operation: AutoRoutingOperation, + ) -> Result<(Vec, usize), String> { + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + let encoded = match operation { + AutoRoutingOperation::LosslessEncode => encode_j2k_lossless_with_accelerator( + lossless_samples(case)?, + &lossless_options(EncodeBackendPreference::Auto), + BackendKind::Metal, + &mut accelerator, + ) + .map(|encoded| (encoded.codestream, encoded.dispatch_report.total())) + .map_err(|error| error.to_string()), + AutoRoutingOperation::LossyEncode => encode_j2k_lossy_with_accelerator( + lossy_samples(case)?, + &lossy_options(EncodeBackendPreference::Auto), + BackendKind::Metal, + &mut accelerator, + ) + .map(|encoded| (encoded.codestream, encoded.dispatch_report.total())) + .map_err(|error| error.to_string()), + _ => Err("invalid encode operation".to_string()), + }?; + Ok(encoded) + } + + fn expected_auto_encode_dispatch(case: &EncodeCase, operation: AutoRoutingOperation) -> bool { + if operation != AutoRoutingOperation::LossyEncode { + return false; + } + let pixels = u64::from(case.width) * u64::from(case.height); + match case.components { + 3 => pixels >= AUTO_RGB8_LARGE_MIN_PIXELS, + _ => false, + } + } + + fn lossless_samples(case: &EncodeCase) -> Result, String> { + J2kLosslessSamples::new( + &case.pixels, + case.width, + case.height, + case.components, + 8, + false, + ) + .map_err(|error| error.to_string()) + } + + fn lossy_samples(case: &EncodeCase) -> Result, String> { + J2kLossySamples::new( + &case.pixels, + case.width, + case.height, + case.components, + 8, + false, + ) + .map_err(|error| error.to_string()) + } + + fn lossless_options(backend: EncodeBackendPreference) -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(3)) + .with_validation(J2kEncodeValidation::External) + } + + fn lossy_options(backend: EncodeBackendPreference) -> J2kLossyEncodeOptions { + let mut options = J2kLossyEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(3)) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel(4.0))) + .with_validation(J2kEncodeValidation::External); + options.psnr_iteration_budget = 1; + options + } + + fn benchmark_roi(dimensions: (u32, u32)) -> Rect { + let width = (dimensions.0 / 2).max(1); + let height = (dimensions.1 / 2).max(1); + Rect { + x: dimensions.0.saturating_sub(width) / 2, + y: dimensions.1.saturating_sub(height) / 2, + w: width, + h: height, + } + } + + fn assert_output_parity( + case_id: &str, + operation: AutoRoutingOperation, + cpu: &[u8], + hybrid: &[u8], + ) { + if cpu == hybrid { + return; + } + let first_difference = cpu + .iter() + .zip(hybrid) + .position(|(cpu, hybrid)| cpu != hybrid) + .map(|index| (index, cpu[index], hybrid[index])); + panic!( + "Metal {} output differs for {case_id}: cpu_len={}, hybrid_len={}, first_difference={first_difference:?}", + operation_label(operation), + cpu.len(), + hybrid.len(), + ); + } + + const fn pixel_format(format: AutoRoutingPixelFormat) -> PixelFormat { + match format { + AutoRoutingPixelFormat::Gray8 => PixelFormat::Gray8, + AutoRoutingPixelFormat::Rgb8 => PixelFormat::Rgb8, + } + } + + fn required_path(name: &str) -> PathBuf { + PathBuf::from(required_env(name)) + } + + fn required_env(name: &str) -> String { + std::env::var(name) + .unwrap_or_else(|_| panic!("{name} must be set for Auto-routing benchmarks")) + } +} diff --git a/crates/j2k-metal/src/batch/execute.rs b/crates/j2k-metal/src/batch/execute.rs index 7065c079..39fcb3b1 100644 --- a/crates/j2k-metal/src/batch/execute.rs +++ b/crates/j2k-metal/src/batch/execute.rs @@ -73,6 +73,47 @@ fn complete_batch_surfaces( true } +pub(super) fn complete_repeated_device_failure( + session: &mut SessionState, + requests: &[QueuedRequest], + source: &crate::Error, +) { + let message = format!("selected repeated Metal batch failed without CPU retry: {source}"); + session.submissions = session.submissions.saturating_add(1); + for request in requests { + session.completed[request.output_slot] = Some(Err(crate::Error::MetalRuntime { + message: message.clone(), + })); + } +} + +fn complete_repeated_device_result( + session: &mut SessionState, + requests: &[QueuedRequest], + decoded: Option, crate::Error>>, +) -> bool { + match decoded { + None => false, + Some(Ok(surfaces)) => { + if !complete_batch_surfaces(session, requests, surfaces) { + complete_repeated_device_failure( + session, + requests, + &crate::Error::MetalStateInvariant { + state: "J2K Metal repeated batch output", + reason: "surface count does not match request count", + }, + ); + } + true + } + Some(Err(error)) => { + complete_repeated_device_failure(session, requests, &error); + true + } + } +} + pub(super) fn process_batch( session: &mut SessionState, grouped: GroupedRequests, @@ -141,22 +182,16 @@ fn process_batch_inner( } if can_decode_requests_as_repeated_full_grayscale_batch(&requests) { - if let Some(Ok(surfaces)) = - decode_repeated_full_grayscale(&requests[0], requests.len(), backend) - { - if complete_batch_surfaces(session, &requests, surfaces) { - return; - } + let decoded = decode_repeated_full_grayscale(&requests[0], requests.len(), backend); + if complete_repeated_device_result(session, &requests, decoded) { + return; } } if can_decode_requests_as_repeated_full_color_batch(&requests) { - if let Some(Ok(surfaces)) = - decode_repeated_full_color(&requests[0], requests.len(), backend) - { - if complete_batch_surfaces(session, &requests, surfaces) { - return; - } + let decoded = decode_repeated_full_color(&requests[0], requests.len(), backend); + if complete_repeated_device_result(session, &requests, decoded) { + return; } } diff --git a/crates/j2k-metal/src/batch/heuristics.rs b/crates/j2k-metal/src/batch/heuristics.rs index 95372c2b..34eeda99 100644 --- a/crates/j2k-metal/src/batch/heuristics.rs +++ b/crates/j2k-metal/src/batch/heuristics.rs @@ -419,10 +419,8 @@ fn can_decode_as_repeated_full_metal_batch(first: &QueuedRequest, next: &QueuedR pub(super) fn is_repeated_full_grayscale_candidate(request: &QueuedRequest) -> bool { matches!(request.op, BatchOp::Full) && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) - && matches!( - request.backend, - BackendRequest::Auto | BackendRequest::Metal - ) + && (request.backend == BackendRequest::Metal + || (request.backend == BackendRequest::Auto && request.fmt == PixelFormat::Gray8)) } pub(super) fn is_repeated_full_color_candidate(request: &QueuedRequest) -> bool { @@ -431,7 +429,8 @@ pub(super) fn is_repeated_full_color_candidate(request: &QueuedRequest) -> bool request.fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 ) - && request.backend == BackendRequest::Metal + && (request.backend == BackendRequest::Metal + || (request.backend == BackendRequest::Auto && request.fmt == PixelFormat::Rgb8)) } pub(super) fn is_distinct_full_grayscale_metal_candidate(request: &QueuedRequest) -> bool { diff --git a/crates/j2k-metal/src/batch/routes.rs b/crates/j2k-metal/src/batch/routes.rs index 83551393..e94ddc23 100644 --- a/crates/j2k-metal/src/batch/routes.rs +++ b/crates/j2k-metal/src/batch/routes.rs @@ -64,11 +64,21 @@ pub(super) fn decode_repeated_full_color( #[cfg(target_os = "macos")] { - Some( - J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| { - decoder.decode_repeated_color_direct_to_device_routed(request.fmt, count, backend) - }), - ) + Some(J2kDecoder::new(request.input.as_ref()).and_then( + |mut decoder| match request.backend { + BackendRequest::Auto => { + decoder.decode_repeated_color_auto_to_device_routed(request.fmt, count, backend) + } + BackendRequest::Metal => decoder.decode_repeated_color_direct_to_device_routed( + request.fmt, + count, + backend, + ), + _ => Err(batch_scheduler_invariant( + "repeated color batch contains an unsupported backend", + )), + }, + )) } #[cfg(not(target_os = "macos"))] diff --git a/crates/j2k-metal/src/batch/tests.rs b/crates/j2k-metal/src/batch/tests.rs index 1d3369bf..befa54b8 100644 --- a/crates/j2k-metal/src/batch/tests.rs +++ b/crates/j2k-metal/src/batch/tests.rs @@ -7,14 +7,15 @@ use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; use crate::{Error, MetalSession}; #[cfg(target_os = "macos")] -use super::execute::process_batch; +use super::execute::{complete_repeated_device_failure, process_batch}; #[cfg(target_os = "macos")] use super::heuristics::GroupedRequests; use super::heuristics::{ - auto_region_scaled_direct_metal_min_dim, can_decode_requests_as_repeated_region_scaled_batch, - group_metal_requests, profile_route_label, same_input_bytes, BatchRoute, - AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT, AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, - AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT, + auto_region_scaled_direct_metal_min_dim, can_decode_requests_as_repeated_full_color_batch, + can_decode_requests_as_repeated_full_grayscale_batch, + can_decode_requests_as_repeated_region_scaled_batch, group_metal_requests, profile_route_label, + same_input_bytes, BatchRoute, AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT, + AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT, AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, }; use super::request::{BatchOp, QueuedRequest}; @@ -38,6 +39,34 @@ fn auto_rgb_region_scaled_request(input: Arc<[u8]>) -> QueuedRequest { ) } +fn auto_full_request(input: Arc<[u8]>, fmt: PixelFormat) -> QueuedRequest { + QueuedRequest::new(input, fmt, BackendRequest::Auto, BatchOp::Full, 0) +} + +#[test] +fn auto_repeated_full_candidates_are_limited_to_measured_formats() { + let shared = Arc::<[u8]>::from([1_u8]); + let requests = |fmt| { + vec![ + auto_full_request(shared.clone(), fmt), + auto_full_request(shared.clone(), fmt), + ] + }; + + assert!(can_decode_requests_as_repeated_full_color_batch(&requests( + PixelFormat::Rgb8 + ))); + assert!(!can_decode_requests_as_repeated_full_color_batch( + &requests(PixelFormat::Rgba8) + )); + assert!(can_decode_requests_as_repeated_full_grayscale_batch( + &requests(PixelFormat::Gray8) + )); + assert!(!can_decode_requests_as_repeated_full_grayscale_batch( + &requests(PixelFormat::Gray16) + )); +} + fn auto_rgb_region_scaled_request_with_max_dim( input: Arc<[u8]>, max_image_dim: u32, @@ -159,6 +188,41 @@ fn slot_release_reports_missing_reserved_capacity_without_panicking() { assert!(state.free_slots.is_empty()); } +#[test] +fn selected_repeated_device_failure_is_reported_without_cpu_retry() { + let shared = Arc::<[u8]>::from([1_u8]); + let requests = (0..2) + .map(|slot| { + let mut request = auto_full_request(shared.clone(), PixelFormat::Rgb8); + request.output_slot = slot; + request + }) + .collect::>(); + let mut session = SessionState { + submissions: 0, + queued: Vec::new(), + completed: (0..requests.len()).map(|_| None).collect(), + free_slots: Vec::new(), + }; + + complete_repeated_device_failure( + &mut session, + &requests, + &Error::MetalKernel { + message: "synthetic dispatch failure".to_string(), + }, + ); + + assert_eq!(session.submissions, 1); + assert!(session.completed.iter().all(|result| { + matches!( + result, + Some(Err(Error::MetalRuntime { message })) + if message.contains("synthetic dispatch failure") + ) + })); +} + #[test] fn auto_region_scaled_grouping_preserves_repeated_rgb_metal_decision() { let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); diff --git a/crates/j2k-metal/src/classic.metal b/crates/j2k-metal/src/classic.metal index c3687b28..9da51757 100644 --- a/crates/j2k-metal/src/classic.metal +++ b/crates/j2k-metal/src/classic.metal @@ -17,6 +17,7 @@ struct J2kClassicCleanupBatchJob { uint sub_band_type; uint style_flags; uint strict; + uint irreversible_midpoint; float dequantization_step; }; @@ -343,6 +344,61 @@ inline void coeff_set_magnitude_refined_tg(threadgroup uchar *states, uint idx) set_state_bit_tg(states, idx, J2K_MAG_REF_SHIFT, uchar(1u)); } +inline float reconstructed_classic_sample( + uint coefficient, + J2kClassicCleanupBatchJob job +) { + const uint magnitude = coefficient & 0x7FFFFFFFu; + const uint decoded_bitplanes = + job.total_bitplanes + job.roi_shift - job.missing_msbs; + float reconstructed; + if (job.irreversible_midpoint != 0u && magnitude != 0u && + job.number_of_coding_passes != 0u) { + const uint final_pass = job.number_of_coding_passes - 1u; + const uint decoded_plane = (final_pass + 2u) / 3u; + if (decoded_bitplanes > decoded_plane) { + uint lowest_decoded_bit = decoded_bitplanes - decoded_plane - 1u; + if (final_pass % 3u == 1u && + (magnitude & (1u << lowest_decoded_bit)) == 0u) { + lowest_decoded_bit += 1u; + } + uint fixed_magnitude = + (magnitude << 1u) | (1u << lowest_decoded_bit); + if (job.roi_shift != 0u && + fixed_magnitude >= (1u << job.roi_shift)) { + fixed_magnitude >>= job.roi_shift; + } + reconstructed = float(fixed_magnitude) * 0.5f; + } else { + reconstructed = float(magnitude); + } + } else { + uint reconstructed_magnitude = magnitude; + if (job.roi_shift != 0u && + reconstructed_magnitude >= (1u << job.roi_shift)) { + reconstructed_magnitude >>= job.roi_shift; + } + reconstructed = float(reconstructed_magnitude); + } + return (coefficient & 0x80000000u) != 0u ? -reconstructed : reconstructed; +} + +inline bool classic_decoded_bitplanes( + J2kClassicCleanupBatchJob job, + thread uint &bitplanes +) { + if (job.total_bitplanes == 0u || job.total_bitplanes > 31u || + job.roi_shift > 31u - job.total_bitplanes) { + return false; + } + const uint coded_bitplanes = job.total_bitplanes + job.roi_shift; + if (job.missing_msbs >= coded_bitplanes) { + return false; + } + bitplanes = coded_bitplanes - job.missing_msbs; + return true; +} + inline void reset_contexts(thread uchar *contexts) { for (uint idx = 0u; idx < 19u; ++idx) { contexts[idx] = uchar(0); @@ -805,12 +861,12 @@ inline bool decode_classic_job( set_classic_status(status, J2K_CLASSIC_STATUS_UNSUPPORTED, 0u); return false; } - if (job.total_bitplanes == 0u || job.total_bitplanes > 31u || job.missing_msbs >= job.total_bitplanes) { + uint bitplanes = 0u; + if (!classic_decoded_bitplanes(job, bitplanes)) { set_classic_status(status, J2K_CLASSIC_STATUS_UNSUPPORTED, 1u); return false; } - const uint bitplanes = job.total_bitplanes - job.missing_msbs; const uint max_coding_passes = bitplanes == 0u ? 0u : 1u + 3u * (bitplanes - 1u); if (job.coded_len == 0u || max_coding_passes == 0u || job.number_of_coding_passes == 0u) { return true; @@ -1073,11 +1129,8 @@ inline bool decode_classic_job( for (uint x = 0u; x < job.width; ++x) { const uint coeff = coefficients[coeff_index(padded_width, x + J2K_CLASSIC_PADDING, y + J2K_CLASSIC_PADDING)]; - int magnitude = int(coeff & 0x7FFFFFFFu); - if ((coeff & 0x80000000u) != 0u) { - magnitude = -magnitude; - } - output[output_row + x] = float(magnitude) * job.dequantization_step; + output[output_row + x] = + reconstructed_classic_sample(coeff, job) * job.dequantization_step; } } } @@ -1106,12 +1159,12 @@ inline bool decode_classic_job_plain( set_classic_status(status, J2K_CLASSIC_STATUS_UNSUPPORTED, 0u); return false; } - if (job.total_bitplanes == 0u || job.total_bitplanes > 31u || job.missing_msbs >= job.total_bitplanes) { + uint bitplanes = 0u; + if (!classic_decoded_bitplanes(job, bitplanes)) { set_classic_status(status, J2K_CLASSIC_STATUS_UNSUPPORTED, 1u); return false; } - const uint bitplanes = job.total_bitplanes - job.missing_msbs; const uint max_coding_passes = bitplanes == 0u ? 0u : 1u + 3u * (bitplanes - 1u); if (job.coded_len == 0u || max_coding_passes == 0u || job.number_of_coding_passes == 0u) { return true; @@ -1307,12 +1360,12 @@ inline bool decode_classic_job_plain_dev( set_classic_status(status, J2K_CLASSIC_STATUS_UNSUPPORTED, 0u); return false; } - if (job.total_bitplanes == 0u || job.total_bitplanes > 31u || job.missing_msbs >= job.total_bitplanes) { + uint bitplanes = 0u; + if (!classic_decoded_bitplanes(job, bitplanes)) { set_classic_status(status, J2K_CLASSIC_STATUS_UNSUPPORTED, 1u); return false; } - const uint bitplanes = job.total_bitplanes - job.missing_msbs; const uint max_coding_passes = bitplanes == 0u ? 0u : 1u + 3u * (bitplanes - 1u); if (job.coded_len == 0u || max_coding_passes == 0u || job.number_of_coding_passes == 0u) { return true; @@ -1495,11 +1548,8 @@ inline bool decode_classic_job_plain_dev( for (uint x = 0u; x < job.width; ++x) { const uint coeff = coefficients[coeff_index(padded_width, x + J2K_CLASSIC_PADDING, y + J2K_CLASSIC_PADDING)]; - int magnitude = int(coeff & 0x7FFFFFFFu); - if ((coeff & 0x80000000u) != 0u) { - magnitude = -magnitude; - } - output[output_row + x] = float(magnitude) * job.dequantization_step; + output[output_row + x] = + reconstructed_classic_sample(coeff, job) * job.dequantization_step; } } } @@ -1524,12 +1574,8 @@ inline void store_classic_job_plain_output_tg( const uint coeff_idx = coeff_index(padded_width, x + J2K_CLASSIC_PADDING, y + J2K_CLASSIC_PADDING); const uint coeff = coefficients[coeff_idx]; - int magnitude = int(coeff & 0x7FFFFFFFu); - if ((coeff & 0x80000000u) != 0u) { - magnitude = -magnitude; - } output[job.output_offset + y * job.output_stride + x] = - float(magnitude) * job.dequantization_step; + reconstructed_classic_sample(coeff, job) * job.dequantization_step; } } @@ -1667,12 +1713,8 @@ kernel void j2k_store_classic_repeated_batched( const uint y = sample_idx / job.width; const uint coeff = coefficients[coeff_index(padded_width, x + J2K_CLASSIC_PADDING, y + J2K_CLASSIC_PADDING)]; - int magnitude = int(coeff & 0x7FFFFFFFu); - if ((coeff & 0x80000000u) != 0u) { - magnitude = -magnitude; - } output[job.output_offset + y * job.output_stride + x] = - float(magnitude) * job.dequantization_step; + reconstructed_classic_sample(coeff, job) * job.dequantization_step; } } diff --git a/crates/j2k-metal/src/classic.rs b/crates/j2k-metal/src/classic.rs index 4c1e641f..742aae18 100644 --- a/crates/j2k-metal/src/classic.rs +++ b/crates/j2k-metal/src/classic.rs @@ -39,13 +39,12 @@ impl MetalClassicBlockDecoder { pub(crate) fn batched_kernel_dispatches(&self) -> usize { self.batched_kernel_dispatches } -} -impl HtCodeBlockDecoder for MetalClassicBlockDecoder { - fn decode_j2k_sub_band( + fn decode_j2k_sub_band_inner( &mut self, job: J2kSubBandDecodeJob<'_>, output: &mut [f32], + irreversible_midpoint: bool, ) -> Result { if job.jobs.len() <= 1 { return Ok(false); @@ -58,35 +57,86 @@ impl HtCodeBlockDecoder for MetalClassicBlockDecoder { .iter() .all(|batch_job| supports_metal_classic_kernel(&batch_job.code_block)) { - compute::decode_classic_cleanup_sub_band(job, output) - .map_err(metal_classic_sub_band_decode_error)?; + compute::decode_classic_cleanup_sub_band_with_midpoint( + job, + output, + irreversible_midpoint, + ) + .map_err(metal_classic_sub_band_decode_error)?; self.batched_kernel_dispatches = self.batched_kernel_dispatches.saturating_add(1); return Ok(true); } + if irreversible_midpoint { + return Ok(false); + } decode_j2k_sub_band_scalar(job, output)?; Ok(true) } - fn decode_j2k_code_block( + fn decode_j2k_code_block_inner( &mut self, job: J2kCodeBlockDecodeJob<'_>, output: &mut [f32], + irreversible_midpoint: bool, ) -> Result { self.blocks_decoded = self.blocks_decoded.saturating_add(1); #[cfg(target_os = "macos")] if supports_metal_classic_kernel(&job) { - compute::decode_classic_cleanup_code_block(job, output) - .map_err(metal_classic_code_block_decode_error)?; + compute::decode_classic_cleanup_code_block_with_midpoint( + job, + output, + irreversible_midpoint, + ) + .map_err(metal_classic_code_block_decode_error)?; self.kernel_dispatches = self.kernel_dispatches.saturating_add(1); return Ok(true); } + if irreversible_midpoint { + return Ok(false); + } decode_j2k_code_block_scalar(job, output)?; Ok(true) } } +impl HtCodeBlockDecoder for MetalClassicBlockDecoder { + fn decode_j2k_sub_band( + &mut self, + job: J2kSubBandDecodeJob<'_>, + output: &mut [f32], + ) -> Result { + self.decode_j2k_sub_band_inner(job, output, false) + } + + fn decode_j2k_sub_band_with_midpoint( + &mut self, + job: J2kSubBandDecodeJob<'_>, + output: &mut [f32], + irreversible_midpoint: bool, + ) -> Result { + self.decode_j2k_sub_band_inner(job, output, irreversible_midpoint) + } + + fn decode_j2k_code_block( + &mut self, + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result { + self.decode_j2k_code_block_inner(job, output, false) + } + + fn decode_j2k_code_block_with_midpoint( + &mut self, + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + irreversible_midpoint: bool, + ) -> Result { + self.decode_j2k_code_block_inner(job, output, irreversible_midpoint) + } +} + #[cfg(target_os = "macos")] fn metal_classic_sub_band_decode_error(_error: crate::Error) -> j2k_native::DecodeError { DecodingError::CodeBlockDecodeFailureWithContext("Metal classic sub-band decode kernel failed") @@ -115,16 +165,17 @@ fn supports_metal_classic_kernel(job: &J2kCodeBlockDecodeJob<'_>) -> bool { if job.number_of_coding_passes == 0 { return false; } - if job.roi_shift != 0 { - return false; - } if job.data.is_empty() { return false; } - if job.total_bitplanes == 0 || job.total_bitplanes > 31 || job.missing_bit_planes >= 31 { + let Some(coded_bitplanes) = job.total_bitplanes.checked_add(job.roi_shift) else { + return false; + }; + if job.total_bitplanes == 0 || coded_bitplanes > 31 || job.missing_bit_planes >= coded_bitplanes + { return false; } - let bitplanes = job.total_bitplanes.saturating_sub(job.missing_bit_planes); + let bitplanes = coded_bitplanes - job.missing_bit_planes; if bitplanes == 0 { return false; } @@ -210,17 +261,21 @@ mod tests { #[cfg(target_os = "macos")] use crate::compute; #[cfg(target_os = "macos")] + use j2k_native::{decode_j2k_code_block_scalar, J2kCodeBlockDecodeJob, J2kCodeBlockSegment}; use j2k_native::{ - decode_j2k_code_block_scalar, HtCodeBlockDecoder, J2kCodeBlockDecodeJob, - J2kCodeBlockSegment, + encode, ColorSpace, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecoder, + Image, }; - use j2k_native::{encode, ColorSpace, DecodeSettings, DecoderContext, EncodeOptions, Image}; #[cfg(target_os = "macos")] fn should_run_metal_runtime() -> bool { j2k_test_support::metal_runtime_gate(module_path!()) } + struct CpuOnlyCodeBlockDecoder; + + impl HtCodeBlockDecoder for CpuOnlyCodeBlockDecoder {} + #[cfg(target_os = "macos")] #[derive(Clone)] struct OwnedClassicJob { @@ -461,6 +516,52 @@ mod tests { } } + #[test] + fn metal_classic_decoder_matches_native_region_for_openjpeg_irreversible_rgb() { + #[cfg(target_os = "macos")] + if !should_run_metal_runtime() { + return; + } + + let image = Image::new( + j2k_test_support::OPENJPEG_IRREVERSIBLE_RGB8_8X8, + &DecodeSettings::default(), + ) + .expect("image"); + let roi = (2, 2, 4, 4); + let mut expected_context = DecoderContext::default(); + let expected = image + .decode_region_components_with_ht_decoder( + &mut expected_context, + roi, + &mut CpuOnlyCodeBlockDecoder, + ) + .expect("native region decode"); + + let mut hooked_context = DecoderContext::default(); + let mut decoder = MetalClassicBlockDecoder::default(); + let actual = image + .decode_region_components_with_ht_decoder(&mut hooked_context, roi, &mut decoder) + .expect("Metal classic region decode"); + + assert_eq!(actual.dimensions(), expected.dimensions()); + for (component, (actual_plane, expected_plane)) in + actual.planes().iter().zip(expected.planes()).enumerate() + { + assert_eq!( + actual_plane.samples(), + expected_plane.samples(), + "Metal classic component {component} must match native region decode" + ); + } + assert!(decoder.blocks_decoded() > 0); + #[cfg(target_os = "macos")] + assert!( + decoder.kernel_dispatches() + decoder.batched_kernel_dispatches() > 0, + "OpenJPEG RGB region must exercise a Metal classic kernel" + ); + } + #[test] fn metal_classic_decoder_batches_multi_block_subbands() { #[cfg(target_os = "macos")] @@ -549,7 +650,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -598,7 +699,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -625,7 +726,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -674,7 +775,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -723,7 +824,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -776,7 +877,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -832,7 +933,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -870,7 +971,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } @@ -898,7 +999,7 @@ mod tests { let mut expected = vec![0.0f32; job.output_len()]; decode_j2k_code_block_scalar(job.as_job(), &mut expected).expect("scalar decode"); let mut actual = vec![0.0f32; job.output_len()]; - compute::decode_classic_cleanup_code_block(job.as_job(), &mut actual) + compute::decode_classic_cleanup_code_block_with_midpoint(job.as_job(), &mut actual, false) .expect("metal decode"); assert_eq!(actual, expected); } diff --git a/crates/j2k-metal/src/compute.rs b/crates/j2k-metal/src/compute.rs index f4228863..49ca756a 100644 --- a/crates/j2k-metal/src/compute.rs +++ b/crates/j2k-metal/src/compute.rs @@ -327,7 +327,7 @@ use self::direct_execute::{ mod decode_cleanup; #[cfg(target_os = "macos")] pub(crate) use self::decode_cleanup::{ - decode_classic_cleanup_code_block, decode_classic_cleanup_sub_band, + decode_classic_cleanup_code_block_with_midpoint, decode_classic_cleanup_sub_band_with_midpoint, decode_ht_cleanup_code_block, decode_ht_cleanup_sub_band, }; #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/compute/abi.rs b/crates/j2k-metal/src/compute/abi.rs index 18d3f12c..4c433cdc 100644 --- a/crates/j2k-metal/src/compute/abi.rs +++ b/crates/j2k-metal/src/compute/abi.rs @@ -163,6 +163,7 @@ pub(crate) struct J2kClassicCleanupBatchJob { pub(crate) sub_band_type: u32, pub(crate) style_flags: u32, pub(crate) strict: u32, + pub(crate) irreversible_midpoint: u32, pub(crate) dequantization_step: f32, } diff --git a/crates/j2k-metal/src/compute/code_block_decoder.rs b/crates/j2k-metal/src/compute/code_block_decoder.rs index 574c77ac..af1a5e60 100644 --- a/crates/j2k-metal/src/compute/code_block_decoder.rs +++ b/crates/j2k-metal/src/compute/code_block_decoder.rs @@ -28,6 +28,16 @@ impl HtCodeBlockDecoder for MetalCodeBlockDecoder { self.classic.decode_j2k_sub_band(job, output) } + fn decode_j2k_sub_band_with_midpoint( + &mut self, + job: J2kSubBandDecodeJob<'_>, + output: &mut [f32], + irreversible_midpoint: bool, + ) -> j2k_native::Result { + self.classic + .decode_j2k_sub_band_with_midpoint(job, output, irreversible_midpoint) + } + fn decode_j2k_code_block( &mut self, job: J2kCodeBlockDecodeJob<'_>, @@ -36,6 +46,16 @@ impl HtCodeBlockDecoder for MetalCodeBlockDecoder { self.classic.decode_j2k_code_block(job, output) } + fn decode_j2k_code_block_with_midpoint( + &mut self, + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + irreversible_midpoint: bool, + ) -> j2k_native::Result { + self.classic + .decode_j2k_code_block_with_midpoint(job, output, irreversible_midpoint) + } + fn decode_sub_band( &mut self, job: HtSubBandDecodeJob<'_>, @@ -71,3 +91,72 @@ impl HtCodeBlockDecoder for MetalCodeBlockDecoder { self.store.decode_store_component(job) } } + +#[cfg(test)] +mod tests { + use super::MetalCodeBlockDecoder; + use j2k_native::{DecodeSettings, DecoderContext, HtCodeBlockDecoder, Image}; + + struct CpuOnlyCodeBlockDecoder; + + impl HtCodeBlockDecoder for CpuOnlyCodeBlockDecoder {} + + #[test] + fn composite_decoder_retains_exact_openjpeg_irreversible_rgb_region_planes() { + #[cfg(target_os = "macos")] + if !j2k_test_support::metal_runtime_gate(module_path!()) { + return; + } + + let image = Image::new( + j2k_test_support::OPENJPEG_IRREVERSIBLE_RGB8_8X8, + &DecodeSettings::default(), + ) + .expect("image"); + let roi = (2, 2, 4, 4); + let mut expected_context = DecoderContext::default(); + let expected = image + .decode_region_components_with_ht_decoder( + &mut expected_context, + roi, + &mut CpuOnlyCodeBlockDecoder, + ) + .expect("native region decode"); + + let mut hooked_context = DecoderContext::default(); + let mut decoder = MetalCodeBlockDecoder::default(); + let actual = image + .decode_region_components_with_ht_decoder(&mut hooked_context, roi, &mut decoder) + .expect("composite Metal region decode"); + + assert_eq!(actual.dimensions(), expected.dimensions()); + for (component, (actual_plane, expected_plane)) in + actual.planes().iter().zip(expected.planes()).enumerate() + { + assert_eq!( + actual_plane.samples(), + expected_plane.samples(), + "composite Metal component {component} must match native decode" + ); + } + + #[cfg(target_os = "macos")] + { + let captured = decoder.mct.take_captured_planes(); + assert_eq!(captured.len(), actual.planes().len()); + for (component, (buffer, plane)) in captured.iter().zip(actual.planes()).enumerate() { + let retained = crate::compute::checked_buffer_slice::( + buffer, + plane.samples().len(), + "composite MCT plane", + ) + .expect("retained MCT plane readback"); + assert_eq!( + retained, + plane.samples(), + "retained Metal component {component} must match host plane" + ); + } + } + } +} diff --git a/crates/j2k-metal/src/compute/decode_cleanup.rs b/crates/j2k-metal/src/compute/decode_cleanup.rs index 997a4c23..a63d03f6 100644 --- a/crates/j2k-metal/src/compute/decode_cleanup.rs +++ b/crates/j2k-metal/src/compute/decode_cleanup.rs @@ -58,9 +58,10 @@ fn validate_classic_sub_band_output( } #[cfg(target_os = "macos")] -pub(crate) fn decode_classic_cleanup_code_block( +pub(crate) fn decode_classic_cleanup_code_block_with_midpoint( job: J2kCodeBlockDecodeJob<'_>, output: &mut [f32], + irreversible_midpoint: bool, ) -> Result<(), Error> { let required_len = required_classic_output_len(job)?; if output.len() < required_len { @@ -117,6 +118,7 @@ pub(crate) fn decode_classic_cleanup_code_block( }, style_flags: classic_style_flags(job.style), strict: u32::from(job.strict), + irreversible_midpoint: u32::from(irreversible_midpoint), dequantization_step: job.dequantization_step, }; dispatch_classic_cleanup_batched(runtime, job.data, &[batch_job], &segments, &decoded)?; @@ -128,9 +130,10 @@ pub(crate) fn decode_classic_cleanup_code_block( } #[cfg(target_os = "macos")] -pub(crate) fn decode_classic_cleanup_sub_band( +pub(crate) fn decode_classic_cleanup_sub_band_with_midpoint( job: J2kSubBandDecodeJob<'_>, output: &mut [f32], + irreversible_midpoint: bool, ) -> Result<(), Error> { validate_classic_sub_band_output(&job, output.len())?; if job.jobs.is_empty() { @@ -221,6 +224,7 @@ pub(crate) fn decode_classic_cleanup_sub_band( }, style_flags: classic_style_flags(block.code_block.style), strict: u32::from(block.code_block.strict), + irreversible_midpoint: u32::from(irreversible_midpoint), dequantization_step: block.code_block.dequantization_step, }); } diff --git a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs index 25b9a9a7..d7461c07 100644 --- a/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs +++ b/crates/j2k-metal/src/compute/decode_dispatch/classic_cleanup/distinct_metadata_tests.rs @@ -164,6 +164,7 @@ fn distinct_classic_batches_honor_empty_and_zero_fill_output_semantics() { sub_band_type: 0, style_flags: 1, strict: 1, + irreversible_midpoint: 0, dequantization_step: 1.0, }; let output = new_shared_buffer_with_slice(&runtime.device, &[11.0_f32; 4])?; @@ -222,6 +223,7 @@ fn distinct_classic_device_failure_keeps_nonzero_source_identity() { sub_band_type: 0, style_flags: 1, strict: 1, + irreversible_midpoint: 0, dequantization_step: 1.0, }; let invalid = J2kClassicCleanupBatchJob { diff --git a/crates/j2k-metal/src/compute/direct_cpu.rs b/crates/j2k-metal/src/compute/direct_cpu.rs index 557eadd6..e2ed10c1 100644 --- a/crates/j2k-metal/src/compute/direct_cpu.rs +++ b/crates/j2k-metal/src/compute/direct_cpu.rs @@ -6,6 +6,8 @@ use j2k_native::{ decode_ht_code_block_scalar_with_workspace, decode_ht_code_block_scalar_with_workspace_profiled, decode_j2k_code_block_scalar_with_workspace, + decode_j2k_code_block_scalar_with_workspace_midpoint, + decode_j2k_code_block_scalar_with_workspace_midpoint_profiled, decode_j2k_code_block_scalar_with_workspace_profiled, HtCodeBlockDecodeJob, HtCodeBlockDecodeProfile, HtCodeBlockDecodeWorkspace, J2kCodeBlockDecodeJob, J2kCodeBlockDecodeProfile, J2kCodeBlockDecodeWorkspace, J2kCodeBlockSegment, J2kCodeBlockStyle, @@ -192,23 +194,23 @@ fn decode_prepared_classic_jobs_on_cpu_with_scratch_impl( if PROFILE { let decode_started = Instant::now(); let mut profile = J2kCodeBlockDecodeProfile::default(); - decode_j2k_code_block_scalar_with_workspace_profiled( - decode_job, - output_window, - &mut scratch.decode, - &mut profile, - ) - .map_err(native_decode_error)?; + let decode = if job.irreversible_midpoint != 0 { + decode_j2k_code_block_scalar_with_workspace_midpoint_profiled + } else { + decode_j2k_code_block_scalar_with_workspace_profiled + }; + decode(decode_job, output_window, &mut scratch.decode, &mut profile) + .map_err(native_decode_error)?; profile_counters .expect("profile counters required for profiled classic decode") .record_classic_block_decode(decode_started, &profile); } else { - decode_j2k_code_block_scalar_with_workspace( - decode_job, - output_window, - &mut scratch.decode, - ) - .map_err(native_decode_error)?; + let decode = if job.irreversible_midpoint != 0 { + decode_j2k_code_block_scalar_with_workspace_midpoint + } else { + decode_j2k_code_block_scalar_with_workspace + }; + decode(decode_job, output_window, &mut scratch.decode).map_err(native_decode_error)?; } } Ok(()) diff --git a/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs b/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs index 11f7a8f9..8ec86204 100644 --- a/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs +++ b/crates/j2k-metal/src/compute/direct_plan_validation/runtime.rs @@ -66,13 +66,16 @@ fn classic_prepared_job_supports_runtime( if job.width > J2K_CLASSIC_MAX_WIDTH || job.height > J2K_CLASSIC_MAX_HEIGHT { return false; } - if job.output_stride < job.width || job.roi_shift != 0 { + if job.output_stride < job.width { return false; } - if job.total_bitplanes == 0 || job.total_bitplanes > 31 || job.missing_msbs >= 31 { + let Some(coded_bitplanes) = job.total_bitplanes.checked_add(job.roi_shift) else { + return false; + }; + if job.total_bitplanes == 0 || coded_bitplanes > 31 || job.missing_msbs >= coded_bitplanes { return false; } - let bitplanes = job.total_bitplanes.saturating_sub(job.missing_msbs); + let bitplanes = coded_bitplanes - job.missing_msbs; if bitplanes == 0 { return false; } @@ -183,6 +186,7 @@ mod tests { sub_band_type: 0, style_flags: 0, strict: 1, + irreversible_midpoint: 0, dequantization_step: 1.0, } } @@ -198,12 +202,15 @@ mod tests { } #[test] - fn classic_runtime_preflight_rejects_unimplemented_roi_shift_and_inconsistent_empty_job() { + fn classic_runtime_preflight_accepts_roi_shift_and_rejects_invalid_bitplanes() { let segment = valid_classic_segment(); let mut job = valid_classic_job(); assert!(classic_prepared_job_supports_runtime(&job, &[segment])); job.roi_shift = 1; + assert!(classic_prepared_job_supports_runtime(&job, &[segment])); + + job.roi_shift = 31; assert!(!classic_prepared_job_supports_runtime(&job, &[segment])); job.roi_shift = 0; diff --git a/crates/j2k-metal/src/compute/direct_prepare/classic/sub_band.rs b/crates/j2k-metal/src/compute/direct_prepare/classic/sub_band.rs index 137e576c..b23a11a5 100644 --- a/crates/j2k-metal/src/compute/direct_prepare/classic/sub_band.rs +++ b/crates/j2k-metal/src/compute/direct_prepare/classic/sub_band.rs @@ -49,6 +49,7 @@ pub(super) fn prepare_classic_sub_band_with_payloads( block_index, block, job.width, + job.irreversible_midpoint, &mut append_payload, )?; } @@ -90,6 +91,7 @@ fn append_classic_sub_band_job( block_index: usize, block: &j2k_native::J2kOwnedCodeBlockBatchJob, output_stride: u32, + irreversible_midpoint: bool, append_payload: &mut impl FnMut(usize, &mut Vec) -> Result, ) -> Result<(), Error> { let coded_offset = u32::try_from(owners.coded_data.len()).map_err(|_| Error::MetalKernel { @@ -103,6 +105,7 @@ fn append_classic_sub_band_job( block_coded_len, segment_offset, output_stride, + irreversible_midpoint, )?); Ok(()) } @@ -140,6 +143,7 @@ fn classic_cleanup_job( block_coded_len: usize, segment_offset: u32, output_stride: u32, + irreversible_midpoint: bool, ) -> Result { Ok(J2kClassicCleanupBatchJob { coded_offset, @@ -172,6 +176,7 @@ fn classic_cleanup_job( }, style_flags: classic_style_flags(block.style), strict: u32::from(block.strict), + irreversible_midpoint: u32::from(irreversible_midpoint), dequantization_step: block.dequantization_step, }) } diff --git a/crates/j2k-metal/src/compute/tests/classic.rs b/crates/j2k-metal/src/compute/tests/classic.rs index b429e77d..81529c7a 100644 --- a/crates/j2k-metal/src/compute/tests/classic.rs +++ b/crates/j2k-metal/src/compute/tests/classic.rs @@ -7,9 +7,10 @@ use super::super::decode_dispatch::{ }; use super::super::{ decode_prepared_classic_sub_band_on_cpu, direct_tier1_input_buffer_prepares_for_test, - prepare_direct_color_plan, prepare_direct_color_plan_for_cpu_upload, - prepare_direct_grayscale_plan, reset_direct_tier1_input_buffer_prepares_for_test, - PreparedClassicSubBand, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, + execute_hybrid_cpu_tier1_direct_color_plan, prepare_direct_color_plan, + prepare_direct_color_plan_for_cpu_upload, prepare_direct_grayscale_plan, + reset_direct_tier1_input_buffer_prepares_for_test, PreparedClassicSubBand, + PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, }; use super::runtime::should_run_metal_runtime; use j2k_native::{ @@ -19,6 +20,7 @@ use j2k_native::{ J2kOwnedSubBandPlan, J2kSubBandDecodeJob, }; use metal::Device; +use std::sync::Arc; #[test] #[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] @@ -46,6 +48,72 @@ fn prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload() { assert_eq!(actual, expected); } +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn prepared_irreversible_classic_sub_band_records_midpoint_reconstruction() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: false, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode 9/7 gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let prepared_sub_band = first_prepared_classic_sub_band(&prepared); + + assert!( + prepared_sub_band + .jobs + .iter() + .all(|job| job.irreversible_midpoint != 0), + "every prepared job in a 9/7 sub-band must retain midpoint reconstruction" + ); +} + +#[test] +#[ignore = "requires Metal runtime; exercised by the fail-closed Metal release lane"] +fn irreversible_hybrid_cpu_tier1_matches_native_decode_exactly() { + let pixels = j2k_test_support::gradient_u8(16, 16, 3); + let bytes = encode( + &pixels, + 16, + 16, + 3, + 8, + false, + &EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode irreversible RGB8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut expected_context = DecoderContext::default(); + let expected = image + .decode_with_context(&mut expected_context) + .expect("native decode"); + let mut plan_context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut plan_context) + .expect("direct color plan"); + let prepared = prepare_direct_color_plan_for_cpu_upload(&plan).expect("prepared color plan"); + + let surface = + execute_hybrid_cpu_tier1_direct_color_plan(Arc::new(prepared), j2k_core::PixelFormat::Rgb8) + .expect("hybrid decode"); + + assert_eq!( + surface.as_bytes().expect("surface bytes").as_ref(), + expected.data + ); +} + #[test] fn cpu_upload_color_prepare_skips_tier1_metal_input_buffers() { if !should_run_metal_runtime() { @@ -211,6 +279,7 @@ fn classic_plain_fast_path_accepts_style_zero_arithmetic_jobs() { sub_band_type: 0, style_flags: 0, strict: 1, + irreversible_midpoint: 0, dequantization_step: 1.0, }]; let segments = [J2kClassicSegment { @@ -245,6 +314,7 @@ fn classic_repeated_plain_fast_path_stays_off_for_wsi_batch_size() { sub_band_type: 0, style_flags: 0, strict: 1, + irreversible_midpoint: 0, dequantization_step: 1.0, }]; let segments = [J2kClassicSegment { diff --git a/crates/j2k-metal/src/decoder/direct_paths.rs b/crates/j2k-metal/src/decoder/direct_paths.rs index c4f81e79..554c1305 100644 --- a/crates/j2k-metal/src/decoder/direct_paths.rs +++ b/crates/j2k-metal/src/decoder/direct_paths.rs @@ -6,9 +6,7 @@ use std::sync::Arc; #[cfg(target_os = "macos")] use j2k_core::{BackendRequest, PixelFormat}; #[cfg(target_os = "macos")] -use j2k_native::{ - DecodeSettings as NativeDecodeSettings, Image as NativeImage, J2kDirectGrayscalePlan, -}; +use j2k_native::{DecodeSettings as NativeDecodeSettings, Image as NativeImage}; #[cfg(target_os = "macos")] use metal::Device; @@ -120,11 +118,6 @@ macro_rules! define_ensure_prepared_direct_plan { }; } -#[cfg(target_os = "macos")] -const AUTO_REPEATED_GRAYSCALE_MIN_DIM: u32 = 512; -#[cfg(target_os = "macos")] -const AUTO_REPEATED_GRAYSCALE_MIN_COUNT: usize = 16; - impl J2kDecoder<'_> { #[cfg(target_os = "macos")] pub(super) fn ensure_native_image(&mut self) -> Result<(), Error> { @@ -272,7 +265,7 @@ impl J2kDecoder<'_> { } #[cfg(target_os = "macos")] - fn decode_repeated_grayscale_cpu_to_surfaces( + fn decode_repeated_cpu_to_surfaces( &mut self, fmt: PixelFormat, count: usize, @@ -287,20 +280,6 @@ impl J2kDecoder<'_> { Ok(surfaces) } - #[cfg(target_os = "macos")] - fn should_auto_use_direct_for_repeated( - plan: &J2kDirectGrayscalePlan, - fmt: PixelFormat, - count: usize, - ) -> bool { - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) || count == 0 { - return false; - } - - let max_dim = plan.dimensions.0.max(plan.dimensions.1); - max_dim >= AUTO_REPEATED_GRAYSCALE_MIN_DIM && count >= AUTO_REPEATED_GRAYSCALE_MIN_COUNT - } - #[cfg(target_os = "macos")] #[doc(hidden)] pub fn decode_repeated_grayscale_direct_to_device( @@ -408,14 +387,15 @@ impl J2kDecoder<'_> { if count == 0 { return Ok(Vec::new()); } - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); - } let dims = self.inner.info().dimensions; - if dims.0.max(dims.1) < AUTO_REPEATED_GRAYSCALE_MIN_DIM - || count < AUTO_REPEATED_GRAYSCALE_MIN_COUNT - { - return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); + let Some(transfer_syntax) = j2k::J2kDecoder::inspect_support(self.bytes) + .ok() + .map(|support| support.transfer_syntax) + else { + return self.decode_repeated_cpu_to_surfaces(fmt, count); + }; + if !crate::routing::auto_repeated_decode_uses_metal(dims, fmt, count, transfer_syntax) { + return self.decode_repeated_cpu_to_surfaces(fmt, count); } let device_registry_id = crate::compute::current_runtime_device_registry_id()?; if self.native_prepared_direct_gray_plan.is_some() @@ -434,7 +414,7 @@ impl J2kDecoder<'_> { ))); }; let Ok(plan) = image.build_direct_grayscale_plan_with_context(native_context) else { - return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); + return self.decode_repeated_cpu_to_surfaces(fmt, count); }; let plan = Arc::new(plan); let prepared = Arc::new(crate::compute::prepare_direct_grayscale_plan( @@ -444,16 +424,41 @@ impl J2kDecoder<'_> { self.native_prepared_direct_gray_plan = Some(prepared); self.native_prepared_direct_gray_device_registry_id = Some(device_registry_id); } - let Some(plan) = self.native_direct_gray_plan.as_ref() else { - return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); + let Some(prepared) = self.native_prepared_direct_gray_plan.as_ref() else { + return self.decode_repeated_cpu_to_surfaces(fmt, count); }; - if Self::should_auto_use_direct_for_repeated(plan, fmt, count) { - let Some(prepared) = self.native_prepared_direct_gray_plan.as_ref() else { - return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); - }; - crate::compute::execute_repeated_prepared_direct_grayscale_plan(prepared, fmt, count) - } else { - self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count) + crate::compute::execute_repeated_prepared_direct_grayscale_plan(prepared, fmt, count) + } + + #[cfg(target_os = "macos")] + pub(crate) fn decode_repeated_color_auto_to_device_routed( + &mut self, + fmt: PixelFormat, + count: usize, + session: Option<&MetalBackendSession>, + ) -> Result, Error> { + if count == 0 { + return Ok(Vec::new()); + } + let Some(transfer_syntax) = j2k::J2kDecoder::inspect_support(self.bytes) + .ok() + .map(|support| support.transfer_syntax) + else { + return self.decode_repeated_cpu_to_surfaces(fmt, count); + }; + if !crate::routing::auto_repeated_decode_uses_metal( + self.inner.info().dimensions, + fmt, + count, + transfer_syntax, + ) { + return self.decode_repeated_cpu_to_surfaces(fmt, count); + } + match self.decode_repeated_color_direct_to_device_routed(fmt, count, session) { + Err(error) if error.is_direct_fallback() => { + self.decode_repeated_cpu_to_surfaces(fmt, count) + } + result => result, } } } diff --git a/crates/j2k-metal/src/direct.rs b/crates/j2k-metal/src/direct.rs index 6de27651..91032a2d 100644 --- a/crates/j2k-metal/src/direct.rs +++ b/crates/j2k-metal/src/direct.rs @@ -141,9 +141,10 @@ fn execute_grayscale_plan_to_plane( fn decode_classic_sub_band(plan: &J2kOwnedSubBandPlan, output: &mut [f32]) -> Result<(), Error> { if let [block] = plan.jobs.as_slice() { let start = block.output_y as usize * plan.width as usize + block.output_x as usize; - return compute::decode_classic_cleanup_code_block( + return compute::decode_classic_cleanup_code_block_with_midpoint( classic_job(block), &mut output[start..], + plan.irreversible_midpoint, ); } @@ -156,13 +157,14 @@ fn decode_classic_sub_band(plan: &J2kOwnedSubBandPlan, output: &mut [f32]) -> Re code_block: classic_job(owned), }) .collect(); - compute::decode_classic_cleanup_sub_band( + compute::decode_classic_cleanup_sub_band_with_midpoint( j2k_native::J2kSubBandDecodeJob { width: plan.width, height: plan.height, jobs: &jobs, }, output, + plan.irreversible_midpoint, ) } diff --git a/crates/j2k-metal/src/encode/stage_accelerator.rs b/crates/j2k-metal/src/encode/stage_accelerator.rs index 873269d0..05152001 100644 --- a/crates/j2k-metal/src/encode/stage_accelerator.rs +++ b/crates/j2k-metal/src/encode/stage_accelerator.rs @@ -7,10 +7,10 @@ use j2k::J2kEncodeStageError; #[cfg(target_os = "macos")] use j2k::{EncodeBackendPreference, J2kLosslessEncodeOptions}; use j2k::{ - EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, - J2kEncodeStageAccelerator, J2kEncodeStageResult, J2kForwardDwt53Job, J2kForwardDwt53Output, - J2kForwardDwt97Job, J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, - J2kHtCodeBlockEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationEncodeJob, + EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kDeinterleaveToF32Job, J2kEncodeContext, + J2kEncodeDispatchReport, J2kEncodeStageAccelerator, J2kEncodeStageResult, J2kForwardDwt53Job, + J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Output, J2kForwardIctJob, + J2kForwardRctJob, J2kHtCodeBlockEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationEncodeJob, J2kQuantizeSubbandJob, J2kTier1CodeBlockEncodeJob, }; #[cfg(target_os = "macos")] @@ -23,7 +23,9 @@ use super::{ MetalEncodeInputStaging, MetalLosslessEncodeTile, }; -const AUTO_HOST_OUTPUT_STAGE_MIN_PIXELS: usize = 512 * 512; +// Minimum qualified cells from verified Auto-routing artifact +// 162a47f7a96b2be88abebc100aab672513af04895532863fa1a293660546f879. +const AUTO_LOSSY_RGB8_MIN_PIXELS: usize = 5_038_848; /// Encode-stage accelerator for JPEG 2000 Metal work. /// @@ -35,6 +37,7 @@ pub struct MetalEncodeStageAccelerator { route_profile: MetalEncodeRouteProfile, parallel_cpu_code_block_fallback: bool, auto_host_output_force_cpu_fallback: bool, + host_output_stages_enabled: bool, deinterleave_attempts: usize, forward_rct_attempts: usize, forward_ict_attempts: usize, @@ -62,6 +65,7 @@ impl Default for MetalEncodeStageAccelerator { route_profile: MetalEncodeRouteProfile::Explicit, parallel_cpu_code_block_fallback: false, auto_host_output_force_cpu_fallback: false, + host_output_stages_enabled: false, deinterleave_attempts: 0, forward_rct_attempts: 0, forward_ict_attempts: 0, @@ -88,6 +92,7 @@ impl Default for MetalEncodeStageAccelerator { enum MetalEncodeRouteProfile { Explicit, AutoHostOutput, + HostOutputEvidence, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -152,6 +157,23 @@ impl MetalEncodeStageAccelerator { } } + /// Create the host-output hybrid route without applying the Auto size gate. + /// + /// This is intended for reproducible route benchmarks and adapter-IUT + /// conformance evidence. It runs the same Metal preparation stages as + /// [`Self::for_auto_host_output`] while keeping code-block coding and + /// packetization on the CPU. It does not change the public Auto policy. + #[must_use] + #[doc(hidden)] + pub fn for_host_output_benchmark() -> Self { + Self { + dispatch_stages: MetalEncodeDispatchStages::AUTO_HOST_OUTPUT_STAGE_DISPATCHES, + route_profile: MetalEncodeRouteProfile::HostOutputEvidence, + parallel_cpu_code_block_fallback: true, + ..Self::default() + } + } + /// Create an accelerator that only attempts the HT code-block stage on Metal. pub fn for_ht_code_block_encode() -> Self { Self { @@ -286,12 +308,25 @@ impl MetalEncodeStageAccelerator { self.packetization_dispatches } - fn auto_host_stage_supported_for_len(&self, len: usize) -> bool { - self.route_profile != MetalEncodeRouteProfile::AutoHostOutput - || len >= AUTO_HOST_OUTPUT_STAGE_MIN_PIXELS + fn host_output_stage_supported(&self) -> bool { + self.route_profile == MetalEncodeRouteProfile::Explicit || self.host_output_stages_enabled } } +pub(super) fn auto_host_output_should_dispatch(context: J2kEncodeContext) -> bool { + if context.reversible || context.bit_depth != 8 || context.signed { + return false; + } + match context.num_components { + 3 => context.num_pixels >= AUTO_LOSSY_RGB8_MIN_PIXELS, + _ => false, + } +} + +pub(super) fn host_output_evidence_should_dispatch(context: J2kEncodeContext) -> bool { + matches!(context.num_components, 1..=4) +} + #[cfg(target_os = "macos")] fn metal_dispatch_result( result: Result<(), crate::Error>, @@ -318,6 +353,18 @@ pub(super) fn metal_dispatch_option( #[doc(hidden)] impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { + fn begin_encode(&mut self, context: J2kEncodeContext) -> J2kEncodeStageResult<()> { + self.auto_host_output_force_cpu_fallback = false; + self.host_output_stages_enabled = match self.route_profile { + MetalEncodeRouteProfile::Explicit => true, + MetalEncodeRouteProfile::AutoHostOutput => auto_host_output_should_dispatch(context), + MetalEncodeRouteProfile::HostOutputEvidence => { + host_output_evidence_should_dispatch(context) + } + }; + Ok(()) + } + fn dispatch_report(&self) -> J2kEncodeDispatchReport { J2kEncodeDispatchReport { deinterleave: self.deinterleave_dispatches, @@ -345,7 +392,7 @@ impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { .dispatch_stages .contains(MetalEncodeDispatchStages::DEINTERLEAVE) || self.auto_host_output_force_cpu_fallback - || !self.auto_host_stage_supported_for_len(job.num_pixels) + || !self.host_output_stage_supported() { let _ = job; return Ok(None); @@ -381,7 +428,7 @@ impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { .dispatch_stages .contains(MetalEncodeDispatchStages::FORWARD_RCT) || self.auto_host_output_force_cpu_fallback - || !self.auto_host_stage_supported_for_len(job.plane0.len()) + || !self.host_output_stage_supported() { let _ = job; return Ok(false); @@ -408,7 +455,7 @@ impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { .dispatch_stages .contains(MetalEncodeDispatchStages::FORWARD_ICT) || self.auto_host_output_force_cpu_fallback - || !self.auto_host_stage_supported_for_len(job.plane0.len()) + || !self.host_output_stage_supported() { let _ = job; return Ok(false); @@ -450,8 +497,7 @@ impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { let _ = job; return Ok(None); } - let sample_count = (job.width as usize).saturating_mul(job.height as usize); - if !self.auto_host_stage_supported_for_len(sample_count) { + if !self.host_output_stage_supported() { let _ = job; return Ok(None); } @@ -492,8 +538,7 @@ impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { let _ = job; return Ok(None); } - let sample_count = (job.width as usize).saturating_mul(job.height as usize); - if !self.auto_host_stage_supported_for_len(sample_count) { + if !self.host_output_stage_supported() { let _ = job; return Ok(None); } @@ -531,7 +576,7 @@ impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { .dispatch_stages .contains(MetalEncodeDispatchStages::QUANTIZE_SUBBAND) || self.auto_host_output_force_cpu_fallback - || !self.auto_host_stage_supported_for_len(job.coefficients.len()) + || !self.host_output_stage_supported() { let _ = job; return Ok(None); diff --git a/crates/j2k-metal/src/encode/tests.rs b/crates/j2k-metal/src/encode/tests.rs index 6879a6c1..565ceb0a 100644 --- a/crates/j2k-metal/src/encode/tests.rs +++ b/crates/j2k-metal/src/encode/tests.rs @@ -1,10 +1,14 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +#[cfg(target_os = "macos")] +use super::stage_accelerator::{ + auto_host_output_should_dispatch, host_output_evidence_should_dispatch, +}; use super::MetalEncodeStageAccelerator; #[cfg(target_os = "macos")] use crate::compute; use j2k::{ - encode_j2k_lossless_with_accelerator, EncodeBackendPreference, EncodedJ2k, + encode_j2k_lossless, encode_j2k_lossless_with_accelerator, EncodeBackendPreference, EncodedJ2k, J2kLosslessEncodeOptions, J2kLosslessSamples, }; #[cfg(target_os = "macos")] @@ -14,7 +18,10 @@ use j2k::{ ReversibleTransform, }; #[cfg(target_os = "macos")] -use j2k::{J2kDeinterleaveToF32Job, J2kForwardDwt53Job, J2kForwardIctJob, J2kQuantizeSubbandJob}; +use j2k::{ + J2kDeinterleaveToF32Job, J2kEncodeContext, J2kForwardDwt53Job, J2kForwardDwt97Job, + J2kForwardIctJob, J2kQuantizeSubbandJob, +}; use j2k::{J2kEncodeDispatchReport, J2kEncodeStageAccelerator, J2kForwardRctJob}; #[cfg(target_os = "macos")] use j2k_core::CodecError; @@ -23,8 +30,9 @@ use j2k_core::DeviceSubmission; use j2k_core::{BackendKind, PixelFormat}; #[cfg(target_os = "macos")] use j2k_native::{ - forward_dwt53_reference, quantize_reversible_reference as quantize_reference, - try_deinterleave_reference, EncodeOptions, J2kCodeBlockStyle, + forward_dwt53_reference, forward_dwt97_reference, forward_ict_reference, + quantize_subband_reference as quantize_reference, try_deinterleave_reference, EncodeOptions, + J2kCodeBlockStyle, }; use j2k_native::{DecodeSettings, Image}; #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/encode/tests/dwt_parity.rs b/crates/j2k-metal/src/encode/tests/dwt_parity.rs index 45ef4876..a1eabf84 100644 --- a/crates/j2k-metal/src/encode/tests/dwt_parity.rs +++ b/crates/j2k-metal/src/encode/tests/dwt_parity.rs @@ -178,3 +178,72 @@ fn metal_forward_dwt97_multi_level_matches_native_encode_output() { assert_metal_dwt97_matches_native_encode(&pixels, width, height, 3); } + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_dwt97_matches_fractional_cpu_reference_exactly() { + fn assert_exact(actual: &[f32], expected: &[f32], label: &str) { + assert_eq!(actual.len(), expected.len(), "{label} length mismatch"); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "{label}[{index}] mismatch: actual={actual:?}, expected={expected:?}" + ); + } + } + + if !should_run_metal_runtime() { + return; + } + + let width = 64; + let height = 48; + let samples = (0..width * height) + .map(|index| { + let byte = + u8::try_from((index * 43 + index / 7 + 91) & 0xff).expect("masked sample fits u8"); + f32::from(byte) * 0.587 - 74.472_99 + }) + .collect::>(); + let expected = + forward_dwt97_reference(&samples, width, height, 3).expect("CPU 9/7 DWT reference"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + let actual = accelerator + .encode_forward_dwt97(J2kForwardDwt97Job { + samples: &samples, + width, + height, + num_levels: 3, + }) + .expect("Metal 9/7 DWT stage") + .expect("Metal 9/7 DWT dispatch"); + + assert_eq!(actual.ll_width, expected.ll_width); + assert_eq!(actual.ll_height, expected.ll_height); + assert_exact(&actual.ll, &expected.ll, "LL"); + assert_eq!(actual.levels.len(), expected.levels.len()); + for (index, (actual, expected)) in actual.levels.iter().zip(&expected.levels).enumerate() { + assert_eq!(actual.width, expected.width, "level {index} width"); + assert_eq!(actual.height, expected.height, "level {index} height"); + assert_eq!( + actual.low_width, expected.low_width, + "level {index} low width" + ); + assert_eq!( + actual.low_height, expected.low_height, + "level {index} low height" + ); + assert_eq!( + actual.high_width, expected.high_width, + "level {index} high width" + ); + assert_eq!( + actual.high_height, expected.high_height, + "level {index} high height" + ); + assert_exact(&actual.hl, &expected.hl, "HL"); + assert_exact(&actual.lh, &expected.lh, "LH"); + assert_exact(&actual.hh, &expected.hh, "HH"); + } +} diff --git a/crates/j2k-metal/src/encode/tests/routing.rs b/crates/j2k-metal/src/encode/tests/routing.rs index d138ff70..72dd168d 100644 --- a/crates/j2k-metal/src/encode/tests/routing.rs +++ b/crates/j2k-metal/src/encode/tests/routing.rs @@ -37,6 +37,92 @@ fn auto_host_output_encode_options_preserve_auto_for_hybrid_path() { assert_eq!(routed.validation, J2kEncodeValidation::External); } +#[cfg(target_os = "macos")] +#[test] +fn auto_lossy_host_output_thresholds_match_verified_external_cells() { + let context = |num_pixels, num_components, reversible, bit_depth, signed| J2kEncodeContext { + num_pixels, + num_components, + bit_depth, + signed, + reversible, + }; + + assert!(!auto_host_output_should_dispatch(context( + 640 * 480, + 1, + false, + 8, + false + ))); + assert!(!auto_host_output_should_dispatch(context( + 3323 * 891, + 1, + false, + 8, + false + ))); + assert!(!auto_host_output_should_dispatch(context( + 3323 * 891, + 1, + true, + 8, + false + ))); + + assert!(!auto_host_output_should_dispatch(context( + 640 * 480, + 3, + false, + 8, + false + ))); + assert!(auto_host_output_should_dispatch(context( + 2592 * 1944, + 3, + false, + 8, + false + ))); + assert!(!auto_host_output_should_dispatch(context( + 2592 * 1944, + 4, + false, + 8, + false + ))); + assert!(!auto_host_output_should_dispatch(context( + 2592 * 1944, + 3, + false, + 16, + false + ))); + assert!(!auto_host_output_should_dispatch(context( + 2592 * 1944, + 3, + false, + 8, + true + ))); +} + +#[cfg(target_os = "macos")] +#[test] +fn host_output_evidence_route_uses_only_supported_component_counts() { + let context = |num_components| J2kEncodeContext { + num_pixels: 64 * 64, + num_components, + bit_depth: 8, + signed: false, + reversible: true, + }; + + assert!(host_output_evidence_should_dispatch(context(1))); + assert!(host_output_evidence_should_dispatch(context(4))); + assert!(!host_output_evidence_should_dispatch(context(5))); +} + #[cfg(target_os = "macos")] #[test] fn auto_classic_host_output_stays_cpu_without_metal_dispatches() { @@ -69,7 +155,52 @@ fn auto_classic_host_output_stays_cpu_without_metal_dispatches() { #[cfg(target_os = "macos")] #[test] -fn auto_classic_large_host_output_dispatches_benchmark_gated_prep_stages_only() { +fn host_output_benchmark_route_forces_small_prep_stages_only() { + if !should_run_metal_runtime() { + return; + } + + let width = 64u32; + let height = 64u32; + let pixels = (0..width * height) + .map(|index| u8::try_from((index * 17 + index / 5) & 0xff).expect("masked pixel fits u8")) + .collect::>(); + let options = lossless_options! { + backend: EncodeBackendPreference::Auto, + block_coding_mode: J2kBlockCodingMode::Classic, + max_decomposition_levels: Some(3), + validation: J2kEncodeValidation::External, + }; + let expected = encode_j2k_lossless( + J2kLosslessSamples::new(&pixels, width, height, 1, 8, false).expect("valid CPU samples"), + &lossless_options! { + backend: EncodeBackendPreference::CpuOnly, + block_coding_mode: J2kBlockCodingMode::Classic, + max_decomposition_levels: Some(3), + validation: J2kEncodeValidation::External, + }, + ) + .expect("CPU lossless encode"); + let mut accelerator = MetalEncodeStageAccelerator::for_host_output_benchmark(); + let actual = encode_j2k_lossless_with_accelerator( + J2kLosslessSamples::new(&pixels, width, height, 1, 8, false).expect("valid hybrid samples"), + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("benchmark host-output encode"); + + assert_eq!(actual.codestream, expected.codestream); + assert_eq!(actual.dispatch_report.deinterleave, 1); + assert_eq!(actual.dispatch_report.forward_dwt53, 1); + assert!(actual.dispatch_report.quantize_subband > 0); + assert_eq!(actual.dispatch_report.tier1_code_block, 0); + assert_eq!(actual.dispatch_report.packetization, 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_classic_large_lossless_host_output_stays_cpu_without_metal_dispatches() { if !should_run_metal_runtime() { return; } @@ -94,12 +225,12 @@ fn auto_classic_large_host_output_dispatches_benchmark_gated_prep_stages_only() BackendKind::Metal, &mut accelerator, ) - .expect("benchmark-gated Auto host-output encode"); + .expect("Auto lossless host-output encode"); assert_eq!(encoded.backend, BackendKind::Cpu); - assert_eq!(accelerator.deinterleave_dispatches(), 1); - assert_eq!(accelerator.forward_dwt53_dispatches(), 1); - assert!(accelerator.quantize_subband_dispatches() > 0); + assert_eq!(accelerator.deinterleave_dispatches(), 0); + assert_eq!(accelerator.forward_dwt53_dispatches(), 0); + assert_eq!(accelerator.quantize_subband_dispatches(), 0); assert_eq!(accelerator.tier1_code_block_dispatches(), 0); assert_eq!(accelerator.packetization_dispatches(), 0); } diff --git a/crates/j2k-metal/src/encode/tests/stage_validation.rs b/crates/j2k-metal/src/encode/tests/stage_validation.rs index 4cf4c74d..8f93b15a 100644 --- a/crates/j2k-metal/src/encode/tests/stage_validation.rs +++ b/crates/j2k-metal/src/encode/tests/stage_validation.rs @@ -53,7 +53,7 @@ fn metal_encode_deinterleave_compute_rejects_invalid_shape_structured() { #[cfg(target_os = "macos")] #[test] -fn metal_quantize_subband_kernel_matches_cpu_reference() { +fn metal_quantize_subband_kernel_matches_cpu_reference_at_boundaries() { #[derive(Clone, Copy)] struct Case { name: &'static str, @@ -67,13 +67,6 @@ fn metal_quantize_subband_kernel_matches_cpu_reference() { return; } - let coefficients = (0_u16..257) - .map(|idx| { - let centered = f32::from(idx) - 128.0; - centered * 0.375 + f32::from(idx % 7) * 0.125 - if idx % 5 == 0 { 0.5 } else { 0.0 } - }) - .collect::>(); - for case in [ Case { name: "reversible", @@ -104,6 +97,23 @@ fn metal_quantize_subband_kernel_matches_cpu_reference() { reversible: false, }, ] { + let mut coefficients = (0_u16..257) + .map(|idx| { + let centered = f32::from(idx) - 128.0; + centered * 0.375 + f32::from(idx % 7) * 0.125 - if idx % 5 == 0 { 0.5 } else { 0.0 } + }) + .collect::>(); + if !case.reversible { + let exponent = i32::from(case.range_bits) - i32::from(case.step_exponent); + let base = 2.0_f32.powi(exponent); + let delta = base * (1.0 + f32::from(case.step_mantissa) / 2048.0); + for magnitude in [1_u16, 2, 5, 30, 255, 4_096] { + let boundary = delta * f32::from(magnitude); + let below = f32::from_bits(boundary.to_bits() - 1); + let above = f32::from_bits(boundary.to_bits() + 1); + coefficients.extend([below, boundary, above, -below, -boundary, -above]); + } + } let expected = quantize_reference( &coefficients, case.step_exponent, @@ -120,7 +130,20 @@ fn metal_quantize_subband_kernel_matches_cpu_reference() { }) .unwrap_or_else(|err| panic!("Metal quantize_subband failed for {}: {err}", case.name)); - assert_eq!(actual, expected, "{}", case.name); + assert_eq!(actual.len(), expected.len(), "{} length", case.name); + if let Some((index, (&actual, &expected))) = actual + .iter() + .zip(&expected) + .enumerate() + .find(|(_, (actual, expected))| actual != expected) + { + panic!( + "{} coefficient[{index}]={:?} (bits={:#010x}): Metal={actual}, CPU={expected}", + case.name, + coefficients[index], + coefficients[index].to_bits() + ); + } } } @@ -321,41 +344,40 @@ fn metal_encode_stage_accelerator_can_leave_forward_rct_on_cpu() { #[cfg(target_os = "macos")] #[test] -fn metal_forward_ict_dispatch_matches_cpu_reference() { - fn forward_ict_reference( - plane0: &[f32], - plane1: &[f32], - plane2: &[f32], - ) -> (Vec, Vec, Vec) { - let mut out0 = Vec::with_capacity(plane0.len()); - let mut out1 = Vec::with_capacity(plane1.len()); - let mut out2 = Vec::with_capacity(plane2.len()); - for ((&r, &g), &b) in plane0.iter().zip(plane1).zip(plane2) { - out0.push(0.299 * r + 0.587 * g + 0.114 * b); - out1.push(-0.16875 * r - 0.33126 * g + 0.5 * b); - out2.push(0.5 * r - 0.41869 * g - 0.08131 * b); - } - (out0, out1, out2) - } - - fn assert_near(actual: &[f32], expected: &[f32], label: &str) { +fn metal_forward_ict_dispatch_matches_cpu_reference_exactly() { + fn assert_exact(actual: &[f32], expected: &[f32], label: &str) { assert_eq!(actual.len(), expected.len(), "{label} length mismatch"); for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { - assert!( - (actual - expected).abs() <= 0.0001, + assert_eq!( + actual.to_bits(), + expected.to_bits(), "{label}[{index}] mismatch: actual={actual}, expected={expected}" ); } } + fn centered_u8(value: u32) -> f32 { + f32::from(u8::try_from(value & 0xff).expect("masked sample fits u8")) - 128.0 + } + if !should_run_metal_runtime() { return; } - let mut plane0 = vec![0.0, 64.0, 128.0, 255.0, -12.5, 42.25]; - let mut plane1 = vec![3.0, 67.0, 131.0, 252.0, 19.75, -8.5]; - let mut plane2 = vec![7.0, 71.0, 135.0, 248.0, 33.5, 128.0]; - let expected = forward_ict_reference(&plane0, &plane1, &plane2); + let mut plane0 = (0_u32..4_096) + .map(|index| centered_u8(index * 17 + index / 7)) + .collect::>(); + let mut plane1 = (0_u32..4_096) + .map(|index| centered_u8(index * 29 + index / 11 + 31)) + .collect::>(); + let mut plane2 = (0_u32..4_096) + .map(|index| centered_u8(index * 43 + index / 13 + 73)) + .collect::>(); + plane0.extend([32.0, 35.0]); + plane1.extend([-7.0, -5.0]); + plane2.extend([-75.0, -74.0]); + let expected = + forward_ict_reference(Vec::from([plane0.clone(), plane1.clone(), plane2.clone()])); let mut accelerator = MetalEncodeStageAccelerator::default(); let dispatched = accelerator @@ -367,9 +389,9 @@ fn metal_forward_ict_dispatch_matches_cpu_reference() { .expect("Metal ICT dispatch"); assert!(dispatched); - assert_near(&plane0, &expected.0, "Y"); - assert_near(&plane1, &expected.1, "Cb"); - assert_near(&plane2, &expected.2, "Cr"); + assert_exact(&plane0, &expected[0], "Y"); + assert_exact(&plane1, &expected[1], "Cb"); + assert_exact(&plane2, &expected[2], "Cr"); assert_eq!(accelerator.forward_ict_attempts(), 1); assert_eq!(accelerator.forward_ict_dispatches(), 1); let report = accelerator.dispatch_report(); diff --git a/crates/j2k-metal/src/fdwt.metal b/crates/j2k-metal/src/fdwt.metal index 1ccc2db1..f31d1c59 100644 --- a/crates/j2k-metal/src/fdwt.metal +++ b/crates/j2k-metal/src/fdwt.metal @@ -280,6 +280,8 @@ kernel void j2k_forward_dwt97_lift_horizontal( constant J2kForwardDwt97Params ¶ms [[buffer(2)]], uint2 gid [[thread_position_in_grid]] ) { +#pragma clang fp reassociate(off) +#pragma clang fp contract(off) (void)unused; if ( gid.x >= params.current_width || @@ -307,7 +309,11 @@ kernel void j2k_forward_dwt97_lift_horizontal( update_high, false ); - data[row_base + gid.x] += params.coefficient * (left + right); + data[row_base + gid.x] = fma( + params.coefficient, + left + right, + data[row_base + gid.x] + ); } kernel void j2k_forward_dwt97_lift_vertical( @@ -316,6 +322,8 @@ kernel void j2k_forward_dwt97_lift_vertical( constant J2kForwardDwt97Params ¶ms [[buffer(2)]], uint2 gid [[thread_position_in_grid]] ) { +#pragma clang fp reassociate(off) +#pragma clang fp contract(off) (void)unused; if ( gid.x >= params.current_width || @@ -344,7 +352,8 @@ kernel void j2k_forward_dwt97_lift_vertical( update_high, false ); - data[gid.y * params.full_width + gid.x] += params.coefficient * (top + bottom); + const uint index = gid.y * params.full_width + gid.x; + data[index] = fma(params.coefficient, top + bottom, data[index]); } kernel void j2k_forward_dwt97_deinterleave_horizontal( diff --git a/crates/j2k-metal/src/mct.metal b/crates/j2k-metal/src/mct.metal index cda1f912..aee5cf5c 100644 --- a/crates/j2k-metal/src/mct.metal +++ b/crates/j2k-metal/src/mct.metal @@ -102,6 +102,8 @@ kernel void j2k_forward_ict( device J2kMctStatus *status [[buffer(4)]], uint gid [[thread_position_in_grid]] ) { +#pragma clang fp reassociate(off) +#pragma clang fp contract(off) if (gid >= params.len) { return; } @@ -110,9 +112,10 @@ kernel void j2k_forward_ict( const float g = plane1[gid]; const float b = plane2[gid]; - plane0[gid] = 0.299f * r + 0.587f * g + 0.114f * b; - plane1[gid] = -0.16875f * r - 0.33126f * g + 0.5f * b; - plane2[gid] = 0.5f * r - 0.41869f * g - 0.08131f * b; + // Match the CPU transform's target-independent nested fused rounding. + plane0[gid] = fma(0.114f, b, fma(0.299f, r, 0.587f * g)); + plane1[gid] = fma(0.5f, b, fma(-0.16875f, r, -0.33126f * g)); + plane2[gid] = fma(-0.08131f, b, fma(0.5f, r, -0.41869f * g)); if (gid == 0) { status->code = J2K_MCT_STATUS_OK; diff --git a/crates/j2k-metal/src/mct.rs b/crates/j2k-metal/src/mct.rs index 530585a3..c01c60de 100644 --- a/crates/j2k-metal/src/mct.rs +++ b/crates/j2k-metal/src/mct.rs @@ -203,6 +203,51 @@ mod tests { ); } + #[test] + fn metal_mct_decoder_matches_native_region_for_openjpeg_irreversible_rgb() { + #[cfg(target_os = "macos")] + if !should_run_metal_runtime() { + return; + } + + let image = Image::new( + j2k_test_support::OPENJPEG_IRREVERSIBLE_RGB8_8X8, + &DecodeSettings::default(), + ) + .expect("image"); + let roi = (2, 2, 4, 4); + let mut expected_context = DecoderContext::default(); + let expected = image + .decode_region_components_with_ht_decoder( + &mut expected_context, + roi, + &mut CpuOnlyCodeBlockDecoder, + ) + .expect("native region decode"); + + let mut hooked_context = DecoderContext::default(); + let mut decoder = MetalMctDecoder::default(); + let actual = image + .decode_region_components_with_ht_decoder(&mut hooked_context, roi, &mut decoder) + .expect("Metal MCT region decode"); + + assert_eq!(actual.dimensions(), expected.dimensions()); + for (component, (actual_plane, expected_plane)) in + actual.planes().iter().zip(expected.planes()).enumerate() + { + assert_eq!( + actual_plane.samples(), + expected_plane.samples(), + "Metal MCT component {component} must match native region decode" + ); + } + #[cfg(target_os = "macos")] + assert!( + decoder.kernel_dispatches() > 0, + "OpenJPEG RGB region must exercise the Metal MCT kernel" + ); + } + #[test] fn metal_mct_decoder_captures_final_rgb_planes_matching_host_output() { #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/src/quantize.metal b/crates/j2k-metal/src/quantize.metal index 3fb80325..53171b2d 100644 --- a/crates/j2k-metal/src/quantize.metal +++ b/crates/j2k-metal/src/quantize.metal @@ -34,7 +34,8 @@ inline int j2k_quantize_sample(float sample, constant J2kQuantizeSubbandParams & } const int sign = sample < 0.0f ? -1 : 1; - const int magnitude = int(floor(fabs(sample) / delta)); + // Fast division can cross a deadzone boundary; match the CPU quotient. + const int magnitude = int(floor(precise::divide(fabs(sample), delta))); return sign * magnitude; } diff --git a/crates/j2k-metal/src/routing.rs b/crates/j2k-metal/src/routing.rs index 1794c626..c6bd9240 100644 --- a/crates/j2k-metal/src/routing.rs +++ b/crates/j2k-metal/src/routing.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use j2k_core::{BackendRequest, PixelFormat}; +use j2k_core::{BackendRequest, CompressedTransferSyntax, PixelFormat}; #[cfg(target_os = "macos")] use j2k_metal_support::metal_kernel_route; use j2k_metal_support::{ @@ -13,6 +13,37 @@ use crate::Error; pub(crate) const AUTO_DECODE_CPU_FALLBACK_REASON: &str = "J2K Metal Auto decode stays on CPU until decode benchmark evidence justifies Metal routing"; +// Minimum qualified cells from verified Auto-routing artifact +// 162a47f7a96b2be88abebc100aab672513af04895532863fa1a293660546f879. +const AUTO_REPEATED_DECODE_MIN_COUNT: usize = 16; +const AUTO_REPEATED_GRAY8_MIN_PIXELS: u64 = 2_960_793; +const AUTO_REPEATED_RGB8_LOSSY_MIN_PIXELS: u64 = 307_200; +const AUTO_REPEATED_RGB8_LOSSLESS_MIN_PIXELS: u64 = 5_038_848; + +pub(crate) fn auto_repeated_decode_uses_metal( + dimensions: (u32, u32), + fmt: PixelFormat, + count: usize, + transfer_syntax: CompressedTransferSyntax, +) -> bool { + if count < AUTO_REPEATED_DECODE_MIN_COUNT { + return false; + } + let pixels = u64::from(dimensions.0) * u64::from(dimensions.1); + match (fmt, transfer_syntax) { + (PixelFormat::Gray8, CompressedTransferSyntax::Jpeg2000Lossy) => { + pixels >= AUTO_REPEATED_GRAY8_MIN_PIXELS + } + (PixelFormat::Rgb8, CompressedTransferSyntax::Jpeg2000Lossy) => { + pixels >= AUTO_REPEATED_RGB8_LOSSY_MIN_PIXELS + } + (PixelFormat::Rgb8, CompressedTransferSyntax::Jpeg2000Lossless) => { + pixels >= AUTO_REPEATED_RGB8_LOSSLESS_MIN_PIXELS + } + _ => false, + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum RouteDecision { CpuHost, @@ -151,6 +182,77 @@ fn j2k_route_decision_profile(decision: RouteDecision) -> MetalRouteProfileLabel mod tests { use super::*; + #[test] + fn auto_repeated_decode_thresholds_match_verified_external_cells() { + assert!(!auto_repeated_decode_uses_metal( + (512, 512), + PixelFormat::Gray8, + 16, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + assert!(auto_repeated_decode_uses_metal( + (3323, 891), + PixelFormat::Gray8, + 16, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + assert!(!auto_repeated_decode_uses_metal( + (3323, 891), + PixelFormat::Gray8, + 16, + CompressedTransferSyntax::Jpeg2000Lossless, + )); + assert!(!auto_repeated_decode_uses_metal( + (3323, 891), + PixelFormat::Gray16, + 16, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + + assert!(!auto_repeated_decode_uses_metal( + (256, 149), + PixelFormat::Rgb8, + 16, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + assert!(auto_repeated_decode_uses_metal( + (640, 480), + PixelFormat::Rgb8, + 16, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + assert!(!auto_repeated_decode_uses_metal( + (640, 480), + PixelFormat::Rgb8, + 16, + CompressedTransferSyntax::Jpeg2000Lossless, + )); + assert!(auto_repeated_decode_uses_metal( + (2592, 1944), + PixelFormat::Rgb8, + 16, + CompressedTransferSyntax::Jpeg2000Lossless, + )); + assert!(!auto_repeated_decode_uses_metal( + (640, 480), + PixelFormat::Rgba8, + 16, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + assert!(!auto_repeated_decode_uses_metal( + (2592, 1944), + PixelFormat::Rgb8, + 15, + CompressedTransferSyntax::Jpeg2000Lossy, + )); + assert!(!auto_repeated_decode_uses_metal( + (2592, 1944), + PixelFormat::Rgb8, + 16, + CompressedTransferSyntax::HtJpeg2000Lossless, + )); + } + #[test] fn cuda_route_reports_unsupported_backend() { assert_eq!( diff --git a/crates/j2k-metal/src/store.rs b/crates/j2k-metal/src/store.rs index 315a8447..fb1ff5b0 100644 --- a/crates/j2k-metal/src/store.rs +++ b/crates/j2k-metal/src/store.rs @@ -169,6 +169,51 @@ mod tests { ); } + #[test] + fn metal_store_decoder_matches_native_region_for_openjpeg_irreversible_rgb() { + #[cfg(target_os = "macos")] + if !should_run_metal_runtime() { + return; + } + + let image = Image::new( + j2k_test_support::OPENJPEG_IRREVERSIBLE_RGB8_8X8, + &DecodeSettings::default(), + ) + .expect("image"); + let roi = (2, 2, 4, 4); + let mut expected_context = DecoderContext::default(); + let expected = image + .decode_region_components_with_ht_decoder( + &mut expected_context, + roi, + &mut CpuOnlyCodeBlockDecoder, + ) + .expect("native region decode"); + + let mut hooked_context = DecoderContext::default(); + let mut decoder = MetalStoreDecoder::default(); + let actual = image + .decode_region_components_with_ht_decoder(&mut hooked_context, roi, &mut decoder) + .expect("Metal store region decode"); + + assert_eq!(actual.dimensions(), expected.dimensions()); + for (component, (actual_plane, expected_plane)) in + actual.planes().iter().zip(expected.planes()).enumerate() + { + assert_eq!( + actual_plane.samples(), + expected_plane.samples(), + "Metal store component {component} must match native region decode" + ); + } + #[cfg(target_os = "macos")] + assert!( + decoder.kernel_dispatches() > 0, + "OpenJPEG RGB region must exercise the Metal store kernel" + ); + } + #[test] fn metal_store_decoder_captures_device_plane_for_full_decode() { #[cfg(target_os = "macos")] diff --git a/crates/j2k-metal/tests/batch_classic_color.rs b/crates/j2k-metal/tests/batch_classic_color.rs index 809e2b96..a7618f90 100644 --- a/crates/j2k-metal/tests/batch_classic_color.rs +++ b/crates/j2k-metal/tests/batch_classic_color.rs @@ -155,7 +155,6 @@ fn assert_classic_rgb_layout( layout, ..BatchDecodeOptions::default() }; - let max_lsb_diff = u8::from(matches!(profile, ClassicRgbProfile::U8Irreversible97)); let inputs = requests .iter() .copied() @@ -230,7 +229,7 @@ fn assert_classic_rgb_layout( j2k_metal_support::checked_buffer_read_vec::(&buffer, 4, output_bytes) .expect("classic RGB output samples") }; - assert_native_samples(&actual, expected_group.samples(), max_lsb_diff); + assert_native_samples(&actual, expected_group.samples(), 0); } } diff --git a/crates/j2k-metal/tests/bench_harness.rs b/crates/j2k-metal/tests/bench_harness.rs index d20db0a5..83c2f91e 100644 --- a/crates/j2k-metal/tests/bench_harness.rs +++ b/crates/j2k-metal/tests/bench_harness.rs @@ -39,12 +39,17 @@ fn bench_sources_under(path: &Path) -> Vec { } #[test] -fn j2k_metal_has_no_legacy_criterion_bench_targets() { +fn j2k_metal_declares_only_the_auto_routing_criterion_bench() { let cargo = cargo_toml(); + assert_eq!( + cargo.matches("[[bench]]").count(), + 1, + "j2k-metal must keep one audited benchmark target" + ); assert!( - !cargo.contains("[[bench]]"), - "j2k-metal bench targets were reset for a clean profiling redesign" + cargo.contains("[[bench]]\nname = \"auto_routing\"\nharness = false\ntest = false"), + "j2k-metal must keep the release-routing benchmark explicit" ); for target in ["device_upload", "compare", "encode_stages", "decode_stages"] { @@ -56,24 +61,24 @@ fn j2k_metal_has_no_legacy_criterion_bench_targets() { } #[test] -fn j2k_metal_has_no_legacy_bench_only_dev_dependencies() { +fn j2k_metal_bench_dependencies_are_limited_to_auto_routing() { let cargo = cargo_toml(); - for dependency in ["criterion", "j2k-compare"] { - assert!( - !cargo.contains(&format!("{dependency} =")), - "legacy bench-only dev dependency must stay removed: {dependency}" - ); - } + assert_eq!(cargo.matches("criterion =").count(), 1); + assert!( + !cargo.contains("j2k-compare ="), + "legacy comparison dependency must stay removed" + ); } #[test] -fn j2k_metal_benches_directory_is_clean_for_redesign() { +fn j2k_metal_benches_directory_contains_only_auto_routing() { let sources = bench_sources_under(&manifest_dir().join("benches")); - assert!( - sources.is_empty(), - "remove stale j2k-metal bench sources before adding new profiling benches: {sources:?}" + assert_eq!( + sources, + [manifest_dir().join("benches/auto_routing.rs")], + "j2k-metal benchmark sources must stay limited to release routing evidence" ); } diff --git a/crates/j2k-metal/tests/device.rs b/crates/j2k-metal/tests/device.rs index d26ef7c7..77f4981a 100644 --- a/crates/j2k-metal/tests/device.rs +++ b/crates/j2k-metal/tests/device.rs @@ -42,7 +42,7 @@ fn should_run_metal_runtime() -> bool { j2k_test_support::metal_runtime_gate(module_path!()) } -fn unsupported_classic_roi_rgb() -> Arc<[u8]> { +fn unsupported_ht_roi_rgb() -> Arc<[u8]> { let pixels = (0..4_u8) .flat_map(|index| [index * 17, index * 29 + 3, index * 41 + 5]) .collect::>(); @@ -53,8 +53,8 @@ fn unsupported_classic_roi_rgb() -> Arc<[u8]> { ..EncodeOptions::default() }; Arc::from( - encode(&pixels, 2, 2, 3, 8, false, &options) - .expect("encode classic RGB8 with unsupported RGN maxshift"), + encode_htj2k(&pixels, 2, 2, 3, 8, false, &options) + .expect("encode HTJ2K RGB8 with unsupported RGN maxshift"), ) } @@ -285,6 +285,42 @@ fn fixture_classic_signed_gray12() -> (Vec, Vec) { ) } +fn fixture_classic_signed_gray4() -> (Vec, Vec) { + let samples = (-8_i16..=7).collect::>(); + let pixels = samples + .iter() + .map(|sample| sample.to_le_bytes()[0]) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + ( + encode(&pixels, 4, 4, 1, 4, true, &options).expect("encode signed classic gray4"), + samples, + ) +} + +fn fixture_classic_signed_gray4_roi() -> (Vec, Vec) { + let samples = (-8_i16..=7).collect::>(); + let pixels = samples + .iter() + .map(|sample| sample.to_le_bytes()[0]) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + roi_component_shifts: vec![7], + ..EncodeOptions::default() + }; + ( + encode(&pixels, 4, 4, 1, 4, true, &options) + .expect("encode signed classic gray4 with ROI maxshift"), + samples, + ) +} + fn fixture_gray8_irreversible() -> Vec { let pixels: Vec = (0..16).collect(); let options = EncodeOptions { diff --git a/crates/j2k-metal/tests/device/batch_sessions.rs b/crates/j2k-metal/tests/device/batch_sessions.rs index 82c24edf..2253cb2e 100644 --- a/crates/j2k-metal/tests/device/batch_sessions.rs +++ b/crates/j2k-metal/tests/device/batch_sessions.rs @@ -176,7 +176,7 @@ fn submitted_shared_batch_continues_after_nonfatal_group_submit_failure() { } let valid_gray = Arc::<[u8]>::from(fixture_ht_gray8()); - let unsupported_roi_rgb = unsupported_classic_roi_rgb(); + let unsupported_roi_rgb = unsupported_ht_roi_rgb(); let options = BatchDecodeOptions::default(); let mut decoder = MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); diff --git a/crates/j2k-metal/tests/device/decode.rs b/crates/j2k-metal/tests/device/decode.rs index c27462ef..d4acd269 100644 --- a/crates/j2k-metal/tests/device/decode.rs +++ b/crates/j2k-metal/tests/device/decode.rs @@ -27,6 +27,70 @@ fn full_classic_grayscale_decode_to_metal_matches_host_decode() { ); } +#[test] +fn full_classic_signed_gray4_decode_to_metal_matches_host_exactly() { + if !should_run_metal_runtime() { + return; + } + + let (bytes, expected) = fixture_classic_signed_gray4(); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let batch = decoder + .decode_batch(vec![EncodedImage::full(Arc::from(bytes))]) + .expect("decode signed classic gray4 batch"); + assert!(batch.errors().is_empty(), "{:?}", batch.errors()); + assert!( + batch.group_errors().is_empty(), + "{:?}", + batch.group_errors() + ); + let surface = &batch.groups()[0].surfaces()[0]; + let actual = surface + .as_bytes() + .expect("signed classic gray4 surface bytes") + .chunks_exact(2) + .map(|sample| i16::from_ne_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(actual, expected); +} + +#[test] +fn full_classic_signed_gray4_roi_decode_to_metal_matches_host_exactly() { + if !should_run_metal_runtime() { + return; + } + + let (bytes, expected) = fixture_classic_signed_gray4_roi(); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let mut decoder = + MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); + let batch = decoder + .decode_batch(vec![EncodedImage::full(Arc::from(bytes))]) + .expect("decode signed classic gray4 ROI batch"); + assert!(batch.errors().is_empty(), "{:?}", batch.errors()); + assert!( + batch.group_errors().is_empty(), + "{:?}", + batch.group_errors() + ); + let surface = &batch.groups()[0].surfaces()[0]; + let actual = surface + .as_bytes() + .expect("signed classic gray4 ROI surface bytes") + .chunks_exact(2) + .map(|sample| i16::from_ne_bytes([sample[0], sample[1]])) + .collect::>(); + assert_eq!(actual, expected); +} + #[test] fn full_htj2k_decode_to_metal_matches_host_decode() { if !should_run_metal_runtime() { @@ -109,6 +173,92 @@ fn full_irreversible_j2k_decode_to_metal_matches_host_decode() { ); } +#[test] +fn full_irreversible_rgb_j2k_decode_to_metal_matches_host_decode_exactly() { + if !should_run_metal_runtime() { + return; + } + + let pixels = j2k_test_support::gradient_u8(16, 16, 3); + let bytes = encode( + &pixels, + 16, + 16, + 3, + 8, + false, + &EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }, + ) + .expect("encode irreversible RGB8"); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; 16 * 16 * 3]; + host_decoder + .decode_into(&mut host, 16 * 3, PixelFormat::Rgb8) + .expect("host decode"); + + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Metal) + .expect("device decode"); + + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.as_bytes().expect("surface byte access"), host); +} + +#[test] +fn openjpeg_irreversible_rgb_roi_decode_to_metal_matches_cpu_exactly() { + if !should_run_metal_runtime() { + return; + } + + let codestream = j2k_test_support::OPENJPEG_IRREVERSIBLE_RGB8_8X8; + let roi = Rect { + x: 2, + y: 2, + w: 4, + h: 4, + }; + let mut decoder = J2kDecoder::new(codestream).expect("decoder"); + let session = MetalBackendSession::system_default().expect("Metal session"); + let mut cpu_decoder = J2kDecoder::new(codestream).expect("CPU decoder"); + let cpu = cpu_decoder + .decode_request_to_device_with_session( + MetalDecodeRequest::region(PixelFormat::Rgb8, roi, BackendRequest::Cpu), + &session, + ) + .expect("CPU surface decode"); + let mut full_decoder = J2kDecoder::new(codestream).expect("full CPU decoder"); + let full = full_decoder + .decode_request_to_device_with_session( + MetalDecodeRequest::full(PixelFormat::Rgb8, BackendRequest::Cpu), + &session, + ) + .expect("full CPU surface decode"); + let full = full.as_bytes().expect("full CPU surface byte access"); + let mut cropped = Vec::with_capacity(4 * 4 * 3); + for y in roi.y..roi.y + roi.h { + let start = (y as usize * 8 + roi.x as usize) * 3; + cropped.extend_from_slice(&full[start..start + roi.w as usize * 3]); + } + assert_eq!(cpu.as_bytes().expect("CPU surface byte access"), cropped); + let surface = decoder + .decode_request_to_device_with_session( + MetalDecodeRequest::region(PixelFormat::Rgb8, roi, BackendRequest::Metal), + &session, + ) + .expect("device decode"); + + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!( + surface.as_bytes().expect("surface byte access"), + cpu.as_bytes().expect("CPU surface byte access") + ); +} + #[test] fn auto_full_grayscale_prefers_cpu_for_small_classic_fixture() { let bytes = fixture_gray8(); @@ -143,11 +293,7 @@ fn auto_repeated_grayscale_keeps_short_512_batch_on_cpu() { } #[test] -fn auto_repeated_grayscale_uses_metal_for_512_batch() { - if !should_run_metal_runtime() { - return; - } - +fn auto_repeated_grayscale_keeps_unqualified_512_batch_on_cpu() { let bytes = fixture_gray8_sized(512, 512); let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); let surfaces = decoder @@ -156,7 +302,7 @@ fn auto_repeated_grayscale_uses_metal_for_512_batch() { assert_eq!(surfaces.len(), 16); assert!(surfaces .iter() - .all(|surface| surface.backend_kind() == BackendKind::Metal)); + .all(|surface| surface.backend_kind() == BackendKind::Cpu)); } #[test] diff --git a/crates/j2k-metal/tests/device/legacy_batch.rs b/crates/j2k-metal/tests/device/legacy_batch.rs index 604eaf66..ea9b36cc 100644 --- a/crates/j2k-metal/tests/device/legacy_batch.rs +++ b/crates/j2k-metal/tests/device/legacy_batch.rs @@ -56,7 +56,7 @@ fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { } #[test] -fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { +fn submitted_auto_512_grayscale_tiles_stay_on_cpu() { if !should_run_metal_runtime() { return; } @@ -88,13 +88,14 @@ fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { for submission in submissions { let surface = submission.wait().expect("surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.backend_kind(), BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); assert_eq!(surface.dimensions(), (512, 512)); } assert_eq!( session.submissions().expect("session submissions"), 1, - "compatible auto grayscale tiles should flush through one repeated Metal batch" + "unqualified repeated Auto tiles should flush through one CPU batch fallback" ); } diff --git a/crates/j2k-metal/tests/device/resident_batch.rs b/crates/j2k-metal/tests/device/resident_batch.rs index a3c70571..4cad1503 100644 --- a/crates/j2k-metal/tests/device/resident_batch.rs +++ b/crates/j2k-metal/tests/device/resident_batch.rs @@ -144,7 +144,7 @@ fn metal_prepared_batch_continues_after_one_group_execution_failure() { MetalBatchDecoder::system_default_with_options(options).expect("persistent Metal decoder"); let prepared = decoder .prepare(vec![ - EncodedImage::full(unsupported_classic_roi_rgb()), + EncodedImage::full(unsupported_ht_roi_rgb()), EncodedImage::full(Arc::from(fixture_ht_gray8())), ]) .expect("prepare two distinct Metal groups"); diff --git a/crates/j2k-ml/Cargo.toml b/crates/j2k-ml/Cargo.toml index d5aeb179..5180a628 100644 --- a/crates/j2k-ml/Cargo.toml +++ b/crates/j2k-ml/Cargo.toml @@ -36,16 +36,16 @@ metal = [ burn-core = { workspace = true } burn-cuda = { workspace = true, optional = true } burn-wgpu = { workspace = true, optional = true } -j2k = { path = "../j2k", version = "=0.8.0" } -j2k-cuda = { path = "../j2k-cuda", version = "=0.8.0", features = ["cuda-runtime"], optional = true } -j2k-metal = { path = "../j2k-metal", version = "=0.8.0", optional = true } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0", optional = true } +j2k = { path = "../j2k", version = "=0.8.1" } +j2k-cuda = { path = "../j2k-cuda", version = "=0.8.1", features = ["cuda-runtime"], optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.8.1", optional = true } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.1", optional = true } thiserror = { workspace = true } [dev-dependencies] criterion = { workspace = true } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [target.'cfg(all(target_arch = "aarch64", target_os = "linux"))'.dev-dependencies] diff --git a/crates/j2k-ml/tests/cuda_batch_sessions.rs b/crates/j2k-ml/tests/cuda_batch_sessions.rs index 739bbde5..d72851c1 100644 --- a/crates/j2k-ml/tests/cuda_batch_sessions.rs +++ b/crates/j2k-ml/tests/cuda_batch_sessions.rs @@ -5,12 +5,15 @@ use std::sync::Arc; use burn_cuda::CudaDevice; -use j2k::{prepare_batch, BatchDecodeOptions, BatchItemError, DecodeSettings, EncodedImage}; -use j2k_ml::{BurnBatchTensor, BurnDecodeError, CudaUploadBurnDecoder}; +use j2k::{ + prepare_batch, BatchDecodeOptions, BatchItemError, CpuBatchDecoder, CpuBatchSamples, + DecodeSettings, EncodedImage, +}; +use j2k_ml::{BurnBatchTensor, CudaUploadBurnDecoder}; use j2k_native::{encode, EncodeOptions}; use j2k_test_support::{cuda_runtime_and_strict_oxide_gate, htj2k_gray8_large_fixture}; -fn unsupported_classic_roi_rgb() -> Arc<[u8]> { +fn classic_roi_rgb() -> Arc<[u8]> { let pixels = (0..4_u8) .flat_map(|index| [index * 17, index * 29 + 3, index * 41 + 5]) .collect::>(); @@ -29,7 +32,7 @@ fn unsupported_classic_roi_rgb() -> Arc<[u8]> { ..EncodeOptions::default() }, ) - .expect("encode classic RGB8 with unsupported RGN maxshift"), + .expect("encode classic RGB8 with RGN maxshift"), ) } @@ -205,43 +208,44 @@ fn staged_upload_session_reuses_events_and_codec_memory_for_one_thousand_batches } #[test] -fn cuda_burn_batch_continues_after_one_group_submit_failure() { - if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA group submit continuation") { +fn cuda_burn_batch_decodes_classic_roi_and_ht_groups_together() { + if !cuda_runtime_and_strict_oxide_gate("j2k-ml CUDA mixed classic/HT batch") { return; } + let options = BatchDecodeOptions::default(); let valid_gray = Arc::<[u8]>::from(htj2k_gray8_large_fixture(8, 8)); - let mut decoder = - CudaUploadBurnDecoder::new(CudaDevice::default(), BatchDecodeOptions::default()); + let mut decoder = CudaUploadBurnDecoder::new(CudaDevice::default(), options); let prepared = decoder .prepare(vec![ - EncodedImage::full(unsupported_classic_roi_rgb()), + EncodedImage::full(classic_roi_rgb()), EncodedImage::full(valid_gray), ]) .expect("prepare two homogeneous CUDA groups"); assert_eq!(prepared.groups().len(), 2); + let mut cpu = CpuBatchDecoder::new(options); + let expected = cpu + .decode_prepared(&prepared) + .expect("decode mixed classic/HT CPU oracle"); let submitted = decoder .submit_prepared(&prepared) - .expect("unsupported group must remain a result-level failure"); - assert_eq!(submitted.len(), 1); - let output = submitted.wait().expect("finish supported CUDA group"); + .expect("submit mixed classic/HT CUDA groups"); + assert_eq!(submitted.len(), 2); + let output = submitted.wait().expect("finish mixed CUDA groups"); assert!(output.errors.is_empty()); - assert_eq!(output.groups.len(), 1); - assert_eq!(output.groups[0].source_indices, [1]); - assert_eq!(output.group_errors.len(), 1); - assert_eq!(output.group_errors[0].source_indices(), &[0]); - let BurnDecodeError::Cuda(j2k_cuda::CudaBatchError::GroupExecution { - source_indices, - source, - .. - }) = output.group_errors[0].source() - else { - panic!("group-local CUDA failure must use the published batch error contract"); - }; - assert_eq!(source_indices, &[0]); - assert!(matches!( - source.as_ref(), - j2k_cuda::Error::UnsupportedCudaRequest { .. } - )); + assert!(output.group_errors.is_empty()); + assert_eq!(output.groups.len(), expected.groups().len()); + for (actual, expected) in output.groups.into_iter().zip(expected.groups()) { + assert_eq!(actual.source_indices, expected.source_indices()); + let (BurnBatchTensor::U8(actual), CpuBatchSamples::U8(expected)) = + (actual.tensor, expected.samples()) + else { + panic!("mixed classic/HT fixture must remain native U8") + }; + assert_eq!( + actual.into_data().into_vec::().expect("CUDA U8 data"), + *expected + ); + } } diff --git a/crates/j2k-ml/tests/metal.rs b/crates/j2k-ml/tests/metal.rs index 0b13dda8..653c990c 100644 --- a/crates/j2k-ml/tests/metal.rs +++ b/crates/j2k-ml/tests/metal.rs @@ -14,7 +14,7 @@ use j2k::{ }; use j2k_core::Colorspace; use j2k_ml::{BurnBatchTensor, BurnDecodeError, MetalUploadBurnDecoder}; -use j2k_native::{encode, EncodeOptions}; +use j2k_native::{encode_htj2k, EncodeOptions}; use j2k_test_support::{ generated_htj2k_rgba_fixture, htj2k_rgb8_97_fixture, metal_runtime_gate, openhtj2k_refinement_fixture, openhtj2k_refinement_odd_fixture, openhtj2k_refinement_pixels, @@ -61,12 +61,12 @@ fn wrap_rgba_jph(codestream: &[u8], alpha: Htj2kRgbaAlpha) -> Vec { .expect("wrap explicit HTJ2K RGBA image") } -fn unsupported_classic_roi_rgb() -> Arc<[u8]> { +fn unsupported_ht_roi_rgb() -> Arc<[u8]> { let pixels = (0..4_u8) .flat_map(|index| [index * 17, index * 29 + 3, index * 41 + 5]) .collect::>(); Arc::from( - encode( + encode_htj2k( &pixels, 2, 2, @@ -80,7 +80,7 @@ fn unsupported_classic_roi_rgb() -> Arc<[u8]> { ..EncodeOptions::default() }, ) - .expect("encode classic RGB8 with unsupported RGN maxshift"), + .expect("encode HTJ2K RGB8 with unsupported RGN maxshift"), ) } diff --git a/crates/j2k-ml/tests/metal/sessions.rs b/crates/j2k-ml/tests/metal/sessions.rs index d89d7abc..fe2ae73b 100644 --- a/crates/j2k-ml/tests/metal/sessions.rs +++ b/crates/j2k-ml/tests/metal/sessions.rs @@ -183,7 +183,7 @@ fn metal_burn_batch_continues_after_one_group_submit_failure() { .expect("paired J2K/Burn Metal session"); let prepared = decoder .prepare(vec![ - EncodedImage::full(unsupported_classic_roi_rgb()), + EncodedImage::full(unsupported_ht_roi_rgb()), EncodedImage::full(valid_gray), ]) .expect("prepare two homogeneous Metal groups"); diff --git a/crates/j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml index 429c92c0..5a6e0eaf 100644 --- a/crates/j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -28,9 +28,9 @@ fearless_simd = { workspace = true, optional = true } libm = { workspace = true } log = { workspace = true, optional = true } rayon = { workspace = true, optional = true } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0", default-features = false } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-types = { path = "../j2k-types", version = "=0.8.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1", default-features = false } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-types = { path = "../j2k-types", version = "=0.8.1" } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-native/src/backend.rs b/crates/j2k-native/src/backend.rs index 49d2a794..2e8d8490 100644 --- a/crates/j2k-native/src/backend.rs +++ b/crates/j2k-native/src/backend.rs @@ -277,6 +277,29 @@ pub trait HtCodeBlockDecoder { Ok(false) } + /// Optionally decode a full classic J2K sub-band with the requested + /// coefficient reconstruction rule. + /// + /// The default delegates reversible work to [`Self::decode_j2k_sub_band`]. + /// It declines irreversible midpoint reconstruction because legacy + /// adapters did not receive enough information to implement that rule. + /// + /// # Errors + /// + /// Returns an error when the backend cannot complete the decode request. + fn decode_j2k_sub_band_with_midpoint( + &mut self, + job: J2kSubBandDecodeJob<'_>, + output: &mut [f32], + irreversible_midpoint: bool, + ) -> Result { + if irreversible_midpoint { + Ok(false) + } else { + self.decode_j2k_sub_band(job, output) + } + } + /// Optionally decode one classic J2K code block. /// /// Implementations should return `Ok(true)` if they handled the request @@ -294,6 +317,29 @@ pub trait HtCodeBlockDecoder { Ok(false) } + /// Optionally decode one classic J2K code block with the requested + /// coefficient reconstruction rule. + /// + /// The default delegates reversible work to [`Self::decode_j2k_code_block`]. + /// It declines irreversible midpoint reconstruction so an older adapter + /// cannot silently emit coefficients with the wrong reconstruction bias. + /// + /// # Errors + /// + /// Returns an error when the backend cannot complete the decode request. + fn decode_j2k_code_block_with_midpoint( + &mut self, + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + irreversible_midpoint: bool, + ) -> Result { + if irreversible_midpoint { + Ok(false) + } else { + self.decode_j2k_code_block(job, output) + } + } + /// Optionally decode a full HTJ2K sub-band in one batch. /// /// Implementations should return `Ok(true)` if they handled the request and diff --git a/crates/j2k-native/src/direct_cpu.rs b/crates/j2k-native/src/direct_cpu.rs index c2f56df1..132504b7 100644 --- a/crates/j2k-native/src/direct_cpu.rs +++ b/crates/j2k-native/src/direct_cpu.rs @@ -4,6 +4,7 @@ use core::ops::Range; use crate::error::{bail, DecodingError, Result}; use crate::j2c::idwt; use crate::math::{floor_f32, round_f32}; +use crate::scalar::decode_j2k_code_block_scalar_with_workspace_midpoint; use crate::{ decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar_with_workspace, try_resize_decode_elements, HtCodeBlockDecodeJob, HtCodeBlockDecodeWorkspace, diff --git a/crates/j2k-native/src/direct_cpu/component.rs b/crates/j2k-native/src/direct_cpu/component.rs index 3aaeb6ef..74c3dcf5 100644 --- a/crates/j2k-native/src/direct_cpu/component.rs +++ b/crates/j2k-native/src/direct_cpu/component.rs @@ -1,11 +1,11 @@ use super::{ bail, decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar_with_workspace, - idwt, try_resize_decode_elements, DecodingError, DirectComponentBandScratch, - DirectComponentPlane, DirectCpuBand, DirectWorkspaceBudget, HtCodeBlockDecodeJob, - HtCodeBlockDecodeWorkspace, HtOwnedSubBandPlan, J2kCodeBlockDecodeJob, - J2kCodeBlockDecodeWorkspace, J2kDirectBandId, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, - J2kDirectIdwtStep, J2kDirectStoreStep, J2kIdwtBand, J2kOwnedSubBandPlan, J2kRect, - J2kSingleDecompositionIdwtJob, Range, Result, Vec, + decode_j2k_code_block_scalar_with_workspace_midpoint, idwt, try_resize_decode_elements, + DecodingError, DirectComponentBandScratch, DirectComponentPlane, DirectCpuBand, + DirectWorkspaceBudget, HtCodeBlockDecodeJob, HtCodeBlockDecodeWorkspace, HtOwnedSubBandPlan, + J2kCodeBlockDecodeJob, J2kCodeBlockDecodeWorkspace, J2kDirectBandId, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, J2kIdwtBand, + J2kOwnedSubBandPlan, J2kRect, J2kSingleDecompositionIdwtJob, Range, Result, Vec, }; pub(super) fn execute_component_plan( @@ -80,11 +80,12 @@ fn execute_classic_sub_band( strict: job.strict, dequantization_step: job.dequantization_step, }; - decode_j2k_code_block_scalar_with_workspace( - code_block, - &mut output[output_range], - &mut workspace, - )?; + let decode = if plan.irreversible_midpoint { + decode_j2k_code_block_scalar_with_workspace_midpoint + } else { + decode_j2k_code_block_scalar_with_workspace + }; + decode(code_block, &mut output[output_range], &mut workspace)?; } Ok(()) } diff --git a/crates/j2k-native/src/direct_cpu/referenced_classic.rs b/crates/j2k-native/src/direct_cpu/referenced_classic.rs index a29fb4c9..18a8b7c6 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_classic.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_classic.rs @@ -3,6 +3,7 @@ //! Parse-free CPU execution for referenced classic JPEG 2000 plans. use crate::error::{bail, DecodingError, Result}; +use crate::scalar::decode_j2k_code_block_scalar_with_workspace_midpoint; use crate::{ decode_j2k_code_block_scalar_with_workspace, try_reserve_decode_elements, J2kCodeBlockDecodeJob, J2kCodeBlockDecodeWorkspace, J2kDirectGrayscalePlan, @@ -278,11 +279,12 @@ fn execute_classic_sub_band_referenced( strict: job.strict, dequantization_step: job.dequantization_step, }; - decode_j2k_code_block_scalar_with_workspace( - code_block, - &mut output[output_range], - workspace, - )?; + let decode = if plan.irreversible_midpoint { + decode_j2k_code_block_scalar_with_workspace_midpoint + } else { + decode_j2k_code_block_scalar_with_workspace + }; + decode(code_block, &mut output[output_range], workspace)?; } Ok(()) } diff --git a/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs b/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs index adf3c135..69996031 100644 --- a/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs +++ b/crates/j2k-native/src/direct_cpu/referenced_staged/entropy.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::error::{bail, DecodingError, Result}; +use crate::scalar::decode_j2k_code_block_scalar_with_workspace_midpoint; use crate::{ decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar_with_workspace, HtCodeBlockDecodeJob, HtCodeBlockPayloadRanges, J2kCodeBlockDecodeJob, J2kDirectGrayscaleStep, @@ -118,7 +119,12 @@ pub fn execute_referenced_classic_entropy_job( job.width, job.height, )?; - decode_j2k_code_block_scalar_with_workspace( + let decode = if sub_band.irreversible_midpoint { + decode_j2k_code_block_scalar_with_workspace_midpoint + } else { + decode_j2k_code_block_scalar_with_workspace + }; + decode( J2kCodeBlockDecodeJob { data, segments: &job.segments, diff --git a/crates/j2k-native/src/direct_plan.rs b/crates/j2k-native/src/direct_plan.rs index 6910607e..9ae6d7c1 100644 --- a/crates/j2k-native/src/direct_plan.rs +++ b/crates/j2k-native/src/direct_plan.rs @@ -82,6 +82,8 @@ pub struct J2kOwnedSubBandPlan { pub width: u32, /// Sub-band height in samples. pub height: u32, + /// Whether classic Tier-1 coefficients use irreversible midpoint reconstruction. + pub irreversible_midpoint: bool, /// Owned code-block jobs for this sub-band. pub jobs: Vec, } diff --git a/crates/j2k-native/src/direct_plan/allocation.rs b/crates/j2k-native/src/direct_plan/allocation.rs index c7dd7bb6..d430e7d5 100644 --- a/crates/j2k-native/src/direct_plan/allocation.rs +++ b/crates/j2k-native/src/direct_plan/allocation.rs @@ -297,6 +297,7 @@ mod tests { }, width: 1, height: 1, + irreversible_midpoint: false, jobs, }, )); diff --git a/crates/j2k-native/src/image.rs b/crates/j2k-native/src/image.rs index 37ad77bb..7ec34bc6 100644 --- a/crates/j2k-native/src/image.rs +++ b/crates/j2k-native/src/image.rs @@ -306,6 +306,20 @@ impl<'a> Image<'a> { &self.color_space } + /// Return the primary JP2 restricted ICC profile, when present. + #[doc(hidden)] + #[must_use] + pub fn primary_icc_profile(&self) -> Option<&[u8]> { + match self + .boxes + .primary_color_specification() + .map(|specification| &specification.color_space) + { + Some(jp2::colr::ColorSpace::Icc(profile)) => Some(profile), + _ => None, + } + } + /// The width of the image. #[must_use] pub fn width(&self) -> u32 { diff --git a/crates/j2k-native/src/inspect.rs b/crates/j2k-native/src/inspect.rs index 64c20431..1f35eca0 100644 --- a/crates/j2k-native/src/inspect.rs +++ b/crates/j2k-native/src/inspect.rs @@ -195,6 +195,7 @@ pub fn inspect_j2k_codestream_header( let _ = read_segment_payload(input, &mut offset, "CAP")?; high_throughput_cap = true; } + 0x30..=0x3F => {} _ => { let _ = read_segment_payload(input, &mut offset, "segment")?; } @@ -507,6 +508,20 @@ mod tests { assert!(header.reversible); } + #[test] + fn inspect_skips_parameterless_reserved_main_header_markers() { + let mut bytes = minimal_codestream(); + let sot = bytes + .windows(2) + .position(|marker| marker == [0xFF, 0x90]) + .expect("SOT marker"); + bytes.splice(sot..sot, [0xFF, 0x30]); + + let header = inspect_j2k_codestream_header(&bytes).expect("header with reserved marker"); + + assert_eq!(header.dimensions, (128, 64)); + } + #[test] fn inspect_rejects_zero_component_sampling() { let mut bytes = minimal_codestream(); diff --git a/crates/j2k-native/src/j2c/bitplane.rs b/crates/j2k-native/src/j2c/bitplane.rs index 639d07a5..1c1c0714 100644 --- a/crates/j2k-native/src/j2c/bitplane.rs +++ b/crates/j2k-native/src/j2c/bitplane.rs @@ -5,6 +5,7 @@ mod bypass; mod context; mod facade; mod observer; +mod reconstruction; mod schedule; mod state; diff --git a/crates/j2k-native/src/j2c/bitplane/bypass.rs b/crates/j2k-native/src/j2c/bitplane/bypass.rs index 9c1d9a8d..a6527a27 100644 --- a/crates/j2k-native/src/j2c/bitplane/bypass.rs +++ b/crates/j2k-native/src/j2c/bitplane/bypass.rs @@ -12,7 +12,7 @@ use super::state::{ }; use crate::reader::BitReader; -// Bypass bit reads can fail in strict mode when the raw segment runs short. +// Bypass bit reads can fail in strict mode when byte stuffing is malformed. pub(super) trait BitDecoder { fn read_bit(&mut self, context: &mut ArithmeticDecoderContext) -> Option; } @@ -27,9 +27,14 @@ impl<'a> BypassDecoder<'a> { impl BitDecoder for BypassDecoder<'_> { fn read_bit(&mut self, _: &mut ArithmeticDecoderContext) -> Option { + // T.800 D.4.1 extends a cleanly exhausted terminated segment with + // 0xFF bytes. In raw bypass mode this supplies one bits indefinitely. + if self.0.at_end() { + return Some(1); + } self.0.read_bits_with_stuffing(1).or({ if self.1 { - // We have too little data, return `None`. + // The encoded bytes ended inside a required stuffing pair. None } else { // If not in strict mode, just pad with ones. Not sure if diff --git a/crates/j2k-native/src/j2c/bitplane/reconstruction.rs b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs new file mode 100644 index 00000000..9d9aa111 --- /dev/null +++ b/crates/j2k-native/src/j2c/bitplane/reconstruction.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::state::Coefficient; +use j2k_codec_math::classic::irreversible_midpoint_bit; + +/// Reconstruct an irreversible coefficient at the centre of its final +/// decoded quantization interval, as permitted by T.800 E.1.1.2. +#[expect( + clippy::cast_precision_loss, + reason = "irreversible JPEG 2000 coefficients enter the codec f32 domain here" +)] +pub(super) fn reconstruct_irreversible_midpoint( + coefficient: Coefficient, + decoded_bitplanes: u8, + number_of_coding_passes: u8, + roi_shift: u8, +) -> f32 { + let signed = coefficient.get_i64(); + let magnitude = signed.unsigned_abs(); + if magnitude == 0 || decoded_bitplanes == 0 || number_of_coding_passes == 0 { + return 0.0; + } + + let Some(lowest_decoded_bit) = irreversible_midpoint_bit( + magnitude, + u32::from(decoded_bitplanes), + u32::from(number_of_coding_passes), + ) else { + // Callers validate pass metadata; keep this shared arithmetic boundary total. + return signed as f32; + }; + + // A doubled unsigned representation preserves the half-bin term and has + // headroom for the decoder's 63-bit coefficient limit. + let mut fixed_magnitude = (u128::from(magnitude) << 1) | (1_u128 << lowest_decoded_bit); + if roi_shift != 0 { + let threshold = 1_u128 << u32::from(roi_shift); + if fixed_magnitude >= threshold { + fixed_magnitude >>= roi_shift; + } + } + + let reconstructed = fixed_magnitude as f32 * 0.5; + if signed < 0 { + -reconstructed + } else { + reconstructed + } +} diff --git a/crates/j2k-native/src/j2c/bitplane/state.rs b/crates/j2k-native/src/j2c/bitplane/state.rs index 7e57dcbe..ea1d05ed 100644 --- a/crates/j2k-native/src/j2c/bitplane/state.rs +++ b/crates/j2k-native/src/j2c/bitplane/state.rs @@ -273,6 +273,20 @@ impl BitPlaneDecodeContext { .take(self.height as usize) } + pub(crate) fn reconstruct_irreversible_midpoint( + &self, + coefficient: Coefficient, + number_of_coding_passes: u8, + roi_shift: u8, + ) -> f32 { + super::reconstruction::reconstruct_irreversible_midpoint( + coefficient, + self.bitplanes, + number_of_coding_passes, + roi_shift, + ) + } + pub(super) fn arithmetic_decoder_context( &mut self, ctx_label: u8, diff --git a/crates/j2k-native/src/j2c/bitplane/tests.rs b/crates/j2k-native/src/j2c/bitplane/tests.rs index 3a83591e..a6d99984 100644 --- a/crates/j2k-native/src/j2c/bitplane/tests.rs +++ b/crates/j2k-native/src/j2c/bitplane/tests.rs @@ -4,11 +4,13 @@ use alloc::{vec, vec::Vec}; use super::super::bitplane_encode; use super::arithmetic::cleanup_candidate_scan_mask; +use super::bypass::{BitDecoder, BypassDecoder}; use super::context::{ context_label_magnitude_refinement_coding_from_state_lazy, context_label_sign_coding_index, context_label_sign_coding_index_normal, }; use super::facade::decode_code_block_segments_validated; +use super::reconstruction::reconstruct_irreversible_midpoint; use super::state::{ BitPlaneDecodeContext, Coefficient, CoefficientState, NeighborSignificances, COEFFICIENTS_PADDING, HAS_MAGNITUDE_REFINEMENT_MASK, SIGNIFICANCE_MASK, @@ -59,6 +61,73 @@ fn classic_coefficient_state_preserves_38_bit_magnitude() { assert_eq!(coefficient.get(), i32::MIN); } +#[test] +fn irreversible_midpoint_reconstruction_tracks_the_last_decoded_pass() { + let mut first_plane = Coefficient::default(); + first_plane.push_bit_at(1, 2); + + assert_eq!( + reconstruct_irreversible_midpoint(first_plane, 3, 1, 0).to_bits(), + 6.0_f32.to_bits() + ); + + first_plane.set_sign(1); + assert_eq!( + reconstruct_irreversible_midpoint(first_plane, 3, 1, 0).to_bits(), + (-6.0_f32).to_bits() + ); + + let zero = Coefficient::default(); + assert_eq!( + reconstruct_irreversible_midpoint(zero, 3, 1, 0).to_bits(), + 0.0_f32.to_bits() + ); + + let mut newly_significant_in_sigprop = Coefficient::default(); + newly_significant_in_sigprop.push_bit_at(1, 1); + assert_eq!( + reconstruct_irreversible_midpoint(newly_significant_in_sigprop, 3, 2, 0).to_bits(), + 3.0_f32.to_bits() + ); + + let mut awaiting_refinement = Coefficient::default(); + awaiting_refinement.push_bit_at(1, 2); + assert_eq!( + reconstruct_irreversible_midpoint(awaiting_refinement, 3, 2, 0).to_bits(), + 6.0_f32.to_bits() + ); + assert_eq!( + reconstruct_irreversible_midpoint(awaiting_refinement, 3, 3, 0).to_bits(), + 5.0_f32.to_bits() + ); +} + +#[test] +fn strict_bypass_decoder_extends_clean_segment_end_with_ones() { + let mut decoder = BypassDecoder::new(&[0b1010_0101], true); + let mut context = crate::j2c::arithmetic_decoder::ArithmeticDecoderContext::default(); + let mut bits = 0u16; + + for _ in 0..10 { + bits = (bits << 1) + | u16::try_from(decoder.read_bit(&mut context).expect("raw bit")) + .expect("one bit fits u16"); + } + + assert_eq!(bits, 0b10_1001_0111); +} + +#[test] +fn strict_bypass_decoder_rejects_missing_stuffed_bit() { + let mut decoder = BypassDecoder::new(&[0xff], true); + let mut context = crate::j2c::arithmetic_decoder::ArithmeticDecoderContext::default(); + + for _ in 0..7 { + assert_eq!(decoder.read_bit(&mut context), Some(1)); + } + assert_eq!(decoder.read_bit(&mut context), None); +} + #[test] fn classic_tier1_round_trips_38_bit_coefficients() { let coefficients = vec![ diff --git a/crates/j2k-native/src/j2c/codestream/auxiliary.rs b/crates/j2k-native/src/j2c/codestream/auxiliary.rs index 440808f9..f7bc9fc4 100644 --- a/crates/j2k-native/src/j2c/codestream/auxiliary.rs +++ b/crates/j2k-native/src/j2c/codestream/auxiliary.rs @@ -1,13 +1,9 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 -use alloc::vec::Vec; -use core::mem::size_of; - use super::progression::read_component_index; -use super::{PpmMarkerData, PpmPacket, RgnMarkerData}; -use crate::error::{MarkerError, Result, ValidationError}; +use super::{PpmMarkerData, RgnMarkerData}; +use crate::error::{MarkerError, Result}; use crate::reader::BitReader; -use crate::try_reserve_decode_elements; mod packet_lengths; @@ -27,10 +23,7 @@ pub(super) fn tlm_marker(reader: &mut BitReader<'_>) -> Option<()> { } /// PPM marker (A.7.4). -pub(super) fn ppm_marker<'a>( - reader: &mut BitReader<'a>, - max_owned_bytes: usize, -) -> Result> { +pub(super) fn ppm_marker<'a>(reader: &mut BitReader<'a>) -> Result> { let segment_len = reader .read_u16() .and_then(|length| length.checked_sub(2)) @@ -42,45 +35,13 @@ pub(super) fn ppm_marker<'a>( .first() .copied() .ok_or(MarkerError::ParseFailure("PPM"))?; - let payload = &ppm_data[1..]; - - let packet_count = visit_ppm_packets(payload, |_| {})?; - let packet_bytes = packet_count - .checked_mul(size_of::>()) - .ok_or(ValidationError::ImageTooLarge)?; - if packet_bytes > max_owned_bytes { - return Err(ValidationError::ImageTooLarge.into()); - } - - let mut packets = Vec::new(); - try_reserve_decode_elements(&mut packets, packet_count)?; - visit_ppm_packets(payload, |data| packets.push(PpmPacket { data }))?; Ok(PpmMarkerData { sequence_idx, - packets, + data: &ppm_data[1..], }) } -fn visit_ppm_packets<'a>(payload: &'a [u8], mut visit: impl FnMut(&'a [u8])) -> Result { - let mut packet_count = 0_usize; - let mut reader = BitReader::new(payload); - // This parser handles complete packet payloads carried by the current PPM - // marker. Continuations across multiple PPM markers are rejected by normal - // length parsing until a multi-marker accumulator is added. - while !reader.at_end() { - let packet_len = reader.read_u16().ok_or(MarkerError::ParseFailure("PPM"))? as usize; - let data = reader - .read_bytes(packet_len) - .ok_or(MarkerError::ParseFailure("PPM"))?; - visit(data); - packet_count = packet_count - .checked_add(1) - .ok_or(ValidationError::ImageTooLarge)?; - } - Ok(packet_count) -} - /// RGN marker (A.6.3). pub(crate) fn rgn_marker(reader: &mut BitReader<'_>, csiz: u16) -> Option { let length = reader.read_u16()?; @@ -114,18 +75,16 @@ pub(crate) fn skip_marker_segment(reader: &mut BitReader<'_>) -> Option<()> { #[cfg(test)] mod tests { use super::*; - use crate::error::DecodeError; #[test] - fn ppm_output_limit_is_checked_before_reservation() { - // Lppm=9: one sequence byte followed by three empty packet records. - let data = [0, 9, 0, 0, 0, 0, 0, 0, 0]; + fn ppm_retains_payload_for_cross_marker_accumulation() { + let data = [0, 7, 3, 0, 0, 1, 0]; let mut reader = BitReader::new(&data); - assert_eq!( - ppm_marker(&mut reader, 2 * size_of::>()).unwrap_err(), - DecodeError::Validation(ValidationError::ImageTooLarge) - ); + let marker = ppm_marker(&mut reader).expect("valid PPM marker segment"); + + assert_eq!(marker.sequence_idx, 3); + assert_eq!(marker.data, [0, 0, 1, 0]); assert_eq!(reader.offset(), data.len()); } } diff --git a/crates/j2k-native/src/j2c/codestream/header.rs b/crates/j2k-native/src/j2c/codestream/header.rs index 232c3971..96b1f2a7 100644 --- a/crates/j2k-native/src/j2c/codestream/header.rs +++ b/crates/j2k-native/src/j2c/codestream/header.rs @@ -16,12 +16,13 @@ use crate::reader::BitReader; mod allocation; mod components; +mod ppm; use allocation::{ - try_extend_progression_changes, try_flatten_packet_lengths, try_flatten_ppm_packets, - try_none_vec, HeaderMarkerBudget, + try_extend_progression_changes, try_flatten_packet_lengths, try_none_vec, HeaderMarkerBudget, }; use components::build_component_infos; +use ppm::try_flatten_ppm_packets; #[expect( clippy::similar_names, @@ -172,10 +173,7 @@ pub(crate) fn read_header<'a>( markers::PPM => { reader.read_marker()?; marker_budget.try_reserve_next(&mut ppm_markers)?; - let marker = ppm_marker(reader, marker_budget.remaining_bytes())?; - marker_budget - .account_capacity::>(marker.packets.capacity())?; - ppm_markers.push(marker); + ppm_markers.push(ppm_marker(reader)?); } markers::CRG => { reader.read_marker()?; diff --git a/crates/j2k-native/src/j2c/codestream/header/allocation.rs b/crates/j2k-native/src/j2c/codestream/header/allocation.rs index 6254e1fe..ee298981 100644 --- a/crates/j2k-native/src/j2c/codestream/header/allocation.rs +++ b/crates/j2k-native/src/j2c/codestream/header/allocation.rs @@ -7,7 +7,7 @@ use core::mem::size_of; use super::super::{ CodingStyleComponent, CodingStyleDefault, ComponentInfo, ComponentSizeInfo, PacketLengthMarker, - PpmMarkerData, PpmPacket, ProgressionChange, QuantizationInfo, StepSize, + ProgressionChange, QuantizationInfo, StepSize, }; use crate::error::{DecodeError, Result, ValidationError}; use crate::{try_reserve_decode_elements, DEFAULT_MAX_DECODE_BYTES}; @@ -224,40 +224,5 @@ pub(super) fn try_flatten_packet_lengths( Ok(packet_lengths) } -pub(super) fn try_flatten_ppm_packets<'a>( - markers: Vec>, - budget: &mut HeaderMarkerBudget, -) -> Result>> { - let marker_capacity = markers.capacity(); - let source_packet_capacity = markers.iter().try_fold(0_usize, |total, marker| { - total - .checked_add(marker.packets.capacity()) - .ok_or_else(allocation_overflow) - })?; - let packet_count = markers.iter().try_fold(0_usize, |total, marker| { - let nonempty_count = marker - .packets - .iter() - .filter(|packet| !packet.data.is_empty()) - .count(); - total - .checked_add(nonempty_count) - .ok_or_else(allocation_overflow) - })?; - let mut packets = Vec::new(); - budget.try_reserve_len(&mut packets, packet_count)?; - for marker in markers { - packets.extend( - marker - .packets - .into_iter() - .filter(|packet| !packet.data.is_empty()), - ); - } - budget.release_capacity::>(marker_capacity)?; - budget.release_capacity::>(source_packet_capacity)?; - Ok(packets) -} - #[cfg(test)] mod tests; diff --git a/crates/j2k-native/src/j2c/codestream/header/ppm.rs b/crates/j2k-native/src/j2c/codestream/header/ppm.rs new file mode 100644 index 00000000..944fda55 --- /dev/null +++ b/crates/j2k-native/src/j2c/codestream/header/ppm.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! PPM tile-part packet-header stream assembly. + +use alloc::vec::Vec; + +use super::super::{PpmMarkerData, PpmPacket}; +use super::allocation::HeaderMarkerBudget; +use crate::error::{MarkerError, Result, ValidationError}; + +pub(super) fn try_flatten_ppm_packets<'a>( + markers: Vec>, + budget: &mut HeaderMarkerBudget, +) -> Result>> { + let marker_capacity = markers.capacity(); + for (expected, marker) in markers.iter().enumerate() { + if u8::try_from(expected).ok() != Some(marker.sequence_idx) { + return Err(MarkerError::ParseFailure("PPM").into()); + } + } + + let mut packets = Vec::new(); + { + let mut cursor = PpmPayloadCursor::new(&markers); + while let Some(packet_len) = cursor.read_u32()? { + let mut remaining = + usize::try_from(packet_len).map_err(|_| ValidationError::ImageTooLarge)?; + if remaining == 0 { + budget.try_reserve_next(&mut packets)?; + packets.push(PpmPacket { + data: &[], + ends_tile_part: true, + }); + continue; + } + while remaining != 0 { + let data = cursor + .take_fragment(remaining) + .ok_or(MarkerError::ParseFailure("PPM"))?; + remaining -= data.len(); + budget.try_reserve_next(&mut packets)?; + packets.push(PpmPacket { + data, + ends_tile_part: remaining == 0, + }); + } + } + } + drop(markers); + budget.release_capacity::>(marker_capacity)?; + Ok(packets) +} + +struct PpmPayloadCursor<'markers, 'data> { + markers: &'markers [PpmMarkerData<'data>], + marker: usize, + offset: usize, +} + +impl<'markers, 'data> PpmPayloadCursor<'markers, 'data> { + const fn new(markers: &'markers [PpmMarkerData<'data>]) -> Self { + Self { + markers, + marker: 0, + offset: 0, + } + } + + fn read_u32(&mut self) -> Result> { + let Some(first) = self.next_byte() else { + return Ok(None); + }; + let mut bytes = [first, 0, 0, 0]; + for byte in &mut bytes[1..] { + *byte = self.next_byte().ok_or(MarkerError::ParseFailure("PPM"))?; + } + Ok(Some(u32::from_be_bytes(bytes))) + } + + fn next_byte(&mut self) -> Option { + loop { + let data = self.markers.get(self.marker)?.data; + if let Some(byte) = data.get(self.offset).copied() { + self.offset += 1; + return Some(byte); + } + self.marker += 1; + self.offset = 0; + } + } + + fn take_fragment(&mut self, max_len: usize) -> Option<&'data [u8]> { + loop { + let data = self.markers.get(self.marker)?.data; + if self.offset == data.len() { + self.marker += 1; + self.offset = 0; + continue; + } + let end = self.offset.saturating_add(max_len).min(data.len()); + let fragment = data.get(self.offset..end)?; + self.offset = end; + return Some(fragment); + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-native/src/j2c/codestream/header/ppm/tests.rs b/crates/j2k-native/src/j2c/codestream/header/ppm/tests.rs new file mode 100644 index 00000000..9668ff77 --- /dev/null +++ b/crates/j2k-native/src/j2c/codestream/header/ppm/tests.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use alloc::vec; + +use super::*; + +#[test] +fn tile_part_data_continues_across_marker_segments() { + let first = [0, 0, 0, 3, 0xaa]; + let second = [0xbb, 0xcc, 0, 0, 0, 1, 0xdd]; + let markers = vec![ + PpmMarkerData { + sequence_idx: 0, + data: &first, + }, + PpmMarkerData { + sequence_idx: 1, + data: &second, + }, + ]; + let mut budget = HeaderMarkerBudget::default(); + budget + .account_capacity::>(markers.capacity()) + .expect("marker allocation fits"); + + let packets = try_flatten_ppm_packets(markers, &mut budget).expect("valid continued PPM"); + + assert_eq!(packets.len(), 3); + assert_eq!(packets[0].data, [0xaa]); + assert!(!packets[0].ends_tile_part); + assert_eq!(packets[1].data, [0xbb, 0xcc]); + assert!(packets[1].ends_tile_part); + assert_eq!(packets[2].data, [0xdd]); + assert!(packets[2].ends_tile_part); +} + +#[test] +fn marker_sequence_must_be_contiguous() { + let payload = [0, 0, 0, 1, 0xaa]; + let markers = vec![PpmMarkerData { + sequence_idx: 1, + data: &payload, + }]; + let mut budget = HeaderMarkerBudget::default(); + budget + .account_capacity::>(markers.capacity()) + .expect("marker allocation fits"); + + assert!(try_flatten_ppm_packets(markers, &mut budget).is_err()); +} diff --git a/crates/j2k-native/src/j2c/codestream/model.rs b/crates/j2k-native/src/j2c/codestream/model.rs index 94b2ad9e..c2ea12a9 100644 --- a/crates/j2k-native/src/j2c/codestream/model.rs +++ b/crates/j2k-native/src/j2c/codestream/model.rs @@ -21,12 +21,13 @@ pub(crate) struct Header<'a> { #[derive(Debug)] pub(crate) struct PpmMarkerData<'a> { pub(crate) sequence_idx: u8, - pub(crate) packets: Vec>, + pub(crate) data: &'a [u8], } #[derive(Debug, Clone)] pub(crate) struct PpmPacket<'a> { pub(crate) data: &'a [u8], + pub(crate) ends_tile_part: bool, } #[derive(Debug)] diff --git a/crates/j2k-native/src/j2c/codestream/tests.rs b/crates/j2k-native/src/j2c/codestream/tests.rs index 2e143c02..a018cfa6 100644 --- a/crates/j2k-native/src/j2c/codestream/tests.rs +++ b/crates/j2k-native/src/j2c/codestream/tests.rs @@ -228,6 +228,8 @@ fn codestream_module_boundaries_stay_focused() { include_str!("header/components.rs"), 120, ), + ("header PPM", include_str!("header/ppm.rs"), 130), + ("header PPM tests", include_str!("header/ppm/tests.rs"), 80), ("markers", include_str!("markers.rs"), 120), ("model", include_str!("model.rs"), 460), ("progression", include_str!("progression.rs"), 90), diff --git a/crates/j2k-native/src/j2c/codestream_write.rs b/crates/j2k-native/src/j2c/codestream_write.rs index 4a00ea3b..6224efc7 100644 --- a/crates/j2k-native/src/j2c/codestream_write.rs +++ b/crates/j2k-native/src/j2c/codestream_write.rs @@ -454,7 +454,8 @@ fn write_rgn_markers(out: &mut Vec, params: &EncodeParams) { write_marker(out, markers::RGN); if params.num_components < 257 { out.extend_from_slice(&5u16.to_be_bytes()); - out.push(u8::try_from(component_index).expect("component index fits in Crgn byte")); + // This branch limits the loop to one-byte Crgn component indices. + out.push(component_index.to_be_bytes()[1]); } else { out.extend_from_slice(&6u16.to_be_bytes()); out.extend_from_slice(&component_index.to_be_bytes()); @@ -865,10 +866,16 @@ mod tests { assert_eq!(marker_length(&out, offsets[0]), u16::MAX); assert_eq!(out[offsets[0] + 4], 0); assert_eq!(out[offsets[1] + 4], 1); - assert_eq!(marker_length(&out, offsets[1]), 12); + assert_eq!(marker_length(&out, offsets[1]), 8); assert_eq!( - &out[offsets[1] + 5..offsets[1] + 14], - &[0, 1, 0x22, 0, 4, 0x33, 0x33, 0x33, 0x33] + &out[offsets[0] + 5..offsets[0] + 9], + &u32::try_from(PPM_PACKET_HEADER_LIMIT + 5) + .unwrap() + .to_be_bytes() + ); + assert_eq!( + &out[offsets[1] + 5..offsets[1] + 10], + &[0x22, 0x33, 0x33, 0x33, 0x33] ); } diff --git a/crates/j2k-native/src/j2c/codestream_write/accounting/tests.rs b/crates/j2k-native/src/j2c/codestream_write/accounting/tests.rs index 7ab156a3..becca3e3 100644 --- a/crates/j2k-native/src/j2c/codestream_write/accounting/tests.rs +++ b/crates/j2k-native/src/j2c/codestream_write/accounting/tests.rs @@ -46,8 +46,8 @@ fn assert_marker_payloads(codestream: &[u8]) { let ppm = marker_offsets(codestream, markers::PPM); assert_eq!(ppm.len(), 1); assert_eq!( - &codestream[ppm[0] + 5..ppm[0] + 12], - &[0x00, 0x02, 0xaa, 0xbb, 0x00, 0x01, 0xcc] + &codestream[ppm[0] + 5..ppm[0] + 16], + &[0x00, 0x00, 0x00, 0x02, 0xaa, 0xbb, 0x00, 0x00, 0x00, 0x01, 0xcc] ); let plt = marker_offsets(codestream, markers::PLT); assert_eq!(plt.len(), 2); diff --git a/crates/j2k-native/src/j2c/codestream_write/packet_markers.rs b/crates/j2k-native/src/j2c/codestream_write/packet_markers.rs index 63661c92..22cc603e 100644 --- a/crates/j2k-native/src/j2c/codestream_write/packet_markers.rs +++ b/crates/j2k-native/src/j2c/codestream_write/packet_markers.rs @@ -9,7 +9,7 @@ use crate::j2c::encode::allocation::checked_add_bytes; use crate::{EncodeError, EncodeResult}; pub(super) const PACKET_HEADER_MARKER_PAYLOAD_LIMIT: usize = u16::MAX as usize - 3; -pub(super) const PPM_PACKET_HEADER_LIMIT: usize = PACKET_HEADER_MARKER_PAYLOAD_LIMIT - 2; +pub(super) const PPM_PACKET_HEADER_LIMIT: usize = PACKET_HEADER_MARKER_PAYLOAD_LIMIT - 4; const MAX_PACKET_MARKERS: usize = u8::MAX as usize + 1; const PLT_CHUNK_SIZE: usize = u16::MAX as usize - 3; const PLM_CHUNK_SIZE: usize = u16::MAX as usize - 7; @@ -43,31 +43,15 @@ pub(super) fn plm_marker_bytes(tiles: &[TilePartData<'_>]) -> EncodeResult]) -> EncodeResult { + if tiles.iter().any(|tile| tile.packet_headers.is_empty()) { + return invalid("PPM encode requires separated packet headers"); + } let mut total = 0usize; - let mut payload = 0usize; let mut marker_count = 0usize; - for header in tiles.iter().flat_map(|tile| tile.packet_headers.iter()) { - if header.len() > PPM_PACKET_HEADER_LIMIT { - return invalid("PPM packet header exceeds marker payload limit"); - } - let entry = header - .len() - .checked_add(2) - .ok_or(EncodeError::ArithmeticOverflow { - what: "PPM marker payload length", - })?; - if payload != 0 - && payload - .checked_add(entry) - .is_none_or(|bytes| bytes > PACKET_HEADER_MARKER_PAYLOAD_LIMIT) - { - total = add_ppm_marker(total, payload, &mut marker_count)?; - payload = 0; - } - payload = checked_add_bytes(payload, entry, "PPM marker payload length")?; - } - if payload != 0 { + let mut cursor = HeaderCursor::default(); + while let Some((end, payload)) = next_ppm_chunk(tiles, cursor)? { total = add_ppm_marker(total, payload, &mut marker_count)?; + cursor = end; } Ok(total) } @@ -112,33 +96,7 @@ pub(super) fn write_ppm_markers(out: &mut Vec, tiles: &[TilePartData<'_>]) - ppm_marker_bytes(tiles)?; let mut cursor = HeaderCursor::default(); let mut sequence = 0usize; - loop { - let mut end = cursor; - let mut payload = 0usize; - loop { - let before = end; - let Some(header) = next_header(tiles, &mut end) else { - break; - }; - let entry = header - .len() - .checked_add(2) - .ok_or(EncodeError::ArithmeticOverflow { - what: "PPM marker payload length", - })?; - if payload != 0 - && payload - .checked_add(entry) - .is_none_or(|bytes| bytes > PACKET_HEADER_MARKER_PAYLOAD_LIMIT) - { - end = before; - break; - } - payload = checked_add_bytes(payload, entry, "PPM marker payload length")?; - } - if payload == 0 { - break; - } + while let Some((end, payload)) = next_ppm_chunk(tiles, cursor)? { write_marker(out, markers::PPM); let marker_len = u16::try_from(payload + 3).map_err(|_| EncodeError::InternalInvariant { @@ -151,15 +109,14 @@ pub(super) fn write_ppm_markers(out: &mut Vec, tiles: &[TilePartData<'_>]) - })?, ); while cursor != end { - let header = next_header(tiles, &mut cursor).ok_or(EncodeError::InternalInvariant { - what: "validated PPM header cursor ended early", - })?; - let header_len = - u16::try_from(header.len()).map_err(|_| EncodeError::InternalInvariant { - what: "validated PPM packet header length exceeds u16", + let header = + next_ppm_header(tiles, &mut cursor)?.ok_or(EncodeError::InternalInvariant { + what: "validated PPM header cursor ended early", })?; - out.extend_from_slice(&header_len.to_be_bytes()); - out.extend_from_slice(header); + if let Some(tile_part_len) = header.tile_part_len { + out.extend_from_slice(&tile_part_len.to_be_bytes()); + } + out.extend_from_slice(header.data); } sequence += 1; } @@ -360,22 +317,83 @@ struct HeaderCursor { header: usize, } -fn next_header<'a>(tiles: &[TilePartData<'a>], cursor: &mut HeaderCursor) -> Option<&'a [u8]> { +struct PpmHeader<'a> { + tile_part_len: Option, + data: &'a [u8], +} + +fn next_ppm_chunk( + tiles: &[TilePartData<'_>], + start: HeaderCursor, +) -> EncodeResult> { + let mut end = start; + let mut payload = 0usize; + loop { + let before = end; + let Some(header) = next_ppm_header(tiles, &mut end)? else { + break; + }; + if header.data.len() > PPM_PACKET_HEADER_LIMIT { + return invalid("PPM packet header exceeds marker payload limit"); + } + let entry_len = header + .data + .len() + .checked_add(usize::from(header.tile_part_len.is_some()) * 4) + .ok_or(EncodeError::ArithmeticOverflow { + what: "PPM marker payload length", + })?; + if payload != 0 + && payload + .checked_add(entry_len) + .is_none_or(|bytes| bytes > PACKET_HEADER_MARKER_PAYLOAD_LIMIT) + { + end = before; + break; + } + payload = checked_add_bytes(payload, entry_len, "PPM marker payload length")?; + } + Ok((payload != 0).then_some((end, payload))) +} + +fn next_ppm_header<'a>( + tiles: &[TilePartData<'a>], + cursor: &mut HeaderCursor, +) -> EncodeResult>> { loop { - let tile = tiles.get(cursor.tile)?; + let Some(tile) = tiles.get(cursor.tile) else { + return Ok(None); + }; if let Some(header) = tile.packet_headers.get(cursor.header) { + let tile_part_len = if cursor.header == 0 { + Some(ppm_tile_part_header_len(tile.packet_headers)?) + } else { + None + }; cursor.header += 1; if cursor.header == tile.packet_headers.len() { cursor.tile += 1; cursor.header = 0; } - return Some(header.as_slice()); + return Ok(Some(PpmHeader { + tile_part_len, + data: header, + })); } cursor.tile += 1; cursor.header = 0; } } +fn ppm_tile_part_header_len(headers: &[Vec]) -> EncodeResult { + let len = headers.iter().try_fold(0usize, |total, header| { + checked_add_bytes(total, header.len(), "PPM tile-part packet headers") + })?; + u32::try_from(len).map_err(|_| EncodeError::InvalidInput { + what: "PPM tile-part packet headers exceed u32", + }) +} + fn invalid(what: &'static str) -> EncodeResult { Err(EncodeError::InvalidInput { what }) } diff --git a/crates/j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs index 0c9e10d9..41905de9 100644 --- a/crates/j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -26,12 +26,14 @@ use crate::profile; use crate::reader::BitReader; use crate::{ add_roi_shift_to_bitplanes, apply_roi_maxshift_inverse_i32, apply_roi_maxshift_inverse_i64, - decode_j2k_code_block_scalar_with_workspace, HtCodeBlockBatchJob, HtCodeBlockDecodeJob, - HtCodeBlockDecoder, HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, HtSubBandDecodeJob, - J2kCodeBlockBatchJob, J2kCodeBlockDecodeJob, J2kCodeBlockDecodeWorkspace, J2kCodeBlockStyle, - J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, - J2kDirectIdwtStep, J2kDirectStoreStep, J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kRect, - J2kStoreComponentJob, J2kSubBandDecodeJob, J2kSubBandType, J2kWaveletTransform, + decode_j2k_code_block_scalar_with_workspace, + decode_j2k_code_block_scalar_with_workspace_midpoint, HtCodeBlockBatchJob, + HtCodeBlockDecodeJob, HtCodeBlockDecoder, HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, + HtSubBandDecodeJob, J2kCodeBlockBatchJob, J2kCodeBlockDecodeJob, J2kCodeBlockDecodeWorkspace, + J2kCodeBlockStyle, J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, J2kOwnedCodeBlockBatchJob, + J2kOwnedSubBandPlan, J2kRect, J2kStoreComponentJob, J2kSubBandDecodeJob, J2kSubBandType, + J2kWaveletTransform, }; use core::mem::size_of; use core::ops::Range; diff --git a/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs b/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs index 9e2bc193..aa61bf19 100644 --- a/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs +++ b/crates/j2k-native/src/j2c/decode/direct_plan/storage/sub_band.rs @@ -34,6 +34,7 @@ pub(super) fn build_grayscale_sub_band_step( ) -> Result> { let SubBandDecodeParameters { dequantization_step, + irreversible_midpoint, num_bitplanes, } = sub_band_decode_parameters(sub_band, resolution, component_info)?; @@ -70,6 +71,7 @@ pub(super) fn build_grayscale_sub_band_step( budget, classic_payloads, dequantization_step, + irreversible_midpoint, num_bitplanes, ) .map(Some) @@ -173,6 +175,7 @@ fn build_classic_sub_band_step( budget: &mut DecodeAllocationBudget, mut classic_payloads: Option<&mut ClassicPayloadCollector<'_>>, dequantization_step: f32, + irreversible_midpoint: bool, num_bitplanes: u8, ) -> Result { let (sub_band_type, style) = @@ -217,6 +220,7 @@ fn build_classic_sub_band_step( rect: J2kRect::from(sub_band.rect), width: sub_band.rect.width(), height: sub_band.rect.height(), + irreversible_midpoint, jobs, }, )) diff --git a/crates/j2k-native/src/j2c/decode/subband.rs b/crates/j2k-native/src/j2c/decode/subband.rs index 6660803b..26c8de8c 100644 --- a/crates/j2k-native/src/j2c/decode/subband.rs +++ b/crates/j2k-native/src/j2c/decode/subband.rs @@ -3,7 +3,8 @@ use super::{ add_roi_shift_to_bitplanes, apply_roi_maxshift_inverse_i32, apply_roi_maxshift_inverse_i64, bitplane, classic_decode_job_parameters, collect_classic_code_block_data, - decode_j2k_code_block_scalar_with_workspace, ht_block_decode, + decode_j2k_code_block_scalar_with_workspace, + decode_j2k_code_block_scalar_with_workspace_midpoint, ht_block_decode, ht_code_block_has_decodable_passes, sub_band_decode_parameters, CodeBlock, ComponentInfo, CpuDecodeParallelism, DecodeAllocationBudget, DecodingError, DecompositionStorage, Header, HtCodeBlockBatchJob, HtCodeBlockDecodeJob, HtCodeBlockDecoder, HtSubBandDecodeJob, @@ -95,6 +96,7 @@ fn decode_sub_band_bitplanes( let sub_band = storage.sub_bands[sub_band_idx].clone(); let SubBandDecodeParameters { dequantization_step, + irreversible_midpoint, num_bitplanes, } = sub_band_decode_parameters(&sub_band, resolution, component_info)?; @@ -187,13 +189,14 @@ fn decode_sub_band_bitplanes( } let base_store = &mut storage.coefficients[sub_band.coefficients.clone()]; - if ht_decoder.decode_j2k_sub_band( + if ht_decoder.decode_j2k_sub_band_with_midpoint( J2kSubBandDecodeJob { width: sub_band.rect.width(), height: sub_band.rect.height(), jobs: &batch_jobs, }, base_store, + irreversible_midpoint, )? { tile_ctx.debug_counters.decoded_code_blocks += batch_jobs.len(); return Ok(()); @@ -231,14 +234,19 @@ fn decode_sub_band_bitplanes( .ok_or(DecodingError::CodeBlockDecodeFailure)? }; let output_slice = &mut base_store[base_idx..base_idx + output_len]; - if ht_decoder.decode_j2k_code_block(job.code_block, output_slice)? { - continue; - } - decode_j2k_code_block_scalar_with_workspace( + if ht_decoder.decode_j2k_code_block_with_midpoint( job.code_block, output_slice, - &mut scalar_workspace, - )?; + irreversible_midpoint, + )? { + continue; + } + let decode = if irreversible_midpoint { + decode_j2k_code_block_scalar_with_workspace_midpoint + } else { + decode_j2k_code_block_scalar_with_workspace + }; + decode(job.code_block, output_slice, &mut scalar_workspace)?; } return Ok(()); @@ -265,6 +273,7 @@ fn decode_sub_band_bitplanes( total_bitplanes: num_bitplanes, roi_shift: component_info.roi_shift, dequantization_step, + irreversible_midpoint, }, &mut budget, )?; @@ -311,11 +320,20 @@ fn decode_sub_band_bitplanes( let out_row = &mut base_store[base_idx..]; for (output, coefficient) in out_row.iter_mut().zip(coefficients.iter().copied()) { - let coefficient = apply_roi_maxshift_inverse_i64( - coefficient.get_i64(), - component_info.roi_shift, - ); - *output = coefficient as f32; + *output = if irreversible_midpoint { + tile_ctx + .bit_plane_decode_context + .reconstruct_irreversible_midpoint( + coefficient, + code_block.number_of_coding_passes, + component_info.roi_shift, + ) + } else { + apply_roi_maxshift_inverse_i64( + coefficient.get_i64(), + component_info.roi_shift, + ) as f32 + }; *output *= dequantization_step; } diff --git a/crates/j2k-native/src/j2c/decode/subband/parallel.rs b/crates/j2k-native/src/j2c/decode/subband/parallel.rs index 10348d26..5b03feaf 100644 --- a/crates/j2k-native/src/j2c/decode/subband/parallel.rs +++ b/crates/j2k-native/src/j2c/decode/subband/parallel.rs @@ -7,6 +7,7 @@ use super::{DecodeAllocationBudget, DecompositionStorage, SubBand}; use crate::error::{bail, DecodingError, Result, ValidationError}; use crate::j2c::bitplane::classic_decode_workspace_bytes; use crate::j2c::ht_block_decode::ht_decode_workspace_bytes; +use crate::scalar::decode_j2k_code_block_scalar_with_workspace_midpoint; use crate::{ decode_ht_code_block_scalar_with_workspace, decode_j2k_code_block_scalar_with_workspace, try_reserve_decode_elements, try_resize_decode_elements, HtCodeBlockDecodeJob, @@ -40,6 +41,7 @@ pub(super) struct ClassicParallelParameters { pub(super) total_bitplanes: u8, pub(super) roi_shift: u8, pub(super) dequantization_step: f32, + pub(super) irreversible_midpoint: bool, } trait DecodedSubBandBlock { @@ -106,7 +108,12 @@ pub(super) fn decode_classic_sub_band_blocks_parallel( .zip(pending_blocks.par_iter()) .zip(workspaces.par_iter_mut()) .try_for_each(|((decoded, pending), workspace)| -> Result<()> { - decode_j2k_code_block_scalar_with_workspace( + let decode = if parameters.irreversible_midpoint { + decode_j2k_code_block_scalar_with_workspace_midpoint + } else { + decode_j2k_code_block_scalar_with_workspace + }; + decode( J2kCodeBlockDecodeJob { data: &pending.combined_data, segments: &pending.segments, diff --git a/crates/j2k-native/src/j2c/decode/subband_params.rs b/crates/j2k-native/src/j2c/decode/subband_params.rs index e4c25ceb..ffca29b5 100644 --- a/crates/j2k-native/src/j2c/decode/subband_params.rs +++ b/crates/j2k-native/src/j2c/decode/subband_params.rs @@ -2,11 +2,12 @@ use super::{ bail, CodeBlock, ComponentInfo, DecodingError, J2kCodeBlockStyle, J2kSubBandType, - QuantizationStyle, Result, SubBand, SubBandType, MAX_BITPLANE_COUNT, + QuantizationStyle, Result, SubBand, SubBandType, WaveletTransform, MAX_BITPLANE_COUNT, }; pub(super) struct SubBandDecodeParameters { pub(super) dequantization_step: f32, + pub(super) irreversible_midpoint: bool, pub(super) num_bitplanes: u8, } @@ -43,6 +44,8 @@ pub(super) fn sub_band_decode_parameters( Ok(SubBandDecodeParameters { dequantization_step, + irreversible_midpoint: component_info.wavelet_transform() + == WaveletTransform::Irreversible97, num_bitplanes: u8::try_from(num_bitplanes).map_err(|_| DecodingError::TooManyBitplanes)?, }) } diff --git a/crates/j2k-native/src/j2c/encode.rs b/crates/j2k-native/src/j2c/encode.rs index 9ad21147..5c9c1bc2 100644 --- a/crates/j2k-native/src/j2c/encode.rs +++ b/crates/j2k-native/src/j2c/encode.rs @@ -25,20 +25,20 @@ use crate::profile; pub(crate) use crate::J2kSubBandType; use crate::{ CpuOnlyJ2kEncodeStageAccelerator, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, - J2kDeinterleaveToF32Job, J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardDwt53Level, - J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Level, J2kForwardDwt97Output, - J2kForwardIctJob, J2kForwardRctJob, J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, - J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, - J2kPacketizationPacketDescriptor, J2kPacketizationResolution, J2kPacketizationSubband, - J2kQuantizeSubbandJob, J2kResidentEncodeInput, J2kResidentHtj2kTileEncodeJob, - J2kTier1CodeBlockEncodeJob, PrecomputedHtj2k53Component, PrecomputedHtj2k53Image, - PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, PreencodedHtj2k97CodeBlock, - PreencodedHtj2k97CompactCodeBlock, PreencodedHtj2k97CompactComponent, - PreencodedHtj2k97CompactImage, PreencodedHtj2k97CompactResolution, - PreencodedHtj2k97CompactSubband, PreencodedHtj2k97Component, PreencodedHtj2k97Image, - PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, PrequantizedHtj2k97Component, - PrequantizedHtj2k97Image, PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, - MAX_J2K_SPEC_COMPONENTS, + J2kDeinterleaveToF32Job, J2kEncodeContext, J2kEncodeStageAccelerator, J2kForwardDwt53Job, + J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Level, + J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, J2kHtSubbandEncodeJob, + J2kHtj2kTileEncodeJob, J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, + J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, J2kPacketizationResolution, + J2kPacketizationSubband, J2kQuantizeSubbandJob, J2kResidentEncodeInput, + J2kResidentHtj2kTileEncodeJob, J2kTier1CodeBlockEncodeJob, PrecomputedHtj2k53Component, + PrecomputedHtj2k53Image, PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, + PreencodedHtj2k97CodeBlock, PreencodedHtj2k97CompactCodeBlock, + PreencodedHtj2k97CompactComponent, PreencodedHtj2k97CompactImage, + PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97Component, PreencodedHtj2k97Image, PreencodedHtj2k97Resolution, + PreencodedHtj2k97Subband, PrequantizedHtj2k97Component, PrequantizedHtj2k97Image, + PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, MAX_J2K_SPEC_COMPONENTS, }; const HT_CPU_PARALLEL_FALLBACK_MIN_JOBS: usize = 4; diff --git a/crates/j2k-native/src/j2c/encode/multitile/tests.rs b/crates/j2k-native/src/j2c/encode/multitile/tests.rs index 9506148b..4eaf4546 100644 --- a/crates/j2k-native/src/j2c/encode/multitile/tests.rs +++ b/crates/j2k-native/src/j2c/encode/multitile/tests.rs @@ -133,10 +133,12 @@ fn direct_packet_owners_match_single_tile_marker_serialization() { let header = read_header(&mut reader, &DecodeSettings::default(), 0, None) .expect("serialized single-tile header"); assert_eq!(header.plm_packet_lengths, packetized.packet_lengths); - assert_eq!(header.ppm_packets.len(), packetized.packet_headers.len()); - for (serialized, direct) in header.ppm_packets.iter().zip(&packetized.packet_headers) { - assert_eq!(serialized.data, direct); - } + assert_eq!(header.ppm_packets.len(), 1); + assert!(header.ppm_packets[0].ends_tile_part); + assert!(header.ppm_packets[0].data.iter().copied().eq(packetized + .packet_headers + .iter() + .flat_map(|header| header.iter().copied()))); let sod = codestream .windows(2) diff --git a/crates/j2k-native/src/j2c/encode/single_tile.rs b/crates/j2k-native/src/j2c/encode/single_tile.rs index eefb25ef..ebe45a05 100644 --- a/crates/j2k-native/src/j2c/encode/single_tile.rs +++ b/crates/j2k-native/src/j2c/encode/single_tile.rs @@ -3,8 +3,8 @@ use super::multitile::{encode_multitile_impl, MultiTileEncodeRequest}; use super::{ packet_encode, profile, write_single_tile_packetized_codestream_for_session, BlockCodingMode, - EncodeComponentSampleInfo, EncodeOptions, EncodeRoiRegion, J2kEncodeStageAccelerator, - NativeEncodePipelineResult, NativeEncodeSession, Vec, + EncodeComponentSampleInfo, EncodeOptions, EncodeRoiRegion, J2kEncodeContext, + J2kEncodeStageAccelerator, NativeEncodePipelineResult, NativeEncodeSession, Vec, }; mod accelerator; @@ -23,14 +23,15 @@ pub(super) use coefficient_source::{OwnedDwtComponent, PackedF32DwtComponent}; use accelerator::{ prepare_accelerated_components, try_encode_complete_ht_tile, AcceleratedComponentRequest, + PreparedComponentTransforms, }; use finalize::{ finalize_accelerated_codestream, finalize_staged_codestream, TransformStageTimings, }; use ownership::{codestream_final_plan_retained_bytes, prepared_transforms_retained_bytes}; use plan::{ - build_single_tile_plan, validate_encode_request, CodestreamFinalPlan, ValidatedEncodeRoute, - ValidatedSingleTileInput, + build_single_tile_plan, validate_encode_request, CodestreamFinalPlan, SingleTilePlan, + ValidatedEncodeRoute, ValidatedSingleTileInput, }; pub(super) use precomputed::{ encode_precomputed_53_single_tile, encode_precomputed_97_single_tile, @@ -238,6 +239,15 @@ fn prepare_validated_single_tile( session, )?; + begin_encode_route( + accelerator, + plan.num_pixels, + num_components, + bit_depth, + signed, + options.reversible, + )?; + if plan.high_bit_exact && options.reversible { let (packetized_tile, plan) = encode_reversible_i64_single_tile_packets(ReversibleI64SingleTileRequest { @@ -313,17 +323,48 @@ fn prepare_validated_single_tile( session, accelerator, )?; + Ok(finish_staged_preparation(encoded, plan, prepared)) +} + +fn finish_staged_preparation( + encoded: EncodedTilePackets, + plan: SingleTilePlan, + prepared: PreparedComponentTransforms, +) -> PreparedSingleTile { let transform_timings = TransformStageTimings { deinterleave: prepared.deinterleave_us, mct: prepared.mct_us, dwt: prepared.dwt_us, }; drop(prepared); - Ok(PreparedSingleTile::Staged { + PreparedSingleTile::Staged { encoded, final_plan: plan.into_codestream_final_plan(), transform_timings, - }) + } +} + +fn begin_encode_route( + accelerator: &mut impl J2kEncodeStageAccelerator, + num_pixels: usize, + num_components: u16, + bit_depth: u8, + signed: bool, + reversible: bool, +) -> NativeEncodePipelineResult<()> { + accelerator + .begin_encode(J2kEncodeContext { + num_pixels, + num_components, + bit_depth, + signed, + reversible, + }) + .map_err(|source| crate::EncodeError::Accelerator { + operation: "encode route selection", + source, + })?; + Ok(()) } fn finalize_prepared_single_tile( diff --git a/crates/j2k-native/src/j2c/encode/single_tile/accelerator/tests.rs b/crates/j2k-native/src/j2c/encode/single_tile/accelerator/tests.rs index 5b7b3cf9..6205dbf9 100644 --- a/crates/j2k-native/src/j2c/encode/single_tile/accelerator/tests.rs +++ b/crates/j2k-native/src/j2c/encode/single_tile/accelerator/tests.rs @@ -1,13 +1,14 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use crate::{ - encode_with_accelerator, EncodeError, EncodeOptions, J2kDeinterleaveToF32Job, + encode_with_accelerator, EncodeError, EncodeOptions, J2kDeinterleaveToF32Job, J2kEncodeContext, J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardIctJob, J2kForwardRctJob, }; use alloc::{vec, vec::Vec}; #[derive(Clone, Copy)] enum FailedStage { + Begin, Deinterleave, Rct, Ict, @@ -18,6 +19,18 @@ struct FailingAccelerator(FailedStage); struct MalformedDeinterleaveAccelerator; +#[derive(Default)] +struct ContextRecordingAccelerator { + context: Option, +} + +impl J2kEncodeStageAccelerator for ContextRecordingAccelerator { + fn begin_encode(&mut self, context: J2kEncodeContext) -> crate::J2kEncodeStageResult<()> { + self.context = Some(context); + Ok(()) + } +} + impl J2kEncodeStageAccelerator for MalformedDeinterleaveAccelerator { fn encode_deinterleave( &mut self, @@ -28,6 +41,16 @@ impl J2kEncodeStageAccelerator for MalformedDeinterleaveAccelerator { } impl J2kEncodeStageAccelerator for FailingAccelerator { + fn begin_encode(&mut self, _context: J2kEncodeContext) -> crate::J2kEncodeStageResult<()> { + if matches!(self.0, FailedStage::Begin) { + Err(crate::J2kEncodeStageError::internal_invariant( + "staged test failure", + )) + } else { + Ok(()) + } + } + fn encode_deinterleave( &mut self, _job: J2kDeinterleaveToF32Job<'_>, @@ -118,12 +141,40 @@ fn assert_stage_error( #[test] fn staged_accelerator_failures_keep_typed_operation_taxonomy() { + assert_stage_error(FailedStage::Begin, "encode route selection", 1, true); assert_stage_error(FailedStage::Deinterleave, "pixel deinterleave", 1, true); assert_stage_error(FailedStage::Rct, "forward RCT", 3, true); assert_stage_error(FailedStage::Ict, "forward ICT", 3, false); assert_stage_error(FailedStage::Dwt53, "forward 5/3 DWT", 1, true); } +#[test] +fn encode_supplies_validated_route_context_before_stage_dispatch() { + let pixels = vec![17_u8; 8 * 4 * 3]; + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + use_mct: true, + ..EncodeOptions::default() + }; + let mut accelerator = ContextRecordingAccelerator::default(); + + encode_with_accelerator(&pixels, 8, 4, 3, 8, false, &options, &mut accelerator) + .expect("encode with route context"); + + assert_eq!( + accelerator.context, + Some(J2kEncodeContext { + num_pixels: 32, + num_components: 3, + bit_depth: 8, + signed: false, + reversible: false, + }) + ); +} + #[test] fn malformed_accelerator_output_keeps_the_accelerator_category() { let pixels = vec![17_u8; 8 * 8]; diff --git a/crates/j2k-native/src/j2c/encode/tile_parts.rs b/crates/j2k-native/src/j2c/encode/tile_parts.rs index 2795a49d..aaf3c6ed 100644 --- a/crates/j2k-native/src/j2c/encode/tile_parts.rs +++ b/crates/j2k-native/src/j2c/encode/tile_parts.rs @@ -87,7 +87,7 @@ pub(super) fn validate_packet_header_marker_payloads( tile_packet_headers: &[&[Vec]], ) -> NativeEncodePipelineResult<()> { const PACKET_HEADER_MARKER_PAYLOAD_LIMIT: usize = u16::MAX as usize - 3; - const PPM_PACKET_HEADER_LIMIT: usize = PACKET_HEADER_MARKER_PAYLOAD_LIMIT - 2; + const PPM_PACKET_HEADER_LIMIT: usize = PACKET_HEADER_MARKER_PAYLOAD_LIMIT - 4; const MAX_PACKET_HEADER_MARKERS: usize = u8::MAX as usize + 1; if !write_ppm && !write_ppt { @@ -106,38 +106,48 @@ pub(super) fn validate_packet_header_marker_payloads( if write_ppm { let mut marker_count = 0usize; let mut payload_len = 0usize; - for header in tile_packet_headers - .iter() - .flat_map(|headers| headers.iter()) - { - if header.len() > PPM_PACKET_HEADER_LIMIT { - return Err(NativeEncodePipelineError::unsupported( - "PPM packet header exceeds marker payload limit", - )); - } - let entry_len = 2usize.checked_add(header.len()).ok_or( - NativeEncodePipelineError::arithmetic_overflow("PPM marker payload length"), - )?; - if payload_len == 0 { - marker_count = marker_count.checked_add(1).ok_or( - NativeEncodePipelineError::arithmetic_overflow("PPM marker count"), - )?; - } else if payload_len - .checked_add(entry_len) - .is_none_or(|len| len > PACKET_HEADER_MARKER_PAYLOAD_LIMIT) - { - marker_count = marker_count.checked_add(1).ok_or( - NativeEncodePipelineError::arithmetic_overflow("PPM marker count"), + for headers in tile_packet_headers { + let tile_part_len = headers.iter().try_fold(0usize, |total, header| { + total.checked_add(header.len()).ok_or( + NativeEncodePipelineError::arithmetic_overflow("PPM tile-part packet headers"), + ) + })?; + u32::try_from(tile_part_len).map_err(|_| { + NativeEncodePipelineError::unsupported("PPM tile-part packet headers exceed u32") + })?; + for (header_index, header) in headers.iter().enumerate() { + if header.len() > PPM_PACKET_HEADER_LIMIT { + return Err(NativeEncodePipelineError::unsupported( + "PPM packet header exceeds marker payload limit", + )); + } + let entry_len = header + .len() + .checked_add(usize::from(header_index == 0) * 4) + .ok_or(NativeEncodePipelineError::arithmetic_overflow( + "PPM marker payload length", + ))?; + if payload_len == 0 { + marker_count = marker_count.checked_add(1).ok_or( + NativeEncodePipelineError::arithmetic_overflow("PPM marker count"), + )?; + } else if payload_len + .checked_add(entry_len) + .is_none_or(|len| len > PACKET_HEADER_MARKER_PAYLOAD_LIMIT) + { + marker_count = marker_count.checked_add(1).ok_or( + NativeEncodePipelineError::arithmetic_overflow("PPM marker count"), + )?; + payload_len = 0; + } + payload_len = payload_len.checked_add(entry_len).ok_or( + NativeEncodePipelineError::arithmetic_overflow("PPM marker payload length"), )?; - payload_len = 0; - } - payload_len = payload_len.checked_add(entry_len).ok_or( - NativeEncodePipelineError::arithmetic_overflow("PPM marker payload length"), - )?; - if marker_count > MAX_PACKET_HEADER_MARKERS { - return Err(NativeEncodePipelineError::unsupported( - "PPM packet headers require more than 256 marker segments", - )); + if marker_count > MAX_PACKET_HEADER_MARKERS { + return Err(NativeEncodePipelineError::unsupported( + "PPM packet headers require more than 256 marker segments", + )); + } } } } diff --git a/crates/j2k-native/src/j2c/encode_tests.rs b/crates/j2k-native/src/j2c/encode_tests.rs index 413e15a0..b11b3675 100644 --- a/crates/j2k-native/src/j2c/encode_tests.rs +++ b/crates/j2k-native/src/j2c/encode_tests.rs @@ -772,7 +772,7 @@ fn ht_target_coding_passes_tracks_ht_quality_layers() { )] fn packet_header_validation_allows_chunked_ppm_and_ppt_payloads() { const MARKER_PAYLOAD_LIMIT: usize = u16::MAX as usize - 3; - let ppm_headers = vec![vec![0_u8; MARKER_PAYLOAD_LIMIT - 2], vec![1_u8; 1]]; + let ppm_headers = vec![vec![0_u8; MARKER_PAYLOAD_LIMIT - 4], vec![1_u8; 1]]; let ppt_headers = vec![vec![2_u8; MARKER_PAYLOAD_LIMIT + 1]]; validate_packet_header_marker_payloads(true, false, &[&ppm_headers]) diff --git a/crates/j2k-native/src/j2c/fdwt.rs b/crates/j2k-native/src/j2c/fdwt.rs index 89be9410..6f67f2c4 100644 --- a/crates/j2k-native/src/j2c/fdwt.rs +++ b/crates/j2k-native/src/j2c/fdwt.rs @@ -11,7 +11,7 @@ use alloc::vec; use alloc::vec::Vec; use core::mem::size_of; -use crate::math::floor_f32; +use crate::math::{floor_f32, mul_add}; use crate::{EncodeError, EncodeResult}; use j2k_codec_math::dwt; @@ -343,14 +343,14 @@ fn forward_lift_97(data: &mut [f32]) { } else { data[last_even] }; - data[i] += ALPHA * (left + right); + data[i] = mul_add(ALPHA, left + right, data[i]); } // Step 2: β update on even (low-pass) samples for i in (0..n).step_by(2) { let left = if i > 0 { data[i - 1] } else { data[1] }; let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += BETA * (left + right); + data[i] = mul_add(BETA, left + right, data[i]); } // Step 3: γ predict on odd samples @@ -361,14 +361,14 @@ fn forward_lift_97(data: &mut [f32]) { } else { data[last_even] }; - data[i] += GAMMA * (left + right); + data[i] = mul_add(GAMMA, left + right, data[i]); } // Step 4: δ update on even samples for i in (0..n).step_by(2) { let left = if i > 0 { data[i - 1] } else { data[1] }; let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += DELTA * (left + right); + data[i] = mul_add(DELTA, left + right, data[i]); } // Step 5 & 6: Scale @@ -514,6 +514,27 @@ mod tests { } } + #[test] + fn forward_lift_97_uses_target_independent_fused_rounding() { + let mut data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + + forward_lift_97(&mut data); + + assert_eq!( + data.iter().map(|value| value.to_bits()).collect::>(), + vec![ + 0x3faa_b4bc, + 0x3e7f_ffff, + 0x4044_b068, + 0x3350_dd84, + 0x409e_49c0, + 0xbe3a_ec76, + 0x40e2_0777, + 0x3f5d_7665, + ] + ); + } + #[test] fn test_forward_dwt_53_single_level() { // 4×4 image diff --git a/crates/j2k-native/src/j2c/forward_mct.rs b/crates/j2k-native/src/j2c/forward_mct.rs index c08c531f..36f0e39f 100644 --- a/crates/j2k-native/src/j2c/forward_mct.rs +++ b/crates/j2k-native/src/j2c/forward_mct.rs @@ -6,7 +6,7 @@ use alloc::vec::Vec; -use crate::math::floor_f32; +use crate::math::{floor_f32, mul_add}; use j2k_codec_math::mct; /// Apply the forward Reversible Color Transform (RCT) in-place. @@ -69,9 +69,21 @@ pub(crate) fn forward_ict(components: &mut [Vec]) { let g0 = *g; let b0 = *b; - let y = mct::ICT_FWD_Y_R * r0 + mct::ICT_FWD_Y_G * g0 + mct::ICT_FWD_Y_B * b0; - let cb = mct::ICT_FWD_CB_R * r0 + mct::ICT_FWD_CB_G * g0 + mct::ICT_FWD_CB_B * b0; - let cr = mct::ICT_FWD_CR_R * r0 + mct::ICT_FWD_CR_G * g0 + mct::ICT_FWD_CR_B * b0; + let y = mul_add( + mct::ICT_FWD_Y_B, + b0, + mul_add(mct::ICT_FWD_Y_R, r0, mct::ICT_FWD_Y_G * g0), + ); + let cb = mul_add( + mct::ICT_FWD_CB_B, + b0, + mul_add(mct::ICT_FWD_CB_R, r0, mct::ICT_FWD_CB_G * g0), + ); + let cr = mul_add( + mct::ICT_FWD_CR_B, + b0, + mul_add(mct::ICT_FWD_CR_R, r0, mct::ICT_FWD_CR_G * g0), + ); *r = y; *g = cb; @@ -145,6 +157,32 @@ mod tests { assert!(approx_eq(comps[2][0], 0.0, 0.01)); } + #[test] + fn forward_ict_uses_target_independent_fused_rounding() { + let mut comps = vec![vec![32.0, 35.0], vec![-7.0, -5.0], vec![-75.0, -74.0]]; + + forward_ict(&mut comps); + + assert_eq!( + [ + comps[0][0].to_bits(), + comps[0][1].to_bits(), + comps[1][0].to_bits(), + comps[1][1].to_bits(), + comps[2][0].to_bits(), + comps[2][1].to_bits(), + ], + [ + 0xc045_d2f3, + 0xbf67_efa2, + 0xc222_5321, + 0xc224_fff3, + 0x41c8_3b8e, + 0x41cc_e214, + ] + ); + } + #[test] fn test_ict_round_trip() { let r = 200.0f32; diff --git a/crates/j2k-native/src/j2c/progression.rs b/crates/j2k-native/src/j2c/progression.rs index 0184d60d..72a73bbe 100644 --- a/crates/j2k-native/src/j2c/progression.rs +++ b/crates/j2k-native/src/j2c/progression.rs @@ -6,11 +6,16 @@ use alloc::vec::Vec; +use super::codestream::ComponentInfo; use super::tile::{ComponentTile, ResolutionTile, Tile}; use crate::error::{DecodingError, Result}; +use crate::{try_resize_decode_elements, ValidationError, DEFAULT_MAX_DECODE_BYTES}; use alloc::boxed::Box; use core::cmp::Ordering; use core::iter; +use core::mem::size_of; + +const PACKETS_PER_INCLUSION_WORD: usize = u64::BITS as usize; #[derive(Default, Copy, Clone, Debug, PartialEq, Hash, Eq)] pub(crate) struct ProgressionData { @@ -27,9 +32,105 @@ pub(crate) struct IteratorInput<'a> { components: (u16, u16), } +struct PacketInclusionMap { + resolution_offsets: Vec, + words: Vec, + max_resolution: usize, + layers: usize, +} + +impl PacketInclusionMap { + fn new(tile: &Tile<'_>) -> Result { + let max_resolution = tile + .component_infos + .iter() + .map(ComponentInfo::num_resolution_levels) + .max() + .map(usize::from) + .ok_or(DecodingError::InvalidProgressionIterator)?; + let layers = usize::from(tile.num_layers); + let slot_count = tile + .component_infos + .len() + .checked_mul(max_resolution) + .ok_or(ValidationError::ImageTooLarge)?; + let mut resolution_offsets = Vec::new(); + try_resize_decode_elements(&mut resolution_offsets, slot_count, usize::MAX)?; + + let mut packet_count = 0_usize; + for (component_index, component) in tile.component_tiles().enumerate() { + for resolution in component.resolution_tiles() { + let slot = component_index + .checked_mul(max_resolution) + .and_then(|base| base.checked_add(usize::from(resolution.resolution))) + .ok_or(ValidationError::ImageTooLarge)?; + resolution_offsets[slot] = packet_count; + let precinct_count = usize::try_from(resolution.num_precincts()) + .map_err(|_| ValidationError::ImageTooLarge)?; + let resolution_packets = precinct_count + .checked_mul(layers) + .ok_or(ValidationError::ImageTooLarge)?; + packet_count = packet_count + .checked_add(resolution_packets) + .ok_or(ValidationError::ImageTooLarge)?; + } + } + + let word_count = packet_count.div_ceil(PACKETS_PER_INCLUSION_WORD); + let retained_bytes = resolution_offsets + .capacity() + .checked_mul(size_of::()) + .and_then(|bytes| { + word_count + .checked_mul(size_of::()) + .and_then(|word_bytes| bytes.checked_add(word_bytes)) + }) + .ok_or(ValidationError::ImageTooLarge)?; + if retained_bytes > DEFAULT_MAX_DECODE_BYTES { + return Err(ValidationError::ImageTooLarge.into()); + } + let mut words = Vec::new(); + try_resize_decode_elements(&mut words, word_count, 0_u64)?; + Ok(Self { + resolution_offsets, + words, + max_resolution, + layers, + }) + } + + fn insert(&mut self, packet: ProgressionData) -> bool { + let Some(slot) = usize::from(packet.component) + .checked_mul(self.max_resolution) + .and_then(|base| base.checked_add(usize::from(packet.resolution))) + else { + return false; + }; + let Some(&resolution_offset) = self.resolution_offsets.get(slot) else { + return false; + }; + let Some(packet_offset) = usize::try_from(packet.precinct) + .ok() + .and_then(|precinct| precinct.checked_mul(self.layers)) + .and_then(|base| base.checked_add(usize::from(packet.layer_num))) + .and_then(|offset| resolution_offset.checked_add(offset)) + else { + return false; + }; + let word_index = packet_offset / PACKETS_PER_INCLUSION_WORD; + let mask = 1_u64 << (packet_offset % PACKETS_PER_INCLUSION_WORD); + let Some(word) = self.words.get_mut(word_index) else { + return false; + }; + let is_new = *word & mask == 0; + *word |= mask; + is_new + } +} + impl<'a> IteratorInput<'a> { - pub(crate) fn new(tile: &'a Tile<'a>) -> Self { - Self::new_with_custom_bounds( + pub(crate) fn new(tile: &'a Tile<'a>) -> Option { + Self::try_new_with_custom_bounds( tile, // Will be clamped automatically. (0, u8::MAX), @@ -38,16 +139,6 @@ impl<'a> IteratorInput<'a> { ) } - pub(crate) fn new_with_custom_bounds( - tile: &'a Tile<'a>, - resolutions: (u8, u8), - layers: (u8, u8), - components: (u16, u16), - ) -> Self { - Self::try_new_with_custom_bounds(tile, resolutions, layers, components) - .expect("valid progression iterator bounds") - } - pub(crate) fn try_new_with_custom_bounds( tile: &'a Tile<'a>, mut resolutions: (u8, u8), @@ -125,7 +216,9 @@ pub(crate) fn progression_iterator<'a>( tile: &'a Tile<'a>, ) -> Result + 'a>> { if tile.progression_changes.is_empty() { - return progression_iterator_for_order(tile.progression_order, IteratorInput::new(tile)); + let iter_input = + IteratorInput::new(tile).ok_or(DecodingError::InvalidProgressionIterator)?; + return progression_iterator_for_order(tile.progression_order, iter_input); } let mut iterators = Vec::new(); @@ -144,7 +237,14 @@ pub(crate) fn progression_iterator<'a>( )?); } - Ok(Box::new(iterators.into_iter().flatten())) + let mut inclusion = PacketInclusionMap::new(tile)?; + + Ok(Box::new( + iterators + .into_iter() + .flatten() + .filter(move |packet| inclusion.insert(*packet)), + )) } fn progression_iterator_for_order<'a>( @@ -402,3 +502,127 @@ pub(crate) fn component_position_resolution_layer_progression( .then_with(|| p.precinct_idx.cmp(&s.precinct_idx)) }) } + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + use crate::j2c::codestream::{ + CodeBlockStyle, CodingStyleComponent, CodingStyleFlags, CodingStyleParameters, + ComponentInfo, ComponentSizeInfo, ProgressionChange, ProgressionOrder, QuantizationInfo, + QuantizationStyle, WaveletTransform, + }; + use crate::j2c::rect::IntRect; + + #[test] + fn overlapping_progression_changes_emit_each_packet_once() { + let tile = Tile { + idx: 0, + tile_parts: vec![], + component_infos: vec![ComponentInfo { + size_info: ComponentSizeInfo { + precision: 8, + signed: false, + horizontal_resolution: 1, + vertical_resolution: 1, + }, + coding_style: CodingStyleComponent { + flags: CodingStyleFlags::default(), + parameters: CodingStyleParameters { + num_decomposition_levels: 1, + num_resolution_levels: 2, + code_block_width: 6, + code_block_height: 6, + code_block_style: CodeBlockStyle::default(), + transformation: WaveletTransform::Reversible53, + precinct_exponents: vec![(15, 15), (15, 15)], + }, + }, + quantization_info: QuantizationInfo { + quantization_style: QuantizationStyle::NoQuantization, + guard_bits: 2, + step_sizes: vec![], + }, + roi_shift: 0, + }], + rect: IntRect::from_ltrb(0, 0, 8, 8), + progression_order: ProgressionOrder::LayerResolutionComponentPosition, + progression_changes: vec![ + ProgressionChange { + resolution_start: 0, + component_start: 0, + layer_end: 2, + resolution_end: 1, + component_end: 1, + progression_order: ProgressionOrder::LayerResolutionComponentPosition, + }, + ProgressionChange { + resolution_start: 0, + component_start: 0, + layer_end: 2, + resolution_end: 2, + component_end: 1, + progression_order: ProgressionOrder::LayerResolutionComponentPosition, + }, + ], + num_layers: 2, + mct: false, + }; + + let packets = progression_iterator(&tile) + .expect("valid progression") + .collect::>(); + + assert_eq!( + packets, + [ + ProgressionData { + layer_num: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + ProgressionData { + layer_num: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ProgressionData { + layer_num: 0, + resolution: 1, + component: 0, + precinct: 0, + }, + ProgressionData { + layer_num: 1, + resolution: 1, + component: 0, + precinct: 0, + }, + ] + ); + } + + #[test] + fn empty_component_set_is_an_invalid_progression_iterator() { + let tile = Tile { + idx: 0, + tile_parts: vec![], + component_infos: vec![], + rect: IntRect::from_ltrb(0, 0, 1, 1), + progression_order: ProgressionOrder::LayerResolutionComponentPosition, + progression_changes: vec![], + num_layers: 1, + mct: false, + }; + + assert!(matches!( + progression_iterator(&tile), + Err(crate::error::DecodeError::Decoding( + DecodingError::InvalidProgressionIterator + )) + )); + } +} diff --git a/crates/j2k-native/src/j2c/quantize.rs b/crates/j2k-native/src/j2c/quantize.rs index c54db7db..534b7d66 100644 --- a/crates/j2k-native/src/j2c/quantize.rs +++ b/crates/j2k-native/src/j2c/quantize.rs @@ -302,14 +302,12 @@ pub(crate) fn quantize_subband( if delta <= 0.0 { return vec![0i32; coefficients.len()]; } - let inv_delta = 1.0 / delta; - coefficients .iter() .map(|&c| { // Deadzone quantization: q = sign(c) * floor(|c| / Δ) let sign = if c < 0.0 { -1 } else { 1 }; - let magnitude = floor_f32(c.abs() * inv_delta) as i32; + let magnitude = floor_f32(c.abs() / delta) as i32; sign * magnitude }) .collect() @@ -356,10 +354,9 @@ pub(crate) fn try_quantize_subband( quantized.resize(coefficients.len(), 0); return Ok(quantized); } - let inv_delta = 1.0 / delta; for &coefficient in coefficients { let sign = if coefficient < 0.0 { -1 } else { 1 }; - let magnitude = floor_f32(coefficient.abs() * inv_delta) as i32; + let magnitude = floor_f32(coefficient.abs() / delta) as i32; quantized.push(sign * magnitude); } Ok(quantized) @@ -397,6 +394,19 @@ mod tests { assert_eq!(result[3], 0); // Below deadzone } + #[test] + fn irreversible_quantization_uses_direct_division_at_integer_boundary() { + let coefficient = f32::from_bits(0x41fa_c8ff); + let step = QuantStepSize { + exponent: 8, + mantissa: 23, + }; + + let result = quantize_subband(&[coefficient, -coefficient], &step, 8, false); + + assert_eq!(result, vec![30, -30]); + } + #[test] fn test_compute_step_sizes_reversible() { let steps = compute_step_sizes(8, 3, true, 1); diff --git a/crates/j2k-native/src/j2c/tile.rs b/crates/j2k-native/src/j2c/tile.rs index aa45498b..530c69b5 100644 --- a/crates/j2k-native/src/j2c/tile.rs +++ b/crates/j2k-native/src/j2c/tile.rs @@ -5,7 +5,7 @@ use alloc::vec::Vec; use super::build::{PrecinctData, SubBandType}; use super::codestream::{markers, ComponentInfo, Header, ProgressionChange, ProgressionOrder}; use super::rect::IntRect; -use crate::error::{bail, MarkerError, Result, ValidationError}; +use crate::error::{bail, MarkerError, Result, TileError, ValidationError}; use crate::reader::BitReader; mod cursor; @@ -210,6 +210,10 @@ pub(crate) fn parse<'a>( )?; } + if main_header.strict && ppm_packet_idx != main_header.ppm_packets.len() { + bail!(TileError::Invalid); + } + if main_header.strict && reader.read_marker()? != markers::EOC { bail!(MarkerError::Expected("EOC")); } @@ -494,6 +498,8 @@ impl<'a> ResolutionTile<'a> { .vertical_resolution, ) .checked_mul(y_stride)?; + let precinct_grid_width = 1_u32.checked_shl(u32::from(self.precinct_exponent_x()))?; + let precinct_grid_height = 1_u32.checked_shl(u32::from(self.precinct_exponent_y()))?; // These variables are used to map the start coordinates of each // precinct _on the reference grid_. Remember that the first @@ -507,15 +513,13 @@ impl<'a> ResolutionTile<'a> { // is divisible, then we can't take the x/y position of the tile // as the start of the precinct, but instead have to advance to the // next multiple. - if !r_x.is_multiple_of(precinct_x_step) - && (self.rect.x0 * (1 << nl_minus_r)).is_multiple_of(precinct_x_step) + if !r_x.is_multiple_of(precinct_x_step) && self.rect.x0.is_multiple_of(precinct_grid_width) { r_x = r_x.checked_next_multiple_of(precinct_x_step)?; } // Same as above. - if !r_y.is_multiple_of(precinct_y_step) - && (self.rect.y0 * (1 << nl_minus_r)).is_multiple_of(precinct_y_step) + if !r_y.is_multiple_of(precinct_y_step) && self.rect.y0.is_multiple_of(precinct_grid_height) { r_y = r_y.checked_next_multiple_of(precinct_y_step)?; } @@ -609,6 +613,54 @@ mod tests { assert_eq!(subband_coordinate(u32::MAX, 33, true), 0); } + #[test] + fn first_subsampled_precinct_uses_the_next_reference_grid_position_when_aligned() { + let component_info = ComponentInfo { + size_info: ComponentSizeInfo { + precision: 8, + signed: false, + horizontal_resolution: 4, + vertical_resolution: 1, + }, + coding_style: CodingStyleComponent { + flags: CodingStyleFlags::default(), + parameters: CodingStyleParameters { + num_decomposition_levels: 1, + num_resolution_levels: 2, + code_block_width: 6, + code_block_height: 6, + code_block_style: CodeBlockStyle::default(), + transformation: WaveletTransform::Reversible53, + precinct_exponents: vec![(0, 0), (1, 1)], + }, + }, + quantization_info: QuantizationInfo { + quantization_style: QuantizationStyle::NoQuantization, + guard_bits: 2, + step_sizes: vec![], + }, + roi_shift: 0, + }; + let tile = Tile { + idx: 0, + tile_parts: vec![], + rect: IntRect::from_ltrb(4, 0, 12, 12), + component_infos: vec![component_info], + progression_order: ProgressionOrder::ResolutionPositionComponentLayer, + progression_changes: vec![], + mct: false, + num_layers: 1, + }; + let component = ComponentTile::new(&tile, &tile.component_infos[0]); + let precincts = ResolutionTile::new(component, 0) + .precincts() + .expect("valid precinct geometry") + .collect::>(); + + assert_eq!(precincts[0].r_x, 8); + assert_eq!(precincts[0].r_y, 0); + } + /// Test case for the example in B.4. #[test] #[expect( diff --git a/crates/j2k-native/src/j2c/tile/tile_part.rs b/crates/j2k-native/src/j2c/tile/tile_part.rs index 22b70c80..aa277648 100644 --- a/crates/j2k-native/src/j2c/tile/tile_part.rs +++ b/crates/j2k-native/src/j2c/tile/tile_part.rs @@ -8,10 +8,7 @@ use super::metadata::{ try_clone_coding_parameters, try_clone_quantization_info, TileMetadataBudget, TileMetadataTransaction, }; -use super::{ - ComponentTile, MergedTilePart, PacketLengthMetadata, ResolutionTile, SeparatedTilePart, Tile, - TilePart, -}; +use super::{MergedTilePart, PacketLengthMetadata, SeparatedTilePart, Tile, TilePart}; use crate::error::{bail, err, DecodingError, MarkerError, Result, TileError, ValidationError}; use crate::j2c::codestream::{self, markers, skip_marker_segment, Header, PacketLengthMarker}; use crate::reader::BitReader; @@ -266,13 +263,7 @@ pub(super) fn parse_tile_part<'a>( let ppt_header_capacity = ppt_headers.capacity(); ppt_headers.sort_by_key(|ppt_header| ppt_header.sequence_idx); - let ppm_header_count = ppm_header_count( - tile, - packet_lengths_present.then_some(temporary_packet_length_count), - &tile_part_header, - main_header, - *ppm_packet_idx, - )?; + let ppm_header_count = ppm_header_count(main_header, *ppm_packet_idx)?; let header_count = ppt_headers .len() .checked_add(ppm_header_count) @@ -369,104 +360,18 @@ fn retain_tile_part_metadata( Ok(()) } -fn ppm_header_count( - tile: &Tile<'_>, - packet_length_count: Option, - tile_part_header: &TilePartHeader, - main_header: &Header<'_>, - ppm_packet_idx: usize, -) -> Result { +fn ppm_header_count(main_header: &Header<'_>, ppm_packet_idx: usize) -> Result { if main_header.ppm_packets.is_empty() { return Ok(0); } - if let Some(packet_length_count) = packet_length_count { - return Ok(packet_length_count); - } - if tile_part_header.num_tile_parts == 1 { - return tile_packet_count(tile); - } - - // Without PLT lengths, this legacy PPM representation has no serialized - // boundary for a multi-part tile. Preserve the former one-entry fallback - // until the parser supports cross-marker Nppm tile-part accumulation. - Ok(usize::from( - main_header.ppm_packets.get(ppm_packet_idx).is_some(), - )) -} - -fn tile_packet_count(tile: &Tile<'_>) -> Result { - if tile.progression_changes.is_empty() { - let component_end = u16::try_from(tile.component_infos.len()) - .map_err(|_| ValidationError::TooManyChannels)?; - let resolution_end = tile - .component_infos - .iter() - .map(|component| component.coding_style.parameters.num_resolution_levels) - .max() - .ok_or(ValidationError::InvalidComponentMetadata)?; - return packet_count_for_bounds(tile, 0, resolution_end, tile.num_layers, 0, component_end); - } - - tile.progression_changes - .iter() - .try_fold(0_usize, |total, change| { - let count = packet_count_for_bounds( - tile, - change.resolution_start, - change.resolution_end, - change.layer_end.min(tile.num_layers), - change.component_start, - change.component_end, - )?; - total - .checked_add(count) - .ok_or(ValidationError::ImageTooLarge.into()) - }) -} - -fn packet_count_for_bounds( - tile: &Tile<'_>, - resolution_start: u8, - resolution_end: u8, - layer_end: u8, - component_start: u16, - component_end: u16, -) -> Result { - let component_len = - u16::try_from(tile.component_infos.len()).map_err(|_| ValidationError::TooManyChannels)?; - let component_end = component_end.min(component_len); - let total_resolution_end = tile - .component_infos + main_header + .ppm_packets + .get(ppm_packet_idx..) + .ok_or(TileError::Invalid)? .iter() - .map(|component| component.coding_style.parameters.num_resolution_levels) - .max() - .ok_or(ValidationError::InvalidComponentMetadata)?; - let resolution_end = resolution_end.min(total_resolution_end); - if resolution_start >= resolution_end || layer_end == 0 || component_start >= component_end { - return Err(DecodingError::InvalidProgressionIterator.into()); - } - - let mut packet_count = 0_usize; - for component_idx in component_start..component_end { - let component = tile - .component_infos - .get(usize::from(component_idx)) - .ok_or(ValidationError::InvalidComponentMetadata)?; - let component_tile = ComponentTile::new(tile, component); - let component_resolution_end = resolution_end.min(component.num_resolution_levels()); - for resolution in resolution_start..component_resolution_end { - let precinct_count = - usize::try_from(ResolutionTile::new(component_tile, resolution).num_precincts()) - .map_err(|_| ValidationError::ImageTooLarge)?; - let layer_packets = precinct_count - .checked_mul(usize::from(layer_end)) - .ok_or(ValidationError::ImageTooLarge)?; - packet_count = packet_count - .checked_add(layer_packets) - .ok_or(ValidationError::ImageTooLarge)?; - } - } - Ok(packet_count) + .position(|packet| packet.ends_tile_part) + .and_then(|offset| offset.checked_add(1)) + .ok_or(TileError::Invalid.into()) } struct TilePartHeader { diff --git a/crates/j2k-native/src/jp2/validation.rs b/crates/j2k-native/src/jp2/validation.rs index 5dce4921..384c5c2c 100644 --- a/crates/j2k-native/src/jp2/validation.rs +++ b/crates/j2k-native/src/jp2/validation.rs @@ -4,7 +4,6 @@ use crate::error::{bail, FormatError, Result}; -use super::cmap::ComponentMappingType; use super::container::Jp2FileKind; use super::{ComponentDescriptor, ImageBoxes}; @@ -47,8 +46,8 @@ pub(super) fn validate_component_precision_metadata( let Some(image_header) = boxes.image_header else { bail!(FormatError::InvalidBox); }; - let resolved_count = resolved_image_component_count(boxes, header); - if resolved_count != usize::from(image_header.components) { + let codestream_count = header.component_infos.len(); + if codestream_count != usize::from(image_header.components) { bail!(FormatError::InvalidBox); } @@ -56,9 +55,11 @@ pub(super) fn validate_component_precision_metadata( if !boxes.bits_per_component.is_empty() { bail!(FormatError::InvalidBox); } - for index in 0..resolved_count { - let component = resolved_image_component_descriptor(boxes, header, index) - .ok_or(FormatError::InvalidBox)?; + for component in &header.component_infos { + let component = component_descriptor_from_size_info( + component.size_info.precision, + component.size_info.signed, + ); if component != descriptor { bail!(FormatError::InvalidBox); } @@ -67,9 +68,12 @@ pub(super) fn validate_component_precision_metadata( if boxes.bits_per_component.len() != usize::from(image_header.components) { bail!(FormatError::InvalidBox); } - for (index, descriptor) in boxes.bits_per_component.iter().enumerate() { - let component = resolved_image_component_descriptor(boxes, header, index) - .ok_or(FormatError::InvalidBox)?; + for (component, descriptor) in header.component_infos.iter().zip(&boxes.bits_per_component) + { + let component = component_descriptor_from_size_info( + component.size_info.precision, + component.size_info.signed, + ); if component != *descriptor { bail!(FormatError::InvalidBox); } @@ -79,62 +83,6 @@ pub(super) fn validate_component_precision_metadata( Ok(()) } -fn resolved_image_component_count(boxes: &ImageBoxes, header: &crate::j2c::Header<'_>) -> usize { - if let Some(component_mapping) = boxes.component_mapping.as_ref() { - return component_mapping.entries.len(); - } - - if let Some(palette) = boxes.palette.as_ref() { - return palette.columns.len(); - } - - header.component_infos.len() -} - -fn resolved_image_component_descriptor( - boxes: &ImageBoxes, - header: &crate::j2c::Header<'_>, - index: usize, -) -> Option { - if let Some(component_mapping) = boxes.component_mapping.as_ref() { - let entry = component_mapping.entries.get(index)?; - return match entry.mapping_type { - ComponentMappingType::Direct => { - let component = header - .component_infos - .get(usize::from(entry.component_index))?; - Some(component_descriptor_from_size_info( - component.size_info.precision, - component.size_info.signed, - )) - } - ComponentMappingType::Palette { column } => { - let palette = boxes.palette.as_ref()?; - let column = palette.columns.get(usize::from(column))?; - Some(component_descriptor_from_size_info( - column.bit_depth, - column.signed, - )) - } - ComponentMappingType::Unknown { .. } => None, - }; - } - - if let Some(palette) = boxes.palette.as_ref() { - let column = palette.columns.get(index)?; - return Some(component_descriptor_from_size_info( - column.bit_depth, - column.signed, - )); - } - - let component = header.component_infos.get(index)?; - Some(component_descriptor_from_size_info( - component.size_info.precision, - component.size_info.signed, - )) -} - fn component_descriptor_from_size_info(bit_depth: u8, signed: bool) -> ComponentDescriptor { ComponentDescriptor { bit_depth, signed } } diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs index d2813a3a..2ea0ae8c 100644 --- a/crates/j2k-native/src/lib.rs +++ b/crates/j2k-native/src/lib.rs @@ -262,7 +262,7 @@ pub use j2k_types::{ sort_packet_descriptors_for_progression, CpuOnlyJ2kEncodeStageAccelerator, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, IrreversibleQuantizationStep, IrreversibleQuantizationSubbandScales, J2kCodeBlockSegment, J2kCodeBlockStyle, - J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, + J2kDeinterleaveToF32Job, J2kEncodeContext, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, J2kEncodeStageError, J2kEncodeStageErrorKind, J2kEncodeStageResult, J2kForwardDwt53Job, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Level, J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, @@ -400,6 +400,8 @@ pub use scalar::{ decode_ht_code_block_scalar_until_phase, decode_ht_code_block_scalar_with_workspace, decode_ht_code_block_scalar_with_workspace_profiled, decode_j2k_code_block_scalar, decode_j2k_code_block_scalar_profiled, decode_j2k_code_block_scalar_with_workspace, + decode_j2k_code_block_scalar_with_workspace_midpoint, + decode_j2k_code_block_scalar_with_workspace_midpoint_profiled, decode_j2k_code_block_scalar_with_workspace_profiled, decode_j2k_sub_band_scalar, encode_ht_code_block_scalar, encode_ht_code_block_scalar_with_passes, encode_j2k_code_block_scalar_with_style, encode_j2k_packetization_scalar, diff --git a/crates/j2k-native/src/math.rs b/crates/j2k-native/src/math.rs index 742c4536..54f905fd 100644 --- a/crates/j2k-native/src/math.rs +++ b/crates/j2k-native/src/math.rs @@ -614,32 +614,9 @@ mod inner { )] #[inline(always)] pub(crate) fn mul_add(a: f32, b: f32, c: f32) -> f32 { - #[cfg(all( - feature = "std", - any( - all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "fma" - ), - all(target_arch = "aarch64", target_feature = "neon") - ) - ))] - { - f32::mul_add(a, b, c) - } - #[cfg(not(all( - feature = "std", - any( - all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "fma" - ), - all(target_arch = "aarch64", target_feature = "neon") - ) - )))] - { - a * b + c - } + // The 9/7 lifting path must use one rounding step on every CPU target so + // its output does not change with compile-time FMA availability. + libm::fmaf(a, b, c) } #[expect( @@ -902,6 +879,22 @@ mod simd_operator_tests { } } +#[cfg(test)] +mod floating_point_tests { + use super::mul_add; + + #[test] + fn scalar_mul_add_has_one_rounding_step_on_every_target() { + let value = mul_add( + f32::from_bits(0x4526_ba09), + f32::from_bits(0xbf41_420c), + f32::from_bits(0x4470_e5c9), + ); + + assert_eq!(value.to_bits(), 0xc483_47a5); + } +} + #[cfg(test)] mod integer_tests { use super::{bit_width_u32, bit_width_u64, ceil_log2_u32, SimdBuffer}; diff --git a/crates/j2k-native/src/scalar.rs b/crates/j2k-native/src/scalar.rs index 4d5ce2ac..9ee2f4cc 100644 --- a/crates/j2k-native/src/scalar.rs +++ b/crates/j2k-native/src/scalar.rs @@ -18,6 +18,8 @@ use self::classic_decode::{checked_code_block_output_layout, CodeBlockOutputLayo pub use self::classic_decode::{ decode_j2k_code_block_scalar, decode_j2k_code_block_scalar_profiled, decode_j2k_code_block_scalar_with_workspace, + decode_j2k_code_block_scalar_with_workspace_midpoint, + decode_j2k_code_block_scalar_with_workspace_midpoint_profiled, decode_j2k_code_block_scalar_with_workspace_profiled, decode_j2k_sub_band_scalar, J2kCodeBlockDecodeProfile, J2kCodeBlockDecodeWorkspace, }; diff --git a/crates/j2k-native/src/scalar/classic_decode.rs b/crates/j2k-native/src/scalar/classic_decode.rs index db0d73d2..df52943a 100644 --- a/crates/j2k-native/src/scalar/classic_decode.rs +++ b/crates/j2k-native/src/scalar/classic_decode.rs @@ -53,6 +53,25 @@ pub fn decode_j2k_code_block_scalar_with_workspace( job: J2kCodeBlockDecodeJob<'_>, output: &mut [f32], workspace: &mut J2kCodeBlockDecodeWorkspace, +) -> Result<()> { + decode_j2k_code_block_scalar_with_workspace_inner(job, output, workspace, false) +} + +/// Adapter scalar classic J2K decoder helper using irreversible midpoint reconstruction. +#[doc(hidden)] +pub fn decode_j2k_code_block_scalar_with_workspace_midpoint( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut J2kCodeBlockDecodeWorkspace, +) -> Result<()> { + decode_j2k_code_block_scalar_with_workspace_inner(job, output, workspace, true) +} + +fn decode_j2k_code_block_scalar_with_workspace_inner( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut J2kCodeBlockDecodeWorkspace, + irreversible_midpoint: bool, ) -> Result<()> { let layout = checked_code_block_output_layout(job.width, job.height, job.output_stride, output.len())?; @@ -78,7 +97,13 @@ pub fn decode_j2k_code_block_scalar_with_workspace( &mut workspace.bit_plane_decode_context, )?; - write_j2k_code_block_output(&workspace.bit_plane_decode_context, job, layout, output); + write_j2k_code_block_output( + &workspace.bit_plane_decode_context, + job, + layout, + output, + irreversible_midpoint, + ); Ok(()) } @@ -119,6 +144,7 @@ fn write_j2k_code_block_output( job: J2kCodeBlockDecodeJob<'_>, layout: CodeBlockOutputLayout, output: &mut [f32], + irreversible_midpoint: bool, ) { for (row_idx, coeff_row) in decode_context .coefficient_rows() @@ -128,8 +154,16 @@ fn write_j2k_code_block_output( let row_start = row_idx * job.output_stride; let output_row = &mut output[row_start..row_start + layout.stride]; for (coefficient, sample) in coeff_row.iter().zip(output_row.iter_mut()) { - let coefficient = apply_roi_maxshift_inverse_i64(coefficient.get_i64(), job.roi_shift); - *sample = coefficient as f32 * job.dequantization_step; + let coefficient = if irreversible_midpoint { + decode_context.reconstruct_irreversible_midpoint( + *coefficient, + job.number_of_coding_passes, + job.roi_shift, + ) + } else { + apply_roi_maxshift_inverse_i64(coefficient.get_i64(), job.roi_shift) as f32 + }; + *sample = coefficient * job.dequantization_step; } } } @@ -184,6 +218,31 @@ pub fn decode_j2k_code_block_scalar_with_workspace_profiled( output: &mut [f32], workspace: &mut J2kCodeBlockDecodeWorkspace, profile: &mut J2kCodeBlockDecodeProfile, +) -> Result<()> { + decode_j2k_code_block_scalar_with_workspace_profiled_inner( + job, output, workspace, profile, false, + ) +} + +/// Profiled scalar classic J2K decode using irreversible midpoint reconstruction. +#[doc(hidden)] +pub fn decode_j2k_code_block_scalar_with_workspace_midpoint_profiled( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut J2kCodeBlockDecodeWorkspace, + profile: &mut J2kCodeBlockDecodeProfile, +) -> Result<()> { + decode_j2k_code_block_scalar_with_workspace_profiled_inner( + job, output, workspace, profile, true, + ) +} + +fn decode_j2k_code_block_scalar_with_workspace_profiled_inner( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut J2kCodeBlockDecodeWorkspace, + profile: &mut J2kCodeBlockDecodeProfile, + irreversible_midpoint: bool, ) -> Result<()> { let layout = checked_code_block_output_layout(job.width, job.height, job.output_stride, output.len())?; @@ -214,7 +273,13 @@ pub fn decode_j2k_code_block_scalar_with_workspace_profiled( profile.add_native_stats(stats); let output_convert_started = profile::profile_now(true); - write_j2k_code_block_output(&workspace.bit_plane_decode_context, job, layout, output); + write_j2k_code_block_output( + &workspace.bit_plane_decode_context, + job, + layout, + output, + irreversible_midpoint, + ); profile.output_convert_us += profile::elapsed_us(output_convert_started); Ok(()) diff --git a/crates/j2k-native/src/tests.rs b/crates/j2k-native/src/tests.rs index 9ef28403..31dc63a3 100644 --- a/crates/j2k-native/src/tests.rs +++ b/crates/j2k-native/src/tests.rs @@ -1744,6 +1744,16 @@ fn direct_color_cpu_rgb8_executor_matches_scaled_region_decode() { }; encode(&pixels, 16, 16, 3, 8, false, &options).expect("encode classic rgb8") }), + ("classic-9/7", { + let pixels = gradient_pixels(16, 16, 3); + let options = EncodeOptions { + reversible: false, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode(&pixels, 16, 16, 3, 8, false, &options) + .expect("encode irreversible classic rgb8") + }), ("htj2k", { let pixels = gradient_pixels(16, 16, 3); let options = EncodeOptions { diff --git a/crates/j2k-native/tests/empty_cmap.rs b/crates/j2k-native/tests/empty_cmap.rs index cd37c6dd..d7eeff75 100644 --- a/crates/j2k-native/tests/empty_cmap.rs +++ b/crates/j2k-native/tests/empty_cmap.rs @@ -547,8 +547,8 @@ fn mixed_palette_jp2h_payload( let mut ihdr = Vec::new(); ihdr.extend_from_slice(&height.to_be_bytes()); ihdr.extend_from_slice(&width.to_be_bytes()); - ihdr.extend_from_slice(&2_u16.to_be_bytes()); - ihdr.extend_from_slice(&[0xff, 7, 0, 0]); + ihdr.extend_from_slice(&1_u16.to_be_bytes()); + ihdr.extend_from_slice(&[7, 7, 0, 0]); let mut palette = Vec::new(); palette.extend_from_slice(&1_u16.to_be_bytes()); @@ -557,7 +557,6 @@ fn mixed_palette_jp2h_payload( let mut jp2h_payload = Vec::new(); jp2h_payload.extend_from_slice(&jp2_box(*b"ihdr", &ihdr)); - jp2h_payload.extend_from_slice(&jp2_box(*b"bpcc", &[7, 0x8f])); jp2h_payload.extend_from_slice(&jp2_box(*b"colr", &[1, 0, 0, 0, 0, 0, 17])); jp2h_payload.extend_from_slice(&jp2_box(*b"pclr", &palette)); if include_component_mapping { @@ -586,7 +585,7 @@ fn high_precision_palette_jp2h_payload(width: u32, height: u32, value: u32) -> V ihdr.extend_from_slice(&height.to_be_bytes()); ihdr.extend_from_slice(&width.to_be_bytes()); ihdr.extend_from_slice(&1_u16.to_be_bytes()); - ihdr.extend_from_slice(&[0xff, 7, 0, 0]); + ihdr.extend_from_slice(&[7, 7, 0, 0]); let mut palette = Vec::new(); palette.extend_from_slice(&1_u16.to_be_bytes()); @@ -595,7 +594,6 @@ fn high_precision_palette_jp2h_payload(width: u32, height: u32, value: u32) -> V let mut jp2h_payload = Vec::new(); jp2h_payload.extend_from_slice(&jp2_box(*b"ihdr", &ihdr)); - jp2h_payload.extend_from_slice(&jp2_box(*b"bpcc", &[24])); jp2h_payload.extend_from_slice(&jp2_box(*b"colr", &[1, 0, 0, 0, 0, 0, 17])); jp2h_payload.extend_from_slice(&jp2_box(*b"pclr", &palette)); jp2h_payload.extend_from_slice(&jp2_box(*b"cmap", &[0, 0, 1, 0])); @@ -606,8 +604,8 @@ fn high_precision_sycc_palette_jp2h_payload(width: u32, height: u32) -> Vec let mut ihdr = Vec::new(); ihdr.extend_from_slice(&height.to_be_bytes()); ihdr.extend_from_slice(&width.to_be_bytes()); - ihdr.extend_from_slice(&3_u16.to_be_bytes()); - ihdr.extend_from_slice(&[0xff, 7, 0, 0]); + ihdr.extend_from_slice(&1_u16.to_be_bytes()); + ihdr.extend_from_slice(&[7, 7, 0, 0]); let mut palette = Vec::new(); palette.extend_from_slice(&1_u16.to_be_bytes()); @@ -618,7 +616,6 @@ fn high_precision_sycc_palette_jp2h_payload(width: u32, height: u32) -> Vec let mut jp2h_payload = Vec::new(); jp2h_payload.extend_from_slice(&jp2_box(*b"ihdr", &ihdr)); - jp2h_payload.extend_from_slice(&jp2_box(*b"bpcc", &[24, 24, 24])); jp2h_payload.extend_from_slice(&jp2_box(*b"colr", &[1, 0, 0, 0, 0, 0, 18])); jp2h_payload.extend_from_slice(&jp2_box(*b"pclr", &palette)); jp2h_payload.extend_from_slice(&jp2_box(*b"cmap", &[0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 2])); diff --git a/crates/j2k-native/tests/plt_decode.rs b/crates/j2k-native/tests/plt_decode.rs index 86dfb12c..36842e3c 100644 --- a/crates/j2k-native/tests/plt_decode.rs +++ b/crates/j2k-native/tests/plt_decode.rs @@ -111,10 +111,10 @@ fn insert_ppm(mut codestream: Vec, header_len: usize) -> Vec { codestream.drain(body_start..body_start + header_len); let mut ppm = vec![0xFF, 0x60]; - let marker_len = u16::try_from(5 + header.len()).expect("PPM marker length"); + let marker_len = u16::try_from(7 + header.len()).expect("PPM marker length"); ppm.extend_from_slice(&marker_len.to_be_bytes()); ppm.push(0); - ppm.extend_from_slice(&u16::try_from(header.len()).unwrap().to_be_bytes()); + ppm.extend_from_slice(&u32::try_from(header.len()).unwrap().to_be_bytes()); ppm.extend_from_slice(&header); codestream.splice(sot_offset..sot_offset, ppm); diff --git a/crates/j2k-t803/Cargo.toml b/crates/j2k-t803/Cargo.toml new file mode 100644 index 00000000..01f1086d --- /dev/null +++ b/crates/j2k-t803/Cargo.toml @@ -0,0 +1,56 @@ +[package] +name = "j2k-t803" +description = "Unpublished ISO/IEC 15444-4 conformance support for the j2k workspace" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +[features] +default = [] +runner = [ + "dep:image", + "dep:j2k", + "dep:j2k-codec-math", + "dep:j2k-compare", + "dep:j2k-core", + "dep:j2k-native", + "dep:zip", +] +cuda-runner = [ + "runner", + "dep:j2k-cuda", + "dep:j2k-cuda-runtime", + "j2k-cuda/cuda-runtime", +] +metal-runner = ["runner", "dep:j2k-metal"] + +[dependencies] +image = { workspace = true, optional = true } +j2k = { path = "../j2k", version = "=0.8.1", optional = true } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1", optional = true } +j2k-compare = { path = "../j2k-compare", version = "=0.2.0", optional = true } +j2k-core = { path = "../j2k-core", version = "=0.8.1", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.8.1", optional = true } +j2k-cuda = { path = "../j2k-cuda", version = "=0.8.1", optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.1", optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.8.1", optional = true } +serde = { workspace = true } +serde_json = { workspace = true, features = ["float_roundtrip"] } +sha2 = { workspace = true } +thiserror = { workspace = true } +toml = { workspace = true } +zip = { workspace = true, optional = true } + +[dev-dependencies] +j2k-test-support = { path = "../j2k-test-support" } + +[lints] +workspace = true + +[[bin]] +name = "j2k-t803-runner" +path = "src/bin/j2k-t803-runner.rs" +required-features = ["runner"] diff --git a/crates/j2k-t803/fuzz/Cargo.lock b/crates/j2k-t803/fuzz/Cargo.lock new file mode 100644 index 00000000..0ecec5a8 --- /dev/null +++ b/crates/j2k-t803/fuzz/Cargo.lock @@ -0,0 +1,775 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "libz-sys", + "miniz_oxide", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "moxcms", + "num-traits", + "png", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "j2k" +version = "0.8.1" +dependencies = [ + "j2k-codec-math", + "j2k-core", + "j2k-native", + "j2k-types", + "moxcms", + "thiserror", +] + +[[package]] +name = "j2k-codec-math" +version = "0.8.1" + +[[package]] +name = "j2k-compare" +version = "0.2.0" +dependencies = [ + "cc", + "image", + "j2k", + "j2k-core", + "j2k-native", + "j2k-test-support", + "openjpeg-sys", +] + +[[package]] +name = "j2k-core" +version = "0.8.1" +dependencies = [ + "thiserror", +] + +[[package]] +name = "j2k-native" +version = "0.8.1" +dependencies = [ + "fearless_simd", + "j2k-codec-math", + "j2k-profile", + "j2k-types", + "libm", + "rayon", +] + +[[package]] +name = "j2k-profile" +version = "0.8.1" + +[[package]] +name = "j2k-t803" +version = "0.8.1" +dependencies = [ + "image", + "j2k", + "j2k-codec-math", + "j2k-compare", + "j2k-core", + "j2k-native", + "serde", + "serde_json", + "sha2", + "thiserror", + "toml", + "zip", +] + +[[package]] +name = "j2k-t803-fuzz" +version = "0.1.0" +dependencies = [ + "j2k-t803", + "libfuzzer-sys", +] + +[[package]] +name = "j2k-test-support" +version = "0.8.1" +dependencies = [ + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "j2k-types" +version = "0.8.1" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "openjpeg-sys" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92682cb92e7b01e2c020cf8ec12f3374558924bb24621df1112d8a7c93d419ef" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/crates/j2k-t803/fuzz/Cargo.toml b/crates/j2k-t803/fuzz/Cargo.toml new file mode 100644 index 00000000..333f4c9d --- /dev/null +++ b/crates/j2k-t803/fuzz/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "j2k-t803-fuzz" +version = "0.1.0" +edition = "2021" +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +j2k-t803 = { path = "..", features = ["runner"] } + +[[bin]] +name = "pgx_fuzz" +path = "fuzz_targets/pgx_fuzz.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "archive_fuzz" +path = "fuzz_targets/archive_fuzz.rs" +test = false +doc = false +bench = false + +[profile.release] +debug = 1 + +[workspace] diff --git a/crates/j2k-t803/fuzz/fuzz_targets/archive_fuzz.rs b/crates/j2k-t803/fuzz/fuzz_targets/archive_fuzz.rs new file mode 100644 index 00000000..d1376896 --- /dev/null +++ b/crates/j2k-t803/fuzz/fuzz_targets/archive_fuzz.rs @@ -0,0 +1,33 @@ +#![no_main] + +use std::{io::Cursor, path::PathBuf, sync::OnceLock}; + +use j2k_t803::runner::{extract_selected_archive, ArchiveLimits}; +use libfuzzer_sys::fuzz_target; + +const MAX_INPUT_BYTES: usize = 4 * 1024 * 1024; +const LIMITS: ArchiveLimits = ArchiveLimits { + max_entries: 128, + max_entry_bytes: 1024 * 1024, + max_total_bytes: 4 * 1024 * 1024, +}; + +fuzz_target!(|data: &[u8]| { + if data.len() > MAX_INPUT_BYTES { + return; + } + let output = extraction_directory(); + let _ = extract_selected_archive(Cursor::new(data), output, &[], LIMITS); +}); + +fn extraction_directory() -> &'static PathBuf { + static OUTPUT: OnceLock = OnceLock::new(); + OUTPUT.get_or_init(|| { + let path = std::env::temp_dir().join(format!( + "j2k-t803-archive-fuzz-{}", + std::process::id() + )); + std::fs::create_dir_all(&path).expect("create bounded archive fuzz output"); + path + }) +} diff --git a/crates/j2k-t803/fuzz/fuzz_targets/pgx_fuzz.rs b/crates/j2k-t803/fuzz/fuzz_targets/pgx_fuzz.rs new file mode 100644 index 00000000..2589e883 --- /dev/null +++ b/crates/j2k-t803/fuzz/fuzz_targets/pgx_fuzz.rs @@ -0,0 +1,12 @@ +#![no_main] + +use j2k_t803::parse_pgx; +use libfuzzer_sys::fuzz_target; + +const MAX_INPUT_BYTES: usize = 4 * 1024 * 1024; + +fuzz_target!(|data: &[u8]| { + if data.len() <= MAX_INPUT_BYTES { + let _ = parse_pgx(data); + } +}); diff --git a/crates/j2k-t803/src/bin/j2k-t803-runner.rs b/crates/j2k-t803/src/bin/j2k-t803-runner.rs new file mode 100644 index 00000000..dcd5a36f --- /dev/null +++ b/crates/j2k-t803/src/bin/j2k-t803-runner.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::process::ExitCode; + +fn main() -> ExitCode { + match j2k_t803::runner::run_cli(std::env::args().skip(1)) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("T.803 runner failed: {error}"); + ExitCode::FAILURE + } + } +} diff --git a/crates/j2k-t803/src/compare.rs b/crates/j2k-t803/src/compare.rs new file mode 100644 index 00000000..bd38b25c --- /dev/null +++ b/crates/j2k-t803/src/compare.rs @@ -0,0 +1,130 @@ +use thiserror::Error; + +/// Inclusive T.803 error bounds for one reference component. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ErrorBounds { + /// Maximum absolute sample error. + pub peak: u64, + /// Maximum mean squared error. + pub mse: f64, +} + +/// Error metrics and their inclusive pass result. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Comparison { + /// Measured maximum absolute sample error. + pub peak: u64, + /// Measured mean squared error. + pub mse: f64, + /// Whether both measured values are within their inclusive bounds. + pub passed: bool, +} + +/// Peak-error metric and its inclusive pass result. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PeakComparison { + /// Measured maximum absolute sample error. + pub peak: u64, + /// Whether the measured value is within the inclusive bound. + pub passed: bool, +} + +/// Error returned when sample arrays cannot be compared. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum ComparisonError { + /// T.803 metrics are undefined for an empty component. + #[error("cannot compare empty components")] + Empty, + /// Reference and decoded components contain different sample counts. + #[error("sample count mismatch: reference {reference}, decoded {decoded}")] + Length { + /// Reference sample count. + reference: usize, + /// Decoded sample count. + decoded: usize, + }, + /// The configured MSE bound is negative, infinite, or NaN. + #[error("MSE bound must be finite and non-negative")] + InvalidMseBound, + /// Accumulating squared error exceeded the metric representation. + #[error("squared-error sum overflowed")] + Overflow, +} + +/// Measure peak error and MSE and apply inclusive T.803 bounds. +pub fn compare_samples( + reference: &[i64], + decoded: &[i64], + bounds: ErrorBounds, +) -> Result { + if !bounds.mse.is_finite() || bounds.mse < 0.0 { + return Err(ComparisonError::InvalidMseBound); + } + let (peak, squared_error) = error_sums(reference, decoded)?; + let mse = mse_from_exact_sum(squared_error, reference.len())?; + Ok(Comparison { + peak, + mse, + passed: peak <= bounds.peak && mse <= bounds.mse, + }) +} + +/// Measure peak error and apply an inclusive peak-only bound. +pub fn compare_peak_samples( + reference: &[i64], + decoded: &[i64], + bound: u64, +) -> Result { + let (peak, _) = error_sums(reference, decoded)?; + Ok(PeakComparison { + peak, + passed: peak <= bound, + }) +} + +fn error_sums(reference: &[i64], decoded: &[i64]) -> Result<(u64, u128), ComparisonError> { + if reference.is_empty() { + return Err(ComparisonError::Empty); + } + if reference.len() != decoded.len() { + return Err(ComparisonError::Length { + reference: reference.len(), + decoded: decoded.len(), + }); + } + let mut peak = 0_u64; + let mut squared_error = 0_u128; + for (&reference, &decoded) in reference.iter().zip(decoded) { + let difference = (i128::from(reference) - i128::from(decoded)).unsigned_abs(); + peak = peak.max(u64::try_from(difference).map_err(|_| ComparisonError::Overflow)?); + squared_error = squared_error + .checked_add( + difference + .checked_mul(difference) + .ok_or(ComparisonError::Overflow)?, + ) + .ok_or(ComparisonError::Overflow)?; + } + Ok((peak, squared_error)) +} + +fn mse_from_exact_sum(squared_error: u128, sample_count: usize) -> Result { + // T.803 defines MSE as a real-valued average. Accumulation stays exact; + // conversion happens once at the required floating-point comparison. + let sample_count = u128::try_from(sample_count).map_err(|_| ComparisonError::Overflow)?; + Ok(u128_as_f64(squared_error)? / u128_as_f64(sample_count)?) +} + +fn u128_as_f64(value: u128) -> Result { + const TWO_POW_64: f64 = 18_446_744_073_709_551_616.0; + let low = u64::try_from(value & u128::from(u64::MAX)).map_err(|_| ComparisonError::Overflow)?; + let high = u64::try_from(value >> 64).map_err(|_| ComparisonError::Overflow)?; + Ok(u64_as_f64(high)? * TWO_POW_64 + u64_as_f64(low)?) +} + +pub(crate) fn u64_as_f64(value: u64) -> Result { + const TWO_POW_32: f64 = 4_294_967_296.0; + let low = u32::try_from(value & u64::from(u32::MAX)).map_err(|_| ComparisonError::Overflow)?; + let high = u32::try_from(value >> 32).map_err(|_| ComparisonError::Overflow)?; + Ok(f64::from(high) * TWO_POW_32 + f64::from(low)) +} diff --git a/crates/j2k-t803/src/encoder.rs b/crates/j2k-t803/src/encoder.rs new file mode 100644 index 00000000..e54fa635 --- /dev/null +++ b/crates/j2k-t803/src/encoder.rs @@ -0,0 +1,867 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Declarative Annex D/F encoder test scope and inventory validation. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use crate::manifest::{validate_sha256, STANDARD}; + +const MATRIX_PATH: &str = "corpus/j2k-conformance/encoder-matrix-v1.toml"; +const REFERENCE_STANDARD: &str = "ISO/IEC 15444-5 / ITU-T T.804"; +const REFERENCE_IMPLEMENTATION: &str = "OpenJPEG"; +const REFERENCE_VERSION: &str = "2.5.3"; + +/// Encoder implementation under test. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum EncoderIut { + /// Portable `j2k` CPU encoder surfaces. + Cpu, + /// `j2k-cuda` adapter encoder surfaces. + Cuda, + /// `j2k-metal` adapter encoder surfaces. + Metal, +} + +/// Compression mode exercised by one encoder case. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum EncoderMode { + /// Reversible Part 1 encode. + Lossless, + /// Irreversible Part 1 encode. + Lossy, +} + +/// Part 1 packet progression selected in COD. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum EncoderProgression { + /// Layer-resolution-component-position. + Lrcp, + /// Resolution-layer-component-position. + Rlcp, + /// Resolution-position-component-layer. + Rpcl, + /// Position-component-resolution-layer. + Pcrl, + /// Component-position-resolution-layer. + Cprl, +} + +/// Marker listed in T.803 Table F.1. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum EncoderMarker { + Soc, + Cap, + Prf, + Cpf, + Sot, + Sod, + Eoc, + Siz, + Cod, + Coc, + Rgn, + Qcd, + Qcc, + Poc, + Tlm, + Plm, + Plt, + Ppm, + Ppt, + Sop, + Eph, + Crg, + Com, +} + +const TABLE_F1_MARKERS: [EncoderMarker; 23] = [ + EncoderMarker::Soc, + EncoderMarker::Cap, + EncoderMarker::Prf, + EncoderMarker::Cpf, + EncoderMarker::Sot, + EncoderMarker::Sod, + EncoderMarker::Eoc, + EncoderMarker::Siz, + EncoderMarker::Cod, + EncoderMarker::Coc, + EncoderMarker::Rgn, + EncoderMarker::Qcd, + EncoderMarker::Qcc, + EncoderMarker::Poc, + EncoderMarker::Tlm, + EncoderMarker::Plm, + EncoderMarker::Plt, + EncoderMarker::Ppm, + EncoderMarker::Ppt, + EncoderMarker::Sop, + EncoderMarker::Eph, + EncoderMarker::Crg, + EncoderMarker::Com, +]; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum EncoderInputKind { + #[default] + Interleaved, + ComponentPlanes, + TypedComponentPlanes, +} + +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(crate) enum EncoderPattern { + #[default] + Gradient, + Checkerboard, + DeterministicNoise, + Impulse, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +#[serde(tag = "kind", content = "value", rename_all = "kebab-case")] +pub(crate) enum EncoderRateTarget { + BitsPerPixel(f64), + Bytes(u64), + PsnrDb(f64), +} + +/// One rectangular maxshift request. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct EncoderRoi { + pub(crate) component: u16, + pub(crate) x: u32, + pub(crate) y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) shift: u8, +} + +/// One stable Annex D encoder test case. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncoderCase { + /// Stable case identifier. + pub id: String, + /// Adapter IUTs to which this case applies. + pub iuts: Vec, + /// Reversible or irreversible coding. + pub mode: EncoderMode, + #[serde(default)] + pub(crate) input: EncoderInputKind, + /// Reference-grid width. + pub width: u32, + /// Reference-grid height. + pub height: u32, + /// Component count. + pub components: u16, + /// Common sample precision for interleaved and homogeneous planar inputs. + pub bit_depth: u8, + /// Common signedness for interleaved and homogeneous planar inputs. + pub signed: bool, + #[serde(default)] + pub(crate) pattern: EncoderPattern, + #[serde(default)] + pub(crate) sampling: Vec<[u8; 2]>, + #[serde(default)] + pub(crate) component_bit_depths: Vec, + #[serde(default)] + pub(crate) component_signedness: Vec, + /// COD packet progression. + pub progression: EncoderProgression, + /// Requested wavelet decomposition levels. + pub decomposition_levels: u8, + #[serde(default = "one_quality_layer")] + pub(crate) lossless_quality_layers: u8, + #[serde(default)] + pub(crate) lossy_rate_target: Option, + #[serde(default)] + pub(crate) lossy_quality_layers: Vec, + #[serde(default)] + pub(crate) minimum_psnr_db: Option, + #[serde(default)] + pub(crate) maximum_rate_overshoot_percent: Option, + #[serde(default)] + pub(crate) tile_size: Option<[u32; 2]>, + #[serde(default)] + pub(crate) tile_part_packet_limit: Option, + #[serde(default)] + pub(crate) precinct_exponents: Vec<[u8; 2]>, + #[serde(default)] + pub(crate) roi: Option, + /// Optional markers this case explicitly requests. + #[serde(default)] + pub markers: Vec, + /// Whether this row participates in the declared pairwise covering array. + #[serde(default)] + pub pairwise: bool, +} + +const fn one_quality_layer() -> u8 { + 1 +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct PairwiseScope { + modes: Vec, + dimensions: Vec<[u32; 2]>, + signedness: Vec, + bit_depths: Vec, + component_counts: Vec, + progressions: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct MatrixInventory { + iut: EncoderIut, + case_count: usize, + case_sha256: String, +} + +/// Versioned, tamper-evident Annex D encoder case matrix. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncoderMatrix { + /// Matrix schema version. + pub schema_version: u32, + /// T.803 edition used to define the procedure. + pub standard: String, + pairwise: PairwiseScope, + inventories: Vec, + /// Cases in stable execution order. + pub cases: Vec, +} + +/// Error returned for malformed encoder matrices or ICS files. +#[derive(Debug, Error)] +pub enum EncoderMatrixError { + /// TOML syntax or schema error. + #[error("invalid encoder evidence TOML: {0}")] + Toml(#[from] toml::de::Error), + /// Semantic or inventory error. + #[error("invalid encoder evidence: {0}")] + Validation(String), + /// Canonical case serialization failed. + #[error("serialize encoder case inventory: {0}")] + Json(#[from] serde_json::Error), +} + +impl EncoderMatrix { + /// Parse and validate a complete encoder matrix. + pub fn parse(text: &str) -> Result { + let matrix = toml::from_str::(text)?; + matrix.validate()?; + Ok(matrix) + } + + fn validate(&self) -> Result<(), EncoderMatrixError> { + if self.schema_version != 1 || self.standard != STANDARD { + return validation("schema or standard does not match T.803 v3"); + } + validate_pairwise_scope(&self.pairwise)?; + + let mut previous_id = None; + for case in &self.cases { + if case.id.is_empty() + || previous_id.is_some_and(|previous| previous >= case.id.as_str()) + { + return validation("case ids must be non-empty, sorted, and unique"); + } + previous_id = Some(case.id.as_str()); + validate_case(case)?; + } + validate_pairwise_coverage(&self.pairwise, &self.cases)?; + validate_boundaries(&self.cases)?; + self.validate_inventories() + } + + fn validate_inventories(&self) -> Result<(), EncoderMatrixError> { + let expected_iuts = [EncoderIut::Cpu, EncoderIut::Cuda, EncoderIut::Metal]; + if self.inventories.len() != expected_iuts.len() + || self + .inventories + .iter() + .map(|inventory| inventory.iut) + .ne(expected_iuts) + { + return validation("matrix inventories must contain CPU, CUDA, and Metal in order"); + } + for inventory in &self.inventories { + validate_sha256(&inventory.case_sha256, "encoder case inventory") + .map_err(|error| EncoderMatrixError::Validation(error.to_string()))?; + let cases = self + .cases + .iter() + .filter(|case| case.iuts.contains(&inventory.iut)) + .collect::>(); + if cases.len() != inventory.case_count { + return validation(format!( + "{:?} case count is {}, expected {}", + inventory.iut, + cases.len(), + inventory.case_count + )); + } + let actual = canonical_case_sha256(&cases)?; + if actual != inventory.case_sha256 { + return validation(format!( + "{:?} case inventory SHA-256 is {actual}, expected {}", + inventory.iut, inventory.case_sha256 + )); + } + } + Ok(()) + } + + fn inventory(&self, iut: EncoderIut) -> Option<&MatrixInventory> { + self.inventories.iter().find(|entry| entry.iut == iut) + } + + #[cfg(feature = "runner")] + pub(crate) fn selected_cases(&self, iut: EncoderIut) -> impl Iterator { + self.cases + .iter() + .filter(move |case| case.iuts.contains(&iut)) + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +enum MarkerUse { + Always, + Conditional, + CallerControlled, + NotProduced, + OutsidePart1, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +struct IcsMarker { + marker: EncoderMarker, + usage: MarkerUse, +} + +/// Published T.803 Annex F implementation compliance statement. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncoderIcs { + /// ICS schema version. + pub schema_version: u32, + /// T.803 edition used by the statement. + pub standard: String, + /// CPU, CUDA adapter, or Metal adapter IUT. + pub iut: EncoderIut, + /// Precise informative-encoder scope statement. + pub scope: String, + /// Public encoder entry points covered by the statement. + pub surfaces: Vec, + matrix_path: String, + matrix_case_count: usize, + matrix_case_sha256: String, + reference_decoder_standard: String, + reference_decoder_implementation: String, + reference_decoder_version: String, + /// Public maximum sample precision; the reference decoder may cover less. + pub public_max_bit_depth: u8, + /// Highest precision included in the T.804 `OpenJPEG` matrix. + pub reference_validated_max_bit_depth: u8, + /// Public maximum component count for the listed surfaces. + pub public_max_components: u16, + /// Whether the listed surface accepts component sampling. + pub component_sampling: bool, + /// Known API ranges not validated by this reference implementation. + pub reference_limitations: Vec, + markers: Vec, +} + +impl EncoderIcs { + /// Parse and validate one Annex F ICS. + pub fn parse(text: &str) -> Result { + let ics = toml::from_str::(text)?; + ics.validate()?; + Ok(ics) + } + + /// Verify that this ICS pins the selected IUT's exact matrix inventory. + pub fn validate_against(&self, matrix: &EncoderMatrix) -> Result<(), EncoderMatrixError> { + self.validate()?; + let inventory = matrix.inventory(self.iut).ok_or_else(|| { + EncoderMatrixError::Validation("IUT inventory is missing".to_string()) + })?; + if self.matrix_case_count != inventory.case_count + || self.matrix_case_sha256 != inventory.case_sha256 + { + return validation("ICS matrix count or SHA-256 does not match the case inventory"); + } + for entry in self + .markers + .iter() + .filter(|entry| entry.usage == MarkerUse::CallerControlled) + { + if !matrix + .cases + .iter() + .any(|case| case.iuts.contains(&self.iut) && case.markers.contains(&entry.marker)) + { + return validation(format!( + "{:?} does not exercise caller-controlled {:?}", + self.iut, entry.marker + )); + } + } + Ok(()) + } + + fn validate(&self) -> Result<(), EncoderMatrixError> { + if self.schema_version != 1 || self.standard != STANDARD { + return validation("ICS schema or standard does not match T.803 v3"); + } + if self.scope.is_empty() + || self.surfaces.is_empty() + || self.surfaces.iter().any(String::is_empty) + { + return validation("ICS scope and surfaces must not be empty"); + } + if self.matrix_path != MATRIX_PATH { + return validation(format!("ICS matrix path must be {MATRIX_PATH}")); + } + validate_sha256(&self.matrix_case_sha256, "ICS matrix inventory") + .map_err(|error| EncoderMatrixError::Validation(error.to_string()))?; + if self.matrix_case_count == 0 + || self.reference_decoder_standard != REFERENCE_STANDARD + || self.reference_decoder_implementation != REFERENCE_IMPLEMENTATION + || self.reference_decoder_version != REFERENCE_VERSION + { + return validation("ICS reference decoder or matrix metadata is invalid"); + } + if self.public_max_bit_depth != 38 + || self.reference_validated_max_bit_depth != 31 + || self.public_max_components != 16_384 + || self.reference_limitations.is_empty() + { + return validation("ICS public and reference-decoder limits are incomplete"); + } + if self.iut != EncoderIut::Cpu && self.component_sampling { + return validation("adapter ICS must not claim a sampled-component surface"); + } + let markers = self + .markers + .iter() + .map(|entry| entry.marker) + .collect::>(); + if markers != TABLE_F1_MARKERS { + return validation("ICS must list every Table F.1 marker in table order"); + } + validate_marker_usage(self.iut, &self.markers) + } + + #[cfg(feature = "runner")] + pub(crate) fn matrix_case_count(&self) -> usize { + self.matrix_case_count + } + + #[cfg(feature = "runner")] + pub(crate) fn matrix_case_sha256(&self) -> &str { + &self.matrix_case_sha256 + } +} + +#[cfg(feature = "runner")] +pub(crate) fn reference_decoder_identity() -> (&'static str, &'static str, &'static str) { + ( + REFERENCE_STANDARD, + REFERENCE_IMPLEMENTATION, + REFERENCE_VERSION, + ) +} + +#[cfg(feature = "runner")] +pub(crate) fn matrix_path() -> &'static str { + MATRIX_PATH +} + +#[cfg(feature = "runner")] +pub(crate) const fn ics_path(iut: EncoderIut) -> &'static str { + match iut { + EncoderIut::Cpu => "corpus/j2k-conformance/encoder-ics-cpu.toml", + EncoderIut::Cuda => "corpus/j2k-conformance/encoder-ics-cuda.toml", + EncoderIut::Metal => "corpus/j2k-conformance/encoder-ics-metal.toml", + } +} + +fn validate_marker_usage(iut: EncoderIut, markers: &[IcsMarker]) -> Result<(), EncoderMatrixError> { + for entry in markers { + let expected = match entry.marker { + EncoderMarker::Soc + | EncoderMarker::Sot + | EncoderMarker::Sod + | EncoderMarker::Eoc + | EncoderMarker::Siz + | EncoderMarker::Cod + | EncoderMarker::Qcd => MarkerUse::Always, + EncoderMarker::Cap | EncoderMarker::Cpf => MarkerUse::OutsidePart1, + EncoderMarker::Coc | EncoderMarker::Qcc => MarkerUse::Conditional, + EncoderMarker::Rgn if iut == EncoderIut::Cpu => MarkerUse::CallerControlled, + EncoderMarker::Tlm + | EncoderMarker::Plm + | EncoderMarker::Plt + | EncoderMarker::Ppm + | EncoderMarker::Ppt + | EncoderMarker::Sop + | EncoderMarker::Eph => MarkerUse::CallerControlled, + EncoderMarker::Prf + | EncoderMarker::Rgn + | EncoderMarker::Poc + | EncoderMarker::Crg + | EncoderMarker::Com => MarkerUse::NotProduced, + }; + if entry.usage != expected { + return validation(format!( + "{:?} has incorrect Table F.1 usage for {:?}", + iut, entry.marker + )); + } + } + Ok(()) +} + +fn validate_case(case: &EncoderCase) -> Result<(), EncoderMatrixError> { + if case.iuts.is_empty() + || !case.iuts.windows(2).all(|pair| pair[0] < pair[1]) + || case.width == 0 + || case.height == 0 + || !(1..=16_384).contains(&case.components) + || !(1..=38).contains(&case.bit_depth) + || case.decomposition_levels > 32 + || case.lossless_quality_layers == 0 + || case.markers.iter().collect::>().len() != case.markers.len() + { + return validation(format!("{} has invalid basic parameters", case.id)); + } + if case + .tile_size + .is_some_and(|[width, height]| width == 0 || height == 0) + || case.tile_part_packet_limit == Some(0) + || case + .precinct_exponents + .iter() + .any(|[width, height]| *width > 15 || *height > 15) + { + return validation(format!("{} has invalid tiling or precinct data", case.id)); + } + if let Some(roi) = case.roi { + let fits = roi.component < case.components + && roi.width > 0 + && roi.height > 0 + && roi.shift > 0 + && roi + .x + .checked_add(roi.width) + .is_some_and(|x1| x1 <= case.width) + && roi + .y + .checked_add(roi.height) + .is_some_and(|y1| y1 <= case.height); + if !fits || !case.markers.contains(&EncoderMarker::Rgn) { + return validation(format!("{} has invalid ROI data", case.id)); + } + } else if case.markers.contains(&EncoderMarker::Rgn) { + return validation(format!("{} requests RGN without an ROI", case.id)); + } + validate_input_surface(case)?; + validate_mode(case) +} + +fn validate_input_surface(case: &EncoderCase) -> Result<(), EncoderMatrixError> { + match case.input { + EncoderInputKind::Interleaved => { + if !case.sampling.is_empty() + || !case.component_bit_depths.is_empty() + || !case.component_signedness.is_empty() + { + return validation(format!("{} mixes interleaved and planar metadata", case.id)); + } + } + EncoderInputKind::ComponentPlanes => { + validate_cpu_planar_case(case)?; + if !case.component_bit_depths.is_empty() || !case.component_signedness.is_empty() { + return validation(format!( + "{} has typed metadata on homogeneous planes", + case.id + )); + } + } + EncoderInputKind::TypedComponentPlanes => { + validate_cpu_planar_case(case)?; + if case.component_bit_depths.len() != usize::from(case.components) + || case.component_signedness.len() != usize::from(case.components) + || case + .component_bit_depths + .iter() + .any(|depth| !(1..=38).contains(depth)) + { + return validation(format!("{} has invalid typed component metadata", case.id)); + } + } + } + Ok(()) +} + +fn validate_mode(case: &EncoderCase) -> Result<(), EncoderMatrixError> { + match case.mode { + EncoderMode::Lossless => { + if case.lossy_rate_target.is_some() + || !case.lossy_quality_layers.is_empty() + || case.minimum_psnr_db.is_some() + || case.maximum_rate_overshoot_percent.is_some() + { + return validation(format!("{} has lossy targets on a lossless case", case.id)); + } + } + EncoderMode::Lossy => { + if case.input != EncoderInputKind::Interleaved || case.roi.is_some() { + return validation(format!( + "{} uses an unsupported lossy matrix surface", + case.id + )); + } + for target in case + .lossy_rate_target + .iter() + .chain(case.lossy_quality_layers.iter()) + { + validate_rate_target(*target, &case.id)?; + } + if case + .minimum_psnr_db + .is_none_or(|value| !value.is_finite() || value <= 0.0) + { + return validation(format!("{} has no finite minimum PSNR gate", case.id)); + } + let has_rate_gate = case + .lossy_rate_target + .iter() + .chain(case.lossy_quality_layers.last()) + .any(|target| { + matches!( + target, + EncoderRateTarget::BitsPerPixel(_) | EncoderRateTarget::Bytes(_) + ) + }); + if has_rate_gate + != case + .maximum_rate_overshoot_percent + .is_some_and(|value| value.is_finite() && (0.0..=100.0).contains(&value)) + { + return validation(format!("{} has an invalid rate overshoot gate", case.id)); + } + } + } + Ok(()) +} + +fn validate_cpu_planar_case(case: &EncoderCase) -> Result<(), EncoderMatrixError> { + if case.mode != EncoderMode::Lossless + || case.iuts != [EncoderIut::Cpu] + || case.sampling.len() != usize::from(case.components) + || case + .sampling + .iter() + .any(|[x_rsiz, y_rsiz]| *x_rsiz == 0 || *y_rsiz == 0) + { + return validation(format!("{} has invalid planar surface metadata", case.id)); + } + Ok(()) +} + +fn validate_rate_target(target: EncoderRateTarget, id: &str) -> Result<(), EncoderMatrixError> { + let valid = match target { + EncoderRateTarget::BitsPerPixel(value) | EncoderRateTarget::PsnrDb(value) => { + value.is_finite() && value > 0.0 + } + EncoderRateTarget::Bytes(value) => value > 0, + }; + if valid { + Ok(()) + } else { + validation(format!("{id} has an invalid lossy rate target")) + } +} + +fn validate_pairwise_scope(scope: &PairwiseScope) -> Result<(), EncoderMatrixError> { + let exact = scope.modes == [EncoderMode::Lossless, EncoderMode::Lossy] + && scope.dimensions == [[32, 32], [63, 47]] + && scope.signedness == [false, true] + && scope.bit_depths == [8, 12] + && scope.component_counts == [1, 3] + && scope.progressions + == [ + EncoderProgression::Lrcp, + EncoderProgression::Rlcp, + EncoderProgression::Rpcl, + EncoderProgression::Pcrl, + EncoderProgression::Cprl, + ]; + if exact { + Ok(()) + } else { + validation("pairwise scope does not match the committed Part 1 coverage contract") + } +} + +fn validate_pairwise_coverage( + scope: &PairwiseScope, + cases: &[EncoderCase], +) -> Result<(), EncoderMatrixError> { + let axes = [ + scope.modes.iter().map(debug_value).collect::>(), + scope + .dimensions + .iter() + .map(|[width, height]| format!("{width}x{height}")) + .collect(), + scope.signedness.iter().map(debug_value).collect(), + scope.bit_depths.iter().map(debug_value).collect(), + scope.component_counts.iter().map(debug_value).collect(), + scope.progressions.iter().map(debug_value).collect(), + ]; + let rows = cases + .iter() + .filter(|case| case.pairwise) + .map(|case| { + [ + debug_value(&case.mode), + format!("{}x{}", case.width, case.height), + debug_value(&case.signed), + debug_value(&case.bit_depth), + debug_value(&case.components), + debug_value(&case.progression), + ] + }) + .collect::>(); + for left in 0..axes.len() { + for right in (left + 1)..axes.len() { + for left_value in &axes[left] { + for right_value in &axes[right] { + if !rows + .iter() + .any(|row| row[left] == *left_value && row[right] == *right_value) + { + return validation(format!( + "pairwise rows do not cover axes {left}/{right} values {left_value}/{right_value}" + )); + } + } + } + } + } + Ok(()) +} + +fn validate_boundaries(cases: &[EncoderCase]) -> Result<(), EncoderMatrixError> { + for bit_depth in [1, 8, 12, 16, 31] { + require( + cases.iter().any(|case| case.bit_depth == bit_depth), + "bit-depth boundary", + )?; + } + for components in [1, 2, 3, 4, 5] { + require( + cases.iter().any(|case| case.components == components), + "component-count boundary", + )?; + } + for level in [0, 1, 2, 3, 5] { + require( + cases.iter().any(|case| case.decomposition_levels == level), + "decomposition-level boundary", + )?; + } + require( + cases.iter().any(|case| (case.width, case.height) == (1, 1)), + "singleton geometry", + )?; + require(cases.iter().any(|case| case.tile_size.is_some()), "tiling")?; + require( + cases + .iter() + .any(|case| case.tile_part_packet_limit.is_some()), + "tile parts", + )?; + require( + cases.iter().any(|case| !case.precinct_exponents.is_empty()), + "precincts", + )?; + require(cases.iter().any(|case| case.roi.is_some()), "ROI maxshift")?; + require( + cases + .iter() + .any(|case| case.input == EncoderInputKind::ComponentPlanes), + "component sampling", + )?; + require( + cases + .iter() + .any(|case| case.input == EncoderInputKind::TypedComponentPlanes), + "mixed typed components", + )?; + for target_kind in ["bits-per-pixel", "bytes", "psnr-db"] { + require( + cases.iter().any(|case| { + case.lossy_rate_target + .iter() + .chain(case.lossy_quality_layers.iter()) + .any(|target| rate_kind(*target) == target_kind) + }), + "lossy rate-target variant", + )?; + } + require( + cases.iter().any(|case| case.lossless_quality_layers > 1) + && cases.iter().any(|case| case.lossy_quality_layers.len() > 1), + "lossless and lossy quality layers", + )?; + Ok(()) +} + +fn require(present: bool, what: &str) -> Result<(), EncoderMatrixError> { + if present { + Ok(()) + } else { + validation(format!("matrix does not cover {what}")) + } +} + +fn rate_kind(target: EncoderRateTarget) -> &'static str { + match target { + EncoderRateTarget::BitsPerPixel(_) => "bits-per-pixel", + EncoderRateTarget::Bytes(_) => "bytes", + EncoderRateTarget::PsnrDb(_) => "psnr-db", + } +} + +fn debug_value(value: &impl std::fmt::Debug) -> String { + format!("{value:?}") +} + +fn canonical_case_sha256(cases: &[&EncoderCase]) -> Result { + let bytes = serde_json::to_vec(cases)?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} + +fn validation(message: impl Into) -> Result { + Err(EncoderMatrixError::Validation(message.into())) +} diff --git a/crates/j2k-t803/src/lib.rs b/crates/j2k-t803/src/lib.rs new file mode 100644 index 00000000..ddf386e0 --- /dev/null +++ b/crates/j2k-t803/src/lib.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![forbid(unsafe_code)] + +mod compare; +mod encoder; +mod manifest; +mod normalize; +mod pgx; +mod report; +#[cfg(feature = "runner")] +pub mod runner; + +pub use compare::{ + compare_peak_samples, compare_samples, Comparison, ComparisonError, ErrorBounds, PeakComparison, +}; +pub use encoder::{ + EncoderCase, EncoderIcs, EncoderIut, EncoderMarker, EncoderMatrix, EncoderMatrixError, + EncoderMode, EncoderProgression, +}; +pub use manifest::{CorpusFile, DecoderCase, Jp2Case, ManifestError, T803Manifest, T803Source}; +pub use normalize::{normalize_component, Component, NormalizationError, NormalizationTarget}; +pub use pgx::{parse_pgx, PgxError, PgxImage}; +pub use report::{ + CaseReport, CaseStatus, DecoderRouteSummary, EncodeRouteStage, EncodeRouteStageName, + EncoderCaseReport, EncoderEvidence, EncoderQualityStatus, EncoderReferenceIdentity, + ExecutionLocation, IutIdentity, NativeComponentOracleEvidence, PlatformIdentity, ReportError, + ReportStatus, RouteKind, RouteStage, RouteStageName, T803Report, +}; diff --git a/crates/j2k-t803/src/manifest.rs b/crates/j2k-t803/src/manifest.rs new file mode 100644 index 00000000..6dfb9cb1 --- /dev/null +++ b/crates/j2k-t803/src/manifest.rs @@ -0,0 +1,326 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + path::{Component, Path}, +}; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub(crate) const STANDARD: &str = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3"; +pub(crate) const SOURCE_URL: &str = "https://www.itu.int/wftp3/public/t/testsignal/SpeImage/T803/v2024_02/T.803v3_15444-4ed4-ElecAtt-codestreams.zip"; +const TABLE_COUNTS: [(&str, usize); 5] = [ + ("C.1", 18), + ("C.4", 8), + ("C.6", 35), + ("C.7", 17), + ("C.8", 3), +]; +const REQUIRED_CODESTREAMS: [&str; 24] = [ + "files/codestreams_profile0/p0_01.j2k", + "files/codestreams_profile0/p0_02.j2k", + "files/codestreams_profile0/p0_03.j2k", + "files/codestreams_profile0/p0_04.j2k", + "files/codestreams_profile0/p0_05.j2k", + "files/codestreams_profile0/p0_06.j2k", + "files/codestreams_profile0/p0_07.j2k", + "files/codestreams_profile0/p0_08.j2k", + "files/codestreams_profile0/p0_09.j2k", + "files/codestreams_profile0/p0_10.j2k", + "files/codestreams_profile0/p0_11.j2k", + "files/codestreams_profile0/p0_12.j2k", + "files/codestreams_profile0/p0_13.j2k", + "files/codestreams_profile0/p0_14.j2k", + "files/codestreams_profile0/p0_15.j2k", + "files/codestreams_profile0/p0_16.j2k", + "files/codestreams_profile1/p1_01.j2k", + "files/codestreams_profile1/p1_02.j2k", + "files/codestreams_profile1/p1_03.j2k", + "files/codestreams_profile1/p1_04.j2k", + "files/codestreams_profile1/p1_05.j2k", + "files/codestreams_profile1/p1_06.j2k", + "files/codestreams_profile1/p1_07.j2k", + "files/codestreams_hifi/hifi_p1_02.j2k", +]; + +/// Pinned source metadata for the external electronic attachment. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct T803Source { + /// Official ITU attachment handle. + pub url: String, + /// Expected SHA-256 of the attachment archive. + pub archive_sha256: String, + /// Expected archive size in bytes. + pub archive_bytes: u64, +} + +/// One externally stored file used by the selected test cases. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CorpusFile { + /// Normalized path below the extracted corpus root. + pub path: String, + /// Expected file SHA-256. + pub sha256: String, +} + +/// One reference-component comparison from an Annex C table. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct DecoderCase { + /// Stable case identifier. + pub id: String, + /// T.803 table containing the case. + pub table: String, + /// Codestream path in the external corpus. + pub codestream: String, + /// PGX reference path in the external corpus. + pub reference: String, + /// Zero-based component to compare. + pub component: usize, + /// Exact decoder resolution reduction level. + pub reduction_levels: u8, + /// Whether the reference component is signed. + pub signed: bool, + /// Reference precision. + pub bit_depth: u8, + /// Reference width. + pub width: u32, + /// Reference height. + pub height: u32, + /// Inclusive peak-error bound. + pub peak: u64, + /// Inclusive MSE bound. + pub mse: f64, +} + +/// One Annex G JP2 reader comparison. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Jp2Case { + /// Stable case identifier. + pub id: String, + /// JP2 input path in the external corpus. + pub input: String, + /// TIFF reference path in the external corpus. + pub reference: String, + /// Expected component count. + pub components: u8, + /// Reference precision. + pub bit_depth: u8, + /// Reference width. + pub width: u32, + /// Reference height. + pub height: u32, + /// Inclusive peak-error bound. + pub peak: u64, +} + +/// Complete, pinned T.803 v3 Part 1 and Annex G selection. +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct T803Manifest { + /// Manifest schema version. + pub schema_version: u32, + /// Standard edition represented by the case data. + pub standard: String, + /// External attachment provenance. + pub source: T803Source, + /// Hash inventory for every selected external file. + pub files: Vec, + /// Annex C J2K decoder comparisons. + pub decoder_cases: Vec, + /// Annex G JP2 reader comparisons. + pub jp2_cases: Vec, +} + +/// Error returned when the pinned manifest is malformed or incomplete. +#[derive(Debug, Error)] +pub enum ManifestError { + /// TOML syntax or schema error. + #[error("invalid T.803 manifest TOML: {0}")] + Toml(#[from] toml::de::Error), + /// Semantic or inventory error. + #[error("invalid T.803 manifest: {0}")] + Validation(String), +} + +impl T803Manifest { + /// Parse and validate a complete T.803 v3 manifest. + pub fn parse(text: &str) -> Result { + let manifest = toml::from_str::(text)?; + manifest.validate()?; + Ok(manifest) + } + + /// Return the number of selected comparisons from one Annex C table. + pub fn table_case_count(&self, table: &str) -> usize { + self.decoder_cases + .iter() + .filter(|case| case.table == table) + .count() + } + + fn validate(&self) -> Result<(), ManifestError> { + if self.schema_version != 1 { + return validation("schema_version must be 1"); + } + if self.standard != STANDARD { + return validation(format!("standard must be {STANDARD:?}")); + } + if self.source.url != SOURCE_URL { + return validation(format!("source URL must be {SOURCE_URL}")); + } + validate_sha256(&self.source.archive_sha256, "archive")?; + if self.source.archive_bytes == 0 { + return validation("archive_bytes must be non-zero"); + } + + let mut inventory = BTreeSet::new(); + for file in &self.files { + validate_path(&file.path)?; + validate_sha256(&file.sha256, &file.path)?; + if !inventory.insert(file.path.as_str()) { + return validation(format!("duplicate file inventory path {}", file.path)); + } + } + + let mut case_ids = BTreeSet::new(); + let mut used_files = BTreeSet::new(); + let mut table_counts = BTreeMap::new(); + let mut codestreams = BTreeSet::new(); + for case in &self.decoder_cases { + validate_case_id(&mut case_ids, &case.id)?; + if !TABLE_COUNTS.iter().any(|(table, _)| *table == case.table) { + return validation(format!("{} has unknown Annex C table", case.id)); + } + validate_inventory_reference(&inventory, &case.codestream)?; + validate_inventory_reference(&inventory, &case.reference)?; + if !has_exact_extension(&case.codestream, "j2k") + || !has_exact_extension(&case.reference, "pgx") + { + return validation(format!("{} has misnamed input or reference", case.id)); + } + if case.width == 0 + || case.height == 0 + || !(1..=32).contains(&case.bit_depth) + || !case.mse.is_finite() + || case.mse < 0.0 + { + return validation(format!("{} has invalid comparison bounds", case.id)); + } + *table_counts.entry(case.table.as_str()).or_insert(0_usize) += 1; + codestreams.insert(case.codestream.as_str()); + used_files.insert(case.codestream.as_str()); + used_files.insert(case.reference.as_str()); + } + + for case in &self.jp2_cases { + validate_case_id(&mut case_ids, &case.id)?; + validate_inventory_reference(&inventory, &case.input)?; + validate_inventory_reference(&inventory, &case.reference)?; + if !has_exact_extension(&case.input, "jp2") + || !has_exact_extension(&case.reference, "tif") + { + return validation(format!("{} has misnamed input or reference", case.id)); + } + if case.components == 0 + || case.width == 0 + || case.height == 0 + || !(1..=32).contains(&case.bit_depth) + { + return validation(format!("{} has invalid comparison shape", case.id)); + } + used_files.insert(case.input.as_str()); + used_files.insert(case.reference.as_str()); + } + + if inventory != used_files { + let unused = inventory + .difference(&used_files) + .copied() + .collect::>(); + return validation(format!( + "file inventory contains unused entries: {unused:?}" + )); + } + for (table, expected) in TABLE_COUNTS { + let actual = table_counts.get(table).copied().unwrap_or_default(); + if actual != expected { + return validation(format!( + "table {table} must contain {expected} cases, found {actual}" + )); + } + } + if self.jp2_cases.len() != 9 { + return validation(format!( + "Annex G must contain 9 cases, found {}", + self.jp2_cases.len() + )); + } + if codestreams.len() != REQUIRED_CODESTREAMS.len() + || REQUIRED_CODESTREAMS + .iter() + .any(|path| !codestreams.contains(path)) + { + return validation("Annex C codestream set is incomplete or contains extra entries"); + } + Ok(()) + } +} + +fn validate_case_id<'a>(ids: &mut BTreeSet<&'a str>, id: &'a str) -> Result<(), ManifestError> { + if id.is_empty() { + return validation("case id must not be empty"); + } + if !ids.insert(id) { + return validation(format!("duplicate case id {id}")); + } + Ok(()) +} + +fn validate_inventory_reference( + inventory: &BTreeSet<&str>, + path: &str, +) -> Result<(), ManifestError> { + validate_path(path)?; + if !inventory.contains(path) { + return validation(format!("{path} is not present in the file inventory")); + } + Ok(()) +} + +pub(crate) fn validate_path(path: &str) -> Result<(), ManifestError> { + let normalized = !path.is_empty() + && !path.contains('\\') + && !path.contains("//") + && !path.split('/').any(|segment| matches!(segment, "." | "..")) + && !Path::new(path).is_absolute() + && Path::new(path) + .components() + .all(|component| matches!(component, Component::Normal(_))); + if !normalized { + return validation(format!("{path:?} must be a relative normalized path")); + } + Ok(()) +} + +pub(crate) fn validate_sha256(value: &str, subject: &str) -> Result<(), ManifestError> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return validation(format!("{subject} must have a lowercase SHA-256")); + } + Ok(()) +} + +fn has_exact_extension(path: &str, extension: &str) -> bool { + Path::new(path).extension() == Some(OsStr::new(extension)) +} + +fn validation(message: impl Into) -> Result { + Err(ManifestError::Validation(message.into())) +} diff --git a/crates/j2k-t803/src/normalize.rs b/crates/j2k-t803/src/normalize.rs new file mode 100644 index 00000000..429d7201 --- /dev/null +++ b/crates/j2k-t803/src/normalize.rs @@ -0,0 +1,138 @@ +use thiserror::Error; + +/// One decoded component before T.803 output normalization. +#[derive(Clone, Copy, Debug)] +pub struct Component<'a> { + /// Component width in samples. + pub width: u32, + /// Component height in samples. + pub height: u32, + /// Nominal component precision. + pub bit_depth: u8, + /// Whether samples are signed. + pub signed: bool, + /// Horizontal and vertical strides used to undo decoder-side replication. + /// + /// Use `(1, 1)` when the decoder retained the codestream component's + /// native sampling grid. + pub post_decode_subsampling: (u8, u8), + /// Canonical integer samples in row-major order. + pub samples: &'a [i64], +} + +/// Reference shape and representation required by one T.803 comparison. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct NormalizationTarget { + /// Reference width in samples. + pub width: u32, + /// Reference height in samples. + pub height: u32, + /// Reference precision. + pub bit_depth: u8, + /// Whether reference samples are signed. + pub signed: bool, +} + +/// Error returned when decoded output cannot be normalized as required. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum NormalizationError { + /// Source metadata or storage is internally inconsistent. + #[error("invalid decoded component: {0}")] + InvalidSource(&'static str), + /// The target cannot be cropped from the source component. + #[error("reference dimensions exceed decoded component dimensions")] + Dimensions, + /// Reference and decoded components disagree on signedness. + #[error("reference signedness differs from decoded component signedness")] + Signedness, + /// The reference precision cannot be obtained through T.803 downscaling. + #[error("reference bit depth exceeds decoded component bit depth")] + BitDepth, + /// The normalized output allocation could not be reserved. + #[error("cannot allocate normalized component")] + Allocation, +} + +/// Apply T.803 clipping, precision reduction, and upper-left cropping. +pub fn normalize_component( + source: Component<'_>, + target: NormalizationTarget, +) -> Result, NormalizationError> { + validate_source(source)?; + let (horizontal_step, vertical_step) = source.post_decode_subsampling; + let sampled_width = source.width.div_ceil(u32::from(horizontal_step)); + let sampled_height = source.height.div_ceil(u32::from(vertical_step)); + if target.width == 0 + || target.height == 0 + || target.width > sampled_width + || target.height > sampled_height + { + return Err(NormalizationError::Dimensions); + } + if target.signed != source.signed { + return Err(NormalizationError::Signedness); + } + if target.bit_depth == 0 || target.bit_depth > source.bit_depth { + return Err(NormalizationError::BitDepth); + } + + let source_width = source.width as usize; + let target_width = target.width as usize; + let target_height = target.height as usize; + let horizontal_step = usize::from(horizontal_step); + let vertical_step = usize::from(vertical_step); + let shift = source.bit_depth - target.bit_depth; + let (minimum, maximum) = sample_range(source.bit_depth, source.signed); + let target_len = target_width + .checked_mul(target_height) + .ok_or(NormalizationError::Dimensions)?; + let mut normalized = Vec::new(); + normalized + .try_reserve_exact(target_len) + .map_err(|_| NormalizationError::Allocation)?; + for row in source + .samples + .chunks_exact(source_width) + .step_by(vertical_step) + .take(target_height) + { + normalized.extend( + row.iter() + .step_by(horizontal_step) + .take(target_width) + .map(|sample| sample.clamp(&minimum, &maximum) >> shift), + ); + } + Ok(normalized) +} + +fn validate_source(source: Component<'_>) -> Result<(), NormalizationError> { + if source.width == 0 + || source.height == 0 + || !(1..=32).contains(&source.bit_depth) + || source.post_decode_subsampling.0 == 0 + || source.post_decode_subsampling.1 == 0 + { + return Err(NormalizationError::InvalidSource( + "dimensions and bit depth must be non-zero", + )); + } + let expected = (source.width as usize) + .checked_mul(source.height as usize) + .ok_or(NormalizationError::InvalidSource("dimensions overflow"))?; + if source.samples.len() != expected { + return Err(NormalizationError::InvalidSource( + "sample length does not match dimensions", + )); + } + Ok(()) +} + +fn sample_range(bit_depth: u8, signed: bool) -> (i64, i64) { + if signed { + let limit = 1_i64 << (bit_depth - 1); + (-limit, limit - 1) + } else { + (0, (1_i64 << bit_depth) - 1) + } +} diff --git a/crates/j2k-t803/src/pgx.rs b/crates/j2k-t803/src/pgx.rs new file mode 100644 index 00000000..af8eedbd --- /dev/null +++ b/crates/j2k-t803/src/pgx.rs @@ -0,0 +1,226 @@ +use thiserror::Error; + +/// A decoded T.803 PGX component. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PgxImage { + /// Component width in samples. + pub width: u32, + /// Component height in samples. + pub height: u32, + /// Nominal sample precision. + pub bit_depth: u8, + /// Whether samples are signed. + pub signed: bool, + /// Canonical integer samples in row-major order. + pub samples: Vec, +} + +/// Error returned for malformed or unsupported PGX data. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum PgxError { + /// The header line is absent or malformed. + #[error("invalid PGX header: {0}")] + Header(&'static str), + /// The declared byte order is not a recognized PGX byte order. + #[error("PGX byte order must be ML or LM")] + ByteOrder, + /// The declared bit depth is outside 1 through 32. + #[error("PGX bit depth must be between 1 and 32")] + BitDepth, + /// A numeric header field is invalid. + #[error("invalid PGX {field}")] + Number { + /// Field being parsed. + field: &'static str, + }, + /// Width or height is zero or cannot be represented safely. + #[error("invalid PGX dimensions")] + Dimensions, + /// The binary payload does not exactly match the declared dimensions. + #[error("PGX payload length is {actual}, expected {expected}")] + PayloadLength { + /// Expected byte count. + expected: usize, + /// Actual byte count. + actual: usize, + }, + /// A signed sample is not sign-extended to its storage boundary. + #[error("PGX signed sample has invalid sign extension")] + SignExtension, + /// An unsigned sample uses bits outside its declared precision. + #[error("PGX unsigned sample exceeds its declared precision")] + Precision, + /// The declared component cannot be allocated. + #[error("cannot allocate PGX component")] + Allocation, +} + +/// Parse the PGX representation used by the T.803 electronic attachment. +/// +/// The parser requires an exact payload length and validates sign extension or +/// zero extension outside the declared precision. It accepts the attachment's +/// declared `ML` and legacy `LM` storage orders and only ASCII spaces as field +/// separators. +pub fn parse_pgx(bytes: &[u8]) -> Result { + let newline = bytes + .iter() + .position(|byte| *byte == b'\n') + .ok_or(PgxError::Header("missing newline"))?; + let header_bytes = bytes[..newline] + .strip_suffix(b"\r") + .unwrap_or(&bytes[..newline]); + let header = + core::str::from_utf8(header_bytes).map_err(|_| PgxError::Header("header is not UTF-8"))?; + if header.is_empty() + || header.starts_with(' ') + || header.ends_with(' ') + || !header.is_ascii() + || header + .bytes() + .any(|byte| byte != b' ' && !(0x21..=0x7e).contains(&byte)) + { + return Err(PgxError::Header("fields must use ASCII spaces")); + } + + let mut fields = header.split(' ').filter(|field| !field.is_empty()); + if fields.next() != Some("PG") { + return Err(PgxError::Header("expected PG ML precision width height")); + } + let byte_order = match fields.next() { + Some("ML") => ByteOrder::Big, + Some("LM") => ByteOrder::Little, + _ => return Err(PgxError::ByteOrder), + }; + + let precision = fields + .next() + .ok_or(PgxError::Header("missing precision field"))?; + let (signed, depth_field) = match precision { + "+" => ( + false, + fields.next().ok_or(PgxError::Header("missing bit depth"))?, + ), + "-" => ( + true, + fields.next().ok_or(PgxError::Header("missing bit depth"))?, + ), + _ => parse_attached_precision(precision)?, + }; + let width_field = fields.next().ok_or(PgxError::Header("missing width"))?; + let height_field = fields.next().ok_or(PgxError::Header("missing height"))?; + if fields.next().is_some() { + return Err(PgxError::Header("too many header fields")); + } + let bit_depth = depth_field.parse::().map_err(|_| PgxError::BitDepth)?; + if !(1..=32).contains(&bit_depth) { + return Err(PgxError::BitDepth); + } + let width = parse_dimension(width_field, "width")?; + let height = parse_dimension(height_field, "height")?; + let sample_count = usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .ok_or(PgxError::Dimensions)?; + let bytes_per_sample = match bit_depth { + 1..=8 => 1, + 9..=16 => 2, + 17..=32 => 4, + _ => unreachable!(), + }; + let expected = sample_count + .checked_mul(bytes_per_sample) + .ok_or(PgxError::Dimensions)?; + let payload = &bytes[newline + 1..]; + if payload.len() != expected { + return Err(PgxError::PayloadLength { + expected, + actual: payload.len(), + }); + } + + let mut samples = Vec::new(); + samples + .try_reserve_exact(sample_count) + .map_err(|_| PgxError::Allocation)?; + for storage in payload.chunks_exact(bytes_per_sample) { + samples.push(decode_sample(storage, bit_depth, signed, byte_order)?); + } + Ok(PgxImage { + width, + height, + bit_depth, + signed, + samples, + }) +} + +#[derive(Clone, Copy)] +enum ByteOrder { + Big, + Little, +} + +fn parse_attached_precision(precision: &str) -> Result<(bool, &str), PgxError> { + let (signed, depth) = match precision.as_bytes().first() { + Some(b'+') => (false, &precision[1..]), + Some(b'-') => (true, &precision[1..]), + Some(byte) if byte.is_ascii_digit() => (false, precision), + _ => return Err(PgxError::Header("invalid precision field")), + }; + if depth.is_empty() { + return Err(PgxError::Header("missing bit depth")); + } + Ok((signed, depth)) +} + +fn parse_dimension(field: &str, name: &'static str) -> Result { + let value = field + .parse::() + .map_err(|_| PgxError::Number { field: name })?; + if value == 0 { + return Err(PgxError::Dimensions); + } + Ok(value) +} + +fn decode_sample( + storage: &[u8], + bit_depth: u8, + signed: bool, + byte_order: ByteOrder, +) -> Result { + let unsigned = match (storage, byte_order) { + ([value], _) => u64::from(*value), + ([a, b], ByteOrder::Big) => u64::from(u16::from_be_bytes([*a, *b])), + ([a, b], ByteOrder::Little) => u64::from(u16::from_le_bytes([*a, *b])), + ([a, b, c, d], ByteOrder::Big) => u64::from(u32::from_be_bytes([*a, *b, *c, *d])), + ([a, b, c, d], ByteOrder::Little) => u64::from(u32::from_le_bytes([*a, *b, *c, *d])), + _ => unreachable!(), + }; + if !signed { + let maximum = (1_u64 << bit_depth) - 1; + if unsigned > maximum { + return Err(PgxError::Precision); + } + return i64::try_from(unsigned).map_err(|_| PgxError::Precision); + } + + let value = match (storage, byte_order) { + ([value], _) => i64::from(i8::from_be_bytes([*value])), + ([a, b], ByteOrder::Big) => i64::from(i16::from_be_bytes([*a, *b])), + ([a, b], ByteOrder::Little) => i64::from(i16::from_le_bytes([*a, *b])), + ([a, b, c, d], ByteOrder::Big) => i64::from(i32::from_be_bytes([*a, *b, *c, *d])), + ([a, b, c, d], ByteOrder::Little) => i64::from(i32::from_le_bytes([*a, *b, *c, *d])), + _ => unreachable!(), + }; + let limit = 1_i64 << (bit_depth - 1); + if (-limit..limit).contains(&value) { + Ok(value) + } else { + Err(PgxError::SignExtension) + } +} diff --git a/crates/j2k-t803/src/report.rs b/crates/j2k-t803/src/report.rs new file mode 100644 index 00000000..a9bf2f7a --- /dev/null +++ b/crates/j2k-t803/src/report.rs @@ -0,0 +1,1112 @@ +use std::{collections::BTreeSet, fmt::Write as _}; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::manifest::{validate_path, validate_sha256, CorpusFile, SOURCE_URL, STANDARD}; +use crate::EncoderMode; + +/// Overall report result. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReportStatus { + /// Every selected case passed. + Pass, + /// At least one selected case failed or errored. + Fail, +} + +/// Result of one selected conformance case. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum CaseStatus { + /// Measured errors are within the inclusive bounds. + Pass, + /// Decode completed but exceeded at least one bound. + Fail, + /// Decode or comparison did not complete. + Error, +} + +/// Auditable classification of the complete execution route. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum RouteKind { + /// All used stages ran on the CPU. + Cpu, + /// Used stages include both CPU and one accelerator. + Hybrid, + /// Every used stage ran on one accelerator. + DeviceNative, +} + +/// Location used by one decoder stage. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExecutionLocation { + /// Host CPU. + Cpu, + /// CUDA device. + Cuda, + /// Metal device. + Metal, + /// Stage was not needed for this route. + NotUsed, +} + +/// Decoder stages disclosed for every case. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum RouteStageName { + /// Container and codestream parsing. + Parsing, + /// Tier-1 entropy decoding. + Tier1, + /// Coefficient dequantization. + Dequantization, + /// Inverse discrete wavelet transform. + Idwt, + /// Multiple-component transform. + Mct, + /// Colour conversion and output normalization. + ColorOutput, + /// Host-to-device transfer. + HostToDevice, + /// Device-to-host transfer. + DeviceToHost, +} + +/// Encoder stages disclosed for every Annex D matrix case. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum EncodeRouteStageName { + /// Input sample unpacking, level shift, or component-plane preparation. + InputPreparation, + /// Forward reversible colour transform. + ForwardRct, + /// Forward irreversible colour transform. + ForwardIct, + /// Forward reversible 5/3 wavelet transform. + ForwardDwt53, + /// Forward irreversible 9/7 wavelet transform. + ForwardDwt97, + /// Irreversible sub-band quantization. + Quantization, + /// Part 1 Tier-1 code-block coding. + Tier1, + /// Tier-2 packet formation and codestream writing. + Packetization, + /// Host-to-device transfer. + HostToDevice, + /// Device-to-host transfer. + DeviceToHost, +} + +/// Execution location for one decoder stage. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RouteStage { + /// Stage being disclosed. + pub stage: RouteStageName, + /// Where the stage ran, or that it was not used. + pub location: ExecutionLocation, +} + +/// Execution location for one encoder stage. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncodeRouteStage { + /// Stage being disclosed. + pub stage: EncodeRouteStageName, + /// Where the stage ran, or that it was not used. + pub location: ExecutionLocation, +} + +/// Identity and claim text for the implementation under test. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct IutIdentity { + /// Crate or adapter name. + pub name: String, + /// Candidate version. + pub version: String, + /// Exact source revision under test. + pub candidate_sha: String, + /// Precise candidate claim; never a generic Part 1 claim. + pub claim: String, +} + +/// Operating-system and hardware identity for one run. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PlatformIdentity { + /// Operating system. + pub os: String, + /// Processor architecture. + pub arch: String, + /// CPU or accelerator hardware description. + pub hardware: String, + /// Accelerator driver, or `not-applicable` for CPU runs. + pub driver: String, +} + +/// Aggregate route counts for the complete selected decoder matrix. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DecoderRouteSummary { + /// Total selected decoder and Annex G cases. + pub total: usize, + /// Cases whose used stages all ran on one accelerator. + pub device_native: usize, + /// Cases whose used stages ran across CPU and one accelerator. + pub hybrid: usize, + /// Cases whose used stages all ran on CPU. + pub cpu: usize, +} + +/// Independent native-component comparison against `OpenJPEG` before T.803 normalization. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct NativeComponentOracleEvidence { + /// Repository-relative official codestream path. + pub codestream_path: String, + /// SHA-256 of the official codestream bytes. + pub codestream_sha256: String, + /// Semantic rule that selected this codestream for the independent audit. + pub selection: String, + /// Independent decoder implementation. + pub implementation: String, + /// Independent decoder version. + pub version: String, + /// Exact library or executable identity. + pub library: String, + /// Number of components compared in codestream order. + pub component_count: usize, + /// Total native samples compared across all components. + pub compared_sample_count: u64, + /// Canonical SHA-256 of production-decoder component metadata and samples. + pub production_components_sha256: String, + /// Canonical SHA-256 of independent-decoder component metadata and samples. + pub openjpeg_components_sha256: String, + /// Whether every component's metadata and samples matched exactly. + pub exact: bool, +} + +/// Metrics, bounds, and route evidence for one selected case. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CaseReport { + /// Stable case identifier from the manifest. + pub id: String, + /// T.803 table containing the case. + pub table: String, + /// Case result. + pub status: CaseStatus, + /// Auditable route classification for this case. + pub route: RouteKind, + /// Measured peak error, when comparison completed. + pub peak: Option, + /// Measured MSE, when required and comparison completed. + pub mse: Option, + /// Inclusive peak bound. + pub allowed_peak: u64, + /// Inclusive MSE bound, absent for Annex G peak-only comparisons. + pub allowed_mse: Option, + /// Diagnostic for an errored case. + pub error: Option, + /// Complete per-stage route disclosure. + pub stages: Vec, +} + +/// Exact T.804 reference implementation used for Annex D testing. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncoderReferenceIdentity { + /// Reference-software standard. + pub standard: String, + /// Reference decoder implementation. + pub implementation: String, + /// Exact implementation version. + pub version: String, +} + +/// Result of the project-defined lossy quality gate. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum EncoderQualityStatus { + /// The separately declared project quality gate passed. + Pass, + /// The separately declared project quality gate failed. + Fail, + /// No lossy quality gate applies, as for lossless cases. + NotApplicable, +} + +/// Result of one informative Annex D/F encoder matrix case. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncoderCaseReport { + /// Stable case identifier from the encoder matrix. + pub id: String, + /// Lossless or lossy encoder mode. + pub mode: EncoderMode, + /// Case result. + pub status: CaseStatus, + /// Auditable route classification. + pub route: RouteKind, + /// Whether the T.804 reference implementation fully decoded the codestream. + pub reference_decode_success: bool, + /// Exact input equality for a lossless case; absent for lossy cases. + pub lossless_exact: Option, + /// Produced codestream size when encoding completed. + pub encoded_bytes: Option, + /// Actual codestream bits per reference-grid pixel. + pub actual_bits_per_pixel: Option, + /// Project quality metric for lossy output; not an Annex D acceptance rule. + pub psnr_db: Option, + /// Whether lossy output was exact, giving mathematically infinite PSNR. + pub psnr_infinite: bool, + /// Result of the separately declared project quality gate. + pub quality_status: EncoderQualityStatus, + /// Human-readable, auditable quality-gate requirement. + pub quality_requirement: Option, + /// Diagnostic when the quality gate fails. + pub quality_error: Option, + /// Diagnostic for a failed or errored case. + pub error: Option, + /// Complete per-stage route disclosure. + pub stages: Vec, +} + +/// Informative encoder evidence attached to one T.803 run. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct EncoderEvidence { + /// Annex F ICS repository path. + pub ics_path: String, + /// SHA-256 of the exact ICS bytes. + pub ics_sha256: String, + /// Encoder matrix repository path. + pub matrix_path: String, + /// Expected cases for this IUT. + pub matrix_case_count: usize, + /// Canonical SHA-256 of this IUT's selected matrix cases. + pub matrix_case_sha256: String, + /// T.804 decoder implementation identity. + pub reference_decoder: EncoderReferenceIdentity, + /// Per-case evidence in matrix order. + pub cases: Vec, + /// Encoder result derived from `cases`. + pub standards_status: ReportStatus, + /// Project quality-gate result derived from `cases`. + pub quality_status: ReportStatus, + /// Combined encoder evidence result. + pub status: ReportStatus, +} + +impl EncoderEvidence { + /// Build validated encoder evidence and derive its final status. + pub fn new( + ics_path: String, + ics_sha256: String, + matrix_path: String, + matrix_case_count: usize, + matrix_case_sha256: String, + reference_decoder: EncoderReferenceIdentity, + cases: Vec, + ) -> Result { + let standards_status = derive_status(cases.iter().map(|case| case.status)); + let quality_status = derive_quality_status(&cases); + let status = combine_status(standards_status, quality_status); + let evidence = Self { + ics_path, + ics_sha256, + matrix_path, + matrix_case_count, + matrix_case_sha256, + reference_decoder, + cases, + standards_status, + quality_status, + status, + }; + evidence.validate()?; + Ok(evidence) + } + + fn validate(&self) -> Result<(), ReportError> { + validate_path(&self.ics_path) + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_path(&self.matrix_path) + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_sha256(&self.ics_sha256, "encoder ICS") + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_sha256(&self.matrix_case_sha256, "encoder matrix") + .map_err(|error| ReportError::Validation(error.to_string()))?; + if self.matrix_case_count == 0 || self.cases.len() != self.matrix_case_count { + return report_error("encoder case count does not match the pinned matrix"); + } + if [ + &self.reference_decoder.standard, + &self.reference_decoder.implementation, + &self.reference_decoder.version, + ] + .into_iter() + .any(String::is_empty) + { + return report_error("encoder reference decoder identity must not be empty"); + } + let mut previous_id = None; + for case in &self.cases { + if case.id.is_empty() + || previous_id.is_some_and(|previous| previous >= case.id.as_str()) + { + return report_error("encoder case ids must be non-empty, sorted, and unique"); + } + previous_id = Some(case.id.as_str()); + validate_encoder_case(case)?; + validate_encode_route(case)?; + } + let standards_status = derive_status(self.cases.iter().map(|case| case.status)); + let quality_status = derive_quality_status(&self.cases); + if self.standards_status != standards_status + || self.quality_status != quality_status + || self.status != combine_status(standards_status, quality_status) + { + return report_error("encoder statuses do not match case results"); + } + Ok(()) + } +} + +/// Versioned JSON and Markdown evidence for one T.803 run. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct T803Report { + /// Report schema version. + pub schema_version: u32, + /// Standard edition used by the run. + pub standard: String, + /// Official attachment handle. + pub source_url: String, + /// Observed attachment SHA-256. + pub source_archive_sha256: String, + /// Implementation under test. + pub iut: IutIdentity, + /// Platform and hardware identity. + pub platform: PlatformIdentity, + /// Enabled build features in sorted order. + pub features: Vec, + /// Selected corpus hashes in path order. + pub corpus: Vec, + /// Independent component-level decoder evidence before T.803 normalization. + pub native_component_oracles: Vec, + /// Aggregate decoder route counts derived from `cases`. + pub decoder_routes: DecoderRouteSummary, + /// Per-case evidence in manifest order. + pub cases: Vec, + /// Informative Annex D/F encoder evidence. + pub encoder: EncoderEvidence, + /// Overall result derived from `cases`. + pub status: ReportStatus, +} + +/// Error returned for invalid evidence or report serialization. +#[derive(Debug, Error)] +pub enum ReportError { + /// Report evidence is incomplete or inconsistent. + #[error("invalid T.803 report: {0}")] + Validation(String), + /// JSON serialization or parsing failed. + #[error("invalid T.803 report JSON: {0}")] + Json(#[from] serde_json::Error), +} + +impl T803Report { + /// Build a report and derive its final status from the case results. + #[expect( + clippy::too_many_arguments, + reason = "the constructor mirrors the validated top-level evidence sections" + )] + pub fn new( + iut: IutIdentity, + platform: PlatformIdentity, + source_archive_sha256: String, + features: Vec, + corpus: Vec, + native_component_oracles: Vec, + cases: Vec, + encoder: EncoderEvidence, + ) -> Result { + let decoder_routes = summarize_routes(&cases); + let decoder_status = derive_status(cases.iter().map(|case| case.status)); + let status = if decoder_status == ReportStatus::Pass && encoder.status == ReportStatus::Pass + { + ReportStatus::Pass + } else { + ReportStatus::Fail + }; + let report = Self { + schema_version: 3, + standard: STANDARD.to_string(), + source_url: SOURCE_URL.to_string(), + source_archive_sha256, + iut, + platform, + features, + corpus, + native_component_oracles, + decoder_routes, + cases, + encoder, + status, + }; + report.validate()?; + Ok(report) + } + + /// Serialize a validated report as stable pretty-printed JSON. + pub fn to_json(&self) -> Result { + self.validate()?; + let mut json = serde_json::to_string_pretty(self)?; + json.push('\n'); + Ok(json) + } + + /// Parse and validate report JSON. + pub fn from_json(json: &str) -> Result { + let report = serde_json::from_str::(json)?; + report.validate()?; + Ok(report) + } + + /// Render a validated report as deterministic Markdown. + pub fn to_markdown(&self) -> Result { + self.validate()?; + let mut markdown = format!( + "# T.803 conformance evidence\n\n- Standard: {}\n- IUT: {} {}\n- Candidate SHA: {}\n- Claim: {}\n- Platform: {} {}\n- Hardware: {}\n- Driver: {}\n- Device-native: {} / {}\n- Hybrid: {} / {}\n- CPU-routed: {} / {}\n- Final status: {}\n", + self.standard, + markdown_cell(&self.iut.name), + markdown_cell(&self.iut.version), + markdown_cell(&self.iut.candidate_sha), + markdown_cell(&self.iut.claim), + markdown_cell(&self.platform.os), + markdown_cell(&self.platform.arch), + markdown_cell(&self.platform.hardware), + markdown_cell(&self.platform.driver), + self.decoder_routes.device_native, + self.decoder_routes.total, + self.decoder_routes.hybrid, + self.decoder_routes.total, + self.decoder_routes.cpu, + self.decoder_routes.total, + report_status_name(self.status), + ); + self.push_native_component_oracles(&mut markdown); + markdown.push_str("\n| Case | Table | Status | Route | Peak (measured / allowed) | MSE (measured / allowed) | Route stages |\n|---|---|---:|---|---:|---:|---|\n"); + self.push_decoder_cases(&mut markdown); + self.push_encoder_evidence(&mut markdown); + Ok(markdown) + } + + fn push_native_component_oracles(&self, markdown: &mut String) { + markdown.push_str("\n## Native component oracle\n"); + for oracle in &self.native_component_oracles { + let _ = write!( + markdown, + "\n- {} {} (`{}`) decoded `{}` (`{}`) before T.803 normalization.\n- Selection: {}.\n- {} components / {} samples: {}.\n- Production SHA-256: `{}`.\n- OpenJPEG SHA-256: `{}`.\n", + markdown_cell(&oracle.implementation), + markdown_cell(&oracle.version), + markdown_cell(&oracle.library), + markdown_cell(&oracle.codestream_path), + oracle.codestream_sha256, + markdown_cell(&oracle.selection), + oracle.component_count, + oracle.compared_sample_count, + if oracle.exact { "exact" } else { "mismatch" }, + oracle.production_components_sha256, + oracle.openjpeg_components_sha256, + ); + } + } + + fn push_decoder_cases(&self, markdown: &mut String) { + for case in &self.cases { + let peak = case + .peak + .map_or_else(|| "error".to_string(), |value| value.to_string()); + let mse = match (case.mse, case.allowed_mse) { + (Some(measured), Some(allowed)) => format!("{measured:.6} / {allowed:.6}"), + (None, None) => "not-applicable".to_string(), + _ => "error".to_string(), + }; + let stages = case + .stages + .iter() + .map(|stage| { + format!( + "{}={}", + stage_name(stage.stage), + location_name(stage.location) + ) + }) + .collect::>() + .join(", "); + let _ = writeln!( + markdown, + "| {} | {} | {} | {} | {} / {} | {} | {} |", + markdown_cell(&case.id), + markdown_cell(&case.table), + case_status_name(case.status), + route_kind_name(case.route), + peak, + case.allowed_peak, + mse, + stages, + ); + } + } + + fn push_encoder_evidence(&self, markdown: &mut String) { + let _ = write!( + markdown, + "\n## Informative Annex D/F encoder evidence\n\n- Procedure: T.803 Annex D/F (informative)\n- ICS: {} (`{}`)\n- Matrix: {} ({} cases, `{}`)\n- Reference decoder: {} {} ({})\n- Standards status: {}\n- Quality-gate status: {}\n- Combined encoder status: {}\n\n| Case | Mode | Standards status | Quality status | Route | Reference decode | Lossless exact | Bytes | Bits/pixel | PSNR | Quality requirement | Route stages |\n|---|---|---:|---:|---|---:|---:|---:|---:|---:|---|---|\n", + markdown_cell(&self.encoder.ics_path), + self.encoder.ics_sha256, + markdown_cell(&self.encoder.matrix_path), + self.encoder.matrix_case_count, + self.encoder.matrix_case_sha256, + markdown_cell(&self.encoder.reference_decoder.implementation), + markdown_cell(&self.encoder.reference_decoder.version), + markdown_cell(&self.encoder.reference_decoder.standard), + report_status_name(self.encoder.standards_status), + report_status_name(self.encoder.quality_status), + report_status_name(self.encoder.status), + ); + for case in &self.encoder.cases { + let stages = case + .stages + .iter() + .map(|stage| { + format!( + "{}={}", + encode_stage_name(stage.stage), + location_name(stage.location) + ) + }) + .collect::>() + .join(", "); + let lossless_exact = case + .lossless_exact + .map_or_else(|| "not-applicable".to_string(), |exact| exact.to_string()); + let encoded_bytes = case + .encoded_bytes + .map_or_else(|| "error".to_string(), |bytes| bytes.to_string()); + let bits_per_pixel = case + .actual_bits_per_pixel + .map_or_else(|| "error".to_string(), |value| format!("{value:.6}")); + let psnr = if case.psnr_infinite { + "infinite".to_string() + } else { + case.psnr_db.map_or_else( + || "not-applicable".to_string(), + |value| format!("{value:.6}"), + ) + }; + let quality_requirement = case + .quality_requirement + .as_deref() + .map_or_else(|| "not-applicable".to_string(), markdown_cell); + let _ = writeln!( + markdown, + "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", + markdown_cell(&case.id), + encoder_mode_name(case.mode), + case_status_name(case.status), + quality_status_name(case.quality_status), + route_kind_name(case.route), + case.reference_decode_success, + lossless_exact, + encoded_bytes, + bits_per_pixel, + psnr, + quality_requirement, + stages, + ); + } + markdown.push_str( + "\nEncoder results above are informative under T.803 and are not decoder compliance claims. Conformance does not establish robustness, security, adoption, or performance.\n", + ); + } + + fn validate(&self) -> Result<(), ReportError> { + if self.schema_version != 3 || self.standard != STANDARD || self.source_url != SOURCE_URL { + return report_error("schema, standard, or source URL does not match T.803 v3"); + } + validate_sha256(&self.source_archive_sha256, "report archive") + .map_err(|error| ReportError::Validation(error.to_string()))?; + if [ + &self.iut.name, + &self.iut.version, + &self.iut.candidate_sha, + &self.iut.claim, + ] + .into_iter() + .any(String::is_empty) + { + return report_error("IUT identity fields must not be empty"); + } + if [ + &self.platform.os, + &self.platform.arch, + &self.platform.hardware, + &self.platform.driver, + ] + .into_iter() + .any(String::is_empty) + { + return report_error("platform identity fields must not be empty"); + } + if !is_strictly_sorted(&self.features) { + return report_error("features must be sorted and unique"); + } + if self.corpus.is_empty() || !is_sorted_by(&self.corpus, |entry| entry.path.as_str()) { + return report_error("corpus hashes must be non-empty, sorted, and unique"); + } + for entry in &self.corpus { + validate_path(&entry.path) + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_sha256(&entry.sha256, &entry.path) + .map_err(|error| ReportError::Validation(error.to_string()))?; + } + if self.native_component_oracles.is_empty() + || !is_sorted_by(&self.native_component_oracles, |oracle| { + oracle.codestream_path.as_str() + }) + { + return report_error( + "native component oracle evidence must be non-empty, sorted, and unique", + ); + } + for oracle in &self.native_component_oracles { + validate_native_component_oracle(oracle)?; + if !self.corpus.iter().any(|entry| { + entry.path == oracle.codestream_path && entry.sha256 == oracle.codestream_sha256 + }) { + return report_error(format!( + "native component oracle {} is not pinned by the report corpus", + oracle.codestream_path + )); + } + } + if self.cases.is_empty() { + return report_error("report must contain at least one case"); + } + if self.decoder_routes != summarize_routes(&self.cases) { + return report_error("decoder route summary does not match per-case routes"); + } + let mut ids = BTreeSet::new(); + for case in &self.cases { + if case.id.is_empty() || !ids.insert(case.id.as_str()) { + return report_error("case ids must be non-empty and unique"); + } + validate_case(case)?; + validate_route(case, case.route)?; + } + self.encoder.validate()?; + let decoder_status = derive_status(self.cases.iter().map(|case| case.status)); + let derived = + if decoder_status == ReportStatus::Pass && self.encoder.status == ReportStatus::Pass { + ReportStatus::Pass + } else { + ReportStatus::Fail + }; + if self.status != derived { + return report_error("final status does not match case results"); + } + Ok(()) + } +} + +fn validate_case(case: &CaseReport) -> Result<(), ReportError> { + let valid_metrics = case + .mse + .into_iter() + .chain(case.allowed_mse) + .all(|value| value.is_finite() && value >= 0.0); + if !valid_metrics { + return report_error(format!("{} has invalid MSE data", case.id)); + } + match case.status { + CaseStatus::Pass | CaseStatus::Fail => { + if case.peak.is_none() + || case.error.is_some() + || (case.allowed_mse.is_some() != case.mse.is_some()) + { + return report_error(format!("{} has incomplete comparison metrics", case.id)); + } + } + CaseStatus::Error => { + if case.error.as_deref().is_none_or(str::is_empty) + || case.peak.is_some() + || case.mse.is_some() + { + return report_error(format!("{} has invalid error evidence", case.id)); + } + } + } + Ok(()) +} + +fn validate_native_component_oracle( + oracle: &NativeComponentOracleEvidence, +) -> Result<(), ReportError> { + validate_path(&oracle.codestream_path) + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_sha256(&oracle.codestream_sha256, &oracle.codestream_path) + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_sha256( + &oracle.production_components_sha256, + "production native components", + ) + .map_err(|error| ReportError::Validation(error.to_string()))?; + validate_sha256( + &oracle.openjpeg_components_sha256, + "OpenJPEG native components", + ) + .map_err(|error| ReportError::Validation(error.to_string()))?; + if oracle.selection.is_empty() + || oracle.implementation != "OpenJPEG" + || oracle.version.is_empty() + || oracle.library.is_empty() + || oracle.component_count <= 4 + || oracle.compared_sample_count < oracle.component_count as u64 + { + return report_error(format!( + "{} has incomplete native component oracle identity or coverage", + oracle.codestream_path + )); + } + if !oracle.exact || oracle.production_components_sha256 != oracle.openjpeg_components_sha256 { + return report_error(format!( + "{} did not match OpenJPEG component-for-component", + oracle.codestream_path + )); + } + Ok(()) +} + +fn validate_encoder_case(case: &EncoderCaseReport) -> Result<(), ReportError> { + for (name, value) in [ + ("bits per pixel", case.actual_bits_per_pixel), + ("PSNR", case.psnr_db), + ] { + if value.is_some_and(|value| !value.is_finite() || value < 0.0) { + return report_error(format!("{} has invalid {name}", case.id)); + } + } + if case.mode == EncoderMode::Lossless && case.psnr_db.is_some() { + return report_error(format!("{} reports PSNR for a lossless case", case.id)); + } + if case.psnr_infinite && case.psnr_db.is_some() { + return report_error(format!("{} reports finite and infinite PSNR", case.id)); + } + if case.mode == EncoderMode::Lossless && case.psnr_infinite { + return report_error(format!("{} reports PSNR for a lossless case", case.id)); + } + if case.mode == EncoderMode::Lossy && case.lossless_exact.is_some() { + return report_error(format!( + "{} reports lossless equality for a lossy case", + case.id + )); + } + match (case.mode, case.quality_status) { + (EncoderMode::Lossless, EncoderQualityStatus::NotApplicable) => { + if case.quality_requirement.is_some() || case.quality_error.is_some() { + return report_error(format!("{} has a lossless quality gate", case.id)); + } + } + (EncoderMode::Lossy, EncoderQualityStatus::Pass) => { + if case + .quality_requirement + .as_deref() + .is_none_or(str::is_empty) + || case.quality_error.is_some() + || (!case.psnr_infinite && case.psnr_db.is_none()) + { + return report_error(format!( + "{} has incomplete passing quality evidence", + case.id + )); + } + } + (EncoderMode::Lossy, EncoderQualityStatus::Fail) => { + if case + .quality_requirement + .as_deref() + .is_none_or(str::is_empty) + || case.quality_error.as_deref().is_none_or(str::is_empty) + { + return report_error(format!( + "{} has incomplete failed quality evidence", + case.id + )); + } + } + _ => return report_error(format!("{} has an invalid quality status", case.id)), + } + match case.status { + CaseStatus::Pass => { + let exact = case.mode == EncoderMode::Lossy || case.lossless_exact == Some(true); + if !case.reference_decode_success + || !exact + || case.encoded_bytes.is_none() + || case.actual_bits_per_pixel.is_none() + || case.error.is_some() + { + return report_error(format!( + "{} has incomplete passing encoder evidence", + case.id + )); + } + } + CaseStatus::Fail => { + if case.encoded_bytes.is_none() + || case.actual_bits_per_pixel.is_none() + || case.error.as_deref().is_none_or(str::is_empty) + { + return report_error(format!("{} has invalid failed encoder evidence", case.id)); + } + } + CaseStatus::Error => { + if case.reference_decode_success + || case.lossless_exact == Some(true) + || case.error.as_deref().is_none_or(str::is_empty) + { + return report_error(format!("{} has invalid encoder error evidence", case.id)); + } + } + } + Ok(()) +} + +fn validate_route(case: &CaseReport, route: RouteKind) -> Result<(), ReportError> { + let required = [ + RouteStageName::Parsing, + RouteStageName::Tier1, + RouteStageName::Dequantization, + RouteStageName::Idwt, + RouteStageName::Mct, + RouteStageName::ColorOutput, + RouteStageName::HostToDevice, + RouteStageName::DeviceToHost, + ]; + let stages = case + .stages + .iter() + .map(|stage| stage.stage) + .collect::>(); + if case.stages.len() != required.len() || stages != required.into_iter().collect() { + return report_error(format!("{} does not disclose every route stage", case.id)); + } + validate_route_locations( + &case.id, + route, + case.stages.iter().map(|stage| stage.location), + ) +} + +fn validate_encode_route(case: &EncoderCaseReport) -> Result<(), ReportError> { + let required = [ + EncodeRouteStageName::InputPreparation, + EncodeRouteStageName::ForwardRct, + EncodeRouteStageName::ForwardIct, + EncodeRouteStageName::ForwardDwt53, + EncodeRouteStageName::ForwardDwt97, + EncodeRouteStageName::Quantization, + EncodeRouteStageName::Tier1, + EncodeRouteStageName::Packetization, + EncodeRouteStageName::HostToDevice, + EncodeRouteStageName::DeviceToHost, + ]; + let stages = case + .stages + .iter() + .map(|stage| stage.stage) + .collect::>(); + if case.stages.len() != required.len() || stages != required.into_iter().collect() { + return report_error(format!( + "{} does not disclose every encoder route stage", + case.id + )); + } + validate_route_locations( + &case.id, + case.route, + case.stages.iter().map(|stage| stage.location), + ) +} + +fn validate_route_locations( + id: &str, + route: RouteKind, + locations: impl Iterator, +) -> Result<(), ReportError> { + let locations = locations + .filter(|location| *location != ExecutionLocation::NotUsed) + .collect::>(); + let uses_cpu = locations.contains(&ExecutionLocation::Cpu); + let device_count = usize::from(locations.contains(&ExecutionLocation::Cuda)) + + usize::from(locations.contains(&ExecutionLocation::Metal)); + let valid = match route { + RouteKind::Cpu => uses_cpu && device_count == 0, + RouteKind::Hybrid => uses_cpu && device_count == 1, + RouteKind::DeviceNative => !uses_cpu && device_count == 1, + }; + if valid { + Ok(()) + } else { + report_error(format!( + "{id} route stages contradict the {} label", + route_kind_name(route) + )) + } +} + +fn is_strictly_sorted(values: &[String]) -> bool { + values.windows(2).all(|pair| pair[0] < pair[1]) +} + +fn is_sorted_by(values: &[T], key: impl Fn(&T) -> &str) -> bool { + values.windows(2).all(|pair| key(&pair[0]) < key(&pair[1])) +} + +fn summarize_routes(cases: &[CaseReport]) -> DecoderRouteSummary { + let mut summary = DecoderRouteSummary { + total: cases.len(), + device_native: 0, + hybrid: 0, + cpu: 0, + }; + for case in cases { + match case.route { + RouteKind::Cpu => summary.cpu += 1, + RouteKind::Hybrid => summary.hybrid += 1, + RouteKind::DeviceNative => summary.device_native += 1, + } + } + summary +} + +fn derive_status(statuses: impl Iterator) -> ReportStatus { + if statuses + .into_iter() + .all(|status| status == CaseStatus::Pass) + { + ReportStatus::Pass + } else { + ReportStatus::Fail + } +} + +fn derive_quality_status(cases: &[EncoderCaseReport]) -> ReportStatus { + if cases + .iter() + .any(|case| case.quality_status == EncoderQualityStatus::Fail) + { + ReportStatus::Fail + } else { + ReportStatus::Pass + } +} + +fn combine_status(left: ReportStatus, right: ReportStatus) -> ReportStatus { + if left == ReportStatus::Pass && right == ReportStatus::Pass { + ReportStatus::Pass + } else { + ReportStatus::Fail + } +} + +fn markdown_cell(value: &str) -> String { + value.replace('|', "\\|").replace('\n', " ") +} + +fn report_status_name(status: ReportStatus) -> &'static str { + match status { + ReportStatus::Pass => "pass", + ReportStatus::Fail => "fail", + } +} + +fn case_status_name(status: CaseStatus) -> &'static str { + match status { + CaseStatus::Pass => "pass", + CaseStatus::Fail => "fail", + CaseStatus::Error => "error", + } +} + +fn quality_status_name(status: EncoderQualityStatus) -> &'static str { + match status { + EncoderQualityStatus::Pass => "pass", + EncoderQualityStatus::Fail => "fail", + EncoderQualityStatus::NotApplicable => "not-applicable", + } +} + +fn route_kind_name(route: RouteKind) -> &'static str { + match route { + RouteKind::Cpu => "cpu", + RouteKind::Hybrid => "hybrid", + RouteKind::DeviceNative => "device-native", + } +} + +fn stage_name(stage: RouteStageName) -> &'static str { + match stage { + RouteStageName::Parsing => "parsing", + RouteStageName::Tier1 => "tier1", + RouteStageName::Dequantization => "dequantization", + RouteStageName::Idwt => "idwt", + RouteStageName::Mct => "mct", + RouteStageName::ColorOutput => "color-output", + RouteStageName::HostToDevice => "host-to-device", + RouteStageName::DeviceToHost => "device-to-host", + } +} + +fn encode_stage_name(stage: EncodeRouteStageName) -> &'static str { + match stage { + EncodeRouteStageName::InputPreparation => "input-preparation", + EncodeRouteStageName::ForwardRct => "forward-rct", + EncodeRouteStageName::ForwardIct => "forward-ict", + EncodeRouteStageName::ForwardDwt53 => "forward-dwt53", + EncodeRouteStageName::ForwardDwt97 => "forward-dwt97", + EncodeRouteStageName::Quantization => "quantization", + EncodeRouteStageName::Tier1 => "tier1", + EncodeRouteStageName::Packetization => "packetization", + EncodeRouteStageName::HostToDevice => "host-to-device", + EncodeRouteStageName::DeviceToHost => "device-to-host", + } +} + +fn encoder_mode_name(mode: EncoderMode) -> &'static str { + match mode { + EncoderMode::Lossless => "lossless", + EncoderMode::Lossy => "lossy", + } +} + +fn location_name(location: ExecutionLocation) -> &'static str { + match location { + ExecutionLocation::Cpu => "cpu", + ExecutionLocation::Cuda => "cuda", + ExecutionLocation::Metal => "metal", + ExecutionLocation::NotUsed => "not-used", + } +} + +fn report_error(message: impl Into) -> Result { + Err(ReportError::Validation(message.into())) +} diff --git a/crates/j2k-t803/src/runner.rs b/crates/j2k-t803/src/runner.rs new file mode 100644 index 00000000..6aa1e01a --- /dev/null +++ b/crates/j2k-t803/src/runner.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Fail-closed external-corpus and IUT runner support. + +mod archive; +mod cache; +mod cases; +mod cli; +mod cpu; +#[cfg(feature = "cuda-runner")] +mod cuda; +mod encoder; +mod evidence; +mod execute; +#[cfg(feature = "metal-runner")] +mod metal; +mod oracle; + +pub use archive::{extract_selected_archive, verify_corpus, ArchiveLimits, RunnerError}; +pub use cli::run_cli; diff --git a/crates/j2k-t803/src/runner/archive.rs b/crates/j2k-t803/src/runner/archive.rs new file mode 100644 index 00000000..21ecc762 --- /dev/null +++ b/crates/j2k-t803/src/runner/archive.rs @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ + collections::{BTreeMap, BTreeSet}, + fmt::Write as _, + fs::{self, File, OpenOptions}, + io::{Read, Seek}, + path::{Component, Path, PathBuf}, +}; + +use sha2::{Digest, Sha256}; +use thiserror::Error; +use zip::ZipArchive; + +use crate::CorpusFile; + +/// Resource limits applied before extracting an external T.803 archive. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ArchiveLimits { + /// Maximum number of ZIP entries, including directories. + pub max_entries: usize, + /// Maximum uncompressed size of one entry. + pub max_entry_bytes: u64, + /// Maximum aggregate uncompressed size. + pub max_total_bytes: u64, +} + +impl Default for ArchiveLimits { + fn default() -> Self { + Self { + max_entries: 2_048, + max_entry_bytes: 64 * 1024 * 1024, + max_total_bytes: 512 * 1024 * 1024, + } + } +} + +/// Error returned by fail-closed corpus acquisition and validation. +#[derive(Debug, Error)] +pub enum RunnerError { + /// Filesystem operation failed. + #[error("{operation} {}: {source}", path.display())] + Io { + /// Operation being attempted. + operation: &'static str, + /// Affected path. + path: PathBuf, + /// Underlying I/O failure. + #[source] + source: std::io::Error, + }, + /// ZIP structure or decompression failed. + #[error("invalid T.803 ZIP archive: {0}")] + Zip(#[from] zip::result::ZipError), + /// Archive or extracted inventory violated the pinned contract. + #[error("invalid T.803 corpus: {0}")] + Validation(String), +} + +/// Validate an archive completely, then extract only the pinned files. +pub fn extract_selected_archive( + reader: R, + output: &Path, + required: &[CorpusFile], + limits: ArchiveLimits, +) -> Result<(), RunnerError> { + ensure_empty_directory(output)?; + let mut archive = ZipArchive::new(reader)?; + if archive.len() > limits.max_entries { + return validation(format!( + "archive contains {} entries, limit is {}", + archive.len(), + limits.max_entries + )); + } + + let required_paths = required + .iter() + .map(|file| (file.path.as_str(), file)) + .collect::>(); + let mut entries = BTreeMap::new(); + let mut seen = BTreeSet::new(); + let mut total_bytes = 0_u64; + for index in 0..archive.len() { + let mut entry = archive.by_index(index)?; + let name = validate_entry_name(&entry)?; + if !seen.insert(name.clone()) { + return validation(format!("archive contains duplicate entry {name:?}")); + } + if entry.is_symlink() { + return validation(format!("archive entry {name:?} is a symlink")); + } + if entry.encrypted() { + return validation(format!("archive entry {name:?} is encrypted")); + } + if entry.size() > limits.max_entry_bytes { + return validation(format!("archive entry {name:?} is too large")); + } + total_bytes = total_bytes + .checked_add(entry.size()) + .ok_or_else(|| RunnerError::Validation("archive size overflow".to_string()))?; + if total_bytes > limits.max_total_bytes { + return validation("archive uncompressed contents are too large"); + } + + if let Some(required_file) = required_paths.get(name.as_str()) { + if entry.is_dir() { + return validation(format!("required file {name:?} is a directory")); + } + let observed = sha256_reader(&mut entry)?; + if observed != required_file.sha256 { + return validation(format!( + "required file {name:?} SHA-256 is {observed}, expected {}", + required_file.sha256 + )); + } + entries.insert(name, index); + } + } + + for file in required { + if !entries.contains_key(&file.path) { + return validation(format!("required file {:?} is missing", file.path)); + } + } + + for file in required { + let index = entries[&file.path]; + let mut entry = archive.by_index(index)?; + let destination = output.join(path_from_manifest(&file.path)?); + let parent = destination.parent().ok_or_else(|| { + RunnerError::Validation(format!("required file {:?} has no parent", file.path)) + })?; + fs::create_dir_all(parent).map_err(|source| RunnerError::Io { + operation: "create corpus directory", + path: parent.to_path_buf(), + source, + })?; + let mut destination_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&destination) + .map_err(|source| RunnerError::Io { + operation: "create extracted corpus file", + path: destination.clone(), + source, + })?; + std::io::copy(&mut entry, &mut destination_file).map_err(|source| RunnerError::Io { + operation: "extract corpus file", + path: destination, + source, + })?; + } + verify_corpus(output, required) +} + +/// Verify hashes and require the extracted tree to contain no extra files. +pub fn verify_corpus(root: &Path, required: &[CorpusFile]) -> Result<(), RunnerError> { + let mut observed = BTreeSet::new(); + collect_files(root, root, &mut observed)?; + let expected = required + .iter() + .map(|file| file.path.as_str()) + .collect::>(); + let observed_refs = observed.iter().map(String::as_str).collect::>(); + if observed_refs != expected { + let missing = expected + .difference(&observed_refs) + .copied() + .collect::>(); + let extra = observed_refs + .difference(&expected) + .copied() + .collect::>(); + return validation(format!( + "extracted inventory mismatch; missing {missing:?}, extra {extra:?}" + )); + } + + for file in required { + let path = root.join(path_from_manifest(&file.path)?); + let observed_hash = sha256_file(&path)?; + if observed_hash != file.sha256 { + return validation(format!( + "{} SHA-256 is {observed_hash}, expected {}", + file.path, file.sha256 + )); + } + } + Ok(()) +} + +fn ensure_empty_directory(path: &Path) -> Result<(), RunnerError> { + let mut entries = fs::read_dir(path).map_err(|source| RunnerError::Io { + operation: "read extraction directory", + path: path.to_path_buf(), + source, + })?; + if entries + .next() + .transpose() + .map_err(|source| RunnerError::Io { + operation: "read extraction directory entry", + path: path.to_path_buf(), + source, + })? + .is_some() + { + return validation(format!( + "extraction directory {} is not empty", + path.display() + )); + } + Ok(()) +} + +fn validate_entry_name( + entry: &zip::read::ZipFile<'_, R>, +) -> Result { + let name = entry.name(); + if name.contains('\\') || entry.enclosed_name().is_none() { + return validation(format!("archive entry {name:?} has an unsafe path")); + } + let normalized = name.strip_suffix('/').unwrap_or(name); + if normalized.is_empty() { + return validation("archive contains an empty path"); + } + let path = Path::new(normalized); + if path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return validation(format!("archive entry {name:?} has an unsafe path")); + } + manifest_path(path) +} + +fn path_from_manifest(path: &str) -> Result { + if path.is_empty() || path.contains('\\') { + return validation(format!("manifest path {path:?} is invalid")); + } + let parsed = Path::new(path); + if parsed + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return validation(format!("manifest path {path:?} is invalid")); + } + Ok(parsed.to_path_buf()) +} + +fn collect_files( + root: &Path, + directory: &Path, + files: &mut BTreeSet, +) -> Result<(), RunnerError> { + let entries = fs::read_dir(directory).map_err(|source| RunnerError::Io { + operation: "read extracted corpus directory", + path: directory.to_path_buf(), + source, + })?; + for entry in entries { + let entry = entry.map_err(|source| RunnerError::Io { + operation: "read extracted corpus entry", + path: directory.to_path_buf(), + source, + })?; + let path = entry.path(); + let metadata = fs::symlink_metadata(&path).map_err(|source| RunnerError::Io { + operation: "inspect extracted corpus entry", + path: path.clone(), + source, + })?; + if metadata.file_type().is_symlink() { + return validation(format!("extracted entry {} is a symlink", path.display())); + } + if metadata.is_dir() { + collect_files(root, &path, files)?; + } else if metadata.is_file() { + let relative = path.strip_prefix(root).map_err(|_| { + RunnerError::Validation("extracted file escaped corpus root".to_string()) + })?; + files.insert(manifest_path(relative)?); + } else { + return validation(format!( + "extracted entry {} is not a regular file", + path.display() + )); + } + } + Ok(()) +} + +fn manifest_path(path: &Path) -> Result { + path.components() + .map(|component| match component { + Component::Normal(value) => value + .to_str() + .map(str::to_string) + .ok_or_else(|| RunnerError::Validation("corpus path is not UTF-8".to_string())), + _ => validation("corpus path is not relative and normalized"), + }) + .collect::, _>>() + .map(|components| components.join("/")) +} + +pub(super) fn sha256_file(path: &Path) -> Result { + let mut file = File::open(path).map_err(|source| RunnerError::Io { + operation: "open corpus file", + path: path.to_path_buf(), + source, + })?; + sha256_reader(&mut file) +} + +fn sha256_reader(reader: &mut impl Read) -> Result { + let mut hasher = Sha256::new(); + let mut buffer = Vec::new(); + buffer + .try_reserve_exact(64 * 1024) + .map_err(|_| RunnerError::Validation("cannot allocate hash buffer".to_string()))?; + buffer.resize(64 * 1024, 0); + loop { + let count = reader.read(&mut buffer).map_err(|source| RunnerError::Io { + operation: "hash corpus data", + path: PathBuf::from(""), + source, + })?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + let digest = hasher.finalize(); + let mut hex = String::new(); + hex.try_reserve_exact(64) + .map_err(|_| RunnerError::Validation("cannot allocate SHA-256 text".to_string()))?; + for byte in digest { + let _ = write!(hex, "{byte:02x}"); + } + Ok(hex) +} + +fn validation(message: impl Into) -> Result { + Err(RunnerError::Validation(message.into())) +} diff --git a/crates/j2k-t803/src/runner/cache.rs b/crates/j2k-t803/src/runner/cache.rs new file mode 100644 index 00000000..719a955e --- /dev/null +++ b/crates/j2k-t803/src/runner/cache.rs @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ + fs::{self, File}, + path::{Path, PathBuf}, + process::Command, +}; + +use crate::T803Manifest; + +use super::archive::{extract_selected_archive, sha256_file, verify_corpus, ArchiveLimits}; + +const ARCHIVE_NAME: &str = "t803-v3.zip"; + +pub(super) fn load_manifest() -> Result { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .join("corpus/j2k-conformance/t803-v3.toml"); + let text = + fs::read_to_string(&path).map_err(|error| format!("read {}: {error}", path.display()))?; + T803Manifest::parse(&text).map_err(|error| error.to_string()) +} + +pub(super) fn archive_path(cache_dir: &Path) -> PathBuf { + cache_dir.join(ARCHIVE_NAME) +} + +pub(super) fn corpus_path(cache_dir: &Path, manifest: &T803Manifest) -> PathBuf { + cache_dir.join(format!("corpus-{}", manifest.source.archive_sha256)) +} + +pub(super) fn verify_cached(cache_dir: &Path) -> Result<(T803Manifest, PathBuf), String> { + let manifest = load_manifest()?; + let archive = archive_path(cache_dir); + if !archive.is_file() { + return Err(format!( + "pinned T.803 archive is absent at {}; run `cargo xtask t803 fetch`", + archive.display() + )); + } + verify_archive(&archive, &manifest)?; + let corpus = corpus_path(cache_dir, &manifest); + if !corpus.is_dir() { + return Err(format!( + "verified T.803 corpus is absent at {}; run `cargo xtask t803 fetch`", + corpus.display() + )); + } + verify_corpus(&corpus, &manifest.files).map_err(|error| error.to_string())?; + Ok((manifest, corpus)) +} + +pub(super) fn fetch(cache_dir: &Path) -> Result<(), String> { + let manifest = load_manifest()?; + fs::create_dir_all(cache_dir) + .map_err(|error| format!("create {}: {error}", cache_dir.display()))?; + let archive = archive_path(cache_dir); + if archive.exists() { + verify_archive(&archive, &manifest)?; + } else { + download_archive(cache_dir, &archive, &manifest)?; + } + + let corpus = corpus_path(cache_dir, &manifest); + if corpus.exists() { + verify_corpus(&corpus, &manifest.files).map_err(|error| error.to_string())?; + return Ok(()); + } + let staging = cache_dir.join(format!( + ".corpus-{}-extracting-{}", + manifest.source.archive_sha256, + std::process::id() + )); + if staging.exists() { + return Err(format!( + "stale T.803 extraction directory exists at {}", + staging.display() + )); + } + fs::create_dir(&staging).map_err(|error| format!("create {}: {error}", staging.display()))?; + let extraction = File::open(&archive) + .map_err(|error| format!("open {}: {error}", archive.display())) + .and_then(|file| { + extract_selected_archive(file, &staging, &manifest.files, ArchiveLimits::default()) + .map_err(|error| error.to_string()) + }); + if let Err(error) = extraction { + let _ = fs::remove_dir_all(&staging); + return Err(error); + } + fs::rename(&staging, &corpus).map_err(|error| { + format!( + "publish extracted corpus {} as {}: {error}", + staging.display(), + corpus.display() + ) + })?; + verify_corpus(&corpus, &manifest.files).map_err(|error| error.to_string()) +} + +fn verify_archive(path: &Path, manifest: &T803Manifest) -> Result<(), String> { + let metadata = + fs::metadata(path).map_err(|error| format!("inspect {}: {error}", path.display()))?; + if !metadata.is_file() { + return Err(format!( + "T.803 archive {} is not a regular file", + path.display() + )); + } + if metadata.len() != manifest.source.archive_bytes { + return Err(format!( + "T.803 archive size is {}, expected {}", + metadata.len(), + manifest.source.archive_bytes + )); + } + let observed = sha256_file(path).map_err(|error| error.to_string())?; + if observed != manifest.source.archive_sha256 { + return Err(format!( + "T.803 archive SHA-256 is {observed}, expected {}", + manifest.source.archive_sha256 + )); + } + Ok(()) +} + +fn download_archive( + cache_dir: &Path, + destination: &Path, + manifest: &T803Manifest, +) -> Result<(), String> { + let partial = cache_dir.join(format!(".{ARCHIVE_NAME}.download-{}", std::process::id())); + let headers = cache_dir.join(format!(".{ARCHIVE_NAME}.headers-{}", std::process::id())); + for path in [&partial, &headers] { + if path.exists() { + return Err(format!( + "stale T.803 download file exists at {}", + path.display() + )); + } + } + let output = Command::new("curl") + .args([ + "--fail", + "--location", + "--silent", + "--show-error", + "--proto", + "=https", + "--max-redirs", + "5", + "--dump-header", + ]) + .arg(&headers) + .arg("--output") + .arg(&partial) + .arg("--write-out") + .arg("%{url_effective}") + .arg(&manifest.source.url) + .output() + .map_err(|error| format!("start curl for official T.803 attachment: {error}"))?; + if !output.status.success() { + cleanup_download(&partial, &headers); + return Err(format!( + "official T.803 attachment download failed with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let effective_url = String::from_utf8(output.stdout) + .map_err(|error| format!("curl returned a non-UTF-8 effective URL: {error}"))?; + let header_text = fs::read_to_string(&headers) + .map_err(|error| format!("read {}: {error}", headers.display()))?; + if let Err(error) = validate_redirects(&header_text, effective_url.trim()) { + cleanup_download(&partial, &headers); + return Err(error); + } + if let Err(error) = verify_archive(&partial, manifest) { + cleanup_download(&partial, &headers); + return Err(error); + } + fs::rename(&partial, destination).map_err(|error| { + format!( + "publish verified T.803 archive {} as {}: {error}", + partial.display(), + destination.display() + ) + })?; + let _ = fs::remove_file(headers); + Ok(()) +} + +fn validate_redirects(headers: &str, effective_url: &str) -> Result<(), String> { + if !approved_itu_https_url(effective_url) { + return Err(format!( + "T.803 download ended outside approved ITU domains: {effective_url:?}" + )); + } + for line in headers.lines() { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + if name.eq_ignore_ascii_case("location") { + let location = value.trim(); + if is_absolute_or_network_url(location) && !approved_itu_redirect(location) { + return Err(format!( + "T.803 download redirects outside approved ITU domains: {location:?}" + )); + } + } + } + Ok(()) +} + +fn approved_itu_https_url(url: &str) -> bool { + let Some(authority_and_path) = url.strip_prefix("https://") else { + return false; + }; + approved_itu_authority(authority_and_path) +} + +fn approved_itu_redirect(url: &str) -> bool { + url.strip_prefix("https://") + .or_else(|| url.strip_prefix("//")) + .is_some_and(approved_itu_authority) +} + +fn approved_itu_authority(authority_and_path: &str) -> bool { + let authority = authority_and_path.split('/').next().unwrap_or_default(); + matches!(authority, "handle.itu.int" | "www.itu.int") +} + +fn is_absolute_or_network_url(url: &str) -> bool { + url.contains("://") || url.starts_with("//") +} + +fn cleanup_download(partial: &Path, headers: &Path) { + let _ = fs::remove_file(partial); + let _ = fs::remove_file(headers); +} + +#[cfg(test)] +mod tests { + use super::validate_redirects; + + #[test] + fn redirect_validation_accepts_only_itu_https_targets() { + validate_redirects( + "HTTP/2 302\r\nlocation: //www.itu.int/rec/T-REC-T.803\r\n", + "https://www.itu.int/rec/T-REC-T.803", + ) + .expect("approved protocol-relative redirect"); + validate_redirects( + "HTTP/2 200\r\n", + "https://www.itu.int/wftp3/public/t/testsignal/SpeImage/T803/v2024_02/T.803v3_15444-4ed4-ElecAtt-codestreams.zip", + ) + .expect("official test-signal attachment URL"); + + for target in [ + "http://www.itu.int/attachment.zip", + "https://www.itu.int.example/attachment.zip", + "//example.test/attachment.zip", + ] { + let headers = format!("HTTP/2 302\r\nlocation: {target}\r\n"); + let error = validate_redirects(&headers, "https://www.itu.int/final") + .expect_err("unapproved redirect must fail"); + assert!(error.contains("outside approved ITU domains")); + } + } +} diff --git a/crates/j2k-t803/src/runner/cases.rs b/crates/j2k-t803/src/runner/cases.rs new file mode 100644 index 00000000..cd3b22cf --- /dev/null +++ b/crates/j2k-t803/src/runner/cases.rs @@ -0,0 +1,881 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{fs, path::Path, sync::Arc}; + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos"), + test +))] +use j2k::{BatchGroupInfo, BatchLayout, DecodeRequest, NativeSampleType}; +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos") +))] +use j2k::{BatchItemError, PreparationDepth, PreparedBatch}; +use j2k::{J2kDecodedNativeComponents, J2kDecoder, J2kNativeComponentPlane, J2kSrgb8Layout}; +use j2k_codec_math::mct; +use j2k_core::Colorspace; +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos"), + test +))] +use j2k_core::Downscale; + +use crate::{ + compare_peak_samples, compare_samples, normalize_component, parse_pgx, CaseReport, CaseStatus, + Component, DecoderCase, ErrorBounds, ExecutionLocation, Jp2Case, NormalizationTarget, + RouteKind, RouteStage, RouteStageName, T803Manifest, +}; + +#[derive(Clone, Debug)] +pub(super) struct RouteEvidence { + pub(super) kind: RouteKind, + pub(super) stages: Vec, +} + +#[derive(Debug)] +pub(super) struct DecodedPlane { + pub(super) dimensions: (u32, u32), + pub(super) bit_depth: u8, + pub(super) signed: bool, + pub(super) sampling: (u8, u8), + pub(super) samples: Vec, +} + +#[derive(Debug)] +pub(super) struct DecodedImage { + pub(super) dimensions: (u32, u32), + pub(super) component_transform: Option, + pub(super) planes: Vec, + pub(super) route: RouteEvidence, +} + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos"), + test +))] +pub(super) fn decoded_interleaved( + info: &BatchGroupInfo, + bytes: &[u8], + component_transform: Option, + route: RouteEvidence, +) -> Result { + if info.layout != BatchLayout::Nhwc { + return Err("T.803 adapter output must use NHWC layout".to_string()); + } + let pixel_count = (info.dimensions.0 as usize) + .checked_mul(info.dimensions.1 as usize) + .ok_or_else(|| "T.803 adapter output dimensions overflow".to_string())?; + let channels = info.color.channels(); + let bytes_per_sample = match info.sample_type { + NativeSampleType::U8 => 1, + NativeSampleType::U16 | NativeSampleType::I16 => 2, + _ => return Err("T.803 adapter output uses an unsupported sample type".to_string()), + }; + let bytes_per_pixel = channels + .checked_mul(bytes_per_sample) + .ok_or_else(|| "T.803 adapter pixel size overflows".to_string())?; + let expected_len = pixel_count + .checked_mul(bytes_per_pixel) + .ok_or_else(|| "T.803 adapter output length overflows".to_string())?; + if bytes.len() != expected_len { + return Err(format!( + "T.803 adapter output length is {}, expected {expected_len}", + bytes.len() + )); + } + + let mut component_samples = Vec::new(); + component_samples + .try_reserve_exact(channels) + .map_err(|_| "cannot allocate T.803 adapter component owners".to_string())?; + for _ in 0..channels { + let mut samples = Vec::new(); + samples + .try_reserve_exact(pixel_count) + .map_err(|_| "cannot allocate T.803 adapter component samples".to_string())?; + component_samples.push(samples); + } + for pixel in bytes.chunks_exact(bytes_per_pixel) { + for (channel, samples) in component_samples.iter_mut().enumerate() { + let start = channel * bytes_per_sample; + let sample = match info.sample_type { + NativeSampleType::U8 => i64::from(pixel[start]), + NativeSampleType::U16 => { + i64::from(u16::from_ne_bytes([pixel[start], pixel[start + 1]])) + } + NativeSampleType::I16 => { + i64::from(i16::from_ne_bytes([pixel[start], pixel[start + 1]])) + } + _ => unreachable!("sample type was validated above"), + }; + samples.push(sample); + } + } + let planes = component_samples + .into_iter() + .map(|samples| DecodedPlane { + dimensions: info.dimensions, + bit_depth: info.precision, + signed: info.signed, + sampling: (1, 1), + samples, + }) + .collect(); + Ok(DecodedImage { + dimensions: info.dimensions, + component_transform, + planes, + route, + }) +} + +pub(super) fn codestream_component_transform(input: &[u8]) -> Result, String> { + let payload = j2k::extract_j2k_codestream_payload(input).map_err(|error| error.to_string())?; + let header = j2k_native::inspect_j2k_codestream_header(payload.codestream()) + .map_err(|error| error.to_string())?; + Ok(header.has_mct.then_some(if header.reversible { + Colorspace::Rct + } else { + Colorspace::Ict + })) +} + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos"), + test +))] +pub(super) const fn reduction_request(reduction_levels: u8) -> Option { + let scale = match reduction_levels { + 0 => return Some(DecodeRequest::Full), + 1 => Downscale::Half, + 2 => Downscale::Quarter, + 3 => Downscale::Eighth, + _ => return None, + }; + Some(DecodeRequest::Reduced { scale }) +} + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos") +))] +pub(super) fn prepared_requires_cpu(prepared: &PreparedBatch) -> Result { + if !prepared.errors().is_empty() { + if prepared.groups().is_empty() + && prepared.errors().iter().all(|error| { + matches!( + error.source, + BatchItemError::NonRepresentableBatchOutput { .. } + ) + }) + { + return Ok(true); + } + return Err(prepared + .errors() + .iter() + .map(ToString::to_string) + .collect::>() + .join("; ")); + } + let [group] = prepared.groups() else { + return Err(format!( + "T.803 adapter preparation produced {} groups for one input", + prepared.groups().len() + )); + }; + let [image] = group.images() else { + return Err(format!( + "T.803 adapter preparation retained {} images for one input", + group.images().len() + )); + }; + Ok(image.preparation_depth() == PreparationDepth::MetadataOnly) +} + +#[derive(Debug)] +pub(super) struct DecodeFailure { + message: String, + route: RouteEvidence, +} + +impl DecodeFailure { + pub(super) fn new(message: impl Into, route: RouteEvidence) -> Self { + Self { + message: message.into(), + route, + } + } +} + +pub(super) fn run_decoder_cases( + manifest: &T803Manifest, + corpus: &Path, + mut decode: impl FnMut(Arc<[u8]>, u8) -> Result, +) -> Vec { + let mut reports = Vec::new(); + let mut start = 0; + while start < manifest.decoder_cases.len() { + let first = &manifest.decoder_cases[start]; + let mut end = start + 1; + while end < manifest.decoder_cases.len() + && manifest.decoder_cases[end].codestream == first.codestream + && manifest.decoder_cases[end].reduction_levels == first.reduction_levels + { + end += 1; + } + let input_path = corpus.join(&first.codestream); + let decoded = fs::read(&input_path) + .map_err(|error| { + DecodeFailure::new( + format!("read {}: {error}", input_path.display()), + cpu_route(false), + ) + }) + .and_then(|input| decode(Arc::from(input), first.reduction_levels)); + for case in &manifest.decoder_cases[start..end] { + reports.push(match &decoded { + Ok(decoded) => { + compare_decoder_case(case, decoded, corpus).unwrap_or_else(|error| { + error_report(case, Some(case.mse), error, decoded.route.clone()) + }) + } + Err(error) => error_report( + case, + Some(case.mse), + error.message.clone(), + error.route.clone(), + ), + }); + } + start = end; + } + reports +} + +pub(super) fn decode_cpu( + input: &[u8], + reduction_levels: u8, +) -> Result { + let component_transform = codestream_component_transform(input) + .map_err(|error| DecodeFailure::new(error, cpu_route(false)))?; + let mut iut = J2kDecoder::new(input) + .map_err(|error| DecodeFailure::new(error.to_string(), cpu_route(false)))?; + let native = iut + .decode_native_components_at_reduction(reduction_levels) + .map_err(|error| { + DecodeFailure::new(error.to_string(), cpu_route(component_transform.is_some())) + })?; + decoded_native_components(&native, component_transform) + .map_err(|error| DecodeFailure::new(error, cpu_route(component_transform.is_some()))) +} + +fn decoded_native_components( + decoded: &J2kDecodedNativeComponents, + component_transform: Option, +) -> Result { + let dimensions = decoded.dimensions(); + let mut planes = Vec::new(); + planes + .try_reserve_exact(decoded.planes().len()) + .map_err(|_| "cannot allocate T.803 decoded component owners".to_string())?; + for plane in decoded.planes() { + planes.push(DecodedPlane { + dimensions: plane.dimensions(), + bit_depth: plane.bit_depth(), + signed: plane.signed(), + sampling: plane.sampling(), + samples: unpack_native_plane(plane)?, + }); + } + Ok(DecodedImage { + dimensions, + component_transform, + planes, + route: cpu_route(component_transform.is_some()), + }) +} + +fn compare_decoder_case( + case: &DecoderCase, + decoded: &DecodedImage, + corpus: &Path, +) -> Result { + let plane = decoded + .planes + .get(case.component) + .ok_or_else(|| format!("decoded output has no component {}", case.component))?; + let decoded_samples = if let (true, Some(component_transform)) = ( + matches!(case.table.as_str(), "C.1" | "C.4"), + decoded.component_transform, + ) { + let planes = decoded.planes.get(..3).ok_or_else(|| { + "multi-component transform output has fewer than three planes".to_string() + })?; + if planes + .iter() + .any(|candidate| candidate.dimensions != plane.dimensions) + { + return Err("multi-component transform plane dimensions differ".to_string()); + } + forward_first_component( + [&planes[0].samples, &planes[1].samples, &planes[2].samples], + component_transform, + )? + } else { + plane.samples.clone() + }; + let normalized = normalize_component( + Component { + width: plane.dimensions.0, + height: plane.dimensions.1, + bit_depth: plane.bit_depth, + signed: plane.signed, + post_decode_subsampling: post_decode_subsampling(plane, decoded), + samples: &decoded_samples, + }, + NormalizationTarget { + width: case.width, + height: case.height, + bit_depth: case.bit_depth, + signed: case.signed, + }, + ) + .map_err(|error| error.to_string())?; + let reference_path = corpus.join(&case.reference); + let reference_bytes = fs::read(&reference_path) + .map_err(|error| format!("read {}: {error}", reference_path.display()))?; + let reference = parse_pgx(&reference_bytes).map_err(|error| error.to_string())?; + if ( + reference.width, + reference.height, + reference.bit_depth, + reference.signed, + ) != (case.width, case.height, case.bit_depth, case.signed) + { + return Err("PGX metadata does not match the pinned case".to_string()); + } + let comparison = compare_samples( + &reference.samples, + &normalized, + ErrorBounds { + peak: case.peak, + mse: case.mse, + }, + ) + .map_err(|error| error.to_string())?; + Ok(CaseReport { + id: case.id.clone(), + table: case.table.clone(), + status: if comparison.passed { + CaseStatus::Pass + } else { + CaseStatus::Fail + }, + route: decoded.route.kind, + peak: Some(comparison.peak), + mse: Some(comparison.mse), + allowed_peak: case.peak, + allowed_mse: Some(case.mse), + error: None, + stages: decoded.route.stages.clone(), + }) +} + +fn post_decode_subsampling(plane: &DecodedPlane, decoded: &DecodedImage) -> (u8, u8) { + let common_sampling = decoded + .planes + .first() + .map(|plane| plane.sampling) + .filter(|first| { + decoded + .planes + .iter() + .all(|candidate| candidate.sampling == *first) + }) + .unwrap_or((1, 1)); + let output_sampling = ( + plane.sampling.0 / common_sampling.0, + plane.sampling.1 / common_sampling.1, + ); + let native_dimensions = ( + decoded.dimensions.0.div_ceil(u32::from(output_sampling.0)), + decoded.dimensions.1.div_ceil(u32::from(output_sampling.1)), + ); + ( + if plane.dimensions.0 == decoded.dimensions.0 && plane.dimensions.0 != native_dimensions.0 { + output_sampling.0 + } else { + 1 + }, + if plane.dimensions.1 == decoded.dimensions.1 && plane.dimensions.1 != native_dimensions.1 { + output_sampling.1 + } else { + 1 + }, + ) +} + +pub(super) fn run_jp2_cases(manifest: &T803Manifest, corpus: &Path) -> Vec { + manifest + .jp2_cases + .iter() + .map(|case| { + compare_jp2_case(case, corpus) + .unwrap_or_else(|(error, route)| error_report_jp2(case, error, route)) + }) + .collect() +} + +fn compare_jp2_case(case: &Jp2Case, corpus: &Path) -> Result { + let route = cpu_route(false); + let compare = || -> Result { + let input_path = corpus.join(&case.input); + let input = fs::read(&input_path) + .map_err(|error| format!("read {}: {error}", input_path.display()))?; + let component_transform = codestream_component_transform(&input)?; + let support = J2kDecoder::inspect_support(&input).map_err(|error| error.to_string())?; + if support.component_count() != u16::from(case.components) { + return Err(format!( + "codestream has {} components, expected {}", + support.component_count(), + case.components + )); + } + let mut iut = J2kDecoder::new(&input).map_err(|error| error.to_string())?; + let case_route = cpu_route(component_transform.is_some()); + let normalized = iut.decode_srgb8().map_err(|error| error.to_string())?; + if normalized.dimensions() != (case.width, case.height) { + return Err(format!( + "decoded dimensions are {:?}, expected {}x{}", + normalized.dimensions(), + case.width, + case.height + )); + } + let reference_path = corpus.join(&case.reference); + let reference = image::open(&reference_path) + .map_err(|error| format!("read {}: {error}", reference_path.display()))? + .into_rgb8(); + if reference.dimensions() != (case.width, case.height) { + return Err("TIFF dimensions do not match the pinned case".to_string()); + } + let reference = reference.into_raw(); + let (expected_samples, actual_samples) = match normalized.layout() { + J2kSrgb8Layout::Gray => { + let mut gray = Vec::new(); + gray.try_reserve_exact(reference.len() / 3) + .map_err(|_| "cannot allocate Annex G grayscale reference".to_string())?; + for pixel in reference.chunks_exact(3) { + if pixel[0] != pixel[1] || pixel[0] != pixel[2] { + return Err( + "grayscale TIFF reference contains non-neutral pixels".to_string() + ); + } + gray.push(i64::from(pixel[0])); + } + let actual_samples = normalized + .data() + .iter() + .map(|&sample| i64::from(sample)) + .collect(); + (gray, actual_samples) + } + J2kSrgb8Layout::Rgb => ( + reference + .iter() + .map(|&sample| i64::from(sample)) + .collect::>(), + normalized + .data() + .iter() + .map(|&sample| i64::from(sample)) + .collect::>(), + ), + J2kSrgb8Layout::Rgba => { + return Err("Annex G case unexpectedly produced alpha".to_string()); + } + _ => return Err("Annex G case produced an unknown sRGB8 layout".to_string()), + }; + let comparison = compare_peak_samples(&expected_samples, &actual_samples, case.peak) + .map_err(|error| error.to_string())?; + Ok(CaseReport { + id: case.id.clone(), + table: "G.1".to_string(), + status: if comparison.passed { + CaseStatus::Pass + } else { + CaseStatus::Fail + }, + route: case_route.kind, + peak: Some(comparison.peak), + mse: None, + allowed_peak: case.peak, + allowed_mse: None, + error: None, + stages: case_route.stages, + }) + }; + compare().map_err(|error| (error, route)) +} + +pub(super) fn unpack_native_plane(plane: &J2kNativeComponentPlane) -> Result, String> { + let bytes_per_sample = usize::from(plane.bytes_per_sample()); + let mut samples = Vec::new(); + samples + .try_reserve_exact(plane.data().len() / bytes_per_sample) + .map_err(|_| "cannot allocate T.803 native component samples".to_string())?; + for bytes in plane.data().chunks_exact(bytes_per_sample) { + let value = match (plane.signed(), bytes) { + (false, [value]) => i64::from(*value), + (true, [value]) => i64::from(i8::from_le_bytes([*value])), + (false, [a, b]) => i64::from(u16::from_le_bytes([*a, *b])), + (true, [a, b]) => i64::from(i16::from_le_bytes([*a, *b])), + (false, [a, b, c, d]) => i64::from(u32::from_le_bytes([*a, *b, *c, *d])), + (true, [a, b, c, d]) => i64::from(i32::from_le_bytes([*a, *b, *c, *d])), + _ => return Err("unsupported native component storage width".to_string()), + }; + samples.push(value); + } + if samples.len() + != (plane.dimensions().0 as usize) + .checked_mul(plane.dimensions().1 as usize) + .ok_or_else(|| "component dimensions overflow".to_string())? + { + return Err("native component storage length does not match dimensions".to_string()); + } + Ok(samples) +} + +fn forward_first_component( + planes: [&[i64]; 3], + colorspace: Colorspace, +) -> Result, String> { + let [red, green, blue] = planes; + if red.len() != green.len() || green.len() != blue.len() { + return Err("multi-component transform plane dimensions differ".to_string()); + } + let mut output = Vec::new(); + output + .try_reserve_exact(red.len()) + .map_err(|_| "cannot allocate T.803 component transform output".to_string())?; + match colorspace { + Colorspace::Rct => { + for ((&red, &green), &blue) in red.iter().zip(green).zip(blue) { + let numerator = red + .checked_add(green.checked_mul(2).ok_or_else(mct_overflow)?) + .and_then(|value| value.checked_add(blue)) + .ok_or_else(mct_overflow)?; + output.push(numerator.div_euclid(4)); + } + } + Colorspace::Ict => { + for ((&red, &green), &blue) in red.iter().zip(green).zip(blue) { + #[expect( + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + reason = "T.803 forward ICT intentionally converts bounded integer components into the JPEG 2000 float domain and rounds the finite result back to an integer reference sample" + )] + let rounded = (mct::ICT_FWD_Y_R * red as f32 + + mct::ICT_FWD_Y_G * green as f32 + + mct::ICT_FWD_Y_B * blue as f32) + .round() as i64; + output.push(rounded); + } + } + _ => return Err("component transform metadata is not RCT or ICT".to_string()), + } + Ok(output) +} + +fn mct_overflow() -> String { + "multi-component transform arithmetic overflow".to_string() +} + +fn error_report( + case: &DecoderCase, + allowed_mse: Option, + error: String, + route: RouteEvidence, +) -> CaseReport { + CaseReport { + id: case.id.clone(), + table: case.table.clone(), + status: CaseStatus::Error, + route: route.kind, + peak: None, + mse: None, + allowed_peak: case.peak, + allowed_mse, + error: Some(error), + stages: route.stages, + } +} + +fn error_report_jp2(case: &Jp2Case, error: String, route: RouteEvidence) -> CaseReport { + CaseReport { + id: case.id.clone(), + table: "G.1".to_string(), + status: CaseStatus::Error, + route: route.kind, + peak: None, + mse: None, + allowed_peak: case.peak, + allowed_mse: None, + error: Some(error), + stages: route.stages, + } +} + +pub(super) fn cpu_route(mct: bool) -> RouteEvidence { + route_evidence(ExecutionLocation::Cpu, None, mct) +} + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos") +))] +pub(super) fn device_route(location: ExecutionLocation, mct: bool) -> RouteEvidence { + route_evidence(ExecutionLocation::Cpu, Some(location), mct) +} + +fn route_evidence( + parsing: ExecutionLocation, + device: Option, + mct: bool, +) -> RouteEvidence { + let execution = device.unwrap_or(ExecutionLocation::Cpu); + RouteEvidence { + kind: if device.is_some() { + RouteKind::Hybrid + } else { + RouteKind::Cpu + }, + stages: Vec::from([ + RouteStage { + stage: RouteStageName::Parsing, + location: parsing, + }, + RouteStage { + stage: RouteStageName::Tier1, + location: execution, + }, + RouteStage { + stage: RouteStageName::Dequantization, + location: execution, + }, + RouteStage { + stage: RouteStageName::Idwt, + location: execution, + }, + RouteStage { + stage: RouteStageName::Mct, + location: if mct { + execution + } else { + ExecutionLocation::NotUsed + }, + }, + RouteStage { + stage: RouteStageName::ColorOutput, + location: execution, + }, + RouteStage { + stage: RouteStageName::HostToDevice, + location: device.unwrap_or(ExecutionLocation::NotUsed), + }, + RouteStage { + stage: RouteStageName::DeviceToHost, + location: device.unwrap_or(ExecutionLocation::NotUsed), + }, + ]), + } +} + +#[cfg(test)] +mod tests { + use j2k::{ + BatchAlpha, BatchCodecRoute, BatchColor, BatchGroupInfo, BatchLayout, + BatchWaveletTransform, DecodeRequest, NativeSampleType, + }; + use j2k_core::{Colorspace, CompressedPayloadKind, CompressedTransferSyntax, Downscale}; + + use super::{ + codestream_component_transform, decoded_interleaved, forward_first_component, + reduction_request, RouteEvidence, + }; + use crate::{ExecutionLocation, RouteKind, RouteStage, RouteStageName}; + use j2k::J2kDecoder; + use j2k_test_support::{minimal_j2k_codestream, wrap_jp2_codestream}; + + fn route() -> RouteEvidence { + let stages = Vec::from([RouteStage { + stage: RouteStageName::Parsing, + location: ExecutionLocation::Cpu, + }]); + RouteEvidence { + kind: RouteKind::Hybrid, + stages, + } + } + + fn info( + color: BatchColor, + sample_type: NativeSampleType, + precision: u8, + signed: bool, + ) -> BatchGroupInfo { + BatchGroupInfo { + dimensions: (2, 1), + color, + alpha: if color == BatchColor::Rgba { + BatchAlpha::Straight + } else { + BatchAlpha::None + }, + precision, + signed, + sample_type, + layout: BatchLayout::Nhwc, + colorspace: Colorspace::Rgb, + route: BatchCodecRoute::Classic, + transform: BatchWaveletTransform::Reversible53, + transfer_syntax: CompressedTransferSyntax::Jpeg2000Lossless, + payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + } + } + + #[test] + fn interleaved_native_bytes_are_split_into_component_planes() { + let decoded = decoded_interleaved( + &info(BatchColor::Rgb, NativeSampleType::U8, 8, false), + &[1, 2, 3, 4, 5, 6], + None, + route(), + ) + .expect("decode interleaved RGB bytes"); + + assert_eq!(decoded.planes[0].samples, [1, 4]); + assert_eq!(decoded.planes[1].samples, [2, 5]); + assert_eq!(decoded.planes[2].samples, [3, 6]); + } + + #[test] + fn interleaved_signed_samples_preserve_native_endianness() { + let bytes = [-2048_i16, 2047] + .into_iter() + .flat_map(i16::to_ne_bytes) + .collect::>(); + let decoded = decoded_interleaved( + &info(BatchColor::Gray, NativeSampleType::I16, 12, true), + &bytes, + None, + route(), + ) + .expect("decode interleaved signed bytes"); + + assert_eq!(decoded.planes[0].samples, [-2048, 2047]); + } + + #[test] + fn interleaved_decode_rejects_layout_or_length_mismatch() { + let mut planar = info(BatchColor::Rgb, NativeSampleType::U8, 8, false); + planar.layout = BatchLayout::Nchw; + assert!(decoded_interleaved(&planar, &[0; 6], None, route()) + .expect_err("planar input must be rejected") + .contains("NHWC")); + assert!(decoded_interleaved( + &info(BatchColor::Rgb, NativeSampleType::U8, 8, false), + &[0; 5], + None, + route(), + ) + .expect_err("short input must be rejected") + .contains("length")); + } + + #[test] + fn adapter_reduction_uses_only_the_public_downscale_range() { + assert_eq!(reduction_request(0), Some(DecodeRequest::Full)); + assert_eq!( + reduction_request(1), + Some(DecodeRequest::Reduced { + scale: Downscale::Half, + }) + ); + assert_eq!( + reduction_request(3), + Some(DecodeRequest::Reduced { + scale: Downscale::Eighth, + }) + ); + assert_eq!(reduction_request(4), None); + } + + #[test] + fn forward_mct_recovers_the_first_codestream_component() { + let red = [100, 3]; + let green = [150, 4]; + let blue = [200, 8]; + + assert_eq!( + forward_first_component([&red, &green, &blue], Colorspace::Rct).expect("forward RCT"), + [150, 4] + ); + assert_eq!( + forward_first_component([&red, &green, &blue], Colorspace::Ict).expect("forward ICT"), + [141, 4] + ); + } + + #[test] + fn codestream_transform_detection_is_independent_of_color_space_inference() { + let mut codestream = minimal_j2k_codestream(); + let siz = codestream + .windows(2) + .position(|marker| marker == [0xff, 0x51]) + .expect("SIZ marker"); + let cod = codestream + .windows(2) + .position(|marker| marker == [0xff, 0x52]) + .expect("COD marker"); + let extra_components = (3..257).flat_map(|_| [0x07, 0x01, 0x01]); + codestream.splice(cod..cod, extra_components); + codestream[siz + 38..siz + 40].copy_from_slice(&257_u16.to_be_bytes()); + let siz_length = u16::from_be_bytes([codestream[siz + 2], codestream[siz + 3]]) + .checked_add(254 * 3) + .expect("expanded SIZ length"); + codestream[siz + 2..siz + 4].copy_from_slice(&siz_length.to_be_bytes()); + + assert_eq!( + J2kDecoder::inspect(&codestream) + .expect("wide-component inspect") + .colorspace, + Colorspace::IccTagged + ); + assert_eq!( + codestream_component_transform(&codestream).expect("COD transform"), + Some(Colorspace::Rct) + ); + let jp2 = wrap_jp2_codestream(&codestream, 128, 64, 257, 8, 16); + assert_eq!( + codestream_component_transform(&jp2).expect("wrapped COD transform"), + Some(Colorspace::Rct) + ); + } + + #[test] + fn forward_mct_rejects_mismatched_plane_lengths() { + let one = [1]; + let two = [2, 3]; + let three = [4]; + let error = forward_first_component([&one, &two, &three], Colorspace::Rct) + .expect_err("mismatched planes must fail"); + assert!(error.contains("dimensions")); + } +} diff --git a/crates/j2k-t803/src/runner/cli.rs b/crates/j2k-t803/src/runner/cli.rs new file mode 100644 index 00000000..8d0d2dfb --- /dev/null +++ b/crates/j2k-t803/src/runner/cli.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::path::PathBuf; + +#[cfg(feature = "cuda-runner")] +use super::cuda; +#[cfg(feature = "metal-runner")] +use super::metal; +use super::{cache, cpu, evidence}; + +const DEFAULT_CACHE_DIR: &str = "target/t803"; + +pub fn run_cli(args: impl IntoIterator) -> Result<(), String> { + let mut args = args.into_iter(); + let command = args.next().ok_or_else(usage)?; + let options = parse_options(args)?; + match command.as_str() { + "fetch" => cache::fetch(&options.cache_dir), + "run" => { + let iut = options + .iut + .as_deref() + .ok_or_else(|| "t803 run requires --iut cpu|cuda|metal".to_string())?; + match iut { + "cpu" => cpu::run(&options.cache_dir, options.output_dir, options.development), + "cuda" => run_cuda(&options), + "metal" => run_metal(&options), + _ => Err(format!("unknown T.803 IUT {iut:?}")), + } + } + "verify" => { + evidence::verify_reports( + &options.cache_dir, + &options.reports, + options.candidate_sha.as_deref(), + evidence::EvidenceScope::parse(options.scope.as_deref().ok_or_else(|| { + "t803 verify requires --scope cpu|cuda|metal|all".to_string() + })?)?, + ) + } + "help" | "-h" | "--help" => Err(usage()), + other => Err(format!("unknown T.803 command {other:?}\n{}", usage())), + } +} + +#[cfg(feature = "cuda-runner")] +fn run_cuda(options: &Options) -> Result<(), String> { + cuda::run( + &options.cache_dir, + options.output_dir.clone(), + options.development, + ) +} + +#[cfg(not(feature = "cuda-runner"))] +fn run_cuda(_options: &Options) -> Result<(), String> { + Err("cuda T.803 adapter runner is not available in this build".to_string()) +} + +#[cfg(feature = "metal-runner")] +fn run_metal(options: &Options) -> Result<(), String> { + metal::run( + &options.cache_dir, + options.output_dir.clone(), + options.development, + ) +} + +#[cfg(not(feature = "metal-runner"))] +fn run_metal(_options: &Options) -> Result<(), String> { + Err("metal T.803 adapter runner is not available in this build".to_string()) +} + +#[derive(Debug)] +struct Options { + cache_dir: PathBuf, + output_dir: Option, + iut: Option, + development: bool, + reports: Vec, + candidate_sha: Option, + scope: Option, +} + +fn parse_options(args: impl IntoIterator) -> Result { + let mut options = Options { + cache_dir: PathBuf::from(DEFAULT_CACHE_DIR), + output_dir: None, + iut: None, + development: false, + reports: Vec::new(), + candidate_sha: None, + scope: None, + }; + let mut args = args.into_iter(); + while let Some(argument) = args.next() { + match argument.as_str() { + "--cache-dir" => options.cache_dir = PathBuf::from(next_value(&mut args, &argument)?), + "--out-dir" => { + options.output_dir = Some(PathBuf::from(next_value(&mut args, &argument)?)); + } + "--iut" => options.iut = Some(next_value(&mut args, &argument)?), + "--development" => options.development = true, + "--report" => options + .reports + .push(PathBuf::from(next_value(&mut args, &argument)?)), + "--candidate-sha" => options.candidate_sha = Some(next_value(&mut args, &argument)?), + "--scope" => options.scope = Some(next_value(&mut args, &argument)?), + "-h" | "--help" => return Err(usage()), + other => return Err(format!("unknown T.803 argument {other:?}\n{}", usage())), + } + } + Ok(options) +} + +fn next_value(args: &mut impl Iterator, option: &str) -> Result { + args.next() + .ok_or_else(|| format!("{option} requires a value")) +} + +fn usage() -> String { + "usage: cargo xtask t803 fetch [--cache-dir DIR]\n cargo xtask t803 run --iut cpu|cuda|metal [--out-dir DIR] [--development] [--cache-dir DIR]\n cargo xtask t803 verify --scope cpu|cuda|metal|all --candidate-sha SHA --report FILE [--report FILE...] [--cache-dir DIR]".to_string() +} diff --git a/crates/j2k-t803/src/runner/cpu.rs b/crates/j2k-t803/src/runner/cpu.rs new file mode 100644 index 00000000..ef235f84 --- /dev/null +++ b/crates/j2k-t803/src/runner/cpu.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{fs, path::Path, path::PathBuf, process::Command}; + +use crate::PlatformIdentity; + +use super::{cases, encoder, execute}; + +const CPU_CLAIM: &str = + "Profile-1 Cclass-1; Profile-1 Cclass-1HF; Annex G JP2 reader (candidate evidence)"; + +pub(super) fn run( + cache_dir: &Path, + output_dir: Option, + development: bool, +) -> Result<(), String> { + let encoder = encoder::run_cpu()?; + let features = Vec::from(["cpu".to_string()]); + execute::run( + cache_dir, + output_dir, + development, + execute::IutConfig { + name: "j2k", + claim: CPU_CLAIM, + report_stem: "cpu", + features, + platform: cpu_platform(), + }, + encoder, + |input, reduction_levels| cases::decode_cpu(&input, reduction_levels), + ) +} + +fn cpu_platform() -> PlatformIdentity { + PlatformIdentity { + os: std::env::consts::OS.to_string(), + arch: std::env::consts::ARCH.to_string(), + hardware: cpu_hardware(), + driver: "not-applicable".to_string(), + } +} + +fn cpu_hardware() -> String { + if cfg!(target_os = "macos") { + for key in ["machdep.cpu.brand_string", "hw.model"] { + if let Ok(output) = Command::new("sysctl").args(["-n", key]).output() { + if output.status.success() { + let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !value.is_empty() { + return value; + } + } + } + } + } + if cfg!(target_os = "linux") { + if let Ok(cpuinfo) = fs::read_to_string("/proc/cpuinfo") { + if let Some(value) = cpuinfo.lines().find_map(|line| { + line.split_once(':') + .filter(|(key, _)| matches!(key.trim(), "model name" | "Hardware")) + .map(|(_, value)| value.trim()) + }) { + if !value.is_empty() { + return value.to_string(); + } + } + } + } + std::env::var("PROCESSOR_IDENTIFIER").unwrap_or_else(|_| "unknown-cpu".to_string()) +} diff --git a/crates/j2k-t803/src/runner/cuda.rs b/crates/j2k-t803/src/runner/cuda.rs new file mode 100644 index 00000000..bc3377e9 --- /dev/null +++ b/crates/j2k-t803/src/runner/cuda.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{path::Path, path::PathBuf, process::Command, sync::Arc}; + +use j2k::{BatchDecodeOptions, BatchLayout, EncodedImage}; +use j2k_core::SurfaceResidency; +use j2k_cuda::{CudaBatchDecoder, CudaSession, Surface}; +use j2k_cuda_runtime::CudaContext; + +use crate::{ExecutionLocation, PlatformIdentity}; + +use super::{cases, encoder, execute}; + +const CUDA_CLAIM: &str = "Profile-1 Cclass-1 adapter IUT; Profile-1 Cclass-1HF adapter IUT; Annex G JP2 reader via j2k CPU stages (candidate evidence)"; + +pub(super) fn run( + cache_dir: &Path, + output_dir: Option, + development: bool, +) -> Result<(), String> { + let mut iut = CudaIut::new()?; + let platform = iut.platform()?; + let encoder = encoder::run_cuda()?; + execute::run( + cache_dir, + output_dir, + development, + execute::IutConfig { + name: "j2k-cuda", + claim: CUDA_CLAIM, + report_stem: "cuda", + features: Vec::from([ + "adapter-iut".to_string(), + "cuda".to_string(), + "production-batch-decode".to_string(), + ]), + platform, + }, + encoder, + move |input, reduction_levels| iut.decode(&input, reduction_levels), + ) +} + +struct CudaIut { + decoder: CudaBatchDecoder, + device_ordinal: usize, +} + +impl CudaIut { + fn new() -> Result { + let context = CudaContext::system_default().map_err(|error| error.to_string())?; + let device_ordinal = context.device_ordinal(); + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let decoder = + CudaBatchDecoder::with_session_and_options(CudaSession::with_context(context), options); + Ok(Self { + decoder, + device_ordinal, + }) + } + + fn platform(&self) -> Result { + let (name, driver) = nvidia_device_identity(self.device_ordinal)?; + Ok(PlatformIdentity { + os: std::env::consts::OS.to_string(), + arch: std::env::consts::ARCH.to_string(), + hardware: format!("{name} (CUDA device {})", self.device_ordinal), + driver: format!("NVIDIA {driver}"), + }) + } + + fn decode( + &mut self, + input: &Arc<[u8]>, + reduction_levels: u8, + ) -> Result { + let Some(request) = cases::reduction_request(reduction_levels) else { + return cases::decode_cpu(input, reduction_levels); + }; + let component_transform = cases::codestream_component_transform(input) + .map_err(|error| cases::DecodeFailure::new(error, cases::cpu_route(false)))?; + let prepared = self + .decoder + .prepare(Vec::from([EncodedImage::new(Arc::clone(input), request)])) + .map_err(|error| { + cases::DecodeFailure::new(error.to_string(), cases::cpu_route(false)) + })?; + match cases::prepared_requires_cpu(&prepared) { + Ok(true) => return cases::decode_cpu(input, reduction_levels), + Ok(false) => {} + Err(error) => { + return Err(cases::DecodeFailure::new(error, cases::cpu_route(false))); + } + } + let info = prepared.groups()[0].info().clone(); + let route = cases::device_route(ExecutionLocation::Cuda, component_transform.is_some()); + let decoded = self + .decoder + .decode_prepared(&prepared) + .map_err(|error| cases::DecodeFailure::new(error.to_string(), route.clone()))?; + if !decoded.errors().is_empty() || !decoded.group_errors().is_empty() { + let errors = decoded + .errors() + .iter() + .map(ToString::to_string) + .chain(decoded.group_errors().iter().map(ToString::to_string)) + .collect::>() + .join("; "); + return Err(cases::DecodeFailure::new(errors, route)); + } + let [group] = decoded.groups() else { + return Err(cases::DecodeFailure::new( + format!( + "CUDA T.803 adapter produced {} groups for one input", + decoded.groups().len() + ), + route, + )); + }; + let [surface] = group.surfaces() else { + return Err(cases::DecodeFailure::new( + format!( + "CUDA T.803 adapter produced {} NHWC surfaces for one input", + group.surfaces().len() + ), + route, + )); + }; + if surface.residency() != SurfaceResidency::CudaResidentDecode { + return Err(cases::DecodeFailure::new( + format!( + "CUDA T.803 adapter returned unexpected {:?} residency", + surface.residency() + ), + route, + )); + } + let bytes = Surface::download_batch_tight(group.surfaces()) + .map_err(|error| cases::DecodeFailure::new(error.to_string(), route.clone()))?; + cases::decoded_interleaved(&info, &bytes, component_transform, route.clone()) + .map_err(|error| cases::DecodeFailure::new(error, route)) + } +} + +fn nvidia_device_identity(device_ordinal: usize) -> Result<(String, String), String> { + let output = Command::new("nvidia-smi") + .args([ + "--query-gpu=name,driver_version", + "--format=csv,noheader,nounits", + "-i", + &device_ordinal.to_string(), + ]) + .output() + .map_err(|error| format!("start nvidia-smi: {error}"))?; + if !output.status.success() { + return Err(format!("nvidia-smi exited with {}", output.status)); + } + parse_nvidia_identity(&String::from_utf8_lossy(&output.stdout)) +} + +fn parse_nvidia_identity(output: &str) -> Result<(String, String), String> { + let line = output + .lines() + .find(|line| !line.trim().is_empty()) + .ok_or_else(|| "nvidia-smi returned an empty value".to_string())?; + let (name, driver) = line + .split_once(',') + .ok_or_else(|| "nvidia-smi returned malformed device identity".to_string())?; + let name = name.trim(); + let driver = driver.trim(); + if name.is_empty() || driver.is_empty() { + return Err("nvidia-smi returned malformed device identity".to_string()); + } + Ok((name.to_string(), driver.to_string())) +} + +#[cfg(test)] +mod tests { + use super::parse_nvidia_identity; + + #[test] + fn parses_selected_nvidia_device_identity() { + assert_eq!( + parse_nvidia_identity("NVIDIA A100-SXM4-80GB, 580.65.06\n").expect("NVIDIA identity"), + ("NVIDIA A100-SXM4-80GB".to_string(), "580.65.06".to_string()) + ); + assert!(parse_nvidia_identity("malformed").is_err()); + assert!(parse_nvidia_identity(" , 580.65.06").is_err()); + } +} diff --git a/crates/j2k-t803/src/runner/encoder.rs b/crates/j2k-t803/src/runner/encoder.rs new file mode 100644 index 00000000..57196cc2 --- /dev/null +++ b/crates/j2k-t803/src/runner/encoder.rs @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +mod evaluate; +mod input; + +use std::{fs, path::Path}; + +#[cfg(all(feature = "metal-runner", target_os = "macos"))] +use j2k::encode_j2k_lossless_with_accelerator; +use j2k::{ + encode_j2k_lossless, encode_j2k_lossless_components, encode_j2k_lossless_typed_components, + encode_j2k_lossless_with_roi_regions, encode_j2k_lossy, EncodeBackendPreference, + J2kBlockCodingMode, J2kEncodeDispatchReport, J2kEncodeValidation, J2kLosslessComponentPlane, + J2kLosslessComponentSamples, J2kLosslessEncodeOptions, J2kLosslessSamples, + J2kLosslessTypedComponentPlane, J2kLosslessTypedComponentSamples, J2kLossyEncodeOptions, + J2kLossySamples, J2kMarkerSegment, J2kProgressionOrder, J2kQualityLayer, J2kRateTarget, + J2kRoiRegion, ReversibleTransform, +}; +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos") +))] +use j2k::{encode_j2k_lossy_with_accelerator, BackendKind}; +#[cfg(feature = "cuda-runner")] +use j2k_cuda::{CudaEncodeStageAccelerator, CudaLosslessEncoder}; +#[cfg(all(feature = "metal-runner", target_os = "macos"))] +use j2k_metal::MetalEncodeStageAccelerator; +use sha2::{Digest, Sha256}; + +use crate::encoder::{ + ics_path, matrix_path, reference_decoder_identity, EncoderCase, EncoderIcs, EncoderInputKind, + EncoderRateTarget, +}; +use crate::{ + EncoderEvidence, EncoderIut, EncoderMatrix, EncoderReferenceIdentity, ExecutionLocation, +}; + +use self::evaluate::{evaluate_case, generation_error}; +use self::input::{generate_input, GeneratedInput}; + +struct EncoderSources { + matrix: EncoderMatrix, + ics: EncoderIcs, + ics_path: &'static str, + ics_sha256: String, +} + +struct EncodedOutput { + codestream: Vec, + dispatch: J2kEncodeDispatchReport, +} + +pub(super) fn run_cpu() -> Result { + run(EncoderIut::Cpu, None, encode_cpu_case) +} + +#[cfg(feature = "cuda-runner")] +pub(super) fn run_cuda() -> Result { + let mut lossless_encoder = CudaLosslessEncoder::new(); + run( + EncoderIut::Cuda, + Some(ExecutionLocation::Cuda), + move |case, input| encode_cuda_case(&mut lossless_encoder, case, input), + ) +} + +#[cfg(all(feature = "metal-runner", target_os = "macos"))] +pub(super) fn run_metal() -> Result { + run( + EncoderIut::Metal, + Some(ExecutionLocation::Metal), + encode_metal_case, + ) +} + +fn run( + iut: EncoderIut, + device: Option, + mut encode: impl FnMut(&EncoderCase, &GeneratedInput) -> Result, +) -> Result { + let sources = load_sources(iut)?; + let (standard, implementation, expected_version) = reference_decoder_identity(); + let actual_version = j2k_compare::openjpeg::version(); + if actual_version != expected_version { + return Err(format!( + "T.804 OpenJPEG version is {actual_version}, expected {expected_version}" + )); + } + let mut reports = Vec::new(); + reports + .try_reserve_exact(sources.ics.matrix_case_count()) + .map_err(|error| format!("allocate encoder case reports: {error}"))?; + for case in sources.matrix.selected_cases(iut) { + let input = match generate_input(case) { + Ok(input) => input, + Err(error) => { + reports.push(generation_error(case, &error, device)); + continue; + } + }; + reports.push(evaluate_case(case, &input, encode(case, &input), device)); + } + EncoderEvidence::new( + sources.ics_path.to_string(), + sources.ics_sha256, + matrix_path().to_string(), + sources.ics.matrix_case_count(), + sources.ics.matrix_case_sha256().to_string(), + EncoderReferenceIdentity { + standard: standard.to_string(), + implementation: implementation.to_string(), + version: actual_version, + }, + reports, + ) + .map_err(|error| error.to_string()) +} + +fn load_sources(iut: EncoderIut) -> Result { + let ics_path = ics_path(iut); + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .ok_or_else(|| "resolve j2k workspace root".to_string())?; + let matrix_text = fs::read_to_string(root.join(matrix_path())) + .map_err(|error| format!("read {}: {error}", matrix_path()))?; + let ics_bytes = + fs::read(root.join(ics_path)).map_err(|error| format!("read {ics_path}: {error}"))?; + let ics_text = std::str::from_utf8(&ics_bytes) + .map_err(|error| format!("read {ics_path} as UTF-8: {error}"))?; + let matrix = EncoderMatrix::parse(&matrix_text).map_err(|error| error.to_string())?; + let ics = EncoderIcs::parse(ics_text).map_err(|error| error.to_string())?; + if ics.iut != iut { + return Err(format!("{ics_path} identifies the wrong encoder IUT")); + } + ics.validate_against(&matrix) + .map_err(|error| error.to_string())?; + Ok(EncoderSources { + matrix, + ics, + ics_path, + ics_sha256: format!("{:x}", Sha256::digest(ics_bytes)), + }) +} + +fn encode_cpu_case(case: &EncoderCase, input: &GeneratedInput) -> Result { + match case.mode { + crate::EncoderMode::Lossless => encode_cpu_lossless(case, input), + crate::EncoderMode::Lossy => { + let samples = J2kLossySamples::new( + &input.interleaved, + case.width, + case.height, + case.components, + case.bit_depth, + case.signed, + ) + .map_err(|error| error.to_string())?; + let encoded = encode_j2k_lossy( + samples, + &lossy_options(case, EncodeBackendPreference::CpuOnly), + ) + .map_err(|error| error.to_string())?; + Ok(EncodedOutput { + codestream: encoded.codestream, + dispatch: encoded.dispatch_report, + }) + } + } +} + +#[cfg(feature = "cuda-runner")] +fn encode_cuda_case( + lossless_encoder: &mut CudaLosslessEncoder, + case: &EncoderCase, + input: &GeneratedInput, +) -> Result { + match case.mode { + crate::EncoderMode::Lossless => { + let samples = interleaved_lossless_samples(case, input)?; + let options = lossless_options(case, EncodeBackendPreference::Auto); + let encoded = lossless_encoder + .encode(samples, &options) + .map_err(|error| error.to_string())? + .into_encoded(); + Ok(EncodedOutput { + codestream: encoded.codestream, + dispatch: encoded.dispatch_report, + }) + } + crate::EncoderMode::Lossy => { + let samples = interleaved_lossy_samples(case, input)?; + let options = lossy_options(case, EncodeBackendPreference::Auto); + let mut accelerator = CudaEncodeStageAccelerator::for_auto_host_output(); + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &options, + BackendKind::Cuda, + &mut accelerator, + ) + .map_err(|error| error.to_string())?; + Ok(EncodedOutput { + codestream: encoded.codestream, + dispatch: encoded.dispatch_report, + }) + } + } +} + +#[cfg(all(feature = "metal-runner", target_os = "macos"))] +fn encode_metal_case(case: &EncoderCase, input: &GeneratedInput) -> Result { + // This lane tests the Metal adapter IUT itself. The separately benchmarked + // public Auto policy may legitimately keep small matrix cases on the CPU. + let mut accelerator = MetalEncodeStageAccelerator::for_host_output_benchmark(); + match case.mode { + crate::EncoderMode::Lossless => { + let samples = interleaved_lossless_samples(case, input)?; + let options = lossless_options(case, EncodeBackendPreference::Auto); + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .map_err(|error| error.to_string())?; + Ok(EncodedOutput { + codestream: encoded.codestream, + dispatch: encoded.dispatch_report, + }) + } + crate::EncoderMode::Lossy => { + let samples = interleaved_lossy_samples(case, input)?; + let options = lossy_options(case, EncodeBackendPreference::Auto); + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .map_err(|error| error.to_string())?; + Ok(EncodedOutput { + codestream: encoded.codestream, + dispatch: encoded.dispatch_report, + }) + } + } +} + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos") +))] +fn interleaved_lossless_samples<'a>( + case: &EncoderCase, + input: &'a GeneratedInput, +) -> Result, String> { + J2kLosslessSamples::new( + &input.interleaved, + case.width, + case.height, + case.components, + case.bit_depth, + case.signed, + ) + .map_err(|error| error.to_string()) +} + +#[cfg(any( + feature = "cuda-runner", + all(feature = "metal-runner", target_os = "macos") +))] +fn interleaved_lossy_samples<'a>( + case: &EncoderCase, + input: &'a GeneratedInput, +) -> Result, String> { + J2kLossySamples::new( + &input.interleaved, + case.width, + case.height, + case.components, + case.bit_depth, + case.signed, + ) + .map_err(|error| error.to_string()) +} + +fn encode_cpu_lossless( + case: &EncoderCase, + input: &GeneratedInput, +) -> Result { + let options = lossless_options(case, EncodeBackendPreference::CpuOnly); + let encoded = match case.input { + EncoderInputKind::Interleaved => { + let samples = J2kLosslessSamples::new( + &input.interleaved, + case.width, + case.height, + case.components, + case.bit_depth, + case.signed, + ) + .map_err(|error| error.to_string())?; + if let Some(roi) = case.roi { + encode_j2k_lossless_with_roi_regions( + samples, + &options, + &[J2kRoiRegion { + component: roi.component, + x: roi.x, + y: roi.y, + width: roi.width, + height: roi.height, + shift: roi.shift, + }], + ) + } else { + encode_j2k_lossless(samples, &options) + } + } + EncoderInputKind::ComponentPlanes => { + let planes = input + .components + .iter() + .map(|component| J2kLosslessComponentPlane { + data: &component.data, + x_rsiz: component.sampling[0], + y_rsiz: component.sampling[1], + }) + .collect::>(); + let samples = J2kLosslessComponentSamples::new( + &planes, + case.width, + case.height, + case.bit_depth, + case.signed, + ) + .map_err(|error| error.to_string())?; + encode_j2k_lossless_components(samples, &options) + } + EncoderInputKind::TypedComponentPlanes => { + let planes = input + .components + .iter() + .map(|component| J2kLosslessTypedComponentPlane { + data: &component.data, + x_rsiz: component.sampling[0], + y_rsiz: component.sampling[1], + bit_depth: component.bit_depth, + signed: component.signed, + }) + .collect::>(); + let samples = J2kLosslessTypedComponentSamples::new(&planes, case.width, case.height) + .map_err(|error| error.to_string())?; + encode_j2k_lossless_typed_components(samples, &options) + } + } + .map_err(|error| error.to_string())?; + Ok(EncodedOutput { + codestream: encoded.codestream, + dispatch: encoded.dispatch_report, + }) +} + +fn lossless_options( + case: &EncoderCase, + backend: EncodeBackendPreference, +) -> J2kLosslessEncodeOptions { + let mut options = J2kLosslessEncodeOptions::default(); + options.backend = backend; + options.block_coding_mode = J2kBlockCodingMode::Classic; + options.progression = progression(case.progression); + options.max_decomposition_levels = Some(case.decomposition_levels); + options.tile_size = case.tile_size.map(|[width, height]| (width, height)); + options.tile_part_packet_limit = case.tile_part_packet_limit; + options.quality_layers = case.lossless_quality_layers; + options.write_tlm = case.markers.contains(&crate::EncoderMarker::Tlm); + options.write_plt = case.markers.contains(&crate::EncoderMarker::Plt); + options.write_plm = case.markers.contains(&crate::EncoderMarker::Plm); + options.write_ppm = case.markers.contains(&crate::EncoderMarker::Ppm); + options.write_ppt = case.markers.contains(&crate::EncoderMarker::Ppt); + options.write_sop = case.markers.contains(&crate::EncoderMarker::Sop); + options.write_eph = case.markers.contains(&crate::EncoderMarker::Eph); + options.reversible_transform = + if case.input == EncoderInputKind::Interleaved && matches!(case.components, 3 | 4) { + ReversibleTransform::Rct53 + } else { + ReversibleTransform::None53 + }; + options.validation = J2kEncodeValidation::External; + options +} + +fn lossy_options(case: &EncoderCase, backend: EncodeBackendPreference) -> J2kLossyEncodeOptions { + let mut options = J2kLossyEncodeOptions::default(); + options.backend = backend; + options.block_coding_mode = J2kBlockCodingMode::Classic; + options.progression = progression(case.progression); + options.max_decomposition_levels = Some(case.decomposition_levels); + options.rate_target = case.lossy_rate_target.map(rate_target); + options.quality_layers = case + .lossy_quality_layers + .iter() + .copied() + .map(rate_target) + .map(J2kQualityLayer::new) + .collect(); + options.tile_size = case.tile_size.map(|[width, height]| (width, height)); + options.tile_part_packet_limit = case.tile_part_packet_limit; + options.precinct_exponents = case + .precinct_exponents + .iter() + .map(|[width, height]| (*width, *height)) + .collect(); + options.marker_segments = marker_segments(case); + options.validation = J2kEncodeValidation::External; + options +} + +fn progression(value: crate::EncoderProgression) -> J2kProgressionOrder { + match value { + crate::EncoderProgression::Lrcp => J2kProgressionOrder::Lrcp, + crate::EncoderProgression::Rlcp => J2kProgressionOrder::Rlcp, + crate::EncoderProgression::Rpcl => J2kProgressionOrder::Rpcl, + crate::EncoderProgression::Pcrl => J2kProgressionOrder::Pcrl, + crate::EncoderProgression::Cprl => J2kProgressionOrder::Cprl, + } +} + +fn rate_target(value: EncoderRateTarget) -> J2kRateTarget { + match value { + EncoderRateTarget::BitsPerPixel(value) => J2kRateTarget::BitsPerPixel(value), + EncoderRateTarget::Bytes(value) => J2kRateTarget::Bytes(value), + EncoderRateTarget::PsnrDb(value) => J2kRateTarget::PsnrDb(value), + } +} + +fn marker_segments(case: &EncoderCase) -> Vec { + case.markers + .iter() + .filter_map(|marker| match marker { + crate::EncoderMarker::Tlm => Some(J2kMarkerSegment::Tlm), + crate::EncoderMarker::Plm => Some(J2kMarkerSegment::Plm), + crate::EncoderMarker::Plt => Some(J2kMarkerSegment::Plt), + crate::EncoderMarker::Ppm => Some(J2kMarkerSegment::Ppm), + crate::EncoderMarker::Ppt => Some(J2kMarkerSegment::Ppt), + crate::EncoderMarker::Sop => Some(J2kMarkerSegment::Sop), + crate::EncoderMarker::Eph => Some(J2kMarkerSegment::Eph), + _ => None, + }) + .collect() +} + +#[cfg(test)] +mod tests { + use crate::{CaseStatus, EncoderIut, EncoderQualityStatus, ReportStatus}; + + #[cfg(all(feature = "metal-runner", target_os = "macos"))] + use super::run_metal; + use super::{generate_input, load_sources, run_cpu}; + + #[test] + fn generated_typed_samples_preserve_declared_component_metadata() { + let sources = load_sources(EncoderIut::Cpu).expect("load CPU matrix and ICS"); + let case = sources + .matrix + .cases + .iter() + .find(|case| case.id == "planar-typed") + .expect("typed case"); + + let input = generate_input(case).expect("generate typed input"); + + assert_eq!(input.components.len(), 3); + assert_eq!(input.components[0].bit_depth, 8); + assert_eq!(input.components[1].bit_depth, 12); + assert!(input.components[1].signed); + assert_eq!(input.components[1].sampling, [2, 1]); + assert_eq!(input.components[2].dimensions, [41, 18]); + } + + #[test] + fn complete_cpu_matrix_decodes_with_t804_openjpeg() { + let evidence = run_cpu().expect("run CPU encoder evidence"); + let failures = evidence + .cases + .iter() + .filter(|case| { + case.status != CaseStatus::Pass || case.quality_status == EncoderQualityStatus::Fail + }) + .collect::>(); + + assert_eq!(evidence.cases.len(), 28); + assert_eq!( + evidence.standards_status, + ReportStatus::Pass, + "{failures:#?}" + ); + assert_eq!(evidence.quality_status, ReportStatus::Pass, "{failures:#?}"); + assert!(evidence + .cases + .iter() + .all(|case| case.status == CaseStatus::Pass)); + assert!(evidence + .cases + .iter() + .all(|case| { case.quality_status != EncoderQualityStatus::Fail })); + let exact_lossy = evidence + .cases + .iter() + .find(|case| case.id == "pairwise-09") + .expect("exact lossy fixture"); + assert!(exact_lossy.psnr_infinite); + assert_eq!(exact_lossy.psnr_db, None); + } + + #[cfg(all(feature = "metal-runner", target_os = "macos"))] + #[test] + fn complete_metal_adapter_matrix_records_routes_and_decodes_with_openjpeg() { + let evidence = run_metal().expect("run Metal encoder evidence"); + let failures = evidence + .cases + .iter() + .filter(|case| { + case.status != CaseStatus::Pass || case.quality_status == EncoderQualityStatus::Fail + }) + .collect::>(); + + assert_eq!(evidence.cases.len(), 25); + assert_eq!(evidence.status, ReportStatus::Pass, "{failures:#?}"); + assert!(evidence.cases.iter().any(|case| { + matches!( + case.route, + crate::RouteKind::Hybrid | crate::RouteKind::DeviceNative + ) + })); + } +} diff --git a/crates/j2k-t803/src/runner/encoder/evaluate.rs b/crates/j2k-t803/src/runner/encoder/evaluate.rs new file mode 100644 index 00000000..adb395a3 --- /dev/null +++ b/crates/j2k-t803/src/runner/encoder/evaluate.rs @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::fmt::Write as _; + +use j2k::J2kEncodeDispatchReport; +use j2k_compare::openjpeg::{self, OpenJpegDecodedImage}; + +use crate::compare::u64_as_f64; +use crate::{ + CaseStatus, EncodeRouteStage, EncodeRouteStageName, EncoderCaseReport, EncoderMarker, + EncoderMode, EncoderQualityStatus, ExecutionLocation, RouteKind, +}; + +use super::{input::GeneratedInput, EncodedOutput}; +use crate::encoder::{EncoderCase, EncoderInputKind, EncoderRateTarget}; + +pub(super) fn evaluate_case( + case: &EncoderCase, + input: &GeneratedInput, + encoded: Result, + device: Option, +) -> EncoderCaseReport { + let encoded = match encoded { + Ok(encoded) => encoded, + Err(error) => { + return error_report(case, error, J2kEncodeDispatchReport::default(), device); + } + }; + let dispatch = encoded.dispatch; + let (route, stages) = route_evidence(case, dispatch, device); + let metrics = match encoded_metrics(case, encoded.codestream.len()) { + Ok(metrics) => metrics, + Err(error) => { + return failed_report(case, route, stages, None, false, None, error); + } + }; + + if let Err(error) = validate_markers(case, &encoded.codestream) { + return failed_report(case, route, stages, Some(metrics), false, None, error); + } + + let decoded = match openjpeg::decode_components(&encoded.codestream) { + Ok(decoded) => decoded, + Err(error) => { + return failed_report( + case, + route, + stages, + Some(metrics), + false, + None, + format!("T.804 OpenJPEG reference decode failed: {error}"), + ); + } + }; + if let Err(error) = validate_metadata(case, input, &decoded) { + return failed_report(case, route, stages, Some(metrics), true, None, error); + } + + if case.mode == EncoderMode::Lossless { + lossless_report(case, input, &decoded, route, stages, metrics) + } else { + lossy_report(case, input, &decoded, route, stages, metrics) + } +} + +fn lossless_report( + case: &EncoderCase, + input: &GeneratedInput, + decoded: &OpenJpegDecodedImage, + route: RouteKind, + stages: Vec, + metrics: EncodedMetrics, +) -> EncoderCaseReport { + let exact = input + .components + .iter() + .zip(&decoded.components) + .all(|(expected, actual)| expected.samples == actual.samples); + if !exact { + return failed_report( + case, + route, + stages, + Some(metrics), + true, + Some(false), + "T.804 OpenJPEG output does not exactly match the lossless input".to_string(), + ); + } + EncoderCaseReport { + id: case.id.clone(), + mode: case.mode, + status: CaseStatus::Pass, + route, + reference_decode_success: true, + lossless_exact: Some(true), + encoded_bytes: Some(metrics.bytes), + actual_bits_per_pixel: Some(metrics.bits_per_pixel), + psnr_db: None, + psnr_infinite: false, + quality_status: EncoderQualityStatus::NotApplicable, + quality_requirement: None, + quality_error: None, + error: None, + stages, + } +} + +fn lossy_report( + case: &EncoderCase, + input: &GeneratedInput, + decoded: &OpenJpegDecodedImage, + route: RouteKind, + stages: Vec, + metrics: EncodedMetrics, +) -> EncoderCaseReport { + let psnr = decoded_psnr(input, decoded); + let quality = evaluate_quality(case, psnr, metrics); + EncoderCaseReport { + id: case.id.clone(), + mode: case.mode, + status: CaseStatus::Pass, + route, + reference_decode_success: true, + lossless_exact: None, + encoded_bytes: Some(metrics.bytes), + actual_bits_per_pixel: Some(metrics.bits_per_pixel), + psnr_db: psnr.db, + psnr_infinite: psnr.infinite, + quality_status: quality.status, + quality_requirement: Some(quality.requirement), + quality_error: quality.error, + error: None, + stages, + } +} + +pub(super) fn generation_error( + case: &EncoderCase, + error: &str, + device: Option, +) -> EncoderCaseReport { + error_report( + case, + format!("generate deterministic encoder input: {error}"), + J2kEncodeDispatchReport::default(), + device, + ) +} + +fn error_report( + case: &EncoderCase, + error: String, + dispatch: J2kEncodeDispatchReport, + device: Option, +) -> EncoderCaseReport { + let (route, stages) = route_evidence(case, dispatch, device); + let (quality_status, quality_requirement, quality_error) = if case.mode == EncoderMode::Lossy { + ( + EncoderQualityStatus::Fail, + Some(quality_requirement(case)), + Some("quality gate could not run because encoding failed".to_string()), + ) + } else { + (EncoderQualityStatus::NotApplicable, None, None) + }; + EncoderCaseReport { + id: case.id.clone(), + mode: case.mode, + status: CaseStatus::Error, + route, + reference_decode_success: false, + lossless_exact: None, + encoded_bytes: None, + actual_bits_per_pixel: None, + psnr_db: None, + psnr_infinite: false, + quality_status, + quality_requirement, + quality_error, + error: Some(error), + stages, + } +} + +#[derive(Clone, Copy)] +struct EncodedMetrics { + bytes: u64, + bits_per_pixel: f64, +} + +fn encoded_metrics(case: &EncoderCase, encoded_bytes: usize) -> Result { + let bytes = u64::try_from(encoded_bytes) + .map_err(|_| "encoded codestream size exceeds the report range".to_string())?; + let bytes_as_f64 = u64_as_f64(bytes).map_err(|error| error.to_string())?; + let pixel_count = f64::from(case.width) * f64::from(case.height); + Ok(EncodedMetrics { + bytes, + bits_per_pixel: bytes_as_f64 * 8.0 / pixel_count, + }) +} + +fn failed_report( + case: &EncoderCase, + route: RouteKind, + stages: Vec, + metrics: Option, + reference_decode_success: bool, + lossless_exact: Option, + error: String, +) -> EncoderCaseReport { + let (quality_status, quality_requirement, quality_error) = if case.mode == EncoderMode::Lossy { + ( + EncoderQualityStatus::Fail, + Some(quality_requirement(case)), + Some("quality gate could not pass because the reference decode failed".to_string()), + ) + } else { + (EncoderQualityStatus::NotApplicable, None, None) + }; + EncoderCaseReport { + id: case.id.clone(), + mode: case.mode, + status: CaseStatus::Fail, + route, + reference_decode_success, + lossless_exact, + encoded_bytes: metrics.map(|value| value.bytes), + actual_bits_per_pixel: metrics.map(|value| value.bits_per_pixel), + psnr_db: None, + psnr_infinite: false, + quality_status, + quality_requirement, + quality_error, + error: Some(error), + stages, + } +} + +fn validate_metadata( + case: &EncoderCase, + expected: &GeneratedInput, + actual: &OpenJpegDecodedImage, +) -> Result<(), String> { + if actual.dimensions != (case.width, case.height) { + return Err(format!( + "T.804 OpenJPEG dimensions are {:?}, expected {}x{}", + actual.dimensions, case.width, case.height + )); + } + if actual.components.len() != expected.components.len() { + return Err(format!( + "T.804 OpenJPEG returned {} components, expected {}", + actual.components.len(), + expected.components.len() + )); + } + for (index, (expected, actual)) in expected + .components + .iter() + .zip(&actual.components) + .enumerate() + { + let expected_dimensions = (expected.dimensions[0], expected.dimensions[1]); + let expected_sampling = ( + u32::from(expected.sampling[0]), + u32::from(expected.sampling[1]), + ); + if actual.dimensions != expected_dimensions + || actual.sampling != expected_sampling + || actual.bit_depth != expected.bit_depth + || actual.signed != expected.signed + || actual.samples.len() != expected.samples.len() + { + return Err(format!( + "T.804 OpenJPEG component {index} metadata differs from the encoder input" + )); + } + } + Ok(()) +} + +#[derive(Clone, Copy)] +struct Psnr { + db: Option, + infinite: bool, +} + +fn decoded_psnr(expected: &GeneratedInput, actual: &OpenJpegDecodedImage) -> Psnr { + let mut squared_error = 0.0; + let mut samples = 0.0_f64; + let mut peak = 0.0_f64; + for (expected, actual) in expected.components.iter().zip(&actual.components) { + peak = peak.max(2_f64.powi(i32::from(expected.bit_depth)) - 1.0); + for (&expected, &actual) in expected.samples.iter().zip(&actual.samples) { + let error = f64::from(expected) - f64::from(actual); + squared_error += error * error; + samples += 1.0; + } + } + if squared_error == 0.0 { + return Psnr { + db: None, + infinite: true, + }; + } + let mse = squared_error / samples; + Psnr { + db: Some(10.0 * (peak * peak / mse).log10()), + infinite: false, + } +} + +struct QualityResult { + status: EncoderQualityStatus, + requirement: String, + error: Option, +} + +fn evaluate_quality(case: &EncoderCase, psnr: Psnr, metrics: EncodedMetrics) -> QualityResult { + let requirement = quality_requirement(case); + let minimum_psnr = case + .minimum_psnr_db + .expect("validated lossy case has minimum PSNR"); + let mut failures = Vec::new(); + if psnr.db.is_some_and(|psnr_db| psnr_db < minimum_psnr) { + let psnr_db = psnr.db.expect("finite PSNR was compared"); + failures.push(format!( + "PSNR {psnr_db:.6} dB is below {minimum_psnr:.6} dB" + )); + } + if let Some((target, overshoot)) = rate_gate(case) { + match target { + EncoderRateTarget::BitsPerPixel(target) => { + let actual = metrics.bits_per_pixel; + let one_byte = 8.0 / (f64::from(case.width) * f64::from(case.height)); + let maximum = target * (1.0 + overshoot / 100.0) + one_byte; + if actual > maximum { + failures.push(format!("rate {actual:.6} bpp exceeds {maximum:.6} bpp")); + } + } + EncoderRateTarget::Bytes(target) => { + let encoded_bytes = metrics.bytes; + match (u64_as_f64(target), u64_as_f64(encoded_bytes)) { + (Ok(target), Ok(actual)) => { + let maximum = target * (1.0 + overshoot / 100.0) + 1.0; + if actual > maximum { + failures.push(format!( + "codestream {encoded_bytes} bytes exceeds {maximum:.0} bytes" + )); + } + } + _ => failures.push("rate gate numeric conversion failed".to_string()), + } + } + EncoderRateTarget::PsnrDb(_) => {} + } + } + if failures.is_empty() { + QualityResult { + status: EncoderQualityStatus::Pass, + requirement, + error: None, + } + } else { + QualityResult { + status: EncoderQualityStatus::Fail, + requirement, + error: Some(failures.join("; ")), + } + } +} + +fn quality_requirement(case: &EncoderCase) -> String { + let minimum_psnr = case.minimum_psnr_db.unwrap_or_default(); + let mut requirement = format!("PSNR >= {minimum_psnr:.6} dB"); + if let Some((target, overshoot)) = rate_gate(case) { + let rate = match target { + EncoderRateTarget::BitsPerPixel(value) => format!("{value:.6} bpp"), + EncoderRateTarget::Bytes(value) => format!("{value} bytes"), + EncoderRateTarget::PsnrDb(_) => return requirement, + }; + let _ = write!( + requirement, + "; rate <= {rate} + {overshoot:.6}% + one-byte rounding" + ); + } + requirement +} + +fn rate_gate(case: &EncoderCase) -> Option<(EncoderRateTarget, f64)> { + let target = case + .lossy_quality_layers + .last() + .copied() + .or(case.lossy_rate_target)?; + let overshoot = case.maximum_rate_overshoot_percent?; + Some((target, overshoot)) +} + +fn validate_markers(case: &EncoderCase, codestream: &[u8]) -> Result<(), String> { + for marker in [ + EncoderMarker::Soc, + EncoderMarker::Siz, + EncoderMarker::Cod, + EncoderMarker::Qcd, + EncoderMarker::Sot, + EncoderMarker::Sod, + EncoderMarker::Eoc, + ] + .into_iter() + .chain(case.markers.iter().copied()) + { + if !contains_marker(codestream, marker) { + return Err(format!( + "encoded codestream is missing requested {marker:?} marker" + )); + } + } + Ok(()) +} + +fn contains_marker(codestream: &[u8], marker: EncoderMarker) -> bool { + let code = match marker { + EncoderMarker::Soc => 0x4F, + EncoderMarker::Cap => 0x50, + EncoderMarker::Prf => 0x56, + EncoderMarker::Cpf => 0x59, + EncoderMarker::Sot => 0x90, + EncoderMarker::Sod => 0x93, + EncoderMarker::Eoc => 0xD9, + EncoderMarker::Siz => 0x51, + EncoderMarker::Cod => 0x52, + EncoderMarker::Coc => 0x53, + EncoderMarker::Rgn => 0x5E, + EncoderMarker::Qcd => 0x5C, + EncoderMarker::Qcc => 0x5D, + EncoderMarker::Poc => 0x5F, + EncoderMarker::Tlm => 0x55, + EncoderMarker::Plm => 0x57, + EncoderMarker::Plt => 0x58, + EncoderMarker::Ppm => 0x60, + EncoderMarker::Ppt => 0x61, + EncoderMarker::Sop => 0x91, + EncoderMarker::Eph => 0x92, + EncoderMarker::Crg => 0x63, + EncoderMarker::Com => 0x64, + }; + codestream.windows(2).any(|bytes| bytes == [0xFF, code]) +} + +fn route_evidence( + case: &EncoderCase, + dispatch: J2kEncodeDispatchReport, + device: Option, +) -> (RouteKind, Vec) { + let device = device + .filter(|location| matches!(location, ExecutionLocation::Cuda | ExecutionLocation::Metal)); + let location = |required: bool, count: usize| { + if count > 0 { + device.unwrap_or(ExecutionLocation::Cpu) + } else if required { + ExecutionLocation::Cpu + } else { + ExecutionLocation::NotUsed + } + }; + let interleaved_colour = + case.input == EncoderInputKind::Interleaved && matches!(case.components, 3 | 4); + let transfer_location = if dispatch.any() { + device.unwrap_or(ExecutionLocation::Cpu) + } else { + ExecutionLocation::NotUsed + }; + let stages = Vec::from([ + EncodeRouteStage { + stage: EncodeRouteStageName::InputPreparation, + location: location(true, dispatch.deinterleave), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::ForwardRct, + location: location( + case.mode == EncoderMode::Lossless && interleaved_colour, + dispatch.forward_rct, + ), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::ForwardIct, + location: location( + case.mode == EncoderMode::Lossy && interleaved_colour, + dispatch.forward_ict, + ), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::ForwardDwt53, + location: location( + case.mode == EncoderMode::Lossless && case.decomposition_levels > 0, + dispatch.forward_dwt53, + ), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::ForwardDwt97, + location: location( + case.mode == EncoderMode::Lossy && case.decomposition_levels > 0, + dispatch.forward_dwt97, + ), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::Quantization, + location: location(true, dispatch.quantize_subband), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::Tier1, + location: location(true, dispatch.tier1_code_block), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::Packetization, + location: location(true, dispatch.packetization), + }, + EncodeRouteStage { + stage: EncodeRouteStageName::HostToDevice, + location: transfer_location, + }, + EncodeRouteStage { + stage: EncodeRouteStageName::DeviceToHost, + location: transfer_location, + }, + ]); + let uses_cpu = stages + .iter() + .any(|stage| stage.location == ExecutionLocation::Cpu); + let uses_device = stages.iter().any(|stage| { + matches!( + stage.location, + ExecutionLocation::Cuda | ExecutionLocation::Metal + ) + }); + let route = match (uses_cpu, uses_device) { + (true, true) => RouteKind::Hybrid, + (false, true) => RouteKind::DeviceNative, + _ => RouteKind::Cpu, + }; + (route, stages) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::encoder::EncoderMatrix; + + #[test] + fn planar_input_does_not_claim_an_interleaved_colour_transform() { + let matrix = EncoderMatrix::parse(include_str!( + "../../../../../corpus/j2k-conformance/encoder-matrix-v1.toml" + )) + .expect("valid committed matrix"); + let case = matrix + .cases + .iter() + .find(|case| case.id == "planar-sampled") + .expect("planar matrix case"); + + let (_, stages) = route_evidence(case, J2kEncodeDispatchReport::default(), None); + let rct = stages + .iter() + .find(|stage| stage.stage == EncodeRouteStageName::ForwardRct) + .expect("RCT disclosure"); + + assert_eq!(rct.location, ExecutionLocation::NotUsed); + } +} diff --git a/crates/j2k-t803/src/runner/encoder/input.rs b/crates/j2k-t803/src/runner/encoder/input.rs new file mode 100644 index 00000000..590ba280 --- /dev/null +++ b/crates/j2k-t803/src/runner/encoder/input.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::encoder::{EncoderCase, EncoderInputKind, EncoderPattern}; + +pub(super) struct GeneratedInput { + pub(super) interleaved: Vec, + pub(super) components: Vec, +} + +pub(super) struct GeneratedComponent { + pub(super) data: Vec, + pub(super) dimensions: [u32; 2], + pub(super) sampling: [u8; 2], + pub(super) bit_depth: u8, + pub(super) signed: bool, + pub(super) samples: Vec, +} + +pub(super) fn generate_input(case: &EncoderCase) -> Result { + let component_count = usize::from(case.components); + let mut components = Vec::new(); + components + .try_reserve_exact(component_count) + .map_err(|error| format!("allocate generated component descriptors: {error}"))?; + for component in 0..component_count { + let sampling = case.sampling.get(component).copied().unwrap_or([1, 1]); + let bit_depth = case + .component_bit_depths + .get(component) + .copied() + .unwrap_or(case.bit_depth); + let signed = case + .component_signedness + .get(component) + .copied() + .unwrap_or(case.signed); + components.push(generate_component( + case, component, sampling, bit_depth, signed, + )?); + } + + let interleaved = if case.input == EncoderInputKind::Interleaved { + interleave(case, &components)? + } else { + Vec::new() + }; + Ok(GeneratedInput { + interleaved, + components, + }) +} + +fn generate_component( + case: &EncoderCase, + component: usize, + sampling: [u8; 2], + bit_depth: u8, + signed: bool, +) -> Result { + let dimensions = [ + case.width.div_ceil(u32::from(sampling[0])), + case.height.div_ceil(u32::from(sampling[1])), + ]; + let sample_count = checked_sample_count(dimensions)?; + let bytes_per_sample = usize::from(bit_depth).div_ceil(8); + let byte_count = sample_count + .checked_mul(bytes_per_sample) + .ok_or_else(|| format!("{} generated component byte count overflows", case.id))?; + let mut data = Vec::new(); + data.try_reserve_exact(byte_count) + .map_err(|error| format!("allocate {} component bytes: {error}", case.id))?; + let mut samples = Vec::new(); + samples + .try_reserve_exact(sample_count) + .map_err(|error| format!("allocate {} component samples: {error}", case.id))?; + for y in 0..dimensions[1] { + for x in 0..dimensions[0] { + let sample = generated_sample(case.pattern, x, y, component, bit_depth, signed); + samples.push(sample); + append_sample(&mut data, sample, bit_depth); + } + } + debug_assert_eq!(data.len(), byte_count); + Ok(GeneratedComponent { + data, + dimensions, + sampling, + bit_depth, + signed, + samples, + }) +} + +fn interleave(case: &EncoderCase, components: &[GeneratedComponent]) -> Result, String> { + let pixel_count = (case.width as usize) + .checked_mul(case.height as usize) + .ok_or_else(|| format!("{} interleaved pixel count overflows", case.id))?; + let bytes_per_sample = usize::from(case.bit_depth).div_ceil(8); + let byte_count = pixel_count + .checked_mul(components.len()) + .and_then(|count| count.checked_mul(bytes_per_sample)) + .ok_or_else(|| format!("{} interleaved byte count overflows", case.id))?; + let mut output = Vec::new(); + output + .try_reserve_exact(byte_count) + .map_err(|error| format!("allocate {} interleaved samples: {error}", case.id))?; + for sample_index in 0..pixel_count { + for component in components { + append_sample(&mut output, component.samples[sample_index], case.bit_depth); + } + } + debug_assert_eq!(output.len(), byte_count); + Ok(output) +} + +fn checked_sample_count(dimensions: [u32; 2]) -> Result { + (dimensions[0] as usize) + .checked_mul(dimensions[1] as usize) + .ok_or_else(|| "generated component sample count overflows".to_string()) +} + +fn generated_sample( + pattern: EncoderPattern, + x: u32, + y: u32, + component: usize, + bit_depth: u8, + signed: bool, +) -> i32 { + let modulus = 1_u64 << bit_depth; + let raw = match pattern { + EncoderPattern::Gradient => { + (u64::from(x) * 17 + u64::from(y) * 31 + component as u64 * 47) % modulus + } + EncoderPattern::Checkerboard => { + if (x + y + u32::try_from(component).unwrap_or_default()) & 1 == 0 { + 0 + } else { + modulus - 1 + } + } + EncoderPattern::DeterministicNoise => { + splitmix64(u64::from(x) | (u64::from(y) << 21) | ((component as u64) << 42)) % modulus + } + EncoderPattern::Impulse => { + if x == 0 && y == 0 { + modulus - 1 + } else if signed { + modulus / 2 + } else { + 0 + } + } + }; + if signed { + let raw = i64::try_from(raw).expect("31-bit generated sample fits i64"); + let midpoint = i64::try_from(modulus / 2).expect("31-bit midpoint fits i64"); + i32::try_from(raw - midpoint).expect("31-bit signed generated sample fits i32") + } else { + i32::try_from(raw).expect("31-bit unsigned generated sample fits i32") + } +} + +fn append_sample(output: &mut Vec, sample: i32, bit_depth: u8) { + let modulus = 1_i64 << bit_depth; + let raw = if sample < 0 { + i64::from(sample) + modulus + } else { + i64::from(sample) + }; + let raw = u64::try_from(raw).expect("generated sample is normalized to a non-negative value"); + let bytes = raw.to_le_bytes(); + output.extend_from_slice(&bytes[..usize::from(bit_depth).div_ceil(8)]); +} + +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9E37_79B9_7F4A_7C15); + value = (value ^ (value >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + value = (value ^ (value >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + value ^ (value >> 31) +} diff --git a/crates/j2k-t803/src/runner/evidence.rs b/crates/j2k-t803/src/runner/evidence.rs new file mode 100644 index 00000000..87d1d626 --- /dev/null +++ b/crates/j2k-t803/src/runner/evidence.rs @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{collections::BTreeSet, fs, path::Path}; + +use sha2::{Digest, Sha256}; + +use crate::encoder::{ics_path, matrix_path, reference_decoder_identity}; +use crate::{ + CaseStatus, EncoderEvidence, EncoderIcs, EncoderIut, EncoderMatrix, EncoderQualityStatus, + ReportStatus, T803Manifest, T803Report, +}; + +use super::cache; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum EvidenceScope { + Cpu, + Cuda, + Metal, + All, +} + +impl EvidenceScope { + pub(super) fn parse(value: &str) -> Result { + match value { + "cpu" => Ok(Self::Cpu), + "cuda" => Ok(Self::Cuda), + "metal" => Ok(Self::Metal), + "all" => Ok(Self::All), + _ => Err(format!( + "unknown T.803 evidence scope {value:?}; expected cpu|cuda|metal|all" + )), + } + } + + const fn argument(self) -> &'static str { + match self { + Self::Cpu => "cpu", + Self::Cuda => "cuda", + Self::Metal => "metal", + Self::All => "all", + } + } + + const fn description(self) -> &'static str { + match self { + Self::Cpu => "CPU", + Self::Cuda => "CUDA adapter", + Self::Metal => "Metal adapter", + Self::All => "aggregate CPU/CUDA/Metal", + } + } +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct ReportLane { + iut: String, + os: String, + arch: String, +} + +impl ReportLane { + fn new(iut: &str, os: &str, arch: &str) -> Self { + Self { + iut: iut.to_string(), + os: os.to_string(), + arch: arch.to_string(), + } + } +} + +pub(super) fn verify_reports( + _cache_dir: &Path, + report_paths: &[impl AsRef], + candidate_sha: Option<&str>, + scope: EvidenceScope, +) -> Result<(), String> { + let candidate_sha = candidate_sha.filter(|sha| is_git_sha(sha)).ok_or_else(|| { + "t803 verify requires --candidate-sha with 40 or 64 lowercase hex digits".to_string() + })?; + let manifest = cache::load_manifest()?; + let expected_lanes = required_lanes(scope); + if report_paths.len() != expected_lanes.len() { + return Err(format!( + "t803 verify --scope {} requires exactly {} report(s): {}", + scope.argument(), + expected_lanes.len(), + expected_lanes + .iter() + .map(lane_name) + .collect::>() + .join(", ") + )); + } + + let mut observed_lanes = BTreeSet::new(); + for report_path in report_paths { + let report_path = report_path.as_ref(); + let json = fs::read_to_string(report_path) + .map_err(|error| format!("read {}: {error}", report_path.display()))?; + let report = T803Report::from_json(&json).map_err(|error| error.to_string())?; + if report.to_json().map_err(|error| error.to_string())? != json { + return Err(format!( + "{} is not canonical deterministic report JSON", + report_path.display() + )); + } + verify_report(&report, &manifest, candidate_sha)?; + let lane = ReportLane::new(&report.iut.name, &report.platform.os, &report.platform.arch); + if !observed_lanes.insert(lane.clone()) { + return Err(format!( + "duplicate T.803 report for {} {} {}", + lane.iut, lane.os, lane.arch + )); + } + let markdown_path = report_path.with_extension("md"); + let markdown = fs::read_to_string(&markdown_path) + .map_err(|error| format!("read {}: {error}", markdown_path.display()))?; + if report.to_markdown().map_err(|error| error.to_string())? != markdown { + return Err(format!( + "{} does not match its canonical JSON evidence", + markdown_path.display() + )); + } + } + verify_required_lanes(scope, &observed_lanes) +} + +fn verify_required_lanes( + scope: EvidenceScope, + observed: &BTreeSet, +) -> Result<(), String> { + let expected = required_lanes(scope); + if observed == &expected { + return Ok(()); + } + let missing = expected + .difference(observed) + .map(lane_name) + .collect::>(); + let unexpected = observed + .difference(&expected) + .map(lane_name) + .collect::>(); + Err(format!( + "T.803 {} report lanes are incomplete; missing: {}; unexpected: {}", + scope.description(), + if missing.is_empty() { + "none".to_string() + } else { + missing.join(", ") + }, + if unexpected.is_empty() { + "none".to_string() + } else { + unexpected.join(", ") + } + )) +} + +fn required_lanes(scope: EvidenceScope) -> BTreeSet { + let cpu = [ + ReportLane::new("j2k", "linux", "x86_64"), + ReportLane::new("j2k", "macos", "aarch64"), + ReportLane::new("j2k", "windows", "x86_64"), + ]; + let cuda = ReportLane::new("j2k-cuda", "linux", "x86_64"); + let metal = ReportLane::new("j2k-metal", "macos", "aarch64"); + match scope { + EvidenceScope::Cpu => cpu.into_iter().collect(), + EvidenceScope::Cuda => BTreeSet::from([cuda]), + EvidenceScope::Metal => BTreeSet::from([metal]), + EvidenceScope::All => cpu.into_iter().chain([cuda, metal]).collect(), + } +} + +fn lane_name(lane: &ReportLane) -> String { + match (lane.iut.as_str(), lane.os.as_str(), lane.arch.as_str()) { + ("j2k", "linux", "x86_64") => "Linux x64 CPU".to_string(), + ("j2k", "macos", "aarch64") => "macOS arm64 CPU".to_string(), + ("j2k", "windows", "x86_64") => "Windows x64 CPU".to_string(), + ("j2k-cuda", "linux", "x86_64") => "Linux x64 CUDA adapter".to_string(), + ("j2k-metal", "macos", "aarch64") => "macOS arm64 Metal adapter".to_string(), + _ => format!("{} {} {}", lane.iut, lane.os, lane.arch), + } +} + +fn verify_report( + report: &T803Report, + manifest: &T803Manifest, + candidate_sha: &str, +) -> Result<(), String> { + if report.status != ReportStatus::Pass { + return Err(format!("{} T.803 report did not pass", report.iut.name)); + } + if report.iut.candidate_sha != candidate_sha { + return Err(format!( + "{} report candidate SHA is {}, expected {candidate_sha}", + report.iut.name, report.iut.candidate_sha + )); + } + if report + .features + .iter() + .any(|feature| feature.contains("development") || feature.contains("dirty")) + { + return Err(format!( + "{} report contains development-only feature evidence", + report.iut.name + )); + } + if report.source_archive_sha256 != manifest.source.archive_sha256 + || report.corpus != manifest.files + { + return Err(format!( + "{} report corpus provenance differs from the pinned manifest", + report.iut.name + )); + } + if !report.iut.claim.contains("Profile-1 Cclass-1") + || !report.iut.claim.contains("Profile-1 Cclass-1HF") + || !report.iut.claim.contains("Annex G JP2 reader") + || report.iut.claim.contains("full Part 1") + { + return Err(format!( + "{} report uses an invalid claim label", + report.iut.name + )); + } + verify_encoder_evidence(&report.iut.name, &report.encoder)?; + + let expected_count = manifest.decoder_cases.len() + manifest.jp2_cases.len(); + if report.cases.len() != expected_count { + return Err(format!( + "{} report contains {} cases, expected {expected_count}", + report.iut.name, + report.cases.len() + )); + } + for (observed, expected) in report.cases.iter().zip(&manifest.decoder_cases) { + if observed.id != expected.id + || observed.table != expected.table + || observed.allowed_peak != expected.peak + || observed.allowed_mse != Some(expected.mse) + { + return Err(format!( + "{} report case {} differs from the pinned decoder matrix", + report.iut.name, observed.id + )); + } + } + for (observed, expected) in report.cases[manifest.decoder_cases.len()..] + .iter() + .zip(&manifest.jp2_cases) + { + if observed.id != expected.id + || observed.table != "G.1" + || observed.allowed_peak != expected.peak + || observed.allowed_mse.is_some() + { + return Err(format!( + "{} report case {} differs from the pinned Annex G matrix", + report.iut.name, observed.id + )); + } + } + Ok(()) +} + +fn verify_encoder_evidence(iut_name: &str, evidence: &EncoderEvidence) -> Result<(), String> { + let iut = match iut_name { + "j2k" => EncoderIut::Cpu, + "j2k-cuda" => EncoderIut::Cuda, + "j2k-metal" => EncoderIut::Metal, + other => return Err(format!("unknown encoder IUT {other:?}")), + }; + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .ok_or_else(|| "resolve j2k workspace root".to_string())?; + let expected_matrix_path = matrix_path(); + let expected_ics_path = ics_path(iut); + let matrix_text = fs::read_to_string(root.join(expected_matrix_path)) + .map_err(|error| format!("read {expected_matrix_path}: {error}"))?; + let matrix = EncoderMatrix::parse(&matrix_text).map_err(|error| error.to_string())?; + let ics_bytes = fs::read(root.join(expected_ics_path)) + .map_err(|error| format!("read {expected_ics_path}: {error}"))?; + let ics_text = std::str::from_utf8(&ics_bytes) + .map_err(|error| format!("read {expected_ics_path} as UTF-8: {error}"))?; + let ics = EncoderIcs::parse(ics_text).map_err(|error| error.to_string())?; + if ics.iut != iut { + return Err(format!( + "{expected_ics_path} identifies the wrong encoder IUT" + )); + } + ics.validate_against(&matrix) + .map_err(|error| error.to_string())?; + + let actual_ics_sha256 = format!("{:x}", Sha256::digest(&ics_bytes)); + if evidence.ics_path != expected_ics_path || evidence.ics_sha256 != actual_ics_sha256 { + return Err(format!( + "{iut_name} encoder ICS SHA-256 or path differs from {expected_ics_path}" + )); + } + if evidence.matrix_path != expected_matrix_path + || evidence.matrix_case_count != ics.matrix_case_count() + || evidence.matrix_case_sha256 != ics.matrix_case_sha256() + { + return Err(format!( + "{iut_name} encoder matrix identity differs from the committed ICS" + )); + } + let (standard, implementation, version) = reference_decoder_identity(); + if evidence.reference_decoder.standard != standard + || evidence.reference_decoder.implementation != implementation + || evidence.reference_decoder.version != version + { + return Err(format!( + "{iut_name} encoder reference decoder is not the pinned T.804 OpenJPEG build" + )); + } + if evidence.standards_status != ReportStatus::Pass + || evidence.quality_status != ReportStatus::Pass + || evidence.status != ReportStatus::Pass + { + return Err(format!("{iut_name} encoder evidence did not pass")); + } + + let expected_cases = matrix.selected_cases(iut).collect::>(); + if evidence.cases.len() != expected_cases.len() { + return Err(format!( + "{iut_name} encoder evidence contains {} cases, expected {}", + evidence.cases.len(), + expected_cases.len() + )); + } + for (observed, expected) in evidence.cases.iter().zip(expected_cases) { + if observed.id != expected.id + || observed.mode != expected.mode + || observed.status != CaseStatus::Pass + || observed.quality_status == EncoderQualityStatus::Fail + { + return Err(format!( + "{iut_name} encoder case {} differs from the committed encoder matrix", + observed.id + )); + } + } + Ok(()) +} + +fn is_git_sha(value: &str) -> bool { + matches!(value.len(), 40 | 64) + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +#[cfg(test)] +mod tests { + use sha2::{Digest, Sha256}; + + use super::*; + use crate::{ + CaseStatus, EncodeRouteStage, EncodeRouteStageName, EncoderCaseReport, EncoderEvidence, + EncoderIcs, EncoderIut, EncoderMatrix, EncoderMode, EncoderQualityStatus, + EncoderReferenceIdentity, ExecutionLocation, RouteKind, + }; + + #[test] + fn encoder_verification_rejects_ics_or_case_inventory_tampering() { + let mut evidence = committed_cpu_evidence(); + verify_encoder_evidence("j2k", &evidence).expect("committed CPU encoder evidence"); + + evidence.ics_sha256 = "0".repeat(64); + let error = verify_encoder_evidence("j2k", &evidence) + .expect_err("the report must pin the exact committed ICS bytes"); + assert!(error.contains("ICS SHA-256"), "{error}"); + + let mut evidence = committed_cpu_evidence(); + evidence.cases[0].mode = match evidence.cases[0].mode { + EncoderMode::Lossless => EncoderMode::Lossy, + EncoderMode::Lossy => EncoderMode::Lossless, + }; + let error = verify_encoder_evidence("j2k", &evidence) + .expect_err("the report must retain the selected matrix case modes"); + assert!(error.contains("encoder matrix"), "{error}"); + } + + #[test] + fn release_evidence_scopes_cpu_and_each_adapter_independently() { + let mut cpu_lanes = BTreeSet::from([ + ReportLane::new("j2k", "linux", "x86_64"), + ReportLane::new("j2k", "macos", "aarch64"), + ReportLane::new("j2k", "windows", "x86_64"), + ]); + verify_required_lanes(EvidenceScope::Cpu, &cpu_lanes).expect("complete CPU evidence lanes"); + + cpu_lanes.remove(&ReportLane::new("j2k", "windows", "x86_64")); + let error = verify_required_lanes(EvidenceScope::Cpu, &cpu_lanes) + .expect_err("Windows CPU evidence is mandatory for the CPU claim"); + assert!(error.contains("Windows x64 CPU"), "{error}"); + + verify_required_lanes( + EvidenceScope::Cuda, + &BTreeSet::from([ReportLane::new("j2k-cuda", "linux", "x86_64")]), + ) + .expect("CUDA evidence is independent of Metal evidence"); + verify_required_lanes( + EvidenceScope::Metal, + &BTreeSet::from([ReportLane::new("j2k-metal", "macos", "aarch64")]), + ) + .expect("Metal evidence is independent of CUDA evidence"); + + let all_lanes = BTreeSet::from([ + ReportLane::new("j2k", "linux", "x86_64"), + ReportLane::new("j2k", "macos", "aarch64"), + ReportLane::new("j2k", "windows", "x86_64"), + ReportLane::new("j2k-cuda", "linux", "x86_64"), + ReportLane::new("j2k-metal", "macos", "aarch64"), + ]); + verify_required_lanes(EvidenceScope::All, &all_lanes) + .expect("aggregate evidence remains available as a convenience"); + } + + fn committed_cpu_evidence() -> EncoderEvidence { + let root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("workspace root"); + let matrix_path = "corpus/j2k-conformance/encoder-matrix-v1.toml"; + let ics_path = "corpus/j2k-conformance/encoder-ics-cpu.toml"; + let matrix = EncoderMatrix::parse( + &fs::read_to_string(root.join(matrix_path)).expect("read encoder matrix"), + ) + .expect("valid encoder matrix"); + let ics_bytes = fs::read(root.join(ics_path)).expect("read CPU ICS"); + let ics = EncoderIcs::parse(std::str::from_utf8(&ics_bytes).expect("UTF-8 ICS")) + .expect("valid CPU ICS"); + ics.validate_against(&matrix).expect("ICS matches matrix"); + let cases = matrix + .selected_cases(EncoderIut::Cpu) + .map(|case| passing_case(&case.id, case.mode)) + .collect(); + let (standard, implementation, version) = crate::encoder::reference_decoder_identity(); + EncoderEvidence::new( + ics_path.to_string(), + format!("{:x}", Sha256::digest(ics_bytes)), + matrix_path.to_string(), + ics.matrix_case_count(), + ics.matrix_case_sha256().to_string(), + EncoderReferenceIdentity { + standard: standard.to_string(), + implementation: implementation.to_string(), + version: version.to_string(), + }, + cases, + ) + .expect("valid CPU encoder evidence") + } + + fn passing_case(id: &str, mode: EncoderMode) -> EncoderCaseReport { + let (lossless_exact, psnr_db, quality_status, quality_requirement) = match mode { + EncoderMode::Lossless => (Some(true), None, EncoderQualityStatus::NotApplicable, None), + EncoderMode::Lossy => ( + None, + Some(40.0), + EncoderQualityStatus::Pass, + Some("test quality gate".to_string()), + ), + }; + EncoderCaseReport { + id: id.to_string(), + mode, + status: CaseStatus::Pass, + route: RouteKind::Cpu, + reference_decode_success: true, + lossless_exact, + encoded_bytes: Some(1), + actual_bits_per_pixel: Some(1.0), + psnr_db, + psnr_infinite: false, + quality_status, + quality_requirement, + quality_error: None, + error: None, + stages: cpu_stages(), + } + } + + fn cpu_stages() -> Vec { + [ + EncodeRouteStageName::InputPreparation, + EncodeRouteStageName::ForwardRct, + EncodeRouteStageName::ForwardIct, + EncodeRouteStageName::ForwardDwt53, + EncodeRouteStageName::ForwardDwt97, + EncodeRouteStageName::Quantization, + EncodeRouteStageName::Tier1, + EncodeRouteStageName::Packetization, + ] + .into_iter() + .map(|stage| EncodeRouteStage { + stage, + location: ExecutionLocation::Cpu, + }) + .chain( + [ + EncodeRouteStageName::HostToDevice, + EncodeRouteStageName::DeviceToHost, + ] + .into_iter() + .map(|stage| EncodeRouteStage { + stage, + location: ExecutionLocation::NotUsed, + }), + ) + .collect() + } +} diff --git a/crates/j2k-t803/src/runner/execute.rs b/crates/j2k-t803/src/runner/execute.rs new file mode 100644 index 00000000..6f2f1faf --- /dev/null +++ b/crates/j2k-t803/src/runner/execute.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{fs, path::Path, path::PathBuf, process::Command, sync::Arc}; + +use crate::{EncoderEvidence, IutIdentity, PlatformIdentity, ReportStatus, T803Report}; + +use super::{cache, cases, oracle}; + +pub(super) struct IutConfig { + pub(super) name: &'static str, + pub(super) claim: &'static str, + pub(super) report_stem: &'static str, + pub(super) features: Vec, + pub(super) platform: PlatformIdentity, +} + +pub(super) fn run( + cache_dir: &Path, + output_dir: Option, + development: bool, + mut config: IutConfig, + encoder: EncoderEvidence, + decode: impl FnMut(Arc<[u8]>, u8) -> Result, +) -> Result<(), String> { + let (manifest, corpus) = cache::verify_cached(cache_dir)?; + let candidate_sha = git_output(&["rev-parse", "HEAD"])?; + let dirty = !git_output(&["status", "--porcelain", "--untracked-files=normal"])?.is_empty(); + if dirty && !development { + return Err( + "the source tree is dirty; commit the exact candidate or pass --development for non-release evidence" + .to_string(), + ); + } + + let mut cases = cases::run_decoder_cases(&manifest, &corpus, decode); + cases.extend(cases::run_jp2_cases(&manifest, &corpus)); + let native_component_oracles = oracle::run(&manifest, &corpus)?; + if dirty { + config + .features + .push("development-dirty-worktree".to_string()); + } + config.features.sort_unstable(); + config.features.dedup(); + let report = T803Report::new( + IutIdentity { + name: config.name.to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + candidate_sha, + claim: config.claim.to_string(), + }, + config.platform, + manifest.source.archive_sha256.clone(), + config.features, + manifest.files.clone(), + native_component_oracles, + cases, + encoder, + ) + .map_err(|error| error.to_string())?; + + let output_dir = output_dir.unwrap_or_else(|| cache_dir.join("reports")); + fs::create_dir_all(&output_dir) + .map_err(|error| format!("create {}: {error}", output_dir.display()))?; + let stem = if dirty { + format!("{}-development", config.report_stem) + } else { + config.report_stem.to_string() + }; + let json_path = output_dir.join(format!("{stem}.json")); + let markdown_path = output_dir.join(format!("{stem}.md")); + fs::write( + &json_path, + report.to_json().map_err(|error| error.to_string())?, + ) + .map_err(|error| format!("write {}: {error}", json_path.display()))?; + fs::write( + &markdown_path, + report.to_markdown().map_err(|error| error.to_string())?, + ) + .map_err(|error| format!("write {}: {error}", markdown_path.display()))?; + + println!("wrote {}", json_path.display()); + println!("wrote {}", markdown_path.display()); + if report.status == ReportStatus::Pass { + Ok(()) + } else { + Err(format!( + "T.803 {} IUT failed; complete evidence was written to {}", + config.name, + json_path.display() + )) + } +} + +fn git_output(args: &[&str]) -> Result { + let output = Command::new("git") + .args(args) + .output() + .map_err(|error| format!("start git {}: {error}", args.join(" ")))?; + if !output.status.success() { + return Err(format!( + "git {} exited with {}", + args.join(" "), + output.status + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} diff --git a/crates/j2k-t803/src/runner/metal.rs b/crates/j2k-t803/src/runner/metal.rs new file mode 100644 index 00000000..109c4ff7 --- /dev/null +++ b/crates/j2k-t803/src/runner/metal.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::path::{Path, PathBuf}; + +#[cfg(target_os = "macos")] +use std::{process::Command, sync::Arc}; + +#[cfg(target_os = "macos")] +use j2k::{BatchDecodeOptions, BatchLayout, EncodedImage}; +#[cfg(target_os = "macos")] +use j2k_core::SurfaceResidency; +#[cfg(target_os = "macos")] +use j2k_metal::MetalBatchDecoder; + +#[cfg(target_os = "macos")] +use crate::{ExecutionLocation, PlatformIdentity}; + +#[cfg(target_os = "macos")] +use super::{cases, encoder, execute}; + +const METAL_CLAIM: &str = "Profile-1 Cclass-1 adapter IUT; Profile-1 Cclass-1HF adapter IUT; Annex G JP2 reader via j2k CPU stages (candidate evidence)"; + +#[cfg(target_os = "macos")] +pub(super) fn run( + cache_dir: &Path, + output_dir: Option, + development: bool, +) -> Result<(), String> { + let mut iut = MetalIut::new()?; + let platform = iut.platform()?; + let encoder = encoder::run_metal()?; + execute::run( + cache_dir, + output_dir, + development, + execute::IutConfig { + name: "j2k-metal", + claim: METAL_CLAIM, + report_stem: "metal", + features: Vec::from([ + "adapter-iut".to_string(), + "metal".to_string(), + "production-batch-decode".to_string(), + ]), + platform, + }, + encoder, + move |input, reduction_levels| iut.decode(&input, reduction_levels), + ) +} + +#[cfg(not(target_os = "macos"))] +pub(super) fn run( + _cache_dir: &Path, + _output_dir: Option, + _development: bool, +) -> Result<(), String> { + Err("the Metal T.803 adapter IUT requires macOS and a real Metal device".to_string()) +} + +#[cfg(target_os = "macos")] +struct MetalIut { + decoder: MetalBatchDecoder, +} + +#[cfg(target_os = "macos")] +impl MetalIut { + fn new() -> Result { + let options = BatchDecodeOptions { + layout: BatchLayout::Nhwc, + ..BatchDecodeOptions::default() + }; + let decoder = MetalBatchDecoder::system_default_with_options(options) + .map_err(|error| error.to_string())?; + Ok(Self { decoder }) + } + + fn platform(&self) -> Result { + let device = self.decoder.backend_session().device(); + Ok(PlatformIdentity { + os: std::env::consts::OS.to_string(), + arch: std::env::consts::ARCH.to_string(), + hardware: format!("{} (registry {})", device.name(), device.registry_id()), + driver: macos_driver_identity()?, + }) + } + + fn decode( + &mut self, + input: &Arc<[u8]>, + reduction_levels: u8, + ) -> Result { + let Some(request) = cases::reduction_request(reduction_levels) else { + return cases::decode_cpu(input, reduction_levels); + }; + let component_transform = cases::codestream_component_transform(input) + .map_err(|error| cases::DecodeFailure::new(error, cases::cpu_route(false)))?; + let prepared = self + .decoder + .prepare(Vec::from([EncodedImage::new(Arc::clone(input), request)])) + .map_err(|error| { + cases::DecodeFailure::new(error.to_string(), cases::cpu_route(false)) + })?; + match cases::prepared_requires_cpu(&prepared) { + Ok(true) => return cases::decode_cpu(input, reduction_levels), + Ok(false) => {} + Err(error) => { + return Err(cases::DecodeFailure::new(error, cases::cpu_route(false))); + } + } + let info = prepared.groups()[0].info().clone(); + let route = cases::device_route(ExecutionLocation::Metal, component_transform.is_some()); + let decoded = self + .decoder + .decode_prepared(&prepared) + .map_err(|error| cases::DecodeFailure::new(error.to_string(), route.clone()))?; + if !decoded.errors().is_empty() || !decoded.group_errors().is_empty() { + let errors = decoded + .errors() + .iter() + .map(ToString::to_string) + .chain(decoded.group_errors().iter().map(ToString::to_string)) + .collect::>() + .join("; "); + return Err(cases::DecodeFailure::new(errors, route)); + } + let [group] = decoded.groups() else { + return Err(cases::DecodeFailure::new( + format!( + "Metal T.803 adapter produced {} groups for one input", + decoded.groups().len() + ), + route, + )); + }; + let [surface] = group.surfaces() else { + return Err(cases::DecodeFailure::new( + format!( + "Metal T.803 adapter produced {} NHWC surfaces for one input", + group.surfaces().len() + ), + route, + )); + }; + if surface.residency() != SurfaceResidency::MetalResidentDecode { + return Err(cases::DecodeFailure::new( + format!( + "Metal T.803 adapter returned unexpected {:?} residency", + surface.residency() + ), + route, + )); + } + let bytes = surface + .as_bytes() + .map_err(|error| cases::DecodeFailure::new(error.to_string(), route.clone()))?; + cases::decoded_interleaved(&info, bytes.as_ref(), component_transform, route.clone()) + .map_err(|error| cases::DecodeFailure::new(error, route)) + } +} + +#[cfg(target_os = "macos")] +fn macos_driver_identity() -> Result { + let version = command_output("sw_vers", &["-productVersion"])?; + let build = command_output("sw_vers", &["-buildVersion"])?; + Ok(format!("macOS {version} build {build} Metal driver")) +} + +#[cfg(target_os = "macos")] +fn command_output(command: &str, args: &[&str]) -> Result { + let output = Command::new(command) + .args(args) + .output() + .map_err(|error| format!("start {command}: {error}"))?; + if !output.status.success() { + return Err(format!("{command} exited with {}", output.status)); + } + let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if value.is_empty() { + return Err(format!("{command} returned an empty value")); + } + Ok(value) +} diff --git a/crates/j2k-t803/src/runner/oracle.rs b/crates/j2k-t803/src/runner/oracle.rs new file mode 100644 index 00000000..9d651fc7 --- /dev/null +++ b/crates/j2k-t803/src/runner/oracle.rs @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{collections::BTreeSet, fs, path::Path}; + +use j2k::{J2kDecodedNativeComponents, J2kDecoder, J2kNativeComponentPlane}; +use j2k_compare::openjpeg; +use sha2::{Digest, Sha256}; + +use crate::{NativeComponentOracleEvidence, T803Manifest}; + +use super::cases; + +const SELECTION: &str = "COD MCT enabled with more than four codestream components"; + +pub(super) fn run( + manifest: &T803Manifest, + corpus: &Path, +) -> Result, String> { + let paths = manifest + .decoder_cases + .iter() + .map(|case| case.codestream.as_str()) + .collect::>(); + let mut evidence = Vec::new(); + for path in paths { + let input_path = corpus.join(path); + let input = fs::read(&input_path) + .map_err(|error| format!("read {}: {error}", input_path.display()))?; + let payload = j2k::extract_j2k_codestream_payload(&input).map_err(|error| { + format!( + "extract codestream payload from {}: {error}", + input_path.display() + ) + })?; + let header = j2k_native::inspect_j2k_codestream_header(payload.codestream()) + .map_err(|error| format!("inspect {}: {error}", input_path.display()))?; + if !header.has_mct || header.components <= 4 { + continue; + } + let codestream_sha256 = manifest + .files + .iter() + .find(|entry| entry.path == path) + .map(|entry| entry.sha256.clone()) + .ok_or_else(|| format!("{path} is absent from the pinned corpus inventory"))?; + evidence.push(compare(path, codestream_sha256, &input)?); + } + if evidence.is_empty() { + return Err(format!( + "the selected decoder matrix contains no codestream satisfying {SELECTION:?}" + )); + } + Ok(evidence) +} + +fn compare( + codestream_path: &str, + codestream_sha256: String, + input: &[u8], +) -> Result { + let mut decoder = J2kDecoder::new(input) + .map_err(|error| format!("production decoder open {codestream_path}: {error}"))?; + let production = decoder + .decode_native_components_at_reduction(0) + .map_err(|error| format!("production decoder decode {codestream_path}: {error}"))?; + let reference = openjpeg::decode_components(input) + .map_err(|error| format!("OpenJPEG decode {codestream_path}: {error}"))?; + validate_native_shapes(codestream_path, &production, &reference)?; + + let mut production_hash = Sha256::new(); + let mut openjpeg_hash = Sha256::new(); + update_image_header( + &mut production_hash, + production.dimensions(), + production.planes().len(), + )?; + update_image_header( + &mut openjpeg_hash, + reference.dimensions, + reference.components.len(), + )?; + let mut compared_sample_count = 0_u64; + for (index, (actual, expected)) in production + .planes() + .iter() + .zip(&reference.components) + .enumerate() + { + let actual_samples = cases::unpack_native_plane(actual)?; + let expected_metadata = ( + expected.dimensions, + expected.sampling, + expected.bit_depth, + expected.signed, + ); + let actual_metadata = ( + actual.dimensions(), + ( + u32::from(actual.sampling().0), + u32::from(actual.sampling().1), + ), + actual.bit_depth(), + actual.signed(), + ); + if actual_metadata != expected_metadata { + return Err(format!( + "{codestream_path} component {index} production metadata {actual_metadata:?} differs from OpenJPEG {expected_metadata:?}" + )); + } + if actual_samples.len() != expected.samples.len() { + return Err(format!( + "{codestream_path} component {index} production returned {} samples, OpenJPEG returned {}", + actual_samples.len(), + expected.samples.len() + )); + } + update_component_header(&mut production_hash, index, actual, actual_samples.len())?; + update_openjpeg_component_header( + &mut openjpeg_hash, + index, + expected, + expected.samples.len(), + )?; + for (sample_index, (&actual_sample, &expected_sample)) in + actual_samples.iter().zip(&expected.samples).enumerate() + { + let expected_sample = i64::from(expected_sample); + if actual_sample != expected_sample { + return Err(format!( + "{codestream_path} component {index} sample {sample_index} is {actual_sample}, OpenJPEG returned {expected_sample}" + )); + } + production_hash.update(actual_sample.to_le_bytes()); + openjpeg_hash.update(expected_sample.to_le_bytes()); + } + compared_sample_count = compared_sample_count + .checked_add(u64::try_from(actual_samples.len()).map_err(|_| { + format!("{codestream_path} component {index} sample count exceeds u64") + })?) + .ok_or_else(|| format!("{codestream_path} total sample count exceeds u64"))?; + } + let production_components_sha256 = format!("{:x}", production_hash.finalize()); + let openjpeg_components_sha256 = format!("{:x}", openjpeg_hash.finalize()); + if production_components_sha256 != openjpeg_components_sha256 { + return Err(format!( + "{codestream_path} canonical native component hashes differ after exact comparison" + )); + } + Ok(NativeComponentOracleEvidence { + codestream_path: codestream_path.to_string(), + codestream_sha256, + selection: SELECTION.to_string(), + implementation: "OpenJPEG".to_string(), + version: openjpeg::version(), + library: openjpeg::library_path().to_string(), + component_count: production.planes().len(), + compared_sample_count, + production_components_sha256: production_components_sha256.clone(), + openjpeg_components_sha256, + exact: true, + }) +} + +fn validate_native_shapes( + codestream_path: &str, + production: &J2kDecodedNativeComponents, + reference: &openjpeg::OpenJpegDecodedImage, +) -> Result<(), String> { + if production.dimensions() != reference.dimensions { + return Err(format!( + "{codestream_path} production dimensions {:?} differ from OpenJPEG {:?}", + production.dimensions(), + reference.dimensions + )); + } + if production.planes().len() != reference.components.len() { + return Err(format!( + "{codestream_path} production returned {} components, OpenJPEG returned {}", + production.planes().len(), + reference.components.len() + )); + } + Ok(()) +} + +fn update_image_header( + hasher: &mut Sha256, + dimensions: (u32, u32), + component_count: usize, +) -> Result<(), String> { + hasher.update(dimensions.0.to_le_bytes()); + hasher.update(dimensions.1.to_le_bytes()); + hasher.update( + u32::try_from(component_count) + .map_err(|_| "native component count exceeds u32".to_string())? + .to_le_bytes(), + ); + Ok(()) +} + +fn update_component_header( + hasher: &mut Sha256, + index: usize, + component: &J2kNativeComponentPlane, + sample_count: usize, +) -> Result<(), String> { + update_component_metadata( + hasher, + index, + component.dimensions(), + ( + u32::from(component.sampling().0), + u32::from(component.sampling().1), + ), + component.bit_depth(), + component.signed(), + sample_count, + ) +} + +fn update_openjpeg_component_header( + hasher: &mut Sha256, + index: usize, + component: &openjpeg::OpenJpegDecodedComponent, + sample_count: usize, +) -> Result<(), String> { + update_component_metadata( + hasher, + index, + component.dimensions, + component.sampling, + component.bit_depth, + component.signed, + sample_count, + ) +} + +fn update_component_metadata( + hasher: &mut Sha256, + index: usize, + dimensions: (u32, u32), + sampling: (u32, u32), + bit_depth: u8, + signed: bool, + sample_count: usize, +) -> Result<(), String> { + hasher.update( + u32::try_from(index) + .map_err(|_| "native component index exceeds u32".to_string())? + .to_le_bytes(), + ); + hasher.update(dimensions.0.to_le_bytes()); + hasher.update(dimensions.1.to_le_bytes()); + hasher.update(sampling.0.to_le_bytes()); + hasher.update(sampling.1.to_le_bytes()); + hasher.update([bit_depth, u8::from(signed)]); + hasher.update( + u64::try_from(sample_count) + .map_err(|_| "native component sample count exceeds u64".to_string())? + .to_le_bytes(), + ); + Ok(()) +} diff --git a/crates/j2k-t803/tests/archive.rs b/crates/j2k-t803/tests/archive.rs new file mode 100644 index 00000000..1716004f --- /dev/null +++ b/crates/j2k-t803/tests/archive.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![cfg(feature = "runner")] + +use std::{ + fs, + io::{Cursor, Write}, + path::{Path, PathBuf}, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use j2k_t803::{ + runner::{extract_selected_archive, verify_corpus, ArchiveLimits}, + CorpusFile, +}; +use zip::{write::SimpleFileOptions, CompressionMethod, ZipWriter}; + +const EXPECTED_SHA256: &str = "cea23dd4b87e8b00d19fb9ccaaef93e97353c7353e2070f3baf05aeb3995dff4"; + +fn corpus_file() -> CorpusFile { + CorpusFile { + path: "files/required.pgx".to_string(), + sha256: EXPECTED_SHA256.to_string(), + } +} + +fn archive(entries: &[(&str, &[u8])]) -> Cursor> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + for (name, bytes) in entries { + writer.start_file(*name, options).expect("start ZIP entry"); + writer.write_all(bytes).expect("write ZIP entry"); + } + writer.finish().expect("finish ZIP") +} + +fn symlink_archive() -> Cursor> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "files/required.pgx", + "elsewhere", + SimpleFileOptions::default(), + ) + .expect("add ZIP symlink"); + writer.finish().expect("finish ZIP") +} + +fn temporary_directory(label: &str) -> PathBuf { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let path = std::env::temp_dir().join(format!( + "j2k-t803-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).expect("create temporary directory"); + path +} + +fn cleanup(path: &Path) { + fs::remove_dir_all(path).expect("remove temporary directory"); +} + +#[test] +fn extraction_writes_only_the_pinned_inventory() { + let output = temporary_directory("selected"); + let zip = archive(&[ + ("files/required.pgx", b"expected"), + ("files/unselected.txt", b"not extracted"), + ]); + + extract_selected_archive(zip, &output, &[corpus_file()], ArchiveLimits::default()) + .expect("safe extraction"); + + assert_eq!( + fs::read(output.join("files/required.pgx")).expect("required file"), + b"expected" + ); + assert!(!output.join("files/unselected.txt").exists()); + verify_corpus(&output, &[corpus_file()]).expect("exact extracted inventory"); + cleanup(&output); +} + +#[test] +fn extraction_rejects_traversal_duplicates_missing_and_oversized_entries() { + let cases = [ + ( + "traversal", + archive(&[("../escape", b"x"), ("files/required.pgx", b"expected")]), + ArchiveLimits::default(), + "unsafe path", + ), + ( + "duplicate", + archive(&[ + ("files/required.pgx", b"expected"), + ("files//required.pgx", b"expected"), + ]), + ArchiveLimits::default(), + "duplicate", + ), + ( + "missing", + archive(&[("files/other.pgx", b"expected")]), + ArchiveLimits::default(), + "missing", + ), + ( + "oversized", + archive(&[("files/required.pgx", b"expected")]), + ArchiveLimits { + max_entries: 8, + max_entry_bytes: 7, + max_total_bytes: 64, + }, + "too large", + ), + ]; + + for (label, zip, limits, expected) in cases { + let output = temporary_directory(label); + let error = extract_selected_archive(zip, &output, &[corpus_file()], limits) + .expect_err("unsafe archive must fail"); + assert!( + error.to_string().contains(expected), + "{error:?} did not mention {expected:?}" + ); + assert!(!output.join("files/required.pgx").exists()); + cleanup(&output); + } +} + +#[test] +fn corpus_verification_rejects_changed_or_extra_files() { + let output = temporary_directory("verify"); + fs::create_dir(output.join("files")).expect("create corpus subdirectory"); + fs::write(output.join("files/required.pgx"), b"changed").expect("write changed file"); + + let changed = verify_corpus(&output, &[corpus_file()]).expect_err("changed hash must fail"); + assert!(changed.to_string().contains("SHA-256")); + + fs::write(output.join("files/required.pgx"), b"expected").expect("restore required file"); + fs::write(output.join("files/extra.pgx"), b"extra").expect("write extra file"); + let extra = verify_corpus(&output, &[corpus_file()]).expect_err("extra file must fail"); + assert!(extra.to_string().contains("extra")); + cleanup(&output); +} + +#[test] +fn extraction_rejects_symlinks() { + let output = temporary_directory("symlink"); + let error = extract_selected_archive( + symlink_archive(), + &output, + &[corpus_file()], + ArchiveLimits::default(), + ) + .expect_err("symlink must fail"); + + assert!(error.to_string().contains("symlink")); + assert!(!output.join("files/required.pgx").exists()); + cleanup(&output); +} diff --git a/crates/j2k-t803/tests/conformance_core.rs b/crates/j2k-t803/tests/conformance_core.rs new file mode 100644 index 00000000..f51e75b9 --- /dev/null +++ b/crates/j2k-t803/tests/conformance_core.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_t803::{ + compare_peak_samples, compare_samples, normalize_component, parse_pgx, Component, ErrorBounds, + NormalizationTarget, +}; + +fn pgx_bytes(header: &[u8], payload: &[u8]) -> Vec { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(header.len() + payload.len()) + .expect("allocate PGX fixture"); + bytes.extend_from_slice(header); + bytes.extend_from_slice(payload); + bytes +} + +#[test] +fn pgx_parses_unsigned_one_byte_samples() { + let bytes = pgx_bytes(b"PG ML +8 2 2\n", &[0, 1, 127, 255]); + + let image = parse_pgx(&bytes).expect("valid PGX"); + + assert_eq!(image.width, 2); + assert_eq!(image.height, 2); + assert_eq!(image.bit_depth, 8); + assert!(!image.signed); + assert_eq!(image.samples, [0, 1, 127, 255]); +} + +#[test] +fn pgx_parses_spaced_sign_crlf_and_signed_big_endian_samples() { + let bytes = pgx_bytes( + b"PG ML - 12 4 1\r\n", + &[0xf8, 0x00, 0xff, 0xff, 0x00, 0x00, 0x07, 0xff], + ); + + let image = parse_pgx(&bytes).expect("valid signed PGX"); + + assert_eq!(image.bit_depth, 12); + assert!(image.signed); + assert_eq!(image.samples, [-2048, -1, 0, 2047]); +} + +#[test] +fn pgx_parses_official_little_endian_reference_storage() { + let bytes = pgx_bytes(b"PG LM 12 2 1\n", &[0x34, 0x02, 0xcd, 0x0a]); + + let image = parse_pgx(&bytes).expect("valid little-endian PGX"); + + assert_eq!(image.samples, [0x234, 0xacd]); +} + +#[test] +fn pgx_parses_official_unsigned_header_with_blank_sign_column() { + let bytes = pgx_bytes(b"PG ML 8 1 1\n", &[0x7f]); + + let image = parse_pgx(&bytes).expect("valid unsigned PGX"); + + assert!(!image.signed); + assert_eq!(image.samples, [0x7f]); +} + +#[test] +fn pgx_parses_four_byte_unsigned_samples() { + let bytes = pgx_bytes(b"PG ML 32 2 1\n", &[0, 0, 0, 1, 0xff, 0xff, 0xff, 0xff]); + + let image = parse_pgx(&bytes).expect("valid 32-bit PGX"); + + assert_eq!(image.samples, [1, 4_294_967_295]); +} + +#[test] +fn pgx_rejects_non_normative_or_malformed_storage() { + let cases: &[(&[u8], &str)] = &[ + (b"PG MM +8 1 1\n\x00", "byte order"), + (b"PG ML +0 1 1\n\x00", "bit depth"), + (b"PG ML +33 1 1\n\x00", "bit depth"), + (b"PG ML +8 0 1\n", "dimensions"), + (b"PG ML +8 1 1\n", "payload length"), + (b"PG ML +8 1 1\n\x00\x01", "payload length"), + (b"PG ML +8 18446744073709551615 2\n", "width"), + (b"PG ML -12 1 1\n\x0f\xff", "sign extension"), + (b"PG ML +12 1 1\n\xf0\x00", "precision"), + (b"PG ML +8\t1 1\n\x00", "header"), + ]; + + for (bytes, expected) in cases { + let error = parse_pgx(bytes).expect_err("malformed PGX must fail"); + assert!( + error.to_string().contains(expected), + "{error:?} did not mention {expected:?}" + ); + } +} + +#[test] +fn normalization_clips_scales_and_crops_the_upper_left() { + let samples = [-600, -5, 7, 511, 100, 200, 300, 400, -1, 0, 1, 2]; + let component = Component { + width: 4, + height: 3, + bit_depth: 10, + signed: true, + post_decode_subsampling: (1, 1), + samples: &samples, + }; + let target = NormalizationTarget { + width: 3, + height: 2, + bit_depth: 8, + signed: true, + }; + + let normalized = normalize_component(component, target).expect("normalization succeeds"); + + assert_eq!(normalized, [-128, -2, 1, 25, 50, 75]); +} + +#[test] +fn normalization_subsamples_replicated_decoder_output_before_cropping() { + let samples = [ + 128, 128, 128, 128, 0, 0, 0, 0, 64, 64, 64, 64, 255, 255, 255, 255, + ]; + let component = Component { + width: 8, + height: 2, + bit_depth: 8, + signed: false, + post_decode_subsampling: (4, 1), + samples: &samples, + }; + let target = NormalizationTarget { + width: 2, + height: 2, + bit_depth: 8, + signed: false, + }; + + let normalized = normalize_component(component, target).expect("normalization succeeds"); + + assert_eq!(normalized, [128, 0, 64, 255]); +} + +#[test] +fn normalization_rejects_incompatible_shape_or_signedness() { + let samples = [0]; + let component = Component { + width: 1, + height: 1, + bit_depth: 8, + signed: false, + post_decode_subsampling: (1, 1), + samples: &samples, + }; + + let shape_error = normalize_component( + component, + NormalizationTarget { + width: 2, + height: 1, + bit_depth: 8, + signed: false, + }, + ) + .expect_err("oversized target must fail"); + assert!(shape_error.to_string().contains("dimensions")); + + let signedness_error = normalize_component( + component, + NormalizationTarget { + width: 1, + height: 1, + bit_depth: 8, + signed: true, + }, + ) + .expect_err("signedness mismatch must fail"); + assert!(signedness_error.to_string().contains("signedness")); +} + +#[test] +fn comparison_uses_inclusive_peak_and_mse_bounds() { + let comparison = compare_samples( + &[0, 4, 8, 12], + &[2, 2, 10, 10], + ErrorBounds { peak: 2, mse: 4.0 }, + ) + .expect("comparable samples"); + + assert_eq!(comparison.peak, 2); + assert!((comparison.mse - 4.0).abs() < f64::EPSILON); + assert!(comparison.passed); + + let too_strict = compare_samples( + &[0, 4, 8, 12], + &[2, 2, 10, 10], + ErrorBounds { peak: 1, mse: 4.0 }, + ) + .expect("comparable samples"); + assert!(!too_strict.passed); +} + +#[test] +fn comparison_rejects_invalid_inputs_and_bounds() { + assert!(compare_samples(&[], &[], ErrorBounds { peak: 0, mse: 0.0 }).is_err()); + assert!(compare_samples(&[0], &[], ErrorBounds { peak: 0, mse: 0.0 }).is_err()); + assert!(compare_samples( + &[0], + &[0], + ErrorBounds { + peak: 0, + mse: f64::NAN, + } + ) + .is_err()); +} + +#[test] +fn comparison_preserves_finite_mse_at_the_i64_error_boundary() { + let comparison = compare_samples( + &[i64::MIN, 0], + &[i64::MAX, 0], + ErrorBounds { + peak: u64::MAX, + mse: f64::MAX, + }, + ) + .expect("the exact u128 sum remains representable as finite f64 MSE"); + + assert_eq!(comparison.peak, u64::MAX); + assert_eq!(comparison.mse.to_bits(), 2.0_f64.powi(127).to_bits()); + assert!(comparison.passed); +} + +#[test] +fn peak_only_comparison_uses_an_inclusive_annex_g_bound() { + assert!( + compare_peak_samples(&[0, 4, 8], &[2, 2, 10], 2) + .expect("comparison") + .passed + ); + assert!( + !compare_peak_samples(&[0, 4, 8], &[3, 2, 10], 2) + .expect("comparison") + .passed + ); + assert!(compare_peak_samples(&[], &[], 0).is_err()); +} diff --git a/crates/j2k-t803/tests/encoder_matrix.rs b/crates/j2k-t803/tests/encoder_matrix.rs new file mode 100644 index 00000000..0b77a45f --- /dev/null +++ b/crates/j2k-t803/tests/encoder_matrix.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{fs, path::Path}; + +use j2k_t803::{EncoderIcs, EncoderIut, EncoderMarker, EncoderMatrix}; + +fn repo_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("workspace root") +} + +fn read(path: &str) -> String { + fs::read_to_string(repo_root().join(path)).expect("read committed encoder evidence source") +} + +#[test] +fn committed_matrix_is_pinned_and_each_ics_matches_its_inventory() { + let matrix = EncoderMatrix::parse(&read("corpus/j2k-conformance/encoder-matrix-v1.toml")) + .expect("valid committed encoder matrix"); + + for (iut, path) in [ + ( + EncoderIut::Cpu, + "corpus/j2k-conformance/encoder-ics-cpu.toml", + ), + ( + EncoderIut::Cuda, + "corpus/j2k-conformance/encoder-ics-cuda.toml", + ), + ( + EncoderIut::Metal, + "corpus/j2k-conformance/encoder-ics-metal.toml", + ), + ] { + let ics = EncoderIcs::parse(&read(path)).expect("valid committed Annex F ICS"); + ics.validate_against(&matrix) + .expect("ICS must pin its exact matrix inventory"); + assert_eq!(ics.iut, iut); + } +} + +#[test] +fn matrix_covers_every_exposed_part1_marker_and_declared_pair() { + let matrix = EncoderMatrix::parse(&read("corpus/j2k-conformance/encoder-matrix-v1.toml")) + .expect("valid committed encoder matrix"); + + for marker in [ + EncoderMarker::Rgn, + EncoderMarker::Tlm, + EncoderMarker::Plm, + EncoderMarker::Plt, + EncoderMarker::Ppm, + EncoderMarker::Ppt, + EncoderMarker::Sop, + EncoderMarker::Eph, + ] { + assert!( + matrix + .cases + .iter() + .any(|case| case.markers.contains(&marker)), + "matrix does not exercise {marker:?}" + ); + } +} + +#[test] +fn matrix_rejects_case_tampering_before_execution() { + let text = read("corpus/j2k-conformance/encoder-matrix-v1.toml"); + let tampered = text.replacen( + "pattern = \"checkerboard\"", + "pattern = \"deterministic-noise\"", + 1, + ); + + let error = EncoderMatrix::parse(&tampered).expect_err("case hash mismatch must fail"); + + assert!(error.to_string().contains("SHA-256")); +} diff --git a/crates/j2k-t803/tests/manifest.rs b/crates/j2k-t803/tests/manifest.rs new file mode 100644 index 00000000..d122050b --- /dev/null +++ b/crates/j2k-t803/tests/manifest.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{fs, path::Path}; + +use j2k_t803::T803Manifest; + +fn repo_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("workspace root") +} + +#[test] +fn committed_manifest_covers_the_complete_part1_and_jp2_scope() { + let text = fs::read_to_string(repo_root().join("corpus/j2k-conformance/t803-v3.toml")) + .expect("read committed T.803 manifest"); + + let manifest = T803Manifest::parse(&text).expect("valid committed manifest"); + + assert_eq!(manifest.decoder_cases.len(), 81); + assert_eq!(manifest.jp2_cases.len(), 9); + assert_eq!(manifest.files.len(), 123); + assert_eq!(manifest.table_case_count("C.1"), 18); + assert_eq!(manifest.table_case_count("C.4"), 8); + assert_eq!(manifest.table_case_count("C.6"), 35); + assert_eq!(manifest.table_case_count("C.7"), 17); + assert_eq!(manifest.table_case_count("C.8"), 3); + assert!(manifest + .decoder_cases + .iter() + .all(|case| !case.codestream.contains("htj2k"))); +} + +#[test] +fn manifest_rejects_unknown_fields_and_untrusted_paths() { + let unknown = minimal_manifest("path = \"files/input.j2k\"\nextra = true"); + assert!(T803Manifest::parse(&unknown) + .expect_err("unknown field must fail") + .to_string() + .contains("unknown field")); + + for path in ["../input.j2k", "/tmp/input.j2k", "files/./input.j2k"] { + let text = minimal_manifest(&format!("path = {path:?}")); + let error = T803Manifest::parse(&text).expect_err("unsafe path must fail"); + assert!(error.to_string().contains("relative normalized path")); + } +} + +#[test] +fn manifest_rejects_bad_hashes_duplicates_and_missing_inventory() { + let bad_hash = minimal_manifest("path = \"files/input.j2k\"\nsha256 = \"ABC\""); + assert!(T803Manifest::parse(&bad_hash) + .expect_err("bad hash must fail") + .to_string() + .contains("SHA-256")); + + let duplicate = format!( + "{}\n[[files]]\npath = \"files/input.j2k\"\nsha256 = \"{}\"\n", + minimal_manifest("path = \"files/input.j2k\""), + "0".repeat(64) + ); + assert!(T803Manifest::parse(&duplicate) + .expect_err("duplicate path must fail") + .to_string() + .contains("duplicate file")); + + let missing = minimal_manifest("path = \"files/unrelated.j2k\""); + assert!(T803Manifest::parse(&missing) + .expect_err("case input absent from inventory must fail") + .to_string() + .contains("not present in the file inventory")); +} + +fn minimal_manifest(file_fields: &str) -> String { + let file_hash = if file_fields.contains("sha256") { + String::new() + } else { + format!("sha256 = \"{}\"", "0".repeat(64)) + }; + format!( + r#"schema_version = 1 +standard = "ISO/IEC 15444-4:2024 / ITU-T T.803 v3" + +[source] +url = "https://www.itu.int/wftp3/public/t/testsignal/SpeImage/T803/v2024_02/T.803v3_15444-4ed4-ElecAtt-codestreams.zip" +archive_sha256 = "{hash}" +archive_bytes = 1 + +[[files]] +{file_fields} +{file_hash} + +[[decoder_cases]] +id = "c1-p0-01-0" +table = "C.1" +codestream = "files/input.j2k" +reference = "files/reference.pgx" +component = 0 +reduction_levels = 0 +signed = false +bit_depth = 8 +width = 1 +height = 1 +peak = 0 +mse = 0.0 + +[[jp2_cases]] +id = "jp2-1" +input = "files/file1.jp2" +reference = "files/jp2_1.tif" +components = 3 +bit_depth = 8 +width = 1 +height = 1 +peak = 4 +"#, + hash = "0".repeat(64) + ) +} diff --git a/crates/j2k-t803/tests/report.rs b/crates/j2k-t803/tests/report.rs new file mode 100644 index 00000000..2cf9ddc8 --- /dev/null +++ b/crates/j2k-t803/tests/report.rs @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k_t803::{ + CaseReport, CaseStatus, CorpusFile, EncodeRouteStage, EncodeRouteStageName, EncoderCaseReport, + EncoderEvidence, EncoderMode, EncoderQualityStatus, EncoderReferenceIdentity, + ExecutionLocation, IutIdentity, NativeComponentOracleEvidence, PlatformIdentity, ReportStatus, + RouteKind, RouteStage, RouteStageName, T803Report, +}; + +fn report(cases: Vec) -> T803Report { + T803Report::new( + IutIdentity { + name: "j2k".to_string(), + version: "0.8.0".to_string(), + candidate_sha: "0123456789abcdef".to_string(), + claim: "Profile-1 Cclass-1 candidate".to_string(), + }, + PlatformIdentity { + os: "linux".to_string(), + arch: "x86_64".to_string(), + hardware: "test cpu".to_string(), + driver: "not-applicable".to_string(), + }, + "ac04b52e1fe38404912036c14f215099ea9a785f38644fbe76ae8f3d1523c86d".to_string(), + ["parallel".to_string(), "simd".to_string()] + .into_iter() + .collect(), + [CorpusFile { + path: "files/codestreams_profile0/p0_13.j2k".to_string(), + sha256: "0".repeat(64), + }] + .into_iter() + .collect(), + Vec::from([oracle_evidence()]), + cases, + encoder_evidence(), + ) + .expect("valid report") +} + +fn oracle_evidence() -> NativeComponentOracleEvidence { + NativeComponentOracleEvidence { + codestream_path: "files/codestreams_profile0/p0_13.j2k".to_string(), + codestream_sha256: "0".repeat(64), + selection: "COD MCT enabled with more than four codestream components".to_string(), + implementation: "OpenJPEG".to_string(), + version: "2.5.3".to_string(), + library: "openjpeg-sys vendored openjp2".to_string(), + component_count: 257, + compared_sample_count: 257, + production_components_sha256: "3".repeat(64), + openjpeg_components_sha256: "3".repeat(64), + exact: true, + } +} + +fn encoder_evidence() -> EncoderEvidence { + EncoderEvidence::new( + "corpus/j2k-conformance/encoder-ics-cpu.toml".to_string(), + "1".repeat(64), + "corpus/j2k-conformance/encoder-matrix-v1.toml".to_string(), + 1, + "2".repeat(64), + EncoderReferenceIdentity { + standard: "ISO/IEC 15444-5 / ITU-T T.804".to_string(), + implementation: "OpenJPEG".to_string(), + version: "2.5.3".to_string(), + }, + Vec::from([EncoderCaseReport { + id: "pairwise-01".to_string(), + mode: EncoderMode::Lossless, + status: CaseStatus::Pass, + route: RouteKind::Cpu, + reference_decode_success: true, + lossless_exact: Some(true), + encoded_bytes: Some(123), + actual_bits_per_pixel: Some(0.960_937_5), + psnr_db: None, + psnr_infinite: false, + quality_status: EncoderQualityStatus::NotApplicable, + quality_requirement: None, + quality_error: None, + error: None, + stages: cpu_encode_stages(), + }]), + ) + .expect("valid encoder evidence") +} + +fn cpu_encode_stages() -> Vec { + [ + EncodeRouteStageName::InputPreparation, + EncodeRouteStageName::ForwardRct, + EncodeRouteStageName::ForwardIct, + EncodeRouteStageName::ForwardDwt53, + EncodeRouteStageName::ForwardDwt97, + EncodeRouteStageName::Quantization, + EncodeRouteStageName::Tier1, + EncodeRouteStageName::Packetization, + ] + .into_iter() + .map(|stage| EncodeRouteStage { + stage, + location: ExecutionLocation::Cpu, + }) + .chain( + [ + EncodeRouteStageName::HostToDevice, + EncodeRouteStageName::DeviceToHost, + ] + .into_iter() + .map(|stage| EncodeRouteStage { + stage, + location: ExecutionLocation::NotUsed, + }), + ) + .collect() +} + +fn passing_case(stages: Vec) -> CaseReport { + CaseReport { + id: "c6-c1p0-01-0".to_string(), + table: "C.6".to_string(), + status: CaseStatus::Pass, + route: RouteKind::Cpu, + peak: Some(0), + mse: Some(0.0), + allowed_peak: 0, + allowed_mse: Some(0.0), + error: None, + stages, + } +} + +fn cpu_stages() -> Vec { + [ + RouteStageName::Parsing, + RouteStageName::Tier1, + RouteStageName::Dequantization, + RouteStageName::Idwt, + RouteStageName::Mct, + RouteStageName::ColorOutput, + ] + .into_iter() + .map(|stage| RouteStage { + stage, + location: ExecutionLocation::Cpu, + }) + .chain( + [RouteStageName::HostToDevice, RouteStageName::DeviceToHost] + .into_iter() + .map(|stage| RouteStage { + stage, + location: ExecutionLocation::NotUsed, + }), + ) + .collect() +} + +#[test] +fn report_json_is_deterministic_versioned_and_round_trips() { + let report = report([passing_case(cpu_stages())].into_iter().collect()); + + let first = report.to_json().expect("serialize report"); + let second = report.to_json().expect("serialize report again"); + + assert_eq!(first, second); + assert!(first.ends_with('\n')); + assert!(first.contains("\"schema_version\": 3")); + assert!(first.contains("ISO/IEC 15444-4:2024 / ITU-T T.803 v3")); + assert!(first.contains("ISO/IEC 15444-5 / ITU-T T.804")); + let reparsed = T803Report::from_json(&first).expect("parse report"); + assert_eq!(reparsed, report); + assert_eq!(reparsed.to_json().expect("reserialize report"), first); +} + +#[test] +fn report_json_round_trip_preserves_difficult_f64_values() { + let mut case = passing_case(cpu_stages()); + case.mse = Some(0.247_063_802_083_333_34); + case.allowed_mse = Some(1.0); + let mut report = report(Vec::from([case])); + report.encoder.cases[0].actual_bits_per_pixel = Some(13.244_893_054_554_193); + + let json = report.to_json().expect("serialize report"); + let reparsed = T803Report::from_json(&json).expect("parse report"); + + assert_eq!(reparsed.to_json().expect("reserialize report"), json); +} + +#[test] +fn report_status_fails_when_any_case_fails_or_errors() { + let mut failed = passing_case(cpu_stages()); + failed.status = CaseStatus::Fail; + failed.peak = Some(1); + failed.error = None; + assert_eq!( + report([failed].into_iter().collect()).status, + ReportStatus::Fail + ); + + let mut error = passing_case(cpu_stages()); + error.status = CaseStatus::Error; + error.peak = None; + error.mse = None; + error.error = Some("decode failed".to_string()); + assert_eq!( + report([error].into_iter().collect()).status, + ReportStatus::Fail + ); +} + +#[test] +fn report_rejects_route_labels_that_hide_cpu_assistance() { + let stages = [ + (RouteStageName::Parsing, ExecutionLocation::Cpu), + (RouteStageName::Tier1, ExecutionLocation::Cuda), + (RouteStageName::Dequantization, ExecutionLocation::Cuda), + (RouteStageName::Idwt, ExecutionLocation::Cuda), + (RouteStageName::Mct, ExecutionLocation::Cuda), + (RouteStageName::ColorOutput, ExecutionLocation::Cpu), + (RouteStageName::HostToDevice, ExecutionLocation::Cuda), + (RouteStageName::DeviceToHost, ExecutionLocation::Cuda), + ] + .into_iter() + .map(|(stage, location)| RouteStage { stage, location }) + .collect(); + + let error = T803Report::new( + IutIdentity { + name: "j2k-cuda".to_string(), + version: "0.8.0".to_string(), + candidate_sha: "abc".to_string(), + claim: "adapter IUT candidate".to_string(), + }, + PlatformIdentity { + os: "linux".to_string(), + arch: "x86_64".to_string(), + hardware: "test gpu".to_string(), + driver: "test driver".to_string(), + }, + "0".repeat(64), + Vec::new(), + [CorpusFile { + path: "files/input.j2k".to_string(), + sha256: "0".repeat(64), + }] + .into_iter() + .collect(), + Vec::from([{ + let mut oracle = oracle_evidence(); + oracle.codestream_path = "files/input.j2k".to_string(); + oracle + }]), + [{ + let mut case = passing_case(stages); + case.route = RouteKind::DeviceNative; + case + }] + .into_iter() + .collect(), + encoder_evidence(), + ) + .expect_err("device-native label must reject CPU assistance"); + + assert!(error.to_string().contains("device-native")); +} + +#[test] +fn markdown_discloses_metrics_bounds_and_route_stages() { + let report = report([passing_case(cpu_stages())].into_iter().collect()); + + let markdown = report.to_markdown().expect("render Markdown"); + + assert!(markdown.contains("Profile-1 Cclass-1 candidate")); + assert!(markdown.contains("Device-native: 0 / 1")); + assert!(markdown.contains("CPU-routed: 1 / 1")); + assert!(markdown.contains("Native component oracle")); + assert!(markdown.contains("257 components / 257 samples: exact")); + assert!(markdown.contains("| c6-c1p0-01-0 | C.6 | pass | cpu | 0 / 0 | 0.000000 / 0.000000 |")); + assert!(markdown.contains("color-output=cpu")); + assert!(markdown.contains("Informative Annex D/F encoder evidence")); + assert!(markdown.contains("OpenJPEG 2.5.3")); + assert!(markdown.contains("Standards status: pass")); + assert!(markdown.contains("Quality-gate status: pass")); + assert!(markdown.contains( + "Conformance does not establish robustness, security, adoption, or performance." + )); +} + +#[test] +fn one_report_can_disclose_cpu_and_hybrid_cases() { + let cpu = passing_case(cpu_stages()); + let mut hybrid = passing_case( + [ + (RouteStageName::Parsing, ExecutionLocation::Cpu), + (RouteStageName::Tier1, ExecutionLocation::Cuda), + (RouteStageName::Dequantization, ExecutionLocation::Cuda), + (RouteStageName::Idwt, ExecutionLocation::Cuda), + (RouteStageName::Mct, ExecutionLocation::Cuda), + (RouteStageName::ColorOutput, ExecutionLocation::Cpu), + (RouteStageName::HostToDevice, ExecutionLocation::Cuda), + (RouteStageName::DeviceToHost, ExecutionLocation::Cuda), + ] + .into_iter() + .map(|(stage, location)| RouteStage { stage, location }) + .collect(), + ); + hybrid.id = "c6-c1p0-02-0".to_string(); + hybrid.route = RouteKind::Hybrid; + + let report = report([cpu, hybrid].into_iter().collect()); + + assert_eq!(report.cases[0].route, RouteKind::Cpu); + assert_eq!(report.cases[1].route, RouteKind::Hybrid); + assert_eq!(report.decoder_routes.device_native, 0); + assert_eq!(report.decoder_routes.hybrid, 1); + assert_eq!(report.decoder_routes.cpu, 1); +} + +#[test] +fn report_rejects_an_oracle_that_does_not_match_component_for_component() { + let mut oracle = oracle_evidence(); + oracle.exact = false; + oracle.openjpeg_components_sha256 = "4".repeat(64); + + let error = T803Report::new( + IutIdentity { + name: "j2k".to_string(), + version: "0.8.0".to_string(), + candidate_sha: "abc".to_string(), + claim: "Profile-1 Cclass-1 candidate".to_string(), + }, + PlatformIdentity { + os: "linux".to_string(), + arch: "x86_64".to_string(), + hardware: "test cpu".to_string(), + driver: "not-applicable".to_string(), + }, + "0".repeat(64), + Vec::new(), + Vec::from([CorpusFile { + path: oracle.codestream_path.clone(), + sha256: oracle.codestream_sha256.clone(), + }]), + Vec::from([oracle]), + Vec::from([passing_case(cpu_stages())]), + encoder_evidence(), + ) + .expect_err("non-exact native component evidence must block the report"); + + assert!(error.to_string().contains("component-for-component")); +} + +#[test] +fn encoder_evidence_accepts_a_metadata_failure_after_reference_decode() { + let mut case = encoder_evidence().cases.remove(0); + case.mode = EncoderMode::Lossy; + case.status = CaseStatus::Fail; + case.reference_decode_success = true; + case.lossless_exact = None; + case.quality_status = EncoderQualityStatus::Fail; + case.quality_requirement = Some("PSNR >= 30 dB".to_string()); + case.quality_error = Some("quality gate could not run".to_string()); + case.error = Some("decoded component metadata differs".to_string()); + + EncoderEvidence::new( + "corpus/j2k-conformance/encoder-ics-cpu.toml".to_string(), + "1".repeat(64), + "corpus/j2k-conformance/encoder-matrix-v1.toml".to_string(), + 1, + "2".repeat(64), + EncoderReferenceIdentity { + standard: "ISO/IEC 15444-5 / ITU-T T.804".to_string(), + implementation: "OpenJPEG".to_string(), + version: "2.5.3".to_string(), + }, + Vec::from([case]), + ) + .expect("a completed reference decode can still expose a standards failure"); +} diff --git a/crates/j2k-t803/tests/runner_cli.rs b/crates/j2k-t803/tests/runner_cli.rs new file mode 100644 index 00000000..a13409a3 --- /dev/null +++ b/crates/j2k-t803/tests/runner_cli.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#![cfg(feature = "runner")] + +use std::{fs, path::PathBuf, process::Command}; + +fn empty_cache(label: &str) -> PathBuf { + let path = std::env::temp_dir().join(format!("j2k-t803-cli-{label}-{}", std::process::id())); + if path.exists() { + fs::remove_dir_all(&path).expect("remove stale test cache"); + } + fs::create_dir(&path).expect("create test cache"); + path +} + +#[test] +fn run_fails_closed_when_the_pinned_corpus_is_absent() { + let cache = empty_cache("missing"); + let output = Command::new(env!("CARGO_BIN_EXE_j2k-t803-runner")) + .args(["run", "--iut", "cpu", "--development", "--cache-dir"]) + .arg(&cache) + .output() + .expect("run T.803 CLI"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("archive is absent"), + "unexpected stderr: {stderr}" + ); + fs::remove_dir_all(cache).expect("remove test cache"); +} + +#[test] +fn verify_requires_an_explicit_independent_evidence_scope() { + let output = Command::new(env!("CARGO_BIN_EXE_j2k-t803-runner")) + .args([ + "verify", + "--candidate-sha", + "0123456789abcdef0123456789abcdef01234567", + "--report", + "missing.json", + ]) + .output() + .expect("run T.803 verify CLI"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("requires --scope cpu|cuda|metal|all"), + "unexpected stderr: {stderr}" + ); +} diff --git a/crates/j2k-test-support/Cargo.toml b/crates/j2k-test-support/Cargo.toml index 5771be7c..dd94d70d 100644 --- a/crates/j2k-test-support/Cargo.toml +++ b/crates/j2k-test-support/Cargo.toml @@ -16,7 +16,10 @@ path = "src/lib.rs" j2k-native-fixtures = ["dep:j2k-native"] [dependencies] -j2k-native = { path = "../j2k-native", version = "=0.8.0", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.8.1", optional = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } [lints.rust] unsafe_code = "forbid" diff --git a/crates/j2k-test-support/src/auto_routing.rs b/crates/j2k-test-support/src/auto_routing.rs new file mode 100644 index 00000000..f92e18c5 --- /dev/null +++ b/crates/j2k-test-support/src/auto_routing.rs @@ -0,0 +1,734 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ + collections::BTreeSet, + fs, + path::{Component, Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +const MAX_MANIFEST_BYTES: u64 = 4 * 1024 * 1024; +const MAX_CASE_BYTES: u64 = 512 * 1024 * 1024; +const MAX_TOTAL_CASE_BYTES: u64 = 2 * 1024 * 1024 * 1024; +const MAX_CASES: usize = 4_096; + +/// A pinned external workload manifest for Auto-routing benchmarks. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AutoRoutingManifest { + pub schema_version: u32, + pub corpus: String, + pub source_url: String, + pub cases: Vec, +} + +/// One hash-pinned input in an Auto-routing workload manifest. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AutoRoutingManifestCase { + pub id: String, + pub path: String, + pub kind: AutoRoutingWorkloadKind, + pub pixel_format: AutoRoutingPixelFormat, + pub sha256: String, +} + +/// Whether a workload is a compressed decode input or an uncompressed encode input. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AutoRoutingWorkloadKind { + Decode, + Encode, +} + +/// Pixel layout used for route-parity comparisons. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AutoRoutingPixelFormat { + Gray8, + Rgb8, +} + +/// One validated, in-memory external workload. +#[derive(Clone, Debug)] +pub struct AutoRoutingWorkload { + pub id: String, + pub path: PathBuf, + pub kind: AutoRoutingWorkloadKind, + pub pixel_format: AutoRoutingPixelFormat, + pub bytes: Vec, +} + +/// A validated manifest, its exact hash, and the inputs it names. +#[derive(Clone, Debug)] +pub struct AutoRoutingWorkloadSet { + pub manifest: AutoRoutingManifest, + pub manifest_sha256: String, + pub workloads: Vec, +} + +/// Validated 8-bit PGM/PPM input for an encode benchmark cell. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AutoRoutingPnm { + pub id: String, + pub pixels: Vec, + pub width: u32, + pub height: u32, + pub components: u16, +} + +/// Accelerator lane that produced route evidence. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AutoRoutingBackend { + Cuda, + Metal, +} + +/// Hardware and software identity for one benchmark lane. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AutoRoutingPlatform { + pub os: String, + pub arch: String, + pub hardware: String, + pub driver: String, +} + +/// Workload class evaluated for a fixed Auto-routing decision. +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AutoRoutingOperation { + FullDecode, + RoiDecode, + ScaledDecode, + BatchDecode, + LosslessEncode, + LossyEncode, +} + +/// Actual execution class of a measured route. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum AutoRoutingExecution { + Cpu, + Hybrid, + DeviceNative, +} + +/// Criterion result identity and exact output produced by one route. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AutoRoutingRoute { + pub criterion_id: String, + pub execution: AutoRoutingExecution, + pub output_sha256: String, +} + +/// CPU, hybrid, and optional device-native measurements for one workload class. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AutoRoutingCell { + pub id: String, + pub operation: AutoRoutingOperation, + pub source: String, + pub workload: String, + pub cpu: AutoRoutingRoute, + pub hybrid: AutoRoutingRoute, + pub strict_device_supported: bool, + pub strict_device: Option, +} + +/// Versioned route evidence emitted beside Criterion estimates. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct AutoRoutingEvidence { + pub schema_version: u32, + pub candidate_sha: String, + pub backend: AutoRoutingBackend, + pub platform: AutoRoutingPlatform, + pub external_manifest_sha256: String, + pub external_case_count: usize, + pub cells: Vec, +} + +/// Load and hash-check every case named by an external Auto-routing manifest. +/// +/// # Errors +/// +/// Returns an error for malformed or oversized manifests, unsafe paths, duplicate +/// case IDs, unsupported layouts, non-regular files, or hash mismatches. +pub fn load_auto_routing_manifest( + manifest_path: &Path, + corpus_root: &Path, +) -> Result { + let manifest_bytes = + read_bounded_regular_file(manifest_path, MAX_MANIFEST_BYTES, "Auto-routing manifest")?; + let manifest_sha256 = sha256_hex(&manifest_bytes); + let manifest: AutoRoutingManifest = + serde_json::from_slice(&manifest_bytes).map_err(|error| { + format!( + "parse Auto-routing manifest {}: {error}", + manifest_path.display() + ) + })?; + validate_manifest_header(&manifest)?; + let canonical_root = corpus_root.canonicalize().map_err(|error| { + format!( + "canonicalize Auto-routing corpus root {}: {error}", + corpus_root.display() + ) + })?; + if !canonical_root.is_dir() { + return Err(format!( + "Auto-routing corpus root {} is not a directory", + corpus_root.display() + )); + } + + let mut ids = BTreeSet::new(); + let mut total_bytes = 0u64; + let mut workloads = Vec::with_capacity(manifest.cases.len()); + for case in &manifest.cases { + workloads.push(load_auto_routing_case( + case, + &canonical_root, + &mut ids, + &mut total_bytes, + )?); + } + + Ok(AutoRoutingWorkloadSet { + manifest, + manifest_sha256, + workloads, + }) +} + +fn load_auto_routing_case( + case: &AutoRoutingManifestCase, + canonical_root: &Path, + ids: &mut BTreeSet, + total_bytes: &mut u64, +) -> Result { + if !is_safe_id(&case.id) || !ids.insert(case.id.clone()) { + return Err("Auto-routing manifest cases must have unique ids".to_string()); + } + if !is_lower_hex(&case.sha256, 64) { + return Err(format!( + "Auto-routing case {} has an invalid SHA-256", + case.id + )); + } + let relative = Path::new(&case.path); + if case.path.is_empty() + || relative.is_absolute() + || !relative + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return Err(format!( + "Auto-routing case {} must use a safe relative path", + case.id + )); + } + let unresolved = canonical_root.join(relative); + let metadata = fs::symlink_metadata(&unresolved).map_err(|error| { + format!( + "read Auto-routing case {} at {}: {error}", + case.id, + unresolved.display() + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() == 0 { + return Err(format!( + "Auto-routing case {} must be a non-empty regular file", + case.id + )); + } + if metadata.len() > MAX_CASE_BYTES { + return Err(format!("Auto-routing case {} is too large", case.id)); + } + *total_bytes = total_bytes + .checked_add(metadata.len()) + .ok_or_else(|| "Auto-routing corpus byte count overflow".to_string())?; + if *total_bytes > MAX_TOTAL_CASE_BYTES { + return Err("Auto-routing corpus exceeds its total byte limit".to_string()); + } + let path = unresolved.canonicalize().map_err(|error| { + format!( + "canonicalize Auto-routing case {} at {}: {error}", + case.id, + unresolved.display() + ) + })?; + if !path.starts_with(canonical_root) { + return Err(format!( + "Auto-routing case {} escapes the corpus root", + case.id + )); + } + let bytes = fs::read(&path).map_err(|error| { + format!( + "read Auto-routing case {} at {}: {error}", + case.id, + path.display() + ) + })?; + let actual_sha256 = sha256_hex(&bytes); + if actual_sha256 != case.sha256 { + return Err(format!( + "Auto-routing case {} SHA-256 mismatch: expected {}, found {actual_sha256}", + case.id, case.sha256 + )); + } + Ok(AutoRoutingWorkload { + id: case.id.clone(), + path, + kind: case.kind, + pixel_format: case.pixel_format, + bytes, + }) +} + +/// Serialize validated route evidence deterministically with a trailing newline. +/// +/// # Errors +/// +/// Returns an error when evidence identity, route labels, output hashes, or the +/// destination path are invalid. +pub fn write_auto_routing_evidence( + output_path: &Path, + evidence: &AutoRoutingEvidence, +) -> Result<(), String> { + validate_evidence(evidence)?; + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent).map_err(|error| { + format!( + "create Auto-routing evidence directory {}: {error}", + parent.display() + ) + })?; + } + let mut json = serde_json::to_string_pretty(evidence) + .map_err(|error| format!("serialize Auto-routing evidence: {error}"))?; + json.push('\n'); + fs::write(output_path, json).map_err(|error| { + format!( + "write Auto-routing evidence {}: {error}", + output_path.display() + ) + }) +} + +/// Return the lowercase SHA-256 digest for an output byte sequence. +#[must_use] +pub fn auto_routing_sha256(bytes: &[u8]) -> String { + sha256_hex(bytes) +} + +/// Parse and cross-check one encode workload as a binary 8-bit PGM or PPM. +/// +/// # Errors +/// +/// Returns an error when the workload is not an encode input, the PNM is +/// malformed, or its declared pixel layout disagrees with the payload. +pub fn load_auto_routing_pnm(workload: &AutoRoutingWorkload) -> Result { + if workload.kind != AutoRoutingWorkloadKind::Encode { + return Err(format!( + "Auto-routing workload {} is not an encode input", + workload.id + )); + } + let image = crate::read_pnm_image(&workload.path).map_err(|error| { + format!( + "read encode workload {} as binary PNM: {error}", + workload.id + ) + })?; + let expected_components = match workload.pixel_format { + AutoRoutingPixelFormat::Gray8 => 1, + AutoRoutingPixelFormat::Rgb8 => 3, + }; + if image.channels != expected_components { + return Err(format!( + "encode workload {} declares {:?} but its PNM has {} components", + workload.id, workload.pixel_format, image.channels + )); + } + Ok(AutoRoutingPnm { + id: workload.id.clone(), + pixels: image.pixels, + width: image.width, + height: image.height, + components: u16::try_from(image.channels) + .map_err(|_| "PNM channel count does not fit u16".to_string())?, + }) +} + +/// Build one CPU-versus-hybrid evidence cell with no device-native claim. +#[must_use] +pub fn auto_routing_route_cell( + workload: &str, + operation: AutoRoutingOperation, + criterion_group_id: &str, + output_sha256: String, +) -> AutoRoutingCell { + AutoRoutingCell { + id: format!("{}-{workload}", auto_routing_operation_label(operation)), + operation, + source: "external".to_string(), + workload: workload.to_string(), + cpu: AutoRoutingRoute { + criterion_id: format!("{criterion_group_id}/cpu"), + execution: AutoRoutingExecution::Cpu, + output_sha256: output_sha256.clone(), + }, + hybrid: AutoRoutingRoute { + criterion_id: format!("{criterion_group_id}/hybrid"), + execution: AutoRoutingExecution::Hybrid, + output_sha256, + }, + strict_device_supported: false, + strict_device: None, + } +} + +/// Append one length-delimited route output to a batch parity buffer. +/// +/// # Errors +/// +/// Returns an error when the output length cannot be represented as `u64`. +pub fn append_auto_routing_output(output: &mut Vec, bytes: &[u8]) -> Result<(), String> { + let len = u64::try_from(bytes.len()).map_err(|_| "route output is too large")?; + output.extend_from_slice(&len.to_le_bytes()); + output.extend_from_slice(bytes); + Ok(()) +} + +/// Stable operation label used by Criterion IDs and verification reports. +#[must_use] +pub const fn auto_routing_operation_label(operation: AutoRoutingOperation) -> &'static str { + match operation { + AutoRoutingOperation::FullDecode => "full-decode", + AutoRoutingOperation::RoiDecode => "roi-decode", + AutoRoutingOperation::ScaledDecode => "scaled-decode", + AutoRoutingOperation::BatchDecode => "batch-decode", + AutoRoutingOperation::LosslessEncode => "lossless-encode", + AutoRoutingOperation::LossyEncode => "lossy-encode", + } +} + +fn validate_manifest_header(manifest: &AutoRoutingManifest) -> Result<(), String> { + if manifest.schema_version != 1 + || manifest.corpus.is_empty() + || manifest.cases.is_empty() + || manifest.cases.len() > MAX_CASES + || !manifest.source_url.starts_with("https://") + || manifest.source_url["https://".len()..].is_empty() + || manifest.source_url.chars().any(char::is_whitespace) + { + return Err("Auto-routing manifest identity or case inventory is invalid".to_string()); + } + Ok(()) +} + +fn validate_evidence(evidence: &AutoRoutingEvidence) -> Result<(), String> { + if evidence.schema_version != 1 + || !is_lower_hex(&evidence.candidate_sha, 40) + || !is_lower_hex(&evidence.external_manifest_sha256, 64) + || evidence.external_case_count == 0 + || evidence.cells.is_empty() + || evidence.cells.len() > MAX_CASES + { + return Err("Auto-routing evidence identity is invalid".to_string()); + } + if [ + &evidence.platform.os, + &evidence.platform.arch, + &evidence.platform.hardware, + &evidence.platform.driver, + ] + .into_iter() + .any(String::is_empty) + { + return Err("Auto-routing platform identity must be complete".to_string()); + } + let expected_platform = match evidence.backend { + AutoRoutingBackend::Cuda => ("linux", "x86_64"), + AutoRoutingBackend::Metal => ("macos", "aarch64"), + }; + if ( + evidence.platform.os.as_str(), + evidence.platform.arch.as_str(), + ) != expected_platform + { + return Err("Auto-routing backend and platform identity do not match".to_string()); + } + + let mut cell_ids = BTreeSet::new(); + let mut criterion_ids = BTreeSet::new(); + for cell in &evidence.cells { + if !is_safe_id(&cell.id) + || !is_safe_id(&cell.workload) + || cell.source != "external" + || !cell_ids.insert(cell.id.as_str()) + { + return Err("Auto-routing cells must have unique safe external ids".to_string()); + } + validate_route( + "CPU route", + &cell.cpu, + AutoRoutingExecution::Cpu, + &mut criterion_ids, + )?; + validate_route( + "hybrid route", + &cell.hybrid, + AutoRoutingExecution::Hybrid, + &mut criterion_ids, + )?; + match (cell.strict_device_supported, &cell.strict_device) { + (false, None) => {} + (true, Some(route)) => validate_route( + "strict-device route", + route, + AutoRoutingExecution::DeviceNative, + &mut criterion_ids, + )?, + _ => return Err("Auto-routing strict-device support is inconsistent".to_string()), + } + if cell.cpu.output_sha256 != cell.hybrid.output_sha256 + || cell + .strict_device + .as_ref() + .is_some_and(|route| route.output_sha256 != cell.cpu.output_sha256) + { + return Err(format!( + "Auto-routing cell {} routes do not produce identical outputs", + cell.id + )); + } + } + Ok(()) +} + +fn validate_route( + label: &str, + route: &AutoRoutingRoute, + expected_execution: AutoRoutingExecution, + criterion_ids: &mut BTreeSet, +) -> Result<(), String> { + if route.execution != expected_execution + || !is_safe_criterion_id(&route.criterion_id) + || !is_lower_hex(&route.output_sha256, 64) + || !criterion_ids.insert(route.criterion_id.clone()) + { + return Err(format!("Auto-routing {label} is invalid or duplicated")); + } + Ok(()) +} + +fn read_bounded_regular_file(path: &Path, max_bytes: u64, label: &str) -> Result, String> { + let metadata = fs::symlink_metadata(path) + .map_err(|error| format!("read {label} {}: {error}", path.display()))?; + if metadata.file_type().is_symlink() + || !metadata.is_file() + || metadata.len() == 0 + || metadata.len() > max_bytes + { + return Err(format!( + "{label} {} must be a non-empty bounded regular file", + path.display() + )); + } + fs::read(path).map_err(|error| format!("read {label} {}: {error}", path.display())) +} + +fn is_safe_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) +} + +fn is_safe_criterion_id(value: &str) -> bool { + !value.is_empty() + && !value.contains('\\') + && Path::new(value) + .components() + .all(|component| matches!(component, Component::Normal(segment) if !segment.is_empty())) +} + +fn is_lower_hex(value: &str, length: usize) -> bool { + value.len() == length + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn sha256_hex(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use serde_json::json; + use sha2::{Digest, Sha256}; + + use super::{ + auto_routing_route_cell, load_auto_routing_manifest, load_auto_routing_pnm, + write_auto_routing_evidence, AutoRoutingBackend, AutoRoutingEvidence, AutoRoutingExecution, + AutoRoutingOperation, AutoRoutingPlatform, + }; + + #[test] + fn manifest_loader_hashes_and_classifies_bounded_external_inputs() { + let root = temp_dir("manifest"); + let decode = root.join("decode/sample.j2k"); + let encode = root.join("encode/sample.ppm"); + fs::create_dir_all(decode.parent().unwrap()).unwrap(); + fs::create_dir_all(encode.parent().unwrap()).unwrap(); + fs::write(&decode, b"decode bytes").unwrap(); + fs::write(&encode, b"P6\n1 1\n255\n\x01\x02\x03").unwrap(); + let manifest = root.join("manifest.json"); + let manifest_value = json!({ + "schema_version": 1, + "corpus": "routing-fixture", + "source_url": "https://example.invalid/routing-fixture", + "cases": [ + { + "id": "decode-case", + "path": "decode/sample.j2k", + "kind": "decode", + "pixel_format": "rgb8", + "sha256": sha256(b"decode bytes") + }, + { + "id": "encode-case", + "path": "encode/sample.ppm", + "kind": "encode", + "pixel_format": "rgb8", + "sha256": sha256(b"P6\n1 1\n255\n\x01\x02\x03") + } + ] + }); + fs::write( + &manifest, + serde_json::to_vec_pretty(&manifest_value).unwrap(), + ) + .unwrap(); + + let loaded = load_auto_routing_manifest(&manifest, &root).unwrap(); + + assert_eq!(loaded.manifest.corpus, "routing-fixture"); + assert_eq!(loaded.workloads.len(), 2); + assert_eq!(loaded.workloads[0].bytes, b"decode bytes"); + assert_eq!(loaded.workloads[1].bytes, b"P6\n1 1\n255\n\x01\x02\x03"); + assert_eq!( + loaded.manifest_sha256, + sha256(&fs::read(&manifest).unwrap()) + ); + let pnm = load_auto_routing_pnm(&loaded.workloads[1]).unwrap(); + assert_eq!((pnm.width, pnm.height, pnm.components), (1, 1, 3)); + assert_eq!(pnm.pixels, [1, 2, 3]); + } + + #[test] + fn manifest_loader_rejects_escape_hash_mismatch_and_duplicate_ids() { + let root = temp_dir("invalid"); + fs::write(root.join("sample.j2k"), b"sample").unwrap(); + let manifest = root.join("manifest.json"); + let base = json!({ + "schema_version": 1, + "corpus": "routing-fixture", + "source_url": "https://example.invalid/routing-fixture", + "cases": [{ + "id": "case", + "path": "../outside.j2k", + "kind": "decode", + "pixel_format": "gray8", + "sha256": sha256(b"sample") + }] + }); + fs::write(&manifest, serde_json::to_vec(&base).unwrap()).unwrap(); + assert!(load_auto_routing_manifest(&manifest, &root) + .unwrap_err() + .contains("safe relative path")); + + let mut mismatch = base.clone(); + mismatch["cases"][0]["path"] = json!("sample.j2k"); + mismatch["cases"][0]["sha256"] = json!("0".repeat(64)); + fs::write(&manifest, serde_json::to_vec(&mismatch).unwrap()).unwrap(); + assert!(load_auto_routing_manifest(&manifest, &root) + .unwrap_err() + .contains("SHA-256 mismatch")); + + let mut duplicate = mismatch; + duplicate["cases"][0]["sha256"] = json!(sha256(b"sample")); + let repeated = duplicate["cases"][0].clone(); + duplicate["cases"].as_array_mut().unwrap().push(repeated); + fs::write(&manifest, serde_json::to_vec(&duplicate).unwrap()).unwrap(); + assert!(load_auto_routing_manifest(&manifest, &root) + .unwrap_err() + .contains("unique ids")); + } + + #[test] + fn evidence_writer_is_deterministic_and_refuses_invalid_route_labels() { + let root = temp_dir("evidence"); + let output = root.join("nested/evidence.json"); + let mut evidence = AutoRoutingEvidence { + schema_version: 1, + candidate_sha: "1".repeat(40), + backend: AutoRoutingBackend::Metal, + platform: AutoRoutingPlatform { + os: "macos".to_string(), + arch: "aarch64".to_string(), + hardware: "Apple M fixture".to_string(), + driver: "fixture driver".to_string(), + }, + external_manifest_sha256: "2".repeat(64), + external_case_count: 2, + cells: vec![auto_routing_route_cell( + "decode-case", + AutoRoutingOperation::FullDecode, + "auto-routing_full-decode_decode-case", + "3".repeat(64), + )], + }; + + write_auto_routing_evidence(&output, &evidence).unwrap(); + let first = fs::read(&output).unwrap(); + write_auto_routing_evidence(&output, &evidence).unwrap(); + assert_eq!(fs::read(&output).unwrap(), first); + assert!(first.ends_with(b"\n")); + + evidence.cells[0].hybrid.execution = AutoRoutingExecution::Cpu; + assert!(write_auto_routing_evidence(&output, &evidence) + .unwrap_err() + .contains("hybrid route")); + } + + fn sha256(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) + } + + fn temp_dir(label: &str) -> std::path::PathBuf { + let root = std::env::temp_dir().join(format!( + "j2k-test-support-auto-routing-{label}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir_all(&root).unwrap(); + root + } +} diff --git a/crates/j2k-test-support/src/fixtures.rs b/crates/j2k-test-support/src/fixtures.rs index bc59dd20..b6a79869 100644 --- a/crates/j2k-test-support/src/fixtures.rs +++ b/crates/j2k-test-support/src/fixtures.rs @@ -22,6 +22,24 @@ pub const JPEG_BASELINE_420_RESTART_32X16: &[u8] = pub const JPEG_BASELINE_420_RESTART_32X16_RGB: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.rgb"); +/// `OpenJPEG` 2.5.4 irreversible 8x8 RGB codestream used for adapter parity tests. +/// +/// The source pixels are the deterministic `patterned_rgb8` formula. `OpenJPEG` +/// encoded them with `opj_compress -I -r 8 -n 4`. +pub const OPENJPEG_IRREVERSIBLE_RGB8_8X8: &[u8] = &[ + 0xff, 0x4f, 0xff, 0x51, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x07, 0x01, 0x01, 0x07, 0x01, 0x01, + 0x07, 0x01, 0x01, 0xff, 0x52, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x01, 0x01, 0x03, 0x04, 0x04, 0x00, + 0x00, 0xff, 0x5c, 0x00, 0x17, 0x42, 0x67, 0x38, 0x67, 0x50, 0x67, 0x50, 0x67, 0x68, 0x50, 0x05, + 0x50, 0x05, 0x50, 0x47, 0x57, 0xd3, 0x57, 0xd3, 0x57, 0x62, 0xff, 0x64, 0x00, 0x25, 0x00, 0x01, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x4f, 0x70, 0x65, 0x6e, 0x4a, + 0x50, 0x45, 0x47, 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x20, 0x32, 0x2e, 0x35, 0x2e, + 0x34, 0xff, 0x90, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x01, 0xff, 0x93, 0xc7, + 0xea, 0x04, 0x06, 0xbf, 0x80, 0x80, 0xa0, 0xfb, 0xc0, 0x80, 0x01, 0x9f, 0xc1, 0xf7, 0x81, 0x00, + 0x04, 0x8f, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0xff, 0xd9, +]; + #[cfg(feature = "j2k-native-fixtures")] mod generated_htj2k; mod jp2; diff --git a/crates/j2k-test-support/src/lib.rs b/crates/j2k-test-support/src/lib.rs index cdd460a6..6c8ed7ae 100644 --- a/crates/j2k-test-support/src/lib.rs +++ b/crates/j2k-test-support/src/lib.rs @@ -8,6 +8,7 @@ use std::{ path::Path, }; +mod auto_routing; mod corpus; mod cuda; mod fixtures; @@ -18,6 +19,14 @@ mod metal; mod metal_shader; mod pixels; +pub use auto_routing::{ + append_auto_routing_output, auto_routing_operation_label, auto_routing_route_cell, + auto_routing_sha256, load_auto_routing_manifest, load_auto_routing_pnm, + write_auto_routing_evidence, AutoRoutingBackend, AutoRoutingCell, AutoRoutingEvidence, + AutoRoutingExecution, AutoRoutingManifest, AutoRoutingManifestCase, AutoRoutingOperation, + AutoRoutingPixelFormat, AutoRoutingPlatform, AutoRoutingPnm, AutoRoutingRoute, + AutoRoutingWorkload, AutoRoutingWorkloadKind, AutoRoutingWorkloadSet, +}; pub use corpus::{collect_jpeg_paths, is_jpeg_path, paths_from_env}; pub use cuda::{ cuda_device_unavailable_is_skip, cuda_jpeg_hardware_decode_gate, @@ -35,7 +44,7 @@ pub use fixtures::{ OpenJphBatchFixture, JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_16X16_RGB, JPEG_BASELINE_420_RESTART_32X16, JPEG_BASELINE_420_RESTART_32X16_RGB, JPEG_BASELINE_422_16X8, JPEG_BASELINE_422_16X8_RGB, JPEG_BASELINE_444_8X8, JPEG_BASELINE_444_8X8_RGB, - JPEG_GRAYSCALE_8X8, JPEG_GRAYSCALE_8X8_GRAY, + JPEG_GRAYSCALE_8X8, JPEG_GRAYSCALE_8X8_GRAY, OPENJPEG_IRREVERSIBLE_RGB8_8X8, }; #[cfg(feature = "j2k-native-fixtures")] pub use fixtures::{ diff --git a/crates/j2k-tilecodec/Cargo.toml b/crates/j2k-tilecodec/Cargo.toml index c8231057..1773f78d 100644 --- a/crates/j2k-tilecodec/Cargo.toml +++ b/crates/j2k-tilecodec/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_tilecodec" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } thiserror = { workspace = true } flate2 = { workspace = true } zstd = { workspace = true } diff --git a/crates/j2k-tilecodec/fuzz/Cargo.lock b/crates/j2k-tilecodec/fuzz/Cargo.lock index d6af2315..2a03faa7 100644 --- a/crates/j2k-tilecodec/fuzz/Cargo.lock +++ b/crates/j2k-tilecodec/fuzz/Cargo.lock @@ -72,14 +72,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.8.0" +version = "0.8.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-tilecodec" -version = "0.8.0" +version = "0.8.1" dependencies = [ "flate2", "j2k-core", diff --git a/crates/j2k-transcode-cuda/Cargo.toml b/crates/j2k-transcode-cuda/Cargo.toml index 37b3ed29..57043a8e 100644 --- a/crates/j2k-transcode-cuda/Cargo.toml +++ b/crates/j2k-transcode-cuda/Cargo.toml @@ -29,10 +29,10 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.0", optional = true } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.1" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.8.1", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-transcode-metal/Cargo.toml b/crates/j2k-transcode-metal/Cargo.toml index bacf54ab..60046ce0 100644 --- a/crates/j2k-transcode-metal/Cargo.toml +++ b/crates/j2k-transcode-metal/Cargo.toml @@ -26,21 +26,21 @@ default = [] bench-internals = ["j2k-transcode/dev-support"] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.0" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.8.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.1" } [target.'cfg(target_os = "macos")'.dependencies] -j2k-metal = { path = "../j2k-metal", version = "=0.8.0" } +j2k-metal = { path = "../j2k-metal", version = "=0.8.1" } metal = { workspace = true } [dev-dependencies] criterion = { workspace = true } rayon = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1" } j2k-test-support = { path = "../j2k-test-support" } j2k-transcode-test-support = { path = "../j2k-transcode-test-support" } diff --git a/crates/j2k-transcode-test-support/Cargo.toml b/crates/j2k-transcode-test-support/Cargo.toml index 55ba23e5..7fce1b5e 100644 --- a/crates/j2k-transcode-test-support/Cargo.toml +++ b/crates/j2k-transcode-test-support/Cargo.toml @@ -13,11 +13,11 @@ name = "j2k_transcode_test_support" path = "src/lib.rs" [dependencies] -j2k-transcode = { path = "../j2k-transcode", version = "=0.8.0", features = ["dev-support"] } -j2k-types = { path = "../j2k-types", version = "=0.8.0" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.8.1", features = ["dev-support"] } +j2k-types = { path = "../j2k-types", version = "=0.8.1" } [dev-dependencies] -j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } [lints.rust] unsafe_code = "forbid" diff --git a/crates/j2k-transcode/Cargo.toml b/crates/j2k-transcode/Cargo.toml index 58a39b5a..a80927eb 100644 --- a/crates/j2k-transcode/Cargo.toml +++ b/crates/j2k-transcode/Cargo.toml @@ -22,12 +22,12 @@ dev-support = [] [dependencies] rayon = { workspace = true } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.0" } -j2k = { path = "../j2k", version = "=0.8.0" } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.8.0", default-features = false } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.8.1" } +j2k = { path = "../j2k", version = "=0.8.1" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.8.1", default-features = false } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-transcode/fuzz/Cargo.lock b/crates/j2k-transcode/fuzz/Cargo.lock index 8eb46a0c..ea1b7d14 100644 --- a/crates/j2k-transcode/fuzz/Cargo.lock +++ b/crates/j2k-transcode/fuzz/Cargo.lock @@ -8,6 +8,12 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "cc" version = "1.2.65" @@ -83,29 +89,30 @@ dependencies = [ [[package]] name = "j2k" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-codec-math", "j2k-core", "j2k-native", "j2k-types", + "moxcms", "thiserror", ] [[package]] name = "j2k-codec-math" -version = "0.8.0" +version = "0.8.1" [[package]] name = "j2k-core" -version = "0.8.0" +version = "0.8.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-jpeg" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -117,7 +124,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.8.0" +version = "0.8.1" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -129,11 +136,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.0" +version = "0.8.1" [[package]] name = "j2k-transcode" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k", "j2k-codec-math", @@ -155,7 +162,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.8.0" +version = "0.8.1" [[package]] name = "jobserver" @@ -195,6 +202,25 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -204,6 +230,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quote" version = "1.0.45" diff --git a/crates/j2k-types/src/lib.rs b/crates/j2k-types/src/lib.rs index 7fcdb3d4..7efeff67 100644 --- a/crates/j2k-types/src/lib.rs +++ b/crates/j2k-types/src/lib.rs @@ -120,6 +120,21 @@ pub struct J2kDeinterleaveToF32Job<'a> { pub signed: bool, } +/// Validated image and coding context supplied before encode-stage dispatch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kEncodeContext { + /// Number of pixels in the encoded image or tile. + pub num_pixels: usize, + /// Number of interleaved source components. + pub num_components: u16, + /// Source sample bit depth. + pub bit_depth: u8, + /// Whether source samples are signed. + pub signed: bool, + /// Whether the codestream uses reversible coding. + pub reversible: bool, +} + /// Adapter forward RCT job for backend experimentation. #[derive(Debug)] pub struct J2kForwardRctJob<'a> { @@ -656,6 +671,14 @@ pub struct CpuOnlyJ2kEncodeStageAccelerator; /// Adapter JPEG 2000 encode-stage accelerator for backend experimentation. pub trait J2kEncodeStageAccelerator { + /// Supply validated context before any encode-stage hook is invoked. + /// + /// Implementations may use this to choose a fixed route for the operation. + /// The default keeps existing accelerators source-compatible. + fn begin_encode(&mut self, _context: J2kEncodeContext) -> J2kEncodeStageResult<()> { + Ok(()) + } + /// Report cumulative backend dispatches completed by this accelerator. fn dispatch_report(&self) -> J2kEncodeDispatchReport { J2kEncodeDispatchReport::default() diff --git a/crates/j2k/Cargo.toml b/crates/j2k/Cargo.toml index b010a989..d44d444b 100644 --- a/crates/j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -20,16 +20,17 @@ name = "j2k" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.0" } -j2k-core = { path = "../j2k-core", version = "=0.8.0" } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } -j2k-types = { path = "../j2k-types", version = "=0.8.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.8.1" } +j2k-core = { path = "../j2k-core", version = "=0.8.1" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } +j2k-types = { path = "../j2k-types", version = "=0.8.1" } +moxcms = { workspace = true } thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.8.0" } +j2k-native = { path = "../j2k-native", version = "=0.8.1" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [[bench]] diff --git a/crates/j2k/fuzz/Cargo.lock b/crates/j2k/fuzz/Cargo.lock index f63510d1..c5548dce 100644 --- a/crates/j2k/fuzz/Cargo.lock +++ b/crates/j2k/fuzz/Cargo.lock @@ -8,6 +8,21 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "cc" version = "1.2.65" @@ -26,6 +41,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -51,6 +75,26 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "either" version = "1.16.0" @@ -69,6 +113,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -81,24 +135,31 @@ dependencies = [ "wasip2", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "j2k" -version = "0.8.0" +version = "0.8.1" dependencies = [ "j2k-codec-math", "j2k-core", "j2k-native", "j2k-types", + "moxcms", "thiserror", ] [[package]] name = "j2k-codec-math" -version = "0.8.0" +version = "0.8.1" [[package]] name = "j2k-core" -version = "0.8.0" +version = "0.8.1" dependencies = [ "thiserror", ] @@ -108,12 +169,13 @@ name = "j2k-fuzz" version = "0.1.0" dependencies = [ "j2k", + "j2k-test-support", "libfuzzer-sys", ] [[package]] name = "j2k-native" -version = "0.8.0" +version = "0.8.1" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -125,11 +187,20 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.0" +version = "0.8.1" + +[[package]] +name = "j2k-test-support" +version = "0.8.1" +dependencies = [ + "serde", + "serde_json", + "sha2", +] [[package]] name = "j2k-types" -version = "0.8.0" +version = "0.8.1" [[package]] name = "jobserver" @@ -163,6 +234,31 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -172,6 +268,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + [[package]] name = "quote" version = "1.0.45" @@ -207,6 +309,60 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -224,6 +380,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -241,15 +408,27 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -264,3 +443,9 @@ name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/j2k/fuzz/Cargo.toml b/crates/j2k/fuzz/Cargo.toml index c4e4f5a7..fffbda7e 100644 --- a/crates/j2k/fuzz/Cargo.toml +++ b/crates/j2k/fuzz/Cargo.toml @@ -10,6 +10,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" j2k = { path = ".." } +j2k-test-support = { path = "../../j2k-test-support" } [[bin]] name = "parse_fuzz" @@ -46,6 +47,13 @@ test = false doc = false bench = false +[[bin]] +name = "srgb8_fuzz" +path = "fuzz_targets/srgb8_fuzz.rs" +test = false +doc = false +bench = false + [profile.release] debug = 1 diff --git a/crates/j2k/fuzz/fuzz_targets/srgb8_fuzz.rs b/crates/j2k/fuzz/fuzz_targets/srgb8_fuzz.rs new file mode 100644 index 00000000..8ea6cea2 --- /dev/null +++ b/crates/j2k/fuzz/fuzz_targets/srgb8_fuzz.rs @@ -0,0 +1,57 @@ +#![no_main] + +use j2k::J2kDecoder; +use j2k_test_support::minimal_j2k_codestream; +use libfuzzer_sys::fuzz_target; + +const MAX_ICC_BYTES: usize = 64 * 1024; + +fuzz_target!(|data: &[u8]| { + if data.len() > MAX_ICC_BYTES { + return; + } + + let jp2 = jp2_with_gray_icc(data); + let Ok(mut decoder) = J2kDecoder::new(&jp2) else { + return; + }; + let _ = decoder.decode_srgb8(); +}); + +fn jp2_with_gray_icc(profile: &[u8]) -> Vec { + let codestream = minimal_j2k_codestream(); + let mut jp2h = Vec::new(); + push_box( + &mut jp2h, + *b"ihdr", + &[0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 7, 7, 0, 0], + ); + let mut color = Vec::with_capacity(profile.len().saturating_add(3)); + color.extend_from_slice(&[2, 0, 0]); + color.extend_from_slice(profile); + push_box(&mut jp2h, *b"colr", &color); + + let mut out = Vec::with_capacity( + codestream + .len() + .saturating_add(jp2h.len()) + .saturating_add(48), + ); + out.extend_from_slice(&[0, 0, 0, 12]); + out.extend_from_slice(b"jP "); + out.extend_from_slice(&[0x0d, 0x0a, 0x87, 0x0a]); + out.extend_from_slice(&[0, 0, 0, 20]); + out.extend_from_slice(b"ftypjp2 \0\0\0\0jp2 "); + push_box(&mut out, *b"jp2h", &jp2h); + push_box(&mut out, *b"jp2c", &codestream); + out +} + +fn push_box(out: &mut Vec, box_type: [u8; 4], payload: &[u8]) { + let Ok(length) = u32::try_from(payload.len().saturating_add(8)) else { + return; + }; + out.extend_from_slice(&length.to_be_bytes()); + out.extend_from_slice(&box_type); + out.extend_from_slice(payload); +} diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs index 3eed2a51..c8581f8f 100644 --- a/crates/j2k/src/decode.rs +++ b/crates/j2k/src/decode.rs @@ -13,10 +13,13 @@ pub use component_handoff::{ }; mod output; mod settings; +mod srgb8; use output::{ can_decode_u8_directly, write_components_u8_output, write_u16_output, write_u8_output, }; pub use settings::DecodeSettings; +pub(crate) use srgb8::decode_image_srgb8; +pub use srgb8::{J2kSrgb8Image, J2kSrgb8Layout}; /// Non-fatal JPEG 2000 decode warning surfaced through decode outcomes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/j2k/src/decode/srgb8.rs b/crates/j2k/src/decode/srgb8.rs new file mode 100644 index 00000000..f8994cc5 --- /dev/null +++ b/crates/j2k/src/decode/srgb8.rs @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Explicit 8-bit sRGB output normalization. + +use crate::{backend::ColorSpace, backend::Image, J2kError}; +use alloc::vec::Vec; +use j2k_core::{ + ensure_allocation_within_cap, try_host_vec_filled, BufferError, HostAllocationError, + DEFAULT_MAX_HOST_ALLOCATION_BYTES, +}; +use moxcms::{CmsError, ColorProfile, DataColorSpace, Layout, ParsingOptions, TransformOptions}; + +const SRGB8_OUTPUT_WHAT: &str = "J2K sRGB8 normalized output"; + +/// Interleaved sample layout returned by [`J2kSrgb8Image`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum J2kSrgb8Layout { + /// One 8-bit sRGB-grey sample per pixel. + Gray, + /// Three interleaved 8-bit sRGB samples per pixel. + Rgb, + /// Three interleaved 8-bit sRGB samples followed by alpha per pixel. + Rgba, +} + +impl J2kSrgb8Layout { + const fn channels(self) -> usize { + match self { + Self::Gray => 1, + Self::Rgb => 3, + Self::Rgba => 4, + } + } +} + +/// Owned, tightly packed 8-bit sRGB decode result. +#[derive(Debug, PartialEq, Eq)] +pub struct J2kSrgb8Image { + dimensions: (u32, u32), + layout: J2kSrgb8Layout, + data: Vec, +} + +impl J2kSrgb8Image { + /// Decoded image dimensions. + #[must_use] + pub const fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Interleaved sample layout. + #[must_use] + pub const fn layout(&self) -> J2kSrgb8Layout { + self.layout + } + + /// Tightly packed 8-bit image samples. + #[must_use] + pub fn data(&self) -> &[u8] { + &self.data + } + + /// Consume the image and return its tightly packed sample storage. + #[must_use] + pub fn into_data(self) -> Vec { + self.data + } +} + +pub(crate) fn decode_image_srgb8<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, +) -> Result { + let retained_image_bytes = image + .retained_allocation_bytes() + .map_err(J2kError::from_native_decode_error)?; + let primary_icc = image.primary_icc_profile(); + let bitmap = image + .decode_with_context(native_context) + .map_err(J2kError::from_native_decode_error)?; + let dimensions = (bitmap.width, bitmap.height); + let has_alpha = bitmap.has_alpha; + let bitmap_profile_bytes = match &bitmap.color_space { + ColorSpace::Icc { profile, .. } => profile.capacity(), + _ => 0, + }; + let retained_bytes = checked_peak_bytes( + retained_image_bytes, + bitmap.data.capacity(), + bitmap_profile_bytes, + )?; + + if let Some(profile) = primary_icc { + return convert_icc_data(&bitmap.data, profile, dimensions, has_alpha, retained_bytes); + } + + match bitmap.color_space { + ColorSpace::Gray if has_alpha => { + expand_gray_alpha(&bitmap.data, dimensions, retained_bytes) + } + ColorSpace::Gray => finish(bitmap.data, dimensions, J2kSrgb8Layout::Gray), + ColorSpace::RGB => finish( + bitmap.data, + dimensions, + if has_alpha { + J2kSrgb8Layout::Rgba + } else { + J2kSrgb8Layout::Rgb + }, + ), + ColorSpace::Icc { profile, .. } => convert_icc_data( + &bitmap.data, + &profile, + dimensions, + has_alpha, + retained_bytes, + ), + ColorSpace::CMYK | ColorSpace::Unknown { .. } => Err(j2k_core::Unsupported { + what: "decode_srgb8 requires grayscale, RGB, or a restricted ICC input profile", + } + .into()), + } +} + +fn convert_icc_data( + source: &[u8], + profile: &[u8], + dimensions: (u32, u32), + has_alpha: bool, + retained_bytes: usize, +) -> Result { + let max_profile_size = profile + .len() + .checked_add(1) + .ok_or_else(output_size_overflow)?; + let source_profile = ColorProfile::new_from_slice_with_options( + profile, + ParsingOptions { + max_profile_size, + max_allowed_clut_size: 0, + max_allowed_trc_size: 65_536, + }, + ) + .map_err(|error| map_profile_error(&error))?; + reject_non_restricted_profile(&source_profile)?; + + match source_profile.color_space { + DataColorSpace::Rgb => transform_rgb( + source, + dimensions, + has_alpha, + retained_bytes, + &source_profile, + ), + DataColorSpace::Gray => transform_gray( + source, + dimensions, + has_alpha, + retained_bytes, + &source_profile, + ), + _ => Err(j2k_core::Unsupported { + what: "decode_srgb8 supports restricted ICC RGB and monochrome input profiles", + } + .into()), + } +} + +fn reject_non_restricted_profile(profile: &ColorProfile) -> Result<(), J2kError> { + if profile.lut_a_to_b_perceptual.is_some() + || profile.lut_a_to_b_colorimetric.is_some() + || profile.lut_a_to_b_saturation.is_some() + || profile.lut_b_to_a_perceptual.is_some() + || profile.lut_b_to_a_colorimetric.is_some() + || profile.lut_b_to_a_saturation.is_some() + || profile.gamut.is_some() + { + return Err(j2k_core::Unsupported { + what: "decode_srgb8 supports restricted matrix/TRC ICC profiles only", + } + .into()); + } + Ok(()) +} + +fn transform_rgb( + source: &[u8], + dimensions: (u32, u32), + has_alpha: bool, + retained_bytes: usize, + source_profile: &ColorProfile, +) -> Result { + let cms_layout = if has_alpha { Layout::Rgba } else { Layout::Rgb }; + let destination = ColorProfile::new_srgb(); + transform_icc( + source, + dimensions, + retained_bytes, + source_profile, + cms_layout, + &destination, + cms_layout, + ) +} + +fn transform_gray( + source: &[u8], + dimensions: (u32, u32), + has_alpha: bool, + retained_bytes: usize, + source_profile: &ColorProfile, +) -> Result { + if has_alpha { + let destination = ColorProfile::new_srgb(); + return transform_icc( + source, + dimensions, + retained_bytes, + source_profile, + Layout::GrayAlpha, + &destination, + Layout::Rgba, + ); + } + + let mut destination = ColorProfile::new_gray_with_gamma(1.0); + destination.gray_trc = ColorProfile::new_srgb().red_trc; + transform_icc( + source, + dimensions, + retained_bytes, + source_profile, + Layout::Gray, + &destination, + Layout::Gray, + ) +} + +fn transform_icc( + source: &[u8], + dimensions: (u32, u32), + retained_bytes: usize, + source_profile: &ColorProfile, + source_layout: Layout, + destination_profile: &ColorProfile, + destination_layout: Layout, +) -> Result { + let output_layout = match destination_layout { + Layout::Gray => J2kSrgb8Layout::Gray, + Layout::Rgb => J2kSrgb8Layout::Rgb, + Layout::Rgba => J2kSrgb8Layout::Rgba, + _ => { + return Err(J2kError::InternalInvariant { + what: "restricted ICC transform has a non-sRGB destination layout", + }); + } + }; + let mut output = allocate_output(dimensions, output_layout, retained_bytes)?; + let transform = source_profile + .create_transform_8bit( + source_layout, + destination_profile, + destination_layout, + TransformOptions::default(), + ) + .map_err(|error| map_transform_error(&error))?; + transform + .transform(source, &mut output) + .map_err(|error| map_transform_error(&error))?; + finish(output, dimensions, output_layout) +} + +fn expand_gray_alpha( + source: &[u8], + dimensions: (u32, u32), + retained_bytes: usize, +) -> Result { + let mut output = allocate_output(dimensions, J2kSrgb8Layout::Rgba, retained_bytes)?; + for (input, pixel) in source.chunks_exact(2).zip(output.chunks_exact_mut(4)) { + pixel.copy_from_slice(&[input[0], input[0], input[0], input[1]]); + } + finish(output, dimensions, J2kSrgb8Layout::Rgba) +} + +fn allocate_output( + dimensions: (u32, u32), + layout: J2kSrgb8Layout, + retained_bytes: usize, +) -> Result, J2kError> { + let len = expected_len(dimensions, layout)?; + checked_peak_bytes(retained_bytes, len, 0)?; + let output = try_host_vec_filled(len, 0_u8).map_err(host_allocation_error)?; + checked_peak_bytes(retained_bytes, output.capacity(), 0)?; + Ok(output) +} + +fn finish( + data: Vec, + dimensions: (u32, u32), + layout: J2kSrgb8Layout, +) -> Result { + let expected = expected_len(dimensions, layout)?; + if data.len() != expected { + return Err(J2kError::InternalInvariant { + what: "normalized sRGB output length does not match its layout", + }); + } + Ok(J2kSrgb8Image { + dimensions, + layout, + data, + }) +} + +fn expected_len(dimensions: (u32, u32), layout: J2kSrgb8Layout) -> Result { + (dimensions.0 as usize) + .checked_mul(dimensions.1 as usize) + .and_then(|pixels| pixels.checked_mul(layout.channels())) + .ok_or_else(output_size_overflow) +} + +fn checked_peak_bytes(first: usize, second: usize, third: usize) -> Result { + let requested = first + .checked_add(second) + .and_then(|bytes| bytes.checked_add(third)) + .ok_or_else(output_size_overflow)?; + ensure_allocation_within_cap( + requested, + DEFAULT_MAX_HOST_ALLOCATION_BYTES, + SRGB8_OUTPUT_WHAT, + ) + .map_err(Into::into) +} + +fn output_size_overflow() -> J2kError { + BufferError::SizeOverflow { + what: SRGB8_OUTPUT_WHAT, + } + .into() +} + +fn host_allocation_error(error: HostAllocationError) -> J2kError { + BufferError::HostAllocationFailed { + bytes: error.requested_bytes(), + what: SRGB8_OUTPUT_WHAT, + } + .into() +} + +fn map_profile_error(error: &CmsError) -> J2kError { + match error { + CmsError::OutOfMemory(bytes) => BufferError::HostAllocationFailed { + bytes: *bytes, + what: "restricted ICC profile", + } + .into(), + _ => J2kError::InvalidIccProfile, + } +} + +fn map_transform_error(error: &CmsError) -> J2kError { + match error { + CmsError::OutOfMemory(bytes) => BufferError::HostAllocationFailed { + bytes: *bytes, + what: "restricted ICC transform", + } + .into(), + _ => j2k_core::Unsupported { + what: "restricted ICC profile cannot be transformed to sRGB", + } + .into(), + } +} diff --git a/crates/j2k/src/error.rs b/crates/j2k/src/error.rs index db1ea4c4..bf12605f 100644 --- a/crates/j2k/src/error.rs +++ b/crates/j2k/src/error.rs @@ -186,6 +186,10 @@ pub enum J2kError { expected: usize, }, + /// The primary JP2 restricted ICC profile is malformed. + #[error("invalid JP2 restricted ICC profile")] + InvalidIccProfile, + /// A facade/cache invariant failed with a static diagnostic. #[error("internal JPEG 2000 invariant failed: {what}")] InternalInvariant { diff --git a/crates/j2k/src/lib.rs b/crates/j2k/src/lib.rs index 4382e421..65d0f08e 100644 --- a/crates/j2k/src/lib.rs +++ b/crates/j2k/src/lib.rs @@ -37,22 +37,23 @@ pub use adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}; pub use j2k_types::{ CpuOnlyJ2kEncodeStageAccelerator, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, IrreversibleQuantizationStep, IrreversibleQuantizationSubbandScales, J2kCodeBlockSegment, - J2kCodeBlockStyle, J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, - J2kEncodeStageError, J2kEncodeStageErrorKind, J2kEncodeStageResult, J2kForwardDwt53Job, - J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Level, - J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, - J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationBlockCodingMode, - J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, - J2kPacketizationProgressionOrder, J2kPacketizationResolution, J2kPacketizationSubband, - J2kQuantizeSubbandJob, J2kResidentEncodeInput, J2kResidentEncodeInputError, - J2kResidentHtj2kTileEncodeJob, J2kSubBandType, J2kTier1CodeBlockEncodeJob, - PrecomputedHtj2k53Component, PrecomputedHtj2k53Image, PrecomputedHtj2k97Component, - PrecomputedHtj2k97Image, PreencodedHtj2k97CodeBlock, PreencodedHtj2k97CompactCodeBlock, - PreencodedHtj2k97CompactComponent, PreencodedHtj2k97CompactImage, - PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, - PreencodedHtj2k97Component, PreencodedHtj2k97Image, PreencodedHtj2k97Resolution, - PreencodedHtj2k97Subband, PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, - PrequantizedHtj2k97Image, PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, + J2kCodeBlockStyle, J2kDeinterleaveToF32Job, J2kEncodeContext, J2kEncodeDispatchReport, + J2kEncodeStageAccelerator, J2kEncodeStageError, J2kEncodeStageErrorKind, J2kEncodeStageResult, + J2kForwardDwt53Job, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, + J2kForwardDwt97Level, J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, + J2kHtCodeBlockEncodeJob, J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, + J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, + J2kPacketizationPacketDescriptor, J2kPacketizationProgressionOrder, J2kPacketizationResolution, + J2kPacketizationSubband, J2kQuantizeSubbandJob, J2kResidentEncodeInput, + J2kResidentEncodeInputError, J2kResidentHtj2kTileEncodeJob, J2kSubBandType, + J2kTier1CodeBlockEncodeJob, PrecomputedHtj2k53Component, PrecomputedHtj2k53Image, + PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, PreencodedHtj2k97CodeBlock, + PreencodedHtj2k97CompactCodeBlock, PreencodedHtj2k97CompactComponent, + PreencodedHtj2k97CompactImage, PreencodedHtj2k97CompactResolution, + PreencodedHtj2k97CompactSubband, PreencodedHtj2k97Component, PreencodedHtj2k97Image, + PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, PrequantizedHtj2k97CodeBlock, + PrequantizedHtj2k97Component, PrequantizedHtj2k97Image, PrequantizedHtj2k97Resolution, + PrequantizedHtj2k97Subband, }; mod view; @@ -68,7 +69,8 @@ pub use batch::{ pub use decode::{ DecodeSettings, J2kComponentPlane, J2kDecodeWarning, J2kDecodedColorSpace, - J2kDecodedComponents, J2kDecodedNativeComponents, J2kNativeComponentPlane, + J2kDecodedComponents, J2kDecodedNativeComponents, J2kNativeComponentPlane, J2kSrgb8Image, + J2kSrgb8Layout, }; pub use parallelism::CpuDecodeParallelism; diff --git a/crates/j2k/src/metadata.rs b/crates/j2k/src/metadata.rs index a74654d4..ca3af067 100644 --- a/crates/j2k/src/metadata.rs +++ b/crates/j2k/src/metadata.rs @@ -19,7 +19,7 @@ pub struct J2kComponentInfo { /// Full parsed JPEG 2000 / HTJ2K support metadata. /// /// This preserves the existing compact [`Info`] summary while exposing fields -/// needed to reason about full Part 1 / Part 15 support surfaces. +/// needed to reason about the implemented Part 1 / Part 15 support surfaces. #[derive(Debug, PartialEq, Eq)] pub struct J2kSupportInfo { /// Backward-compatible metadata summary used by shared codec traits. diff --git a/crates/j2k/src/parse/boxes.rs b/crates/j2k/src/parse/boxes.rs index 63e62265..44fea29d 100644 --- a/crates/j2k/src/parse/boxes.rs +++ b/crates/j2k/src/parse/boxes.rs @@ -408,12 +408,11 @@ fn validate_component_metadata( metadata: &J2kFileMetadata, siz: &super::ParsedSiz, ) -> Result<(), J2kError> { - let source = resolved_component_source(metadata, siz); - let resolved_count = resolved_component_count(source, metadata, siz); - if resolved_count != usize::from(ihdr.components) { + let codestream_count = siz.component_info.len(); + if codestream_count != usize::from(ihdr.components) { return Err(J2kError::InvalidBox { offset: ihdr.offset, - what: "ihdr component count must match resolved JP2 image components", + what: "ihdr component count must match codestream components", }); } @@ -424,17 +423,11 @@ fn validate_component_metadata( what: "bpcc must not be present when ihdr bpc is explicit", }); } - for index in 0..resolved_count { - let component = resolved_component_at(source, metadata, siz, index).ok_or( - J2kError::InvalidBox { - offset: ihdr.offset, - what: "JP2 component metadata could not be resolved", - }, - )?; - if !same_component_precision(component, descriptor) { + for component in &siz.component_info { + if !same_component_precision(*component, descriptor) { return Err(J2kError::InvalidBox { offset: ihdr.offset, - what: "ihdr bpc must match resolved JP2 image component precision", + what: "ihdr bpc must match codestream component precision", }); } } @@ -445,17 +438,11 @@ fn validate_component_metadata( what: "bpcc component count must match ihdr component count", }); } - for (index, descriptor) in metadata.bits_per_component.iter().enumerate() { - let component = resolved_component_at(source, metadata, siz, index).ok_or( - J2kError::InvalidBox { - offset: ihdr.offset, - what: "JP2 component metadata could not be resolved", - }, - )?; - if !same_component_precision(component, *descriptor) { + for (component, descriptor) in siz.component_info.iter().zip(&metadata.bits_per_component) { + if !same_component_precision(*component, *descriptor) { return Err(J2kError::InvalidBox { offset: ihdr.offset, - what: "bpcc entries must match resolved JP2 image component precision", + what: "bpcc entries must match codestream component precision", }); } } @@ -464,105 +451,6 @@ fn validate_component_metadata( Ok(()) } -#[derive(Clone, Copy)] -enum ResolvedComponentSource { - Codestream, - Palette, - Mappings, -} - -fn resolved_component_source( - metadata: &J2kFileMetadata, - siz: &super::ParsedSiz, -) -> ResolvedComponentSource { - if metadata.component_mappings.is_empty() { - if metadata.palette.is_some() { - return ResolvedComponentSource::Palette; - } - return ResolvedComponentSource::Codestream; - } - - let resolvable = metadata - .component_mappings - .iter() - .all(|mapping| match mapping.mapping_type { - J2kComponentMappingType::Direct => siz - .component_info - .get(usize::from(mapping.component_index)) - .is_some(), - J2kComponentMappingType::Palette { column } => metadata - .palette - .as_ref() - .and_then(|palette| palette.columns.get(usize::from(column))) - .is_some(), - J2kComponentMappingType::Unknown { .. } => false, - }); - if resolvable { - ResolvedComponentSource::Mappings - } else { - ResolvedComponentSource::Codestream - } -} - -fn resolved_component_count( - source: ResolvedComponentSource, - metadata: &J2kFileMetadata, - siz: &super::ParsedSiz, -) -> usize { - match source { - ResolvedComponentSource::Codestream => siz.component_info.len(), - ResolvedComponentSource::Palette => metadata - .palette - .as_ref() - .map_or(0, |palette| palette.columns.len()), - ResolvedComponentSource::Mappings => metadata.component_mappings.len(), - } -} - -fn resolved_component_at( - source: ResolvedComponentSource, - metadata: &J2kFileMetadata, - siz: &super::ParsedSiz, - index: usize, -) -> Option { - match source { - ResolvedComponentSource::Codestream => siz.component_info.get(index).copied(), - ResolvedComponentSource::Palette => metadata - .palette - .as_ref()? - .columns - .get(index) - .copied() - .map(component_from_palette_column), - ResolvedComponentSource::Mappings => { - let mapping = metadata.component_mappings.get(index)?; - match mapping.mapping_type { - J2kComponentMappingType::Direct => siz - .component_info - .get(usize::from(mapping.component_index)) - .copied(), - J2kComponentMappingType::Palette { column } => metadata - .palette - .as_ref()? - .columns - .get(usize::from(column)) - .copied() - .map(component_from_palette_column), - J2kComponentMappingType::Unknown { .. } => None, - } - } - } -} - -fn component_from_palette_column(column: J2kPaletteColumn) -> J2kComponentInfo { - J2kComponentInfo { - bit_depth: column.bit_depth, - signed: column.signed, - x_rsiz: 1, - y_rsiz: 1, - } -} - fn same_component_precision(left: J2kComponentInfo, right: J2kComponentInfo) -> bool { left.bit_depth == right.bit_depth && left.signed == right.signed } diff --git a/crates/j2k/src/parse/boxes/tests/component_validation.rs b/crates/j2k/src/parse/boxes/tests/component_validation.rs index 52f72a15..536e9af1 100644 --- a/crates/j2k/src/parse/boxes/tests/component_validation.rs +++ b/crates/j2k/src/parse/boxes/tests/component_validation.rs @@ -2,10 +2,7 @@ use j2k_core::TileLayout; -use super::super::{ - resolved_component_at, resolved_component_count, resolved_component_source, - validate_component_metadata, validate_ihdr_matches_codestream, Jp2ImageHeader, -}; +use super::super::{validate_component_metadata, validate_ihdr_matches_codestream, Jp2ImageHeader}; use crate::parse::ParsedSiz; use crate::{ J2kComponentInfo, J2kComponentMapping, J2kComponentMappingType, J2kError, J2kFileMetadata, @@ -82,64 +79,47 @@ fn image_header_validation_rejects_only_dimension_mismatches() { } #[test] -fn component_resolution_prefers_valid_mappings_then_palette_then_codestream() { - let siz = siz(vec![component(8, false), component(12, true)]); +fn palette_channels_do_not_replace_ihdr_codestream_component_metadata() { + let siz = siz(vec![component(8, false)]); let mut metadata = metadata(); - let source = resolved_component_source(&metadata, &siz); - assert_eq!(resolved_component_count(source, &metadata, &siz), 2); - assert_eq!( - resolved_component_at(source, &metadata, &siz, 1), - Some(component(12, true)) - ); - metadata.palette = Some(J2kPaletteMetadata { - columns: vec![J2kPaletteColumn { - bit_depth: 6, - signed: false, - }], + columns: vec![ + J2kPaletteColumn { + bit_depth: 6, + signed: false, + }, + J2kPaletteColumn { + bit_depth: 7, + signed: false, + }, + J2kPaletteColumn { + bit_depth: 8, + signed: false, + }, + ], entries: Vec::new(), }); - let source = resolved_component_source(&metadata, &siz); - assert_eq!(resolved_component_count(source, &metadata, &siz), 1); - assert_eq!( - resolved_component_at(source, &metadata, &siz, 0), - Some(component(6, false)) - ); - - metadata.component_mappings = vec![ - J2kComponentMapping { - component_index: 1, - mapping_type: J2kComponentMappingType::Direct, - }, - J2kComponentMapping { + metadata.component_mappings = (0..3) + .map(|column| J2kComponentMapping { component_index: 0, - mapping_type: J2kComponentMappingType::Palette { column: 0 }, - }, - ]; - let source = resolved_component_source(&metadata, &siz); - assert_eq!(resolved_component_count(source, &metadata, &siz), 2); - assert_eq!( - resolved_component_at(source, &metadata, &siz, 0), - Some(component(12, true)) - ); - assert_eq!( - resolved_component_at(source, &metadata, &siz, 1), - Some(component(6, false)) - ); + mapping_type: J2kComponentMappingType::Palette { column }, + }) + .collect(); - metadata.component_mappings[0].mapping_type = J2kComponentMappingType::Unknown { - value: 9, - column: 0, - }; - let fallback = resolved_component_source(&metadata, &siz); - assert_eq!( - resolved_component_at(fallback, &metadata, &siz, 0), - Some(component(8, false)) + assert!( + validate_component_metadata(header(1, Some(component(8, false))), &metadata, &siz,).is_ok() ); + assert!(matches!( + validate_component_metadata(header(3, Some(component(8, false))), &metadata, &siz), + Err(J2kError::InvalidBox { + what: "ihdr component count must match codestream components", + .. + }) + )); } #[test] -fn explicit_ihdr_precision_must_match_every_resolved_component_and_forbids_bpcc() { +fn explicit_ihdr_precision_must_match_every_codestream_component_and_forbids_bpcc() { let uniform = siz(vec![component(8, false), component(8, false)]); let empty = metadata(); assert!( @@ -150,7 +130,7 @@ fn explicit_ihdr_precision_must_match_every_resolved_component_and_forbids_bpcc( assert!(matches!( validate_component_metadata(header(2, Some(component(8, false))), &empty, &mixed), Err(J2kError::InvalidBox { - what: "ihdr bpc must match resolved JP2 image component precision", + what: "ihdr bpc must match codestream component precision", .. }) )); @@ -186,7 +166,7 @@ fn variable_precision_requires_complete_matching_bpcc_metadata() { assert!(matches!( validate_component_metadata(header(2, None), &metadata, &siz), Err(J2kError::InvalidBox { - what: "bpcc entries must match resolved JP2 image component precision", + what: "bpcc entries must match codestream component precision", .. }) )); @@ -194,7 +174,7 @@ fn variable_precision_requires_complete_matching_bpcc_metadata() { assert!(matches!( validate_component_metadata(header(3, None), &metadata, &siz), Err(J2kError::InvalidBox { - what: "ihdr component count must match resolved JP2 image components", + what: "ihdr component count must match codestream components", .. }) )); diff --git a/crates/j2k/src/view.rs b/crates/j2k/src/view.rs index 3e836cb4..5f5f7036 100644 --- a/crates/j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -21,6 +21,7 @@ use j2k_core::{ mod deep_scale; mod rows; +mod srgb8; mod traits; /// Borrowed parse result for a JP2 or raw JPEG 2000 / HTJ2K codestream. diff --git a/crates/j2k/src/view/deep_scale.rs b/crates/j2k/src/view/deep_scale.rs index 8869e81c..5c6d8ccb 100644 --- a/crates/j2k/src/view/deep_scale.rs +++ b/crates/j2k/src/view/deep_scale.rs @@ -4,7 +4,12 @@ use super::{ backend_image_with_reduction, decode_image_region_into_with_native_context, decode_warnings_for_image, validate_buffer, validate_region, J2kDecoder, }; -use crate::{decode::J2kDecodeOutcome, scratch::J2kScratchPool, J2kError}; +use crate::{ + decode::{J2kDecodeOutcome, J2kDecodedNativeComponents}, + scratch::J2kScratchPool, + view::component_handoff_image_bytes, + J2kError, +}; use j2k_core::{PixelFormat, Rect, Unsupported}; const UNREPRESENTABLE_REDUCTION: &str = "requested reduction exceeds supported image geometry"; @@ -36,6 +41,44 @@ fn scaled_covering_pow2(rect: Rect, denominator: u32) -> Rect { } impl J2kDecoder<'_> { + /// Decode owned native component planes after discarding an exact number + /// of JPEG 2000 resolution levels. + /// + /// A reduction of zero delegates to [`Self::decode_native_components`]. + /// Each additional level halves both axes using the codestream's wavelet + /// resolution ladder; this method does not resample a full-resolution + /// output. + /// + /// # Errors + /// Returns [`J2kError`] when the reduction cannot be represented, exceeds + /// any component's available resolution ladder, is not honored exactly by + /// the native backend, or decode validation fails. + pub fn decode_native_components_at_reduction( + &mut self, + reduction_levels: u8, + ) -> Result { + if reduction_levels == 0 { + return self.decode_native_components(); + } + + let denominator = reduction_denominator(reduction_levels)?; + let expected_dims = ( + self.info.dimensions.0.div_ceil(denominator), + self.info.dimensions.1.div_ceil(denominator), + ); + let image = backend_image_with_reduction(self.bytes, self.settings, reduction_levels)?; + if (image.width(), image.height()) != expected_dims { + return Err(unsupported(INEXACT_REDUCTION)); + } + + let retained_image_bytes = component_handoff_image_bytes(&image)?; + let mut native_context = self.scaled_decode_native_context(); + let decoded = image + .decode_native_components_with_context(&mut native_context) + .map_err(J2kError::from_native_decode_error)?; + J2kDecodedNativeComponents::try_from_native(decoded, retained_image_bytes) + } + /// Decode a source-coordinate region after discarding an exact number of /// JPEG 2000 resolution levels. /// diff --git a/crates/j2k/src/view/srgb8.rs b/crates/j2k/src/view/srgb8.rs new file mode 100644 index 00000000..978ea114 --- /dev/null +++ b/crates/j2k/src/view/srgb8.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use super::J2kDecoder; +use crate::{decode::decode_image_srgb8, J2kError, J2kSrgb8Image}; + +impl J2kDecoder<'_> { + /// Decode the full image and normalize its colour data to 8-bit sRGB. + /// + /// JP2 palette mapping, component sampling, channel definitions, and + /// enumerated sRGB-YCC conversion are applied before this output is + /// produced. Restricted ICC input profiles are converted with the pinned + /// colour-management implementation. + /// + /// # Errors + /// Returns [`J2kError`] when decoding fails, a primary ICC profile is + /// malformed, the colour space is unsupported, or the bounded output + /// allocation cannot be made. + pub fn decode_srgb8(&mut self) -> Result { + self.ensure_image()?; + let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { + return Err(J2kError::internal_backend("internal image cache missing")); + }; + decode_image_srgb8(image, native_context) + } +} diff --git a/crates/j2k/src/wrap/plan.rs b/crates/j2k/src/wrap/plan.rs index d1924a5f..8896e868 100644 --- a/crates/j2k/src/wrap/plan.rs +++ b/crates/j2k/src/wrap/plan.rs @@ -23,7 +23,6 @@ pub(super) struct WrapPlan<'a> { pub(super) parsed: &'a ParsedImageInfo, pub(super) metadata: J2kFileBoxMetadata<'a>, pub(super) colors: ColorSelection<'a>, - pub(super) components: ResolvedComponents<'a>, pub(super) component_count: u16, pub(super) image_header_bpc: u8, pub(super) bpcc_payload_len: Option, @@ -44,18 +43,21 @@ impl<'a> WrapPlan<'a> { ) -> Result { let components = ResolvedComponents::new(parsed, metadata)?; components.validate_precisions()?; - let component_count = u16::try_from(components.len()).map_err(|_| { + let codestream_components = ResolvedComponents::Codestream(&parsed.components); + let component_count = u16::try_from(codestream_components.len()).map_err(|_| { J2kError::Unsupported(Unsupported { - what: "JP2/JPH resolved image component count exceeds u16", + what: "JP2/JPH codestream component count exceeds u16", }) })?; - let uses_bpcc = components.uses_bpcc()?; + let uses_bpcc = codestream_components.uses_bpcc()?; let image_header_bpc = if uses_bpcc { 0xff } else { - components.component(0).map_or(0xff, component_bpc) + codestream_components + .component(0) + .map_or(0xff, component_bpc) }; - let bpcc_payload_len = uses_bpcc.then_some(components.len()); + let bpcc_payload_len = uses_bpcc.then_some(codestream_components.len()); let palette_payload_len = metadata.palette.map(validate_palette); let palette_payload_len = palette_payload_len.transpose()?; let component_mapping_payload_len = component_mapping_payload_len(parsed, metadata)?; @@ -107,7 +109,6 @@ impl<'a> WrapPlan<'a> { parsed, metadata, colors, - components, component_count, image_header_bpc, bpcc_payload_len, diff --git a/crates/j2k/src/wrap/writer.rs b/crates/j2k/src/wrap/writer.rs index f859aef3..7810aab5 100644 --- a/crates/j2k/src/wrap/writer.rs +++ b/crates/j2k/src/wrap/writer.rs @@ -7,7 +7,7 @@ use alloc::vec::Vec; use super::{ allocation::allocate_output, color::PlannedColorSpec, - metadata::{component_bpc, ChannelDefinitionPlan, ResolvedComponents}, + metadata::{component_bpc, ChannelDefinitionPlan}, plan::WrapPlan, JP2_COMPRESSION_TYPE, JP2_SIGNATURE_PAYLOAD, }; @@ -31,7 +31,7 @@ pub(super) fn write(plan: &WrapPlan<'_>, retained_bytes: usize) -> Result) -> Result fn write_bits_per_component( writer: &mut CheckedWriter, - components: ResolvedComponents<'_>, + components: &[crate::J2kComponentInfo], payload_len: usize, ) -> Result<(), J2kError> { writer.box_header(*b"bpcc", payload_len)?; - for index in 0..components.len() { - let component = components - .component(index) - .ok_or(J2kError::InternalInvariant { - what: "validated BPCC component became unresolved", - })?; + for &component in components { writer.byte(component_bpc(component))?; } Ok(()) diff --git a/crates/j2k/tests/deep_scale.rs b/crates/j2k/tests/deep_scale.rs index 4fa13bc9..04c9f50c 100644 --- a/crates/j2k/tests/deep_scale.rs +++ b/crates/j2k/tests/deep_scale.rs @@ -248,3 +248,63 @@ fn deep_scaled_decode_preserves_lenient_recovery_warning() { vec![J2kDecodeWarning::LenientMetadataRecovery] ); } + +#[test] +fn native_components_at_reduction_match_the_packed_production_path() { + let bytes = encode_rgb_fixture(65, 33, 4, None); + let mut decoder = J2kDecoder::new(&bytes).expect("native component decoder"); + + let native = decoder + .decode_native_components_at_reduction(3) + .expect("native 1/8 decode"); + + assert_eq!(native.dimensions(), (9, 5)); + assert_eq!(native.planes().len(), 3); + assert!(native.planes().iter().all(|plane| { + plane.dimensions() == (9, 5) + && plane.bit_depth() == 8 + && !plane.signed() + && plane.bytes_per_sample() == 1 + })); + + let mut packed = vec![0_u8; 9 * 5 * 3]; + let mut packed_decoder = J2kDecoder::new(&bytes).expect("packed decoder"); + packed_decoder + .decode_region_scaled_pow2_into( + &mut J2kScratchPool::new(), + &mut packed, + 9 * 3, + PixelFormat::Rgb8, + Rect { + x: 0, + y: 0, + w: 65, + h: 33, + }, + 3, + ) + .expect("packed 1/8 decode"); + let interleaved = (0..45) + .flat_map(|index| native.planes().iter().map(move |plane| plane.data()[index])) + .collect::>(); + assert_eq!(interleaved, packed); +} + +#[test] +fn native_component_reduction_zero_delegates_and_excess_levels_fail() { + let bytes = encode_rgb_fixture(32, 17, 3, None); + let mut full_decoder = J2kDecoder::new(&bytes).expect("full decoder"); + let full = full_decoder + .decode_native_components() + .expect("full native decode"); + let mut zero_decoder = J2kDecoder::new(&bytes).expect("zero-level decoder"); + let zero = zero_decoder + .decode_native_components_at_reduction(0) + .expect("zero-level native decode"); + assert_eq!(zero, full); + + let error = zero_decoder + .decode_native_components_at_reduction(4) + .expect_err("reduction beyond the wavelet ladder must fail"); + assert!(matches!(error, J2kError::Unsupported(_))); +} diff --git a/crates/j2k/tests/iso_conformance.rs b/crates/j2k/tests/iso_conformance.rs deleted file mode 100644 index 29510aa9..00000000 --- a/crates/j2k/tests/iso_conformance.rs +++ /dev/null @@ -1,232 +0,0 @@ -// SPDX-License-Identifier: MIT OR Apache-2.0 - -use std::{ - env, fs, - path::{Component, Path, PathBuf}, -}; - -use j2k_native::{DecodeSettings, Image}; - -const CONFORMANCE_ENV: &str = "J2K_ISO_CONFORMANCE_DIR"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Classification { - Blocking, - KnownLimitation, - Investigate, - OutOfScope, -} - -#[derive(Debug)] -struct Vector { - id: String, - path: PathBuf, - classification: Classification, - features: String, - reason: String, -} - -fn repo_root() -> &'static Path { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .and_then(Path::parent) - .expect("workspace root") -} - -fn manifest_path() -> PathBuf { - repo_root().join("corpus/j2k-conformance/manifest.tsv") -} - -fn load_manifest() -> Vec { - let text = fs::read_to_string(manifest_path()).expect("read J2K conformance manifest"); - text.lines() - .enumerate() - .filter_map(|(line_idx, line)| { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - return None; - } - Some(parse_manifest_line(line_idx + 1, line)) - }) - .collect() -} - -fn parse_manifest_line(line_number: usize, line: &str) -> Vector { - let fields: Vec<_> = line.split('\t').collect(); - assert_eq!( - fields.len(), - 5, - "manifest line {line_number} must contain id, path, classification, features, reason" - ); - let path = PathBuf::from(fields[1]); - assert!( - !path.is_absolute() - && !path - .components() - .any(|component| matches!(component, Component::ParentDir)), - "manifest line {line_number} path must stay relative to the ISO vector root" - ); - Vector { - id: fields[0].to_string(), - path, - classification: parse_classification(line_number, fields[2]), - features: fields[3].to_string(), - reason: fields[4].to_string(), - } -} - -fn parse_classification(line_number: usize, value: &str) -> Classification { - match value { - "blocking" => Classification::Blocking, - "known-limitation" => Classification::KnownLimitation, - "investigate" => Classification::Investigate, - "out-of-scope" => Classification::OutOfScope, - _ => panic!("manifest line {line_number} has invalid classification {value:?}"), - } -} - -#[test] -fn iso_conformance_manifest_is_release_classified() { - let vectors = load_manifest(); - assert!( - vectors - .iter() - .any(|vector| vector.classification == Classification::Blocking), - "manifest must contain at least one blocking vector" - ); - for vector in vectors { - assert!(!vector.id.is_empty(), "vector id must not be empty"); - assert!( - !vector.features.is_empty(), - "{} must list exercised features", - vector.id - ); - if matches!( - vector.classification, - Classification::KnownLimitation | Classification::OutOfScope - ) { - assert!( - !vector.reason.is_empty(), - "{} non-blocking row must document the deferred feature", - vector.id - ); - } - assert_ne!( - vector.classification, - Classification::Investigate, - "{} must be classified before release signoff", - vector.id - ); - } -} - -#[test] -fn iso_conformance_manifest_blocks_release_shipped_features() { - let vectors = load_manifest(); - for required_feature in [ - "part1-core", - "part15-core", - "poc", - "precincts", - "progression-orders", - "tlm", - "plt", - "sop", - "eph", - ] { - assert!( - vectors.iter().any(|vector| { - vector.classification == Classification::Blocking - && vector - .features - .split(';') - .any(|feature| feature == required_feature) - }), - "manifest must include a blocking vector for shipped feature {required_feature}" - ); - } - assert!( - vectors.iter().any(|vector| { - vector.features.split(';').any(|feature| feature == "plm") - && (vector.classification == Classification::Blocking - || (vector.classification == Classification::KnownLimitation - && vector - .features - .split(';') - .any(|feature| feature == "conformance-coverage-gap"))) - }), - "manifest must include a blocking PLM vector or document the ISO coverage gap" - ); -} - -#[test] -fn iso_conformance_flags_missing_blocking_vectors_as_signoff_failures() { - let vector_root = env::temp_dir().join(format!("j2k-missing-blocking-{}", std::process::id())); - fs::create_dir_all(&vector_root).expect("create temporary vector root"); - let vectors = vec![Vector { - id: "missing_blocking".to_string(), - path: PathBuf::from("part1/missing.j2k"), - classification: Classification::Blocking, - features: "part1-core".to_string(), - reason: "blocking vector required".to_string(), - }]; - - let missing = missing_blocking_vectors(&vectors, &vector_root); - - fs::remove_dir_all(&vector_root).expect("remove temporary vector root"); - assert_eq!(missing, vec!["missing_blocking".to_string()]); -} - -fn missing_blocking_vectors(vectors: &[Vector], vector_root: &Path) -> Vec { - vectors - .iter() - .filter(|vector| vector.classification == Classification::Blocking) - .filter(|vector| !vector_root.join(&vector.path).exists()) - .map(|vector| vector.id.clone()) - .collect() -} - -#[test] -fn env_gated_iso_conformance_blocks_only_shipped_features() { - let Some(vector_root) = env::var_os(CONFORMANCE_ENV).map(PathBuf::from) else { - return; - }; - let vectors = load_manifest(); - let missing_blocking = missing_blocking_vectors(&vectors, &vector_root); - assert!( - missing_blocking.is_empty(), - "blocking ISO conformance vectors are missing from {}: {}", - vector_root.display(), - missing_blocking.join(", ") - ); - - for vector in vectors { - match vector.classification { - Classification::Investigate => { - panic!("{} is still investigate at release signoff", vector.id); - } - Classification::KnownLimitation => { - eprintln!( - "known limitation {}: {} ({})", - vector.id, vector.reason, vector.features - ); - } - Classification::OutOfScope => { - eprintln!( - "out of scope {}: {} ({})", - vector.id, vector.reason, vector.features - ); - } - Classification::Blocking => { - let path = vector_root.join(&vector.path); - let bytes = fs::read(&path) - .unwrap_or_else(|err| panic!("read blocking vector {}: {err}", vector.id)); - let image = Image::new(&bytes, &DecodeSettings::default()) - .unwrap_or_else(|err| panic!("parse blocking vector {}: {err}", vector.id)); - image - .decode_native() - .unwrap_or_else(|err| panic!("decode blocking vector {}: {err}", vector.id)); - } - } - } -} diff --git a/crates/j2k/tests/srgb8.rs b/crates/j2k/tests/srgb8.rs new file mode 100644 index 00000000..4e5c75c1 --- /dev/null +++ b/crates/j2k/tests/srgb8.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use j2k::{ + wrap_j2k_codestream, J2kDecoder, J2kError, J2kFileColorSpec, J2kFileWrapOptions, J2kSrgb8Layout, +}; +use j2k_core::Colorspace; +use j2k_native::{encode, EncodeOptions}; + +fn encode_fixture(pixels: &[u8], width: u32, height: u32, components: u16) -> Vec { + encode( + pixels, + width, + height, + components, + 8, + false, + &EncodeOptions { + reversible: true, + ..EncodeOptions::default() + }, + ) + .expect("encode fixture") +} + +#[test] +fn srgb8_exposes_private_rgb_storage_through_accessors() { + let pixels = [5, 17, 29, 101, 151, 211]; + let codestream = encode_fixture(&pixels, 2, 1, 3); + + let image = J2kDecoder::new(&codestream) + .expect("decoder") + .decode_srgb8() + .expect("sRGB decode"); + + assert_eq!(image.dimensions(), (2, 1)); + assert_eq!(image.layout(), J2kSrgb8Layout::Rgb); + assert_eq!(image.data(), pixels); + assert_eq!(image.into_data(), pixels); +} + +#[test] +fn srgb8_preserves_srgb_gray_and_alpha_layouts() { + let gray = [0, 63, 127, 255]; + let gray_codestream = encode_fixture(&gray, 2, 2, 1); + let gray_image = J2kDecoder::new(&gray_codestream) + .expect("gray decoder") + .decode_srgb8() + .expect("gray decode"); + assert_eq!(gray_image.layout(), J2kSrgb8Layout::Gray); + assert_eq!(gray_image.data(), gray); + + let rgba = [11, 29, 47, 67, 89, 107, 131, 149]; + let rgba_codestream = encode_fixture(&rgba, 2, 1, 4); + let jp2 = wrap_j2k_codestream( + &rgba_codestream, + J2kFileWrapOptions::jp2().with_color(J2kFileColorSpec::Enumerated(Colorspace::SRgb)), + ) + .expect("wrap RGBA JP2"); + let rgba_image = J2kDecoder::new(&jp2) + .expect("RGBA decoder") + .decode_srgb8() + .expect("RGBA decode"); + assert_eq!(rgba_image.layout(), J2kSrgb8Layout::Rgba); + assert_eq!(rgba_image.data(), rgba); +} + +#[test] +fn srgb8_rejects_a_malformed_primary_icc_profile() { + let codestream = encode_fixture(&[17, 31, 47], 1, 1, 3); + let jp2 = wrap_j2k_codestream( + &codestream, + J2kFileWrapOptions::jp2().with_color(J2kFileColorSpec::IccProfile(b"not-an-icc-profile")), + ) + .expect("wrap malformed ICC fixture"); + + let error = J2kDecoder::new(&jp2) + .expect("container remains structurally decodable") + .decode_srgb8() + .expect_err("malformed ICC must not be treated as RGB"); + + assert_eq!(error, J2kError::InvalidIccProfile); +} + +#[test] +fn srgb8_converts_a_restricted_rgb_icc_profile() { + let romm_rgb = [128, 64, 32, 200, 120, 80, 32, 160, 220]; + let codestream = encode_fixture(&romm_rgb, 3, 1, 3); + let profile = moxcms::ColorProfile::new_pro_photo_rgb() + .encode() + .expect("encode restricted ProPhoto RGB profile"); + let jp2 = wrap_j2k_codestream( + &codestream, + J2kFileWrapOptions::jp2().with_color(J2kFileColorSpec::IccProfile(&profile)), + ) + .expect("wrap restricted ICC fixture"); + + let image = J2kDecoder::new(&jp2) + .expect("ICC decoder") + .decode_srgb8() + .expect("ICC to sRGB conversion"); + + // Independent Little CMS 2.17 reference values for this pinned profile. + let reference = [191_u8, 53, 29, 255, 114, 89, 0, 192, 234]; + assert_eq!(image.layout(), J2kSrgb8Layout::Rgb); + assert!( + image + .data() + .iter() + .zip(reference) + .all(|(&actual, expected)| actual.abs_diff(expected) <= 2), + "actual {:?}, reference {reference:?}", + image.data() + ); +} + +#[test] +fn srgb8_converts_a_restricted_monochrome_icc_profile() { + let gamma_18 = [0_u8, 32, 64, 128, 192, 255]; + let codestream = encode_fixture(&gamma_18, 6, 1, 1); + let profile = moxcms::ColorProfile::new_gray_with_gamma(1.8) + .encode() + .expect("encode restricted monochrome profile"); + let jp2 = wrap_j2k_codestream( + &codestream, + J2kFileWrapOptions::jp2().with_color(J2kFileColorSpec::IccProfile(&profile)), + ) + .expect("wrap monochrome ICC fixture"); + + let image = J2kDecoder::new(&jp2) + .expect("ICC decoder") + .decode_srgb8() + .expect("monochrome ICC to sRGB-grey conversion"); + + let reference = [0_u8, 43, 81, 146, 203, 255]; + assert_eq!(image.layout(), J2kSrgb8Layout::Gray); + assert!( + image + .data() + .iter() + .zip(reference) + .all(|(&actual, expected)| actual.abs_diff(expected) <= 1), + "actual {:?}, reference {reference:?}", + image.data() + ); +} diff --git a/crates/j2k/tests/wrap.rs b/crates/j2k/tests/wrap.rs index 403c8bd8..e71f0a76 100644 --- a/crates/j2k/tests/wrap.rs +++ b/crates/j2k/tests/wrap.rs @@ -54,7 +54,7 @@ fn jp2_with_palette_mapping(codestream: &[u8]) -> Vec { let mut ihdr = Vec::new(); ihdr.extend_from_slice(&2_u32.to_be_bytes()); ihdr.extend_from_slice(&2_u32.to_be_bytes()); - ihdr.extend_from_slice(&3_u16.to_be_bytes()); + ihdr.extend_from_slice(&1_u16.to_be_bytes()); ihdr.extend_from_slice(&[7, 7, 0, 0]); push_box(&mut jp2h, b"ihdr", &ihdr); @@ -108,7 +108,7 @@ fn jp2_with_signed_palette_mapping(codestream: &[u8]) -> Vec { ihdr.extend_from_slice(&2_u32.to_be_bytes()); ihdr.extend_from_slice(&2_u32.to_be_bytes()); ihdr.extend_from_slice(&1_u16.to_be_bytes()); - ihdr.extend_from_slice(&[0x87, 7, 0, 0]); + ihdr.extend_from_slice(&[7, 7, 0, 0]); push_box(&mut jp2h, b"ihdr", &ihdr); let mut colr = Vec::new(); diff --git a/docs/architecture.md b/docs/architecture.md index c591d7d2..d9930b78 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,6 +39,7 @@ still-image correctness. Keep row-level status synchronized with | `j2k-test-support`, `j2k-transcode-test-support` | dev helper | Shared fixture, benchmark input, and transcode oracle helpers for tests, benches, and examples. | | `j2k-alloc-probe` | dev helper | Serial process-wide measurement of successful allocation calls and gross requested bytes at real codec boundaries. | | `j2k-compare` | tooling | Comparator tooling. | +| `j2k-t803` | conformance tooling | Unpublished T.803 corpus, comparison, report, and adapter-IUT runner support. | | `xtask` | workspace tool | Repository automation under `xtask/`. | ## Dependency rules @@ -71,6 +72,7 @@ j2k-jpeg-cuda -> j2k-core, j2k-cuda-runtime, j2k-jpeg, j2k-profile j2k-jpeg-metal -> j2k-core, j2k-jpeg, j2k-metal-support, j2k-profile j2k-tilecodec -> j2k-core j2k-compare -> j2k-core, j2k, j2k-native, j2k-test-support +j2k-t803 -> j2k, j2k-codec-math, j2k-compare, j2k-core, j2k-cuda, j2k-cuda-runtime, j2k-metal, j2k-native j2k-transcode -> j2k-codec-math, j2k-core, j2k, j2k-native, j2k-jpeg, j2k-profile j2k-metal-support -> j2k-core j2k-cuda-runtime -> j2k-codec-math, j2k-core diff --git a/docs/benchmark-corpora.md b/docs/benchmark-corpora.md index d11b5a11..ab778cd1 100644 --- a/docs/benchmark-corpora.md +++ b/docs/benchmark-corpora.md @@ -13,7 +13,7 @@ its pinned manifest and summarized by the `adoption-report` subcommand. | Corpus | Use | Source | Repo handling | | --- | --- | --- | --- | -| ISO JPEG 2000 conformance files | Compliance-style claims for JPEG 2000 Part 1 and HTJ2K Part 15. | ISO/IEC 15444-4 / ITU-T T.803 electronic attachments. | User-supplied. Do not commit unless licensing permits. Track expected vectors in `corpus/j2k-conformance/manifest.tsv`. | +| ISO JPEG 2000 conformance files | Exact-reference evidence for the selected JPEG 2000 Part 1 decoder classes and Annex G JP2 reader. | ISO/IEC 15444-4:2024 / ITU-T T.803 v3 electronic attachment. | Copyrighted external input. Never commit or upload the corpus. The official URL, archive digest, and exact selected-file inventory are pinned in `corpus/j2k-conformance/t803-v3.toml`. | | OpenJPEG test data | Regression and interoperability corpus with real JP2/J2K edge cases. | `https://github.com/uclouvain/openjpeg-data` | User-supplied clone path. | | OpenJPH / HTJ2K fixtures | HTJ2K-specific interoperability and JPH/J2K variants. | `https://github.com/aous72/OpenJPH` and released OpenJPH test assets. | User-supplied clone/path; small license-compatible fixtures may be committed with notices. | | jpylyzer test files | JP2 parser/metadata robustness, including valid and invalid files. | `https://github.com/openpreserve/jpylyzer-test-files` | User-supplied clone/path. Invalid files should be used for robustness tests, not throughput comparisons. | @@ -279,6 +279,51 @@ comparator can enforce the shared three-resolution profile. Use ISO conformance attachments, OpenJPEG data, OpenJPH data, and jpylyzer parser fixtures that should not be re-encoded by this repo. +## Auto-routing workload manifest + +Hybrid promotion uses a smaller, strict JSON manifest rather than the adoption +TSVs. Schema version 1 contains a corpus label, an HTTPS provenance URL, and a +non-empty `cases` array. Each case has a unique safe `id`, a relative `path`, a +`kind` of `decode` or `encode`, a `pixel_format` of `gray8` or `rgb8`, and the +lowercase SHA-256 of the exact input bytes. Decode cases are J2K/JP2 inputs; +encode cases are binary 8-bit PGM/PPM files. + +```json +{ + "schema_version": 1, + "corpus": "release-routing-workloads-2026", + "source_url": "https://example.org/pinned-corpus-record", + "cases": [ + { + "id": "decode-natural-01", + "path": "decode/natural-01.jp2", + "kind": "decode", + "pixel_format": "rgb8", + "sha256": "<64 lowercase hexadecimal characters>" + }, + { + "id": "encode-natural-01", + "path": "encode/natural-01.ppm", + "kind": "encode", + "pixel_format": "rgb8", + "sha256": "<64 lowercase hexadecimal characters>" + } + ] +} +``` + +The loader rejects absolute or escaping paths, symlinks, duplicates, hash +drift, malformed PNM, oversized cases, and incomplete inventories. A +representative release manifest should cover more than one favorable image and +span small/large, gray/RGB, lossless/lossy, full, ROI, scaled, and batch work. +The manifest and inputs remain external; the verified report records the exact +manifest hash and every exercised workload ID. + +On a self-hosted runner, dispatch `GPU benchmarks` with `suite=routing` and set +the repository variables `J2K_AUTO_ROUTING_MANIFEST` and +`J2K_AUTO_ROUTING_ROOT` to the manifest and corpus root available on that host. +The workflow fails closed when either is absent. + ## Running All Available Corpora Place or symlink each decoded corpus of J2K/JP2/JPH files into separate diff --git a/docs/benchmark-evidence.md b/docs/benchmark-evidence.md index b616eb82..b9d55fe7 100644 --- a/docs/benchmark-evidence.md +++ b/docs/benchmark-evidence.md @@ -27,6 +27,133 @@ external bundle and identify any missing evidence. Generated repo-local fixtures and passing codec self-checks remain implementation evidence; use manifest-backed external rows for adoption-facing speed reports. +## Fixed Auto-routing promotion evidence + +`BackendRequest::Auto` uses committed thresholds; it does not calibrate on a +user's machine. A hybrid workload cell may be promoted only after CPU, hybrid, +and any supported strict-device route produce identical bytes on the same +manifest-pinned external input. Its end-to-end median must be at least 10% +faster than every competitor and its Criterion 95% confidence interval must not +overlap any competing interval. + +CUDA and Metal collect all six required operations with their production APIs: +full decode, ROI decode, scaled decode, batch decode, lossless encode, and lossy +encode. Route evidence records the exact candidate SHA, manifest SHA-256, +hardware and driver identity, execution label, output SHA-256, and Criterion ID. +These adapters currently disclose CPU-assisted routes as `hybrid`; they do not +claim a strict device-native route when parsing, entropy decode, output, or +codestream assembly still runs on CPU. + +After a hardware run, verify the raw evidence against the exact manifest and +Criterion estimates: + +```bash +cargo xtask auto-routing verify \ + --evidence target/gpu-benchmark/auto-routing/evidence.json \ + --external-manifest "$J2K_AUTO_ROUTING_MANIFEST" \ + --criterion-root target/criterion \ + --out target/gpu-benchmark/auto-routing/verified.json +``` + +The verifier derives each promotion decision; benchmark input cannot request a +promotion. It rejects missing operations or external cases, route/output +mismatches, unsafe Criterion paths, unsupported confidence levels, changed +estimate files, and candidate/platform inconsistencies. The verified artifact +hash covers the raw evidence, manifest, and every referenced estimate. + +A local two-input Metal smoke on August 4, 2026 exercised the pipeline but was +not a representative external release corpus. It promoted zero cells: the +decode routes were slower, and the measured lossless and lossy encode medians +were only about 4.2% and 7.8% faster than CPU. No `Auto` threshold was changed +from that diagnostic. + +### External CUDA routing development run - 2026-08-05 + +The uninterrupted CUDA matrix used the same 12 external cases from +`uclouvain/openjpeg-data` commit +`39524bd3a601d90ed8e0177559400d23945f96a9` and manifest SHA-256 +`f07072f5d0313c0249e2df5df2310cd5c6c5a4b3414a933537fabf2362d2065c`. +It ran all 36 cells with Cargo `release-bench`, Criterion 0.95 confidence +intervals, ten samples, a one-second warm-up, and a three-second target on an +AMD Ryzen 7 5800X3D with an NVIDIA GeForce RTX 4070 SUPER, driver 596.49, +Linux x86-64, and CUDA 13.2. + +The verifier accepted every cell and promoted 18 decode cells. The fixed +policy uses the measured output-work thresholds. RGB8 reversible promotes full +output at 256 x 149, ROI output at 128 x 74, and half-scale output at 1296 x +972. RGB8 irreversible promotes full output at 640 x 480 and ROI or half-scale +output at 320 x 240. Gray8 reversible promotes only full output at 640 x 480. +Gray8 irreversible promotes full output at 3323 x 891, ROI output at 1661 x +445, and half-scale output at 1662 x 446. Qualified repeated-input batches use +the measured full-image thresholds at count 16. + +The policy applies only to raw Part 1 codestreams with the measured source +component/output-format pair. It does not extrapolate to JP2 color +normalization, other scale factors, HTJ2K, higher depths, RGBA, distinct-input +batches, unmeasured operations, smaller output work, or shapes below either +measured dimension. All 12 encode cells stayed on CPU because CUDA-assisted +encode was slower than CPU in this end-to-end matrix. + +The verified artifact's internal SHA-256, recorded beside the thresholds, is +`ded1eb045f9673e5bbe64dc873be3ba227ecb61ec11b6c9ad53653dbcc993f44`. +The raw evidence file SHA-256 is +`ad0b434dbd64f669d58054f4a25f9272f741bf2a689f6f70816d98fe87c02e61`; +the serialized verified file SHA-256 is +`a565d47f81ed32588e551167a91c0df68daff3d7ebd6ff8588fbe4d8ab27ac79`. +Every compared route produced the same output SHA-256 before timing results +were considered. No strict device-native route exists for these public +surfaces, so the competitive comparison was CPU versus the truthfully labelled +hybrid route. + +This was a dirty-tree development run whose recorded candidate SHA is the base +`6400fcd4c9f8cf9708563d62411eadf158f94282`. It supports the fixed policy but +is not exact-release-SHA evidence; the complete matrix must be rerun after +candidate freeze before publication. + +### External Metal routing development run - 2026-08-04 + +The full routing matrix was then run against 12 external decode/encode cases +from `uclouvain/openjpeg-data` commit +`39524bd3a601d90ed8e0177559400d23945f96a9`. The external manifest SHA-256 is +`f07072f5d0313c0249e2df5df2310cd5c6c5a4b3414a933537fabf2362d2065c`. +The run used Cargo `release-bench`, Criterion 0.95 confidence intervals, ten +samples, a one-second warm-up, and a three-second target measurement on an +Apple M4 Pro with a 16-core GPU and 48 GB RAM, macOS 26.5.2 build `25F84`, and +Metal compiler `32023.883`. + +The verifier accepted all 36 workload cells and promoted four. Times below are +Criterion medians; each promoted hybrid interval was wholly below the CPU +interval. + +| Cell | CPU median | Hybrid median | Speedup | +| --- | ---: | ---: | ---: | +| Repeated RGB8 irreversible decode, 640 x 480, batch 16 | 68.364 ms | 43.232 ms | 36.762% | +| Repeated Gray8 irreversible decode, 3323 x 891, batch 16 | 265.409 ms | 149.856 ms | 43.538% | +| Repeated RGB8 reversible decode, 2592 x 1944, batch 16 | 2429.216 ms | 179.528 ms | 92.610% | +| RGB8 irreversible encode, 2592 x 1944 | 813.508 ms | 716.208 ms | 11.961% | + +The verified artifact's internal SHA-256 is +`162a47f7a96b2be88abebc100aab672513af04895532863fa1a293660546f879`. +The raw evidence file SHA-256 is +`3b2ffad6fe3ebb42e2182612946a5c87ebf0b267e25c38bfb5c9d07c11aa6e7d`. +These hashes are the evidence anchors recorded beside the fixed thresholds in +the routing code. + +The batch rows intentionally reuse one encoded input 16 times. They establish +decode-once/repeated-output routing only; they are not distinct-image batch +throughput claims. `Auto` therefore promotes only repeated Part 1 Gray8/RGB8 +requests in the measured reversible/irreversible classes and size ranges. +Single-image, ROI, scaled, HTJ2K, higher-depth, signed, RGBA, and unmeasured +lossless/lossy cells stay on CPU. Lossless encode and Gray8 lossy encode also +stay on CPU; the latter measured only a 6.8% improvement in the canonical +profile. + +This was a dirty-tree development run whose recorded candidate SHA is the base +`6400fcd4c9f8cf9708563d62411eadf158f94282`, not an exact release candidate. +It supports implementation of the pending fixed policy, but it is not formal +release evidence. The complete matrix must be rerun and reverified from the +exact clean candidate SHA before publication. + ## Local Regression Guard - 2026-07-07 Host: diff --git a/docs/env-vars.md b/docs/env-vars.md index ed22a4fe..8dc1073f 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -76,6 +76,7 @@ override it. | `J2K_REQUIRE_KAKADU` | Makes optional Kakadu fixture/encoder comparator rows fail instead of skip when `kdu_expand` or `kdu_compress` is unavailable. Intended only for proprietary CLI/file-output context rows. | Skip unavailable Kakadu path unless explicitly included | Benchmark | | `J2K_REQUIRE_LIBJPEG_TURBO` | Makes libjpeg-turbo comparison tests fail instead of skip when the bench feature/tooling is unavailable. | Skip unavailable comparator path | Test/CI | | `J2K_REQUIRE_CUDA_RUNTIME` | Makes CUDA tests and benchmarks require a usable CUDA runtime instead of skipping. | Skip runtime-only CUDA paths | Test/CI | +| `J2K_CUDA_LOSSY_PARITY_PNM` | Path to an external RGB PNM fixture used by required CUDA irreversible color-transform, DWT 9/7, and lossy encode parity tests. | Optional generated fixture unless the release runner supplies a file | Test/CI | | `J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE` | Requires CUDA JPEG hardware decode coverage in relevant CUDA tests/benches. | Hardware decode may skip | Test/CI | | `J2K_REQUIRE_METAL_RUNTIME` | Runs runtime-only Metal tests and makes them require a usable Metal runtime instead of default-skipping. | Skip runtime-only Metal paths | Test/CI | | `J2K_REQUIRE_CUDA_BENCH` | Makes CUDA benchmark probes fail instead of skip when CUDA is unavailable or does not dispatch. | Skip unavailable CUDA benchmark paths | Benchmark | @@ -184,6 +185,12 @@ override it. | `J2K_ADOPTION_MANIFEST` | Required repository variable or environment override for the manual GPU benchmark workflow decode fixture manifest passed to `--manifest`. | None; adoption runs fail closed when unset | Benchmark/CI | | `J2K_ADOPTION_ENCODE_FIXTURES` | Required repository variable or environment override for the manual GPU benchmark workflow staged PNM encode fixture directory path-list passed to `--encode-fixtures`. | None; adoption runs fail closed when unset | Benchmark/CI | | `J2K_ADOPTION_ENCODE_MANIFEST` | Required repository variable or environment override for the manual GPU benchmark workflow staged PNM encode manifest passed to `--encode-manifest`. | None; adoption runs fail closed when unset | Benchmark/CI | +| `J2K_AUTO_ROUTING_MANIFEST` | Path to the hash-pinned schema-v1 JSON workload manifest used by the CUDA and Metal Auto-routing Criterion benches. | None; routing runs fail closed when unset | Benchmark/CI | +| `J2K_AUTO_ROUTING_ROOT` | Corpus root against which every relative path in `J2K_AUTO_ROUTING_MANIFEST` is resolved and hash-checked. | None; routing runs fail closed when unset | Benchmark/CI | +| `J2K_AUTO_ROUTING_EVIDENCE` | Output path for raw route-parity evidence written beside Criterion estimates. | None; routing runs fail closed when unset | Benchmark/CI | +| `J2K_AUTO_ROUTING_CANDIDATE_SHA` | Exact lowercase 40-hex commit identity recorded in route evidence. | None; routing runs fail closed when unset | Benchmark/CI | +| `J2K_AUTO_ROUTING_HARDWARE` | Non-empty accelerator hardware identity recorded in route evidence. | None; routing runs fail closed when unset | Benchmark/CI | +| `J2K_AUTO_ROUTING_DRIVER` | Non-empty driver/toolchain identity recorded in route evidence. | None; routing runs fail closed when unset | Benchmark/CI | | `J2K_CUDA_PROFILE_BATCH_SIZE` | Batch size for the CUDA HTJ2K decode profile example. | Example default | Benchmark | | `J2K_CUDA_PROFILE_ITERATIONS` | Iteration count for the CUDA HTJ2K decode profile example. | Example default | Benchmark | diff --git a/docs/public-support.md b/docs/public-support.md index 1669c419..5a9e09a9 100644 --- a/docs/public-support.md +++ b/docs/public-support.md @@ -1,8 +1,11 @@ # Public J2K/HTJ2K Support Matrix -This document is the handoff point for full JPEG 2000 Part 1 and HTJ2K Part 15 -support. Keep it synchronized with `corpus/j2k-conformance/manifest.tsv`, -repo-local self-checks, and adoption benchmark publication gates. Run: +This document is the handoff point for the implemented JPEG 2000 Part 1 and +HTJ2K Part 15 feature boundary. It is not a standards-conformance claim. Keep +it synchronized with `corpus/j2k-conformance/support-inventory.tsv`, repo-local +self-checks, and adoption benchmark publication gates. Candidate ISO/IEC +15444-4 evidence and its exact-SHA release gates are tracked in +[`docs/t803-conformance.md`](t803-conformance.md). Run: ```bash cargo xtask public-support diff --git a/docs/release.md b/docs/release.md index c487322a..022ea8e1 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,6 +1,9 @@ # Release Policy The `j2k` 0.8.0 public crate release is published and security-supported. +Version `0.8.1` is the prepared release candidate and carries the +release-scoped T.803 decoder evidence described in +[`T.803 conformance`](t803-conformance.md). Runtime backend selection defaults to `Auto`; CPU remains the portable baseline while supported device paths are selected only with validation and benchmark evidence. @@ -9,7 +12,8 @@ evidence. | Version | Distribution state | Security support | | --- | --- | --- | -| `0.8.0` | Published on crates.io from annotated tag `v0.8.0` after clean-consumer, exact-SHA, benchmark, CUDA, and Metal gates passed. | Latest supported release. | +| `0.8.1` | Prepared candidate; publication is allowed only from annotated tag `v0.8.1` after all exact-SHA CPU, CUDA, Metal, package, and release-integrity gates pass. | Becomes the latest supported release when published. | +| `0.8.0` | Published on crates.io from annotated tag `v0.8.0`. | Latest supported release. | | `0.7.5` | Previous crates.io release. Its `j2k-ml` CPU feature works, but its CUDA and Metal features have the clean-consumer defect described below. | Supported, with the stated `j2k-ml` accelerator exception. | | `0.7.3` | Previous published release line. | Supported. | | `0.7.2` | Previous published release line. | Supported. | @@ -18,12 +22,17 @@ evidence. | `0.6.x` | Previous published release line. | Supported for security fixes during the pre-1.0 transition. | | `<0.6` | Historical releases. | Unsupported. | -Version `0.8.0` is published from annotated tag `v0.8.0`, which peels to commit +Version `0.8.1` is published only from annotated tag `v0.8.1`. The tag must +peel to the exact candidate SHA recorded by all three CPU T.803 reports and the +CUDA and Metal adapter reports. The tag-triggered workflow verifies those five +report contents before publishing any crate; the GitHub release attachments, +tag, workflow runs, and crates.io records are the publication evidence. + +Version `0.8.0` was published from annotated tag `v0.8.0`, which peels to commit `53e0ad3d4f75f492af55413e0dab5a5834bd09c6`. The [tag-triggered publish workflow](https://github.com/frames-sg/j2k/actions/runs/30425822681) validated all 19 registry targets and published the release to crates.io. -GitHub Pages is served directly from `main/docs`; the tag, workflow run, and -crates.io records are the publication evidence. +GitHub Pages is served directly from `main/docs`. Version `0.7.5` is an explicitly reviewed source-compatibility exception to the normal patch policy. Its wrapper-removal migrations are recorded under @@ -39,7 +48,7 @@ released public APIs. Do not recommend the defective 0.7.5 accelerator features. Any Burn community notice still requires the post-publication exact-version consumer and benchmark evidence listed in the notice draft. -Version `0.8.0` is intentionally source- and behavior-incompatible +Version `0.8.0` was intentionally source- and behavior-incompatible with `0.7.5`: decode entry points become strict by default, explicit leniency is limited to the documented JP2/JPH metadata recoveries, warnings report actual recovery rather than lenient configuration, and @@ -48,11 +57,14 @@ actual recovery rather than lenient configuration, and [reviewed API report](../engineering/reviewed-public-api-diff-0.8.0.md) records the release's generated signature diff, and the adjacent [review configuration](../engineering/public-api-review-0.8.0.yml) contains the -exact source- and behavior-break ledger with migrations. The semver gate allows -only the completed `0.8.0` transition to compare against `v0.7.5`. The baseline -version, tag, and peeled commit must rotate to published `v0.8.0`, and the -one-time transition allowance must be removed, before any later candidate is -prepared. +exact source- and behavior-break ledger with migrations. + +Version `0.8.1` compares directly with published `v0.8.0`. Its +[reviewed API report](../engineering/reviewed-public-api-diff-0.8.1.md) and +[review configuration](../engineering/public-api-review-0.8.1.yml) record an +additive-only public surface: exact-resolution and sRGB/ICC decode APIs, +encode-stage context, and the shared irreversible midpoint calculation. The +one-time `0.7.5` to `0.8.0` transition allowance has been removed. Version `0.7.3` retained the API contract introduced by `0.7.1`, which intentionally contracted parts of the published pre-1.0 `0.6.2` API. It does @@ -81,6 +93,16 @@ Both offline candidate gates run from that clean commit. A failure or any tracked correction invalidates `RC_SHA`; commit the correction, choose a new candidate SHA, and rerun the local and exact-SHA evidence. +ISO/IEC 15444-4:2024 / ITU-T T.803 v3 claim eligibility is scoped independently. +CPU wording requires exact-SHA reports from Linux x86-64, macOS arm64, and +Windows x86-64. CUDA and Metal adapter wording each requires that adapter's own +exact-SHA real-hardware report. Every report in the selected scope must contain +all selected cases with no skips. An unavailable adapter blocks only its own +claim; it does not invalidate or suppress a complete CPU result. The optional +`--scope all` verifier is a coordinated-release convenience, not the definition +of CPU compliance. Current status is recorded in +[`docs/t803-conformance.md`](t803-conformance.md). + During remediation, the changelog keeps a real `## [Unreleased]` heading and a structured staged-version line. As the final release-preparation edit before candidate freeze, replace that heading with `## [] - YYYY-MM-DD` using the @@ -98,7 +120,7 @@ have completed: ```bash test "$(git rev-parse origin/main)" = "$RC_SHA" -cargo xtask release-status --sha "$RC_SHA" +cargo xtask release-status --sha "$RC_SHA" --scope all ``` Any tracked edit creates a new candidate: commit it, choose a new `RC_SHA`, and @@ -272,8 +294,8 @@ integrity mode so it cannot bypass those source and metadata checks. The public-support gate verifies that the JPEG 2000 Part 1, JP2, HTJ2K Part 15, JPH, known-limitation, and publication-gate rows remain synchronized with tests -and the conformance manifest before a release can claim full scoped codec -support. +and their support inventory. That implementation boundary does not substitute +for the T.803 exact-reference gate or authorize a conformance claim. ## Required gates @@ -295,6 +317,16 @@ After the candidate is frozen and committed, hosted CI must pass for exactly - bounded fuzz run - coverage via `cargo xtask coverage` - hosted macOS Metal compilation and pure tests via `cargo xtask metal-compile` +- exact-reference T.803 CPU reports on Linux x86-64, macOS arm64, and Windows + x86-64 when the release declares CPU Profile/Cclass wording, with all selected + cases present and passing +- an exact-reference CUDA or Metal adapter-IUT report from real hardware when + the release declares wording for that adapter, with CPU/device/hybrid stages + disclosed per case; compilation alone is never adapter conformance evidence +- packaged clean-consumer checks for `j2k`, `j2k-cuda`, and `j2k-metal` +- route-parity tests for every fixed `Auto` decision; any newly promoted hybrid + threshold additionally requires verified external Criterion evidence and its + artifact hash Changed-line coverage records production Rust across CPU and accelerator crates. The host lane enforces 80% across all changed production Rust and an diff --git a/docs/stable-api-1.0.implementation-public-api.txt b/docs/stable-api-1.0.implementation-public-api.txt index 9a96f9a3..53b3a7d3 100644 --- a/docs/stable-api-1.0.implementation-public-api.txt +++ b/docs/stable-api-1.0.implementation-public-api.txt @@ -70,6 +70,7 @@ pub use j2k::IrreversibleQuantizationSubbandScales pub use j2k::J2kCodeBlockSegment pub use j2k::J2kCodeBlockStyle pub use j2k::J2kDeinterleaveToF32Job +pub use j2k::J2kEncodeContext pub use j2k::J2kEncodeDispatchReport pub use j2k::J2kEncodeStageAccelerator pub use j2k::J2kEncodeStageError @@ -1055,6 +1056,7 @@ pub fn j2k_metal::J2kDecoder<'a>::submit_scaled_to_device(&mut self, &mut Self:: pub fn j2k_metal::J2kDecoder<'a>::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result pub fn j2k_metal::MetalBackendSession::backend_kind(&self) -> j2k_core::backend::BackendKind pub fn j2k_metal::MetalBackendSession::uses_command_queue(&self, &metal::commandqueue::CommandQueueRef) -> core::result::Result +pub fn j2k_metal::MetalEncodeStageAccelerator::begin_encode(&mut self, j2k_types::J2kEncodeContext) -> j2k_types::stage_error::J2kEncodeStageResult<()> pub fn j2k_metal::MetalEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport pub fn j2k_metal::MetalEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> j2k_types::stage_error::J2kEncodeStageResult>>> pub fn j2k_metal::MetalEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> j2k_types::stage_error::J2kEncodeStageResult> @@ -1068,6 +1070,7 @@ pub fn j2k_metal::MetalEncodeStageAccelerator::encode_packetization(&mut self, j pub fn j2k_metal::MetalEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> j2k_types::stage_error::J2kEncodeStageResult>> pub fn j2k_metal::MetalEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> j2k_types::stage_error::J2kEncodeStageResult> pub fn j2k_metal::MetalEncodeStageAccelerator::encode_tier1_code_blocks(&mut self, &[j2k_types::J2kTier1CodeBlockEncodeJob<'_>]) -> j2k_types::stage_error::J2kEncodeStageResult>> +pub fn j2k_metal::MetalEncodeStageAccelerator::for_host_output_benchmark() -> Self pub fn j2k_metal::MetalEncodeStageAccelerator::prefer_parallel_cpu_code_block_fallback(&self) -> bool pub fn j2k_metal::MetalLosslessEncodeBatchStats::new() -> Self pub fn j2k_metal::MetalLosslessEncodeStageStats::add_assign(&mut self, Self) @@ -2251,7 +2254,9 @@ pub fn j2k_native::HtCodeBlockDecodeWorkspace::coefficient_capacity(&self) -> us pub fn j2k_native::HtCodeBlockDecoder::decode_code_block(&mut self, j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::Result<()> pub fn j2k_native::HtCodeBlockDecoder::decode_inverse_mct(&mut self, j2k_native::J2kInverseMctJob<'_>) -> j2k_native::Result pub fn j2k_native::HtCodeBlockDecoder::decode_j2k_code_block(&mut self, j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_j2k_code_block_with_midpoint(&mut self, j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], bool) -> j2k_native::Result pub fn j2k_native::HtCodeBlockDecoder::decode_j2k_sub_band(&mut self, j2k_native::J2kSubBandDecodeJob<'_>, &mut [f32]) -> j2k_native::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_j2k_sub_band_with_midpoint(&mut self, j2k_native::J2kSubBandDecodeJob<'_>, &mut [f32], bool) -> j2k_native::Result pub fn j2k_native::HtCodeBlockDecoder::decode_single_decomposition_idwt(&mut self, j2k_native::J2kSingleDecompositionIdwtJob<'_>, &mut [f32]) -> j2k_native::Result pub fn j2k_native::HtCodeBlockDecoder::decode_store_component(&mut self, j2k_native::J2kStoreComponentJob<'_>) -> j2k_native::Result pub fn j2k_native::HtCodeBlockDecoder::decode_sub_band(&mut self, j2k_native::HtSubBandDecodeJob<'_>, &mut [f32]) -> j2k_native::Result @@ -2272,6 +2277,7 @@ pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients(&self) -> j2k_na pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::Result pub fn j2k_native::Image<'a>::new_with_reduction(&'a [u8], &j2k_native::DecodeSettings, u8) -> j2k_native::Result pub fn j2k_native::Image<'a>::new_with_retained_baseline(&'a [u8], &j2k_native::DecodeSettings, usize) -> j2k_native::Result +pub fn j2k_native::Image<'a>::primary_icc_profile(&self) -> core::option::Option<&[u8]> pub fn j2k_native::Image<'a>::retained_allocation_bytes(&self) -> j2k_native::Result pub fn j2k_native::Image<'a>::supports_direct_device_plane_reuse(&self) -> bool pub fn j2k_native::J2kCodeBlockDecodeProfile::new() -> Self @@ -2324,6 +2330,8 @@ pub fn j2k_native::decode_ht_sigprop_benchmark_state(&mut j2k_native::HtSigPropB pub fn j2k_native::decode_j2k_code_block_scalar(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::Result<()> pub fn j2k_native::decode_j2k_code_block_scalar_profiled(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeProfile) -> j2k_native::Result<()> pub fn j2k_native::decode_j2k_code_block_scalar_with_workspace(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::decode_j2k_code_block_scalar_with_workspace_midpoint(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeWorkspace) -> j2k_native::Result<()> +pub fn j2k_native::decode_j2k_code_block_scalar_with_workspace_midpoint_profiled(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeWorkspace, &mut j2k_native::J2kCodeBlockDecodeProfile) -> j2k_native::Result<()> pub fn j2k_native::decode_j2k_code_block_scalar_with_workspace_profiled(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeWorkspace, &mut j2k_native::J2kCodeBlockDecodeProfile) -> j2k_native::Result<()> pub fn j2k_native::decode_j2k_sub_band_scalar(j2k_native::J2kSubBandDecodeJob<'_>, &mut [f32]) -> j2k_native::Result<()> pub fn j2k_native::encode_ht_code_block_scalar(&[i32], u32, u32, u8) -> j2k_native::EncodeResult @@ -2569,6 +2577,7 @@ pub j2k_native::J2kOwnedCodeBlockBatchJob::total_bitplanes: u8 pub j2k_native::J2kOwnedCodeBlockBatchJob::width: u32 pub j2k_native::J2kOwnedSubBandPlan::band_id: j2k_native::J2kDirectBandId pub j2k_native::J2kOwnedSubBandPlan::height: u32 +pub j2k_native::J2kOwnedSubBandPlan::irreversible_midpoint: bool pub j2k_native::J2kOwnedSubBandPlan::jobs: alloc::vec::Vec pub j2k_native::J2kOwnedSubBandPlan::rect: j2k_native::J2kRect pub j2k_native::J2kOwnedSubBandPlan::width: u32 @@ -2787,6 +2796,7 @@ pub use j2k_native::J2kCodeBlockSegment pub use j2k_native::J2kCodeBlockStyle pub use j2k_native::J2kCodestreamRange pub use j2k_native::J2kDeinterleaveToF32Job +pub use j2k_native::J2kEncodeContext pub use j2k_native::J2kEncodeDispatchReport pub use j2k_native::J2kEncodeStageAccelerator pub use j2k_native::J2kEncodeStageError @@ -2855,6 +2865,7 @@ pub const fn j2k_types::J2kResidentEncodeInput::width(self) -> u32 pub const fn j2k_types::J2kResidentEncodeInputError::reason(&self) -> &'static str pub const j2k_types::MAX_JPEG2000_PART1_COMPONENTS: u16 pub const j2k_types::MAX_JPEG2000_PART1_SAMPLE_BIT_DEPTH: u8 +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::begin_encode(&mut self, j2k_types::J2kEncodeContext) -> j2k_types::J2kEncodeStageResult<()> pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> j2k_types::J2kEncodeStageResult>>> pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> j2k_types::J2kEncodeStageResult> @@ -3341,12 +3352,14 @@ pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::reused: bool pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::scanned_count: usize pub j2k_cuda_runtime::CudaClassicCodeBlockJob::dequantization_step: f32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::height: u32 +pub j2k_cuda_runtime::CudaClassicCodeBlockJob::irreversible_midpoint: bool pub j2k_cuda_runtime::CudaClassicCodeBlockJob::missing_bitplanes: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::number_of_coding_passes: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::output_offset: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::output_stride: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::payload_len: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::payload_offset: u64 +pub j2k_cuda_runtime::CudaClassicCodeBlockJob::roi_shift: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::segment_count: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::segment_start: u32 pub j2k_cuda_runtime::CudaClassicCodeBlockJob::strict: bool diff --git a/docs/stable-api-1.0.md b/docs/stable-api-1.0.md index 997daa7d..661e73d5 100644 --- a/docs/stable-api-1.0.md +++ b/docs/stable-api-1.0.md @@ -41,15 +41,18 @@ therefore remain in the reviewed inventory. Do not use `#[doc(hidden)]` as a compatibility escape hatch. The published 0.7.5 artifact recorded both ordinary and hidden-enabled passes -with the same generator, rustdoc, and target pins. The published 0.8.0 semver -report compares its ordinary inventory with 0.7.5 and also records -each package's complete hidden-inventory count and fingerprint. +with the same generator, rustdoc, and target pins. The historical 0.8.0 semver +report compares its ordinary inventory with 0.7.5. The 0.8.1 report compares +the candidate directly with published 0.8.0. Both reports also record each +package's complete hidden-inventory count and fingerprint. Every semver invocation collects both live passes, compares both committed companions, and requires exact ordinary added/removed fingerprints plus the hidden -count/fingerprint in `engineering/public-api-review-0.8.0.yml`. +count/fingerprint in `engineering/public-api-review-0.8.1.yml`. Nonempty hidden inventories also require a package-specific hidden rationale. -The review file also contains the reviewed 0.7.5-to-0.8.0 break ledger. +The 0.8.0 review file contains the reviewed 0.7.5-to-0.8.0 break ledger. The +0.8.1 review file has no break-ledger entries because its generated diff is +additive only. Source-break entries must enumerate every exact removed API item, package, summary, and migration. Validation requires that inventory to equal the generated removed-item set: an omitted, duplicate, unknown, or stale item fails @@ -72,14 +75,18 @@ contract expectations. Manual prose in this file must not duplicate that inventory. The completed 0.7.5-to-0.8.0 comparison is in the generated [`0.8.0` reviewed API report](../engineering/reviewed-public-api-diff-0.8.0.md). That report became release evidence after source freeze and the exact-SHA -local, hosted, Metal, and CUDA gates completed. +local, hosted, Metal, and CUDA gates completed. The current comparison is in +the generated +[`0.8.1` reviewed API report](../engineering/reviewed-public-api-diff-0.8.1.md). The currently published stable contract is the `0.8.x` line. Version `0.8.0` intentionally changed the strict-decoding behavior and one warning variant under Cargo's pre-1.0 compatibility rules. It does not claim source or behavior compatibility with `0.7.x`; its exact breaks and migrations are in the review -file. Version `0.7.0` similarly contracted parts of the pre-1.0 `0.6.2` API and -did not claim source compatibility with `0.6.x`. +file. Version `0.8.1` adds exact-resolution and sRGB/ICC decode APIs plus +encode-stage context without removing or changing a 0.8.0 item. Version +`0.7.0` similarly contracted parts of the pre-1.0 `0.6.2` API and did not claim +source compatibility with `0.6.x`. ## Stability tiers @@ -123,12 +130,11 @@ exception applied only to `0.7.5`. The completed transition lock was intentionally narrow: `0.8.0` was the only candidate permitted to compare against `v0.7.5` as an intentional pre-1.0 -break. The checked-in release evidence retains that comparison, but it rejects -`0.8.1` or any later candidate while the older baseline remains configured. -Before preparing any follow-up, rotate the semver baseline to the published -`v0.8.0` tag, version, and peeled commit, then disable the transition lock. -Subsequent patch-line checks must compare with the real 0.8 contract instead -of continuing to receive permission for 0.7-to-0.8 breakage. +break. The checked-in historical evidence retains that comparison. The active +baseline is now published `v0.8.0` at its pinned peeled commit, and the one-time +transition lock is disabled. The 0.8.1 and later patch-line checks therefore +compare with the real 0.8 contract instead of inheriting permission for +0.7-to-0.8 breakage. Before `1.0`, a minor release may intentionally change the contract only under the same generated evidence, explicit break-ledger, and migration requirements. diff --git a/docs/stable-api-1.0.public-api.txt b/docs/stable-api-1.0.public-api.txt index a6e28f28..afe11af2 100644 --- a/docs/stable-api-1.0.public-api.txt +++ b/docs/stable-api-1.0.public-api.txt @@ -26,6 +26,7 @@ It is the item-level companion to `docs/stable-api-1.0.md`: every public module, #[non_exhaustive] pub enum j2k::J2kDecodedColorSpace #[non_exhaustive] pub enum j2k::J2kError #[non_exhaustive] pub enum j2k::J2kFileColorSpec<'a> +#[non_exhaustive] pub enum j2k::J2kSrgb8Layout #[non_exhaustive] pub enum j2k::NonRepresentableReason #[non_exhaustive] pub enum j2k::PreparationDepth #[non_exhaustive] pub struct j2k::J2kLosslessEncodeOptions @@ -67,6 +68,7 @@ impl j2k::J2kNativeComponentPlane impl j2k::J2kQualityLayer impl j2k::J2kRowDecodeOptions impl j2k::J2kScratchPool +impl j2k::J2kSrgb8Image impl j2k::J2kSupportInfo impl j2k::J2kToHtj2kOptions impl j2k::PreparedBatch @@ -150,6 +152,8 @@ pub const fn j2k::J2kRowDecodeOptions::new(u32) -> Self pub const fn j2k::J2kRowDecodeOptions::new_with_max_stripe_bytes(u32, usize) -> Self pub const fn j2k::J2kRowDecodeOptions::with_max_stripe_bytes(self, usize) -> Self pub const fn j2k::J2kScratchPool::new() -> Self +pub const fn j2k::J2kSrgb8Image::dimensions(&self) -> (u32, u32) +pub const fn j2k::J2kSrgb8Image::layout(&self) -> j2k::J2kSrgb8Layout pub const fn j2k::J2kToHtj2kOptions::new(j2k_core::passthrough::CompressedPayloadKind, j2k::J2kProgressionOrder, j2k::J2kEncodeValidation) -> Self pub const fn j2k::PreparedBatch::options(&self) -> j2k::BatchDecodeOptions pub const fn j2k::PreparedBatchGroup::options(&self) -> j2k::BatchDecodeOptions @@ -235,9 +239,11 @@ pub fn j2k::J2kDecodedNativeComponents::color_space(&self) -> &j2k::J2kDecodedCo pub fn j2k::J2kDecodedNativeComponents::dimensions(&self) -> (u32, u32) pub fn j2k::J2kDecodedNativeComponents::has_alpha(&self) -> bool pub fn j2k::J2kDecodedNativeComponents::planes(&self) -> &[j2k::J2kNativeComponentPlane] +pub fn j2k::J2kDecoder<'_>::decode_native_components_at_reduction(&mut self, u8) -> core::result::Result pub fn j2k::J2kDecoder<'_>::decode_region_scaled_pow2_into(&mut self, &mut j2k::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, u8) -> core::result::Result, j2k::J2kError> pub fn j2k::J2kDecoder<'_>::decode_rows_u16_bounded>(&mut self, &mut R, j2k::J2kRowDecodeOptions) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> pub fn j2k::J2kDecoder<'_>::decode_rows_u8_bounded>(&mut self, &mut R, j2k::J2kRowDecodeOptions) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub fn j2k::J2kDecoder<'_>::decode_srgb8(&mut self) -> core::result::Result pub fn j2k::J2kDecoder<'a>::cpu_decode_parallelism(&self) -> j2k::CpuDecodeParallelism pub fn j2k::J2kDecoder<'a>::decode_components(&mut self) -> core::result::Result, j2k::J2kError> pub fn j2k::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, j2k::J2kError> @@ -290,6 +296,8 @@ pub fn j2k::J2kNativeComponentPlane::dimensions(&self) -> (u32, u32) pub fn j2k::J2kNativeComponentPlane::sampling(&self) -> (u8, u8) pub fn j2k::J2kNativeComponentPlane::signed(&self) -> bool pub fn j2k::J2kRowDecodeOptions::default() -> Self +pub fn j2k::J2kSrgb8Image::data(&self) -> &[u8] +pub fn j2k::J2kSrgb8Image::into_data(self) -> alloc::vec::Vec pub fn j2k::J2kSupportInfo::component_count(&self) -> u16 pub fn j2k::J2kSupportInfo::has_component_subsampling(&self) -> bool pub fn j2k::J2kSupportInfo::has_mixed_bit_depths(&self) -> bool @@ -507,6 +515,7 @@ pub j2k::J2kError::InvalidBox::offset: usize pub j2k::J2kError::InvalidBox::what: &'static str pub j2k::J2kError::InvalidCod pub j2k::J2kError::InvalidCod::what: &'static str +pub j2k::J2kError::InvalidIccProfile pub j2k::J2kError::InvalidMarker pub j2k::J2kError::InvalidMarker::marker: u8 pub j2k::J2kError::InvalidMarker::offset: usize @@ -650,6 +659,9 @@ pub j2k::J2kRoiRegion::shift: u8 pub j2k::J2kRoiRegion::width: u32 pub j2k::J2kRoiRegion::x: u32 pub j2k::J2kRoiRegion::y: u32 +pub j2k::J2kSrgb8Layout::Gray +pub j2k::J2kSrgb8Layout::Rgb +pub j2k::J2kSrgb8Layout::Rgba pub j2k::J2kSupportInfo::components: alloc::vec::Vec pub j2k::J2kSupportInfo::file_metadata: core::option::Option pub j2k::J2kSupportInfo::info: j2k_core::types::Info @@ -729,6 +741,7 @@ pub struct j2k::J2kQualityLayer pub struct j2k::J2kRoiRegion pub struct j2k::J2kRowDecodeOptions pub struct j2k::J2kScratchPool +pub struct j2k::J2kSrgb8Image pub struct j2k::J2kSupportInfo pub struct j2k::J2kToHtj2kReport pub struct j2k::J2kView<'a> @@ -1277,6 +1290,7 @@ pub unsafe trait j2k_core::accelerator::GpuAbi: core::marker::Copy + 'static impl core::fmt::Display for j2k_codec_math::jpeg::CanonicalHuffmanError impl j2k_codec_math::dwt::Dwt53LinearRow impl j2k_codec_math::dwt::Dwt53LinearTap +pub const fn j2k_codec_math::classic::irreversible_midpoint_bit(u64, u32, u32) -> core::option::Option pub const fn j2k_codec_math::dwt::Dwt53LinearTap::sample_index(self) -> usize pub const fn j2k_codec_math::dwt::Dwt53LinearTap::weight(self) -> f64 pub const fn j2k_codec_math::dwt::max_decomposition_levels(u32, u32) -> u8 @@ -3970,6 +3984,7 @@ pub fn j2k_types::IrreversibleQuantizationSubbandScales::default() -> Self pub fn j2k_types::J2kEncodeDispatchReport::any(self) -> bool pub fn j2k_types::J2kEncodeDispatchReport::saturating_delta(self, Self) -> Self pub fn j2k_types::J2kEncodeDispatchReport::total(self) -> usize +pub fn j2k_types::J2kEncodeStageAccelerator::begin_encode(&mut self, j2k_types::J2kEncodeContext) -> j2k_types::J2kEncodeStageResult<()> pub fn j2k_types::J2kEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport pub fn j2k_types::J2kEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> j2k_types::J2kEncodeStageResult>>> pub fn j2k_types::J2kEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> j2k_types::J2kEncodeStageResult> @@ -4029,6 +4044,11 @@ pub j2k_types::J2kDeinterleaveToF32Job::num_components: u16 pub j2k_types::J2kDeinterleaveToF32Job::num_pixels: usize pub j2k_types::J2kDeinterleaveToF32Job::pixels: &'a [u8] pub j2k_types::J2kDeinterleaveToF32Job::signed: bool +pub j2k_types::J2kEncodeContext::bit_depth: u8 +pub j2k_types::J2kEncodeContext::num_components: u16 +pub j2k_types::J2kEncodeContext::num_pixels: usize +pub j2k_types::J2kEncodeContext::reversible: bool +pub j2k_types::J2kEncodeContext::signed: bool pub j2k_types::J2kEncodeDispatchReport::deinterleave: usize pub j2k_types::J2kEncodeDispatchReport::forward_dwt53: usize pub j2k_types::J2kEncodeDispatchReport::forward_dwt97: usize @@ -4265,6 +4285,7 @@ pub struct j2k_types::J2kCodeBlockSegment pub struct j2k_types::J2kCodeBlockStyle pub struct j2k_types::J2kCodestreamRange pub struct j2k_types::J2kDeinterleaveToF32Job<'a> +pub struct j2k_types::J2kEncodeContext pub struct j2k_types::J2kEncodeDispatchReport pub struct j2k_types::J2kForwardDwt53Job<'a> pub struct j2k_types::J2kForwardDwt53Level diff --git a/docs/t803-conformance.md b/docs/t803-conformance.md new file mode 100644 index 00000000..1ecb1856 --- /dev/null +++ b/docs/t803-conformance.md @@ -0,0 +1,139 @@ +# ISO/IEC 15444-4 / ITU-T T.803 Conformance + +Status: **0.8.1 release-scoped** + +Formal decoder claim: + +- `j2k` CPU IUT: **Profile-1 Cclass-1 compliant; Profile-1 Cclass-1HF + compliant; Annex G JP2 reader compliant.** +- `j2k-cuda` adapter IUT: **Profile-1 Cclass-1 compliant and Profile-1 + Cclass-1HF compliant as an adapter IUT**, with every CPU, CUDA, hybrid, and + transfer stage disclosed per case. +- `j2k-metal` adapter IUT: **Profile-1 Cclass-1 compliant and Profile-1 + Cclass-1HF compliant as an adapter IUT**, with every CPU, Metal, hybrid, and + transfer stage disclosed per case. + +The implemented harness targets ISO/IEC 15444-4:2024 / ITU-T T.803 v3. Part 4 +defines JPEG 2000 conformance-testing procedures and reference comparisons; it +is not another codestream syntax or a performance benchmark. These claims are +limited to release `0.8.1` and require the attached reports to identify the +same immutable release SHA as the annotated tag. + +## Claimed decoder scope + +| IUT | Release wording | Route boundary | +| --- | --- | --- | +| `j2k` CPU | Profile-1 Cclass-1 compliant; Profile-1 Cclass-1HF compliant; Annex G JP2 reader compliant | CPU implementation under test. | +| `j2k-cuda` | Profile-1 Cclass-1 compliant adapter IUT; Profile-1 Cclass-1HF compliant adapter IUT | Parsing, Tier-1, transforms, output, and transfers are reported per case as CPU, CUDA, or not used. | +| `j2k-metal` | Profile-1 Cclass-1 compliant adapter IUT; Profile-1 Cclass-1HF compliant adapter IUT | Parsing, Tier-1, transforms, output, and transfers are reported per case as CPU, Metal, or not used. | + +CPU assistance is permitted for the adapter IUTs. Any such route is labelled +`hybrid`; it is not described as device-native. Annex G JP2 color and component +normalization currently runs through disclosed CPU stages for the GPU adapters. +JPX / Part 2 is outside this scope, except for JP2-compatible JPX input required +by Annex G. + +The project does not use a generic “full Part 1 compliant” label. The exact +Profile/Cclass wording above is tied to the published reports for one immutable +release SHA. + +## Release result + +The macOS arm64, Linux x86-64, and Windows x86-64 CPU reports each pass all 90 +selected decoder/JP2 cases with zero skips. Both real-hardware adapter reports +record **0/90 device-native, 48/90 hybrid, and 42/90 CPU-routed cases**: CUDA +on an NVIDIA GeForce RTX 4070 SUPER and Metal on an Apple M4 Pro. All selected +outputs are within their applicable bounds, but neither adapter result is +device-native conformance evidence. + +The CPU Annex D/F encoder matrix passes 28 of 28 cases. The CUDA and Metal +matrices each pass 25 of 25; CUDA records 24 hybrid encoder routes and one +CPU-routed case, while Metal records 23 hybrid routes and two CPU-routed cases. +These encoder results are informative evidence, not the formal decoder claim. + +The former `c1-c0p0-13` failure was an IUT harness defect. The codestream has +257 components and enables the reversible component transform. T.803 B.2.5 +requires Cclass-0 comparison before inverse MCT, so its first-component +reference is 1; the Cclass-1 component-0 reference after inverse RCT is 0. The +harness had incorrectly inferred MCT use from a display colorspace, which is +unknown for this component count. It now reads the COD transform flag and +transform kind through the existing codestream inspector, reconstructs the +pre-MCT component for Cclass-0, and reports the MCT stage from the same +metadata. A 257-component regression test prevents the colorspace inference +from returning. + +The report now independently decodes every selected codestream whose COD +enables MCT and whose SIZ declares more than four components. For `p0_13.j2k`, +the production decoder and vendored OpenJPEG 2.5.3 matched component metadata +and samples exactly for all 257 components before any T.803 normalization. Both +canonical native-output hashes are +`a01808e0cbf14288274188c8bebb5ef8c2aa46304eca964a2ac71bed1713c1fd`. +As a second manual check, OpenJPEG CLI 2.5.4 emitted 257 PGX components with +zero sample mismatches; the concatenated one-sample component payload SHA-256 +was `54acfbfedc4d8da40f76f275e1a98f10af8ef1fb9fb39e5a67a00aabcbe6597c`. + +The investigation independently confirmed byte-identical `p0_13.j2k`, +`c0p0_13.pgx`, and `c1p0_13-0.pgx` payloads in ITU's current attachment, +ISO's 2024 electronic insert, and ITU's 2002 suite. No corpus mapping, hash, +dimension, precision, signedness, reduction, crop, tolerance, or comparison +arithmetic was changed to obtain the passing result. + +## Evidence commands + +The official corpus is fetched only from the URL and archive digest pinned in +`corpus/j2k-conformance/t803-v3.toml`: + +```bash +cargo xtask t803 fetch +cargo xtask t803 run --iut cpu +cargo xtask t803 run --iut cuda +cargo xtask t803 run --iut metal +``` + +`fetch` rejects unapproved redirects, archive or file hash drift, unsafe archive +entries, duplicate paths, unexpected required-case names, and resource-limit +violations. The copyrighted corpus stays under `target/t803/`. Only versioned +JSON/Markdown reports and hashes may be retained. + +Release eligibility is scoped independently. CPU wording requires the three +CPU operating-system reports; each adapter wording requires only that +adapter's real-hardware report. An unavailable adapter blocks its own claim, +not the CPU claim: + +```bash +cargo xtask t803 verify --scope cpu --candidate-sha "$RC_SHA" \ + --report path/to/cpu-linux.json \ + --report path/to/cpu-macos.json \ + --report path/to/cpu-windows.json +cargo xtask t803 verify --scope cuda --candidate-sha "$RC_SHA" \ + --report path/to/cuda.json +cargo xtask t803 verify --scope metal --candidate-sha "$RC_SHA" \ + --report path/to/metal.json +``` + +`--scope all` verifies all five reports together. Release `0.8.1` uses that +coordinated scope in the tag-publish workflow; independently, an unavailable +adapter invalidates only its own adapter claim and does not erase a complete +CPU result. + +All 90 selected decoder/JP2 cases must be present with no skips, every report +must pass, source and corpus hashes must match, and the IUT/platform/route +identity must match the required lane. Reports are rejected when a route labels +CPU-assisted work as device-native. + +## Encoder evidence + +The CPU, CUDA, and Metal Annex F implementation compliance statements and the +stable pairwise/boundary matrix live in `corpus/j2k-conformance/`. Selected +codestreams are decoded by the pinned T.804 OpenJPEG reference implementation. +Reference-decode success is the Annex D legality result; lossless output must +also match the source exactly. Lossy rate and PSNR checks are separate project +quality gates. + +Encoder testing is informative under T.803 and is not the same formal claim as +decoder compliance. Accelerator dispatch and fallback stages are reported for +every encoder case. + +T.803 does not establish robustness, security, adoption, or performance. Those +properties require their own fuzzing, security review, external workload, and +benchmark evidence. diff --git a/engineering/public-api-review-0.8.1.yml b/engineering/public-api-review-0.8.1.yml new file mode 100644 index 00000000..45944c1f --- /dev/null +++ b/engineering/public-api-review-0.8.1.yml @@ -0,0 +1,130 @@ +version: 3 +baseline_tag: v0.8.0 +baseline_version: 0.8.0 +candidate_version: "0.8.1" +break_ledger: [] +reviews: + j2k-core: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 189 + hidden_fingerprint: "fnv1a64:b87aa26da547e491" + rationale: "Reviewed the complete ordinary j2k-core surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-core inventory; its decode contracts, planning types, validation boundaries, and typed failures are unchanged from v0.8.0." + j2k-profile: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 117 + hidden_fingerprint: "fnv1a64:a1546f56ae38f2e3" + rationale: "Reviewed the complete ordinary j2k-profile surface against v0.8.0; its documented surface remains the crate module root only." + hidden_rationale: "Reviewed the complete hidden profiling inventory; capture ownership, controls, and callable profiling surfaces are unchanged from v0.8.0." + j2k-types: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:dfa9d35be523f07f" + hidden_count: 55 + hidden_fingerprint: "fnv1a64:e1a4e67c2f8431a4" + rationale: "Reviewed the additive J2kEncodeContext fields and begin_encode hook against v0.8.0. They supply validated per-operation shape and coding information before stage dispatch; the hook has a source-compatible default implementation and removes no existing contract." + hidden_rationale: "Reviewed the complete hidden j2k-types inventory, including the matching CPU-only begin_encode implementation; no hidden item was removed or changed incompatibly." + j2k-codec-math: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:ba8bc063b9c0e287" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Reviewed the additive irreversible_midpoint_bit helper. It exposes the shared checked half-bin position calculation used by scalar and accelerator classic JPEG 2000 decoding and returns None for empty or inconsistent state." + j2k-cuda-runtime: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 1091 + hidden_fingerprint: "fnv1a64:29aed9f592c03b03" + rationale: "Reviewed the complete ordinary j2k-cuda-runtime surface against v0.8.0; no ordinary public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden CUDA runtime inventory, including the added classic-code-block midpoint and ROI fields that preserve scalar/device differential semantics; no hidden item was removed." + j2k-metal-support: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 11 + hidden_fingerprint: "fnv1a64:671e50d4d121b419" + rationale: "Reviewed the complete ordinary j2k-metal-support surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden Metal support inventory; checked allocation, command, completion, buffer ownership, and audited unsafe boundaries are unchanged from v0.8.0." + j2k-native: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 670 + hidden_fingerprint: "fnv1a64:140051aa454ad8e5" + rationale: "Reviewed the complete ordinary j2k-native surface against v0.8.0; no ordinary public API item was added, removed, or changed." + hidden_rationale: "Reviewed the added hidden exact-reduction, ICC-profile access, midpoint reconstruction, ROI, and sub-band planning surfaces. They support the production decoder and adapter parity paths without removing or changing the existing hidden contracts." + j2k-jpeg: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 550 + hidden_fingerprint: "fnv1a64:d2bd76732f32e066" + rationale: "Reviewed the complete ordinary j2k-jpeg surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG inventory, including codec context, batch, decode, and typed error boundaries; it is unchanged from v0.8.0." + j2k-tilecodec: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 32 + hidden_fingerprint: "fnv1a64:eb677842559d9b0e" + rationale: "Reviewed the complete ordinary j2k-tilecodec surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-tilecodec inventory, including codec, scratch, and source-preserving error boundaries; it is unchanged from v0.8.0." + j2k: + removed_fingerprint: "none" + added_fingerprint: "fnv1a64:8f65112e24e99498" + hidden_count: 105 + hidden_fingerprint: "fnv1a64:439bc4b61c1ec5b3" + rationale: "Reviewed all fourteen additive j2k items against v0.8.0. The exact-reduction APIs decode through the codestream resolution ladder; J2kSrgb8Image provides owned Gray, RGB, or RGBA output with explicit ICC failures and private storage accessors; no existing API is removed or changed." + hidden_rationale: "Reviewed the complete hidden j2k inventory, including the added J2kEncodeContext re-export used by accelerator stage adapters; retained plans, validation, and typed error boundaries remain compatible with v0.8.0." + j2k-transcode: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 455 + hidden_fingerprint: "fnv1a64:fdd61f6fdda4a9ab" + rationale: "Reviewed the complete ordinary j2k-transcode surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden transcode inventory, including transforms, packetization, resident handoff, accounting, and source-preserving errors; it is unchanged from v0.8.0." + j2k-transcode-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 17 + hidden_fingerprint: "fnv1a64:3e3e69d35c4a62a5" + rationale: "Reviewed the complete ordinary j2k-transcode-cuda surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden CUDA transcode inventory, including routing, resident handoff, allocation caps, and source-preserving failures; it is unchanged from v0.8.0." + j2k-jpeg-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 83 + hidden_fingerprint: "fnv1a64:8c83730972447ab6" + rationale: "Reviewed the complete ordinary j2k-jpeg-metal surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG Metal inventory, including resident-image ownership, session lifetime, and typed Metal failures; it is unchanged from v0.8.0." + j2k-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 301 + hidden_fingerprint: "fnv1a64:c6b5bbbede58c77e" + rationale: "Reviewed the complete ordinary j2k-metal surface against v0.8.0; no ordinary public API item was added, removed, or changed." + hidden_rationale: "Reviewed the added hidden per-encode context hook and host-output benchmark constructor. They select and report an encode route without changing the existing public Metal adapter contract." + j2k-transcode-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 49 + hidden_fingerprint: "fnv1a64:5bf8a22a33753a2d" + rationale: "Reviewed the complete ordinary j2k-transcode-metal surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden Metal transcode inventory, including transforms, buffers, code blocks, resident handoff, fallback, and typed source chains; it is unchanged from v0.8.0." + j2k-jpeg-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 124 + hidden_fingerprint: "fnv1a64:0b31ccb3e7974bc4" + rationale: "Reviewed the complete ordinary j2k-jpeg-cuda surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG CUDA inventory, including session-bound execution, allocation accounting, and typed runtime failures; it is unchanged from v0.8.0." + j2k-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 181 + hidden_fingerprint: "fnv1a64:6bb25d5241c4e86a" + rationale: "Reviewed the complete ordinary j2k-cuda surface against v0.8.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden J2K CUDA inventory, including completion uncertainty, session usability, device identity, event ordering, pool diagnostics, and guarded interop; it is unchanged from v0.8.0." + j2k-ml: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Reviewed the complete j2k-ml ordinary and hidden inventories against v0.8.0; no public API item was added, removed, or changed." diff --git a/engineering/reviewed-public-api-diff-0.8.1.md b/engineering/reviewed-public-api-diff-0.8.1.md new file mode 100644 index 00000000..7465f0a4 --- /dev/null +++ b/engineering/reviewed-public-api-diff-0.8.1.md @@ -0,0 +1,274 @@ +# Reviewed public API diff for j2k 0.8.1 + +This report is generated by `cargo xtask semver --write-report`. Normal `cargo xtask semver` regenerates it in memory and fails if this committed file is stale. Every ordinary added/removed fingerprint and every full rustdoc-hidden candidate-inventory fingerprint requires an exact reviewed entry in `engineering/public-api-review-0.8.1.yml`; report regeneration never updates that review config. + +- Baseline registry version: `0.8.0` +- Baseline source snapshot: `v0.8.0` peeled to `53e0ad3d4f75f492af55413e0dab5a5834bd09c6` +- Candidate version: `0.8.1` +- Tool pins: Rust `1.96`, `cargo-semver-checks 0.48.0`, `cargo-public-api 0.52.0`, rustdoc `nightly-2026-06-28`, target `aarch64-apple-darwin` + +## Summary + +| Package | Baseline | Candidate | Computed release type | Added | Removed/changed | Removed fingerprint | Added fingerprint | Rustdoc-hidden items | Hidden inventory fingerprint | +| --- | --- | --- | --- | ---: | ---: | --- | --- | ---: | --- | +| `j2k-core` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 189 | `fnv1a64:b87aa26da547e491` | +| `j2k-profile` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 117 | `fnv1a64:a1546f56ae38f2e3` | +| `j2k-types` | `0.8.0` | `0.8.1` | `minor` | 7 | 0 | `none` | `fnv1a64:dfa9d35be523f07f` | 55 | `fnv1a64:e1a4e67c2f8431a4` | +| `j2k-codec-math` | `0.8.0` | `0.8.1` | `minor` | 1 | 0 | `none` | `fnv1a64:ba8bc063b9c0e287` | 0 | `none` | +| `j2k-cuda-runtime` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 1091 | `fnv1a64:29aed9f592c03b03` | +| `j2k-metal-support` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 11 | `fnv1a64:671e50d4d121b419` | +| `j2k-native` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 670 | `fnv1a64:140051aa454ad8e5` | +| `j2k-jpeg` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 550 | `fnv1a64:d2bd76732f32e066` | +| `j2k-tilecodec` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 32 | `fnv1a64:eb677842559d9b0e` | +| `j2k` | `0.8.0` | `0.8.1` | `minor` | 14 | 0 | `none` | `fnv1a64:8f65112e24e99498` | 105 | `fnv1a64:439bc4b61c1ec5b3` | +| `j2k-transcode` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 455 | `fnv1a64:fdd61f6fdda4a9ab` | +| `j2k-transcode-cuda` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 17 | `fnv1a64:3e3e69d35c4a62a5` | +| `j2k-jpeg-metal` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 83 | `fnv1a64:8c83730972447ab6` | +| `j2k-metal` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 301 | `fnv1a64:c6b5bbbede58c77e` | +| `j2k-transcode-metal` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 49 | `fnv1a64:5bf8a22a33753a2d` | +| `j2k-jpeg-cuda` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 124 | `fnv1a64:0b31ccb3e7974bc4` | +| `j2k-cuda` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 181 | `fnv1a64:6bb25d5241c4e86a` | +| `j2k-ml` | `0.8.0` | `0.8.1` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | + +## Published-package details + +### `j2k-core` + +Baseline items: 487. Candidate items: 487. Computed release type: `minor`. Rustdoc-hidden candidate items: 189. Full hidden-inventory fingerprint: `fnv1a64:b87aa26da547e491`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-profile` + +Baseline items: 1. Candidate items: 1. Computed release type: `minor`. Rustdoc-hidden candidate items: 117. Full hidden-inventory fingerprint: `fnv1a64:a1546f56ae38f2e3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-types` + +Baseline items: 365. Candidate items: 372. Computed release type: `minor`. Rustdoc-hidden candidate items: 55. Full hidden-inventory fingerprint: `fnv1a64:e1a4e67c2f8431a4`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +pub fn j2k_types::J2kEncodeStageAccelerator::begin_encode(&mut self, j2k_types::J2kEncodeContext) -> j2k_types::J2kEncodeStageResult<()> +pub j2k_types::J2kEncodeContext::bit_depth: u8 +pub j2k_types::J2kEncodeContext::num_components: u16 +pub j2k_types::J2kEncodeContext::num_pixels: usize +pub j2k_types::J2kEncodeContext::reversible: bool +pub j2k_types::J2kEncodeContext::signed: bool +pub struct j2k_types::J2kEncodeContext +``` + +### `j2k-codec-math` + +Baseline items: 100. Candidate items: 101. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +pub const fn j2k_codec_math::classic::irreversible_midpoint_bit(u64, u32, u32) -> core::option::Option +``` + +### `j2k-cuda-runtime` + +Baseline items: 100. Candidate items: 100. Computed release type: `minor`. Rustdoc-hidden candidate items: 1091. Full hidden-inventory fingerprint: `fnv1a64:29aed9f592c03b03`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal-support` + +Baseline items: 178. Candidate items: 178. Computed release type: `minor`. Rustdoc-hidden candidate items: 11. Full hidden-inventory fingerprint: `fnv1a64:671e50d4d121b419`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-native` + +Baseline items: 377. Candidate items: 377. Computed release type: `minor`. Rustdoc-hidden candidate items: 670. Full hidden-inventory fingerprint: `fnv1a64:140051aa454ad8e5`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg` + +Baseline items: 586. Candidate items: 586. Computed release type: `minor`. Rustdoc-hidden candidate items: 550. Full hidden-inventory fingerprint: `fnv1a64:d2bd76732f32e066`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-tilecodec` + +Baseline items: 31. Candidate items: 31. Computed release type: `minor`. Rustdoc-hidden candidate items: 32. Full hidden-inventory fingerprint: `fnv1a64:eb677842559d9b0e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k` + +Baseline items: 763. Candidate items: 777. Computed release type: `minor`. Rustdoc-hidden candidate items: 105. Full hidden-inventory fingerprint: `fnv1a64:439bc4b61c1ec5b3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +```text +#[non_exhaustive] pub enum j2k::J2kSrgb8Layout +impl j2k::J2kSrgb8Image +pub const fn j2k::J2kSrgb8Image::dimensions(&self) -> (u32, u32) +pub const fn j2k::J2kSrgb8Image::layout(&self) -> j2k::J2kSrgb8Layout +pub fn j2k::J2kDecoder<'_>::decode_native_components_at_reduction(&mut self, u8) -> core::result::Result +pub fn j2k::J2kDecoder<'_>::decode_region_scaled_pow2_into(&mut self, &mut j2k::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, u8) -> core::result::Result, j2k::J2kError> +pub fn j2k::J2kDecoder<'_>::decode_srgb8(&mut self) -> core::result::Result +pub fn j2k::J2kSrgb8Image::data(&self) -> &[u8] +pub fn j2k::J2kSrgb8Image::into_data(self) -> alloc::vec::Vec +pub j2k::J2kError::InvalidIccProfile +pub j2k::J2kSrgb8Layout::Gray +pub j2k::J2kSrgb8Layout::Rgb +pub j2k::J2kSrgb8Layout::Rgba +pub struct j2k::J2kSrgb8Image +``` + +### `j2k-transcode` + +Baseline items: 426. Candidate items: 426. Computed release type: `minor`. Rustdoc-hidden candidate items: 455. Full hidden-inventory fingerprint: `fnv1a64:fdd61f6fdda4a9ab`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-cuda` + +Baseline items: 49. Candidate items: 49. Computed release type: `minor`. Rustdoc-hidden candidate items: 17. Full hidden-inventory fingerprint: `fnv1a64:3e3e69d35c4a62a5`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-metal` + +Baseline items: 170. Candidate items: 170. Computed release type: `minor`. Rustdoc-hidden candidate items: 83. Full hidden-inventory fingerprint: `fnv1a64:8c83730972447ab6`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal` + +Baseline items: 272. Candidate items: 272. Computed release type: `minor`. Rustdoc-hidden candidate items: 301. Full hidden-inventory fingerprint: `fnv1a64:c6b5bbbede58c77e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-metal` + +Baseline items: 99. Candidate items: 99. Computed release type: `minor`. Rustdoc-hidden candidate items: 49. Full hidden-inventory fingerprint: `fnv1a64:5bf8a22a33753a2d`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-cuda` + +Baseline items: 71. Candidate items: 71. Computed release type: `minor`. Rustdoc-hidden candidate items: 124. Full hidden-inventory fingerprint: `fnv1a64:0b31ccb3e7974bc4`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda` + +Baseline items: 247. Candidate items: 247. Computed release type: `minor`. Rustdoc-hidden candidate items: 181. Full hidden-inventory fingerprint: `fnv1a64:6bb25d5241c4e86a`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-ml` + +Baseline items: 101. Candidate items: 101. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. diff --git a/scripts/github_actions_verify.py b/scripts/github_actions_verify.py index c940a2d0..d692ab66 100755 --- a/scripts/github_actions_verify.py +++ b/scripts/github_actions_verify.py @@ -5,14 +5,18 @@ from __future__ import annotations import argparse +import io import json import os import re +import stat import sys import urllib.error import urllib.parse import urllib.request +import zipfile from dataclasses import dataclass +from pathlib import Path, PurePosixPath from typing import Any, Callable, Iterable, Mapping, Sequence @@ -20,6 +24,9 @@ PAGE_SIZE = 100 MAX_PAGES = 1_000 MAX_TAG_DEPTH = 16 +MAX_T803_ARTIFACT_ARCHIVE_BYTES = 8 * 1024 * 1024 +MAX_T803_REPORT_BYTES = 4 * 1024 * 1024 +T803_SCOPES = ("cpu", "cuda", "metal", "all") SHA_PATTERN = re.compile(r"[0-9a-fA-F]{40}\Z") CUDA_PREFIXES = ( @@ -34,7 +41,12 @@ "crates/j2k-metal/", "crates/j2k-transcode-metal/", ) -SHARED_GPU_PREFIXES = ("crates/j2k-profile/",) +SHARED_GPU_PREFIXES = ( + "corpus/j2k-conformance/", + "crates/j2k-profile/", + "crates/j2k-t803/", + "xtask/src/auto_routing", +) SHARED_GPU_EXACT_PATHS = frozenset( { ".github/CODEOWNERS", @@ -45,6 +57,7 @@ ".github/workflows/publish.yml", "scripts/ci_plan.py", "scripts/github_actions_verify.py", + "xtask/src/t803.rs", } ) CUDA_QUICK_JOB = "CUDA quick validation" @@ -58,6 +71,41 @@ class VerificationError(RuntimeError): """An expected verification condition was not met.""" +class _CredentialSafeRedirectHandler(urllib.request.HTTPRedirectHandler): + """Follow artifact redirects without forwarding credentials cross-origin.""" + + def redirect_request( + self, + request: urllib.request.Request, + file_pointer: Any, + code: int, + message: str, + headers: Any, + new_url: str, + ) -> urllib.request.Request | None: + redirected = super().redirect_request( + request, file_pointer, code, message, headers, new_url + ) + if redirected is None: + return None + old = urllib.parse.urlsplit(request.full_url) + new = urllib.parse.urlsplit(redirected.full_url) + if (old.scheme.lower(), old.netloc.lower()) != ( + new.scheme.lower(), + new.netloc.lower(), + ): + credential_names = { + "authorization", + "cookie", + "proxy-authorization", + } + for header_map in (redirected.headers, redirected.unredirected_hdrs): + for name in tuple(header_map): + if name.lower() in credential_names: + del header_map[name] + return redirected + + def _dict(value: Any, context: str) -> Mapping[str, Any]: if not isinstance(value, dict): raise VerificationError(f"malformed GitHub response: {context} must be an object") @@ -120,7 +168,7 @@ def __init__( repository: str, token: str, *, - opener: Callable[..., Any] = urllib.request.urlopen, + opener: Callable[..., Any] | None = None, ) -> None: if not token: raise VerificationError("GitHub API token is not configured") @@ -130,7 +178,9 @@ def __init__( self._base_url = api_url.rstrip("/") self._repository = "/".join(urllib.parse.quote(part, safe="") for part in repo_parts) self._token = token - self._opener = opener + self._opener = opener or urllib.request.build_opener( + _CredentialSafeRedirectHandler() + ).open def get_json( self, path: str, params: Mapping[str, str | int] | None = None @@ -147,6 +197,68 @@ def get_optional_json( return self._get_json(path, params, allow_not_found=True) + def download_bytes(self, path: str, *, maximum_bytes: int) -> bytes: + if not path.startswith("/"): + raise VerificationError("internal API path must begin with a slash") + if maximum_bytes <= 0: + raise VerificationError("internal download limit must be positive") + url = f"{self._base_url}/repos/{self._repository}{path}" + request = urllib.request.Request( + url, + headers={ + "Accept": "application/octet-stream", + "Authorization": f"Bearer {self._token}", + "X-GitHub-Api-Version": API_VERSION, + }, + ) + try: + with self._opener(request, timeout=30) as response: + final_url = response.geturl() + parsed = urllib.parse.urlsplit(final_url) + if ( + parsed.scheme != "https" + or not parsed.netloc + or parsed.username is not None + or parsed.password is not None + ): + raise VerificationError( + "artifact download ended at an unsafe URL" + ) + content_length = response.headers.get("Content-Length") + if content_length is not None: + try: + declared = int(content_length) + except ValueError: + raise VerificationError( + "artifact download returned an invalid Content-Length" + ) from None + if declared <= 0 or declared > maximum_bytes: + raise VerificationError( + "artifact download exceeds the configured size limit" + ) + raw = response.read(maximum_bytes + 1) + except VerificationError: + raise + except urllib.error.HTTPError as error: + error.close() + raise VerificationError( + f"GitHub API artifact download failed with HTTP {error.code} for {path}" + ) from None + except urllib.error.URLError as error: + reason = type(error.reason).__name__ + raise VerificationError( + f"GitHub API artifact download failed for {path} ({reason})" + ) from None + except (OSError, TimeoutError) as error: + raise VerificationError( + f"GitHub API artifact download failed for {path} ({type(error).__name__})" + ) from None + if not raw or len(raw) > maximum_bytes: + raise VerificationError( + "artifact download is empty or exceeds the configured size limit" + ) + return raw + def _get_json( self, path: str, @@ -314,6 +426,219 @@ def fetch_run_jobs(api: GitHubApi, run_id: int) -> list[Mapping[str, Any]]: raise VerificationError("workflow job pagination exceeded the safety limit") +@dataclass(frozen=True) +class WorkflowArtifact: + artifact_id: int + name: str + size_in_bytes: int + + +@dataclass(frozen=True) +class T803ArtifactSpec: + run_id: int + artifact_name: str + report_stem: str + + +def t803_artifact_specs( + candidate_sha: str, + ci_run_id: int, + gpu_run_id: int | None, + *, + scope: str = "all", +) -> tuple[T803ArtifactSpec, ...]: + sha = normalize_sha(candidate_sha, "T.803 artifact candidate SHA") + scope = normalize_t803_scope(scope) + if ci_run_id <= 0: + raise VerificationError("T.803 CI artifact run ID must be positive") + if scope in ("cuda", "metal", "all") and ( + gpu_run_id is None or gpu_run_id <= 0 + ): + raise VerificationError("selected T.803 GPU scope requires a positive GPU run ID") + + specs: list[T803ArtifactSpec] = [] + if scope in ("cpu", "all"): + specs.extend( + ( + T803ArtifactSpec( + ci_run_id, f"j2k-t803-cpu-linux-x86_64-{sha}", "cpu" + ), + T803ArtifactSpec( + ci_run_id, f"j2k-t803-cpu-macos-aarch64-{sha}", "cpu" + ), + T803ArtifactSpec( + ci_run_id, f"j2k-t803-cpu-windows-x86_64-{sha}", "cpu" + ), + ) + ) + if scope in ("cuda", "all"): + assert gpu_run_id is not None + specs.append( + T803ArtifactSpec( + gpu_run_id, f"j2k-t803-cuda-linux-x86_64-{sha}", "cuda" + ) + ) + if scope in ("metal", "all"): + assert gpu_run_id is not None + specs.append( + T803ArtifactSpec( + gpu_run_id, f"j2k-t803-metal-macos-aarch64-{sha}", "metal" + ) + ) + return tuple(specs) + + +def normalize_t803_scope(scope: str) -> str: + if scope not in T803_SCOPES: + raise VerificationError( + f"T.803 scope must be one of {', '.join(T803_SCOPES)}" + ) + return scope + + +def fetch_run_artifacts(api: GitHubApi, run_id: int) -> list[WorkflowArtifact]: + if run_id <= 0: + raise VerificationError("workflow artifact run ID must be positive") + artifacts: list[WorkflowArtifact] = [] + for page in range(1, MAX_PAGES + 1): + payload = _dict( + api.get_json( + f"/actions/runs/{run_id}/artifacts", + {"per_page": PAGE_SIZE, "page": page}, + ), + "workflow artifacts", + ) + raw_artifacts = _list(payload.get("artifacts"), "workflow artifacts list") + for index, raw_artifact in enumerate(raw_artifacts): + artifact = _dict(raw_artifact, f"workflow artifact {index}") + artifact_id = _integer(artifact.get("id"), "workflow artifact id") + name = _string(artifact.get("name"), "workflow artifact name") + expired = _boolean(artifact.get("expired"), "workflow artifact expired") + size = _integer( + artifact.get("size_in_bytes"), "workflow artifact size_in_bytes" + ) + workflow_run = _dict( + artifact.get("workflow_run"), "workflow artifact workflow_run" + ) + artifact_run_id = _integer( + workflow_run.get("id"), "workflow artifact run id" + ) + if artifact_id <= 0 or size <= 0: + raise VerificationError("workflow artifact id and size must be positive") + if artifact_run_id != run_id: + raise VerificationError( + f"workflow artifact {name} belongs to run {artifact_run_id}, expected {run_id}" + ) + if expired: + raise VerificationError(f"workflow artifact {name} has expired") + artifacts.append(WorkflowArtifact(artifact_id, name, size)) + if len(raw_artifacts) < PAGE_SIZE: + return artifacts + raise VerificationError("workflow artifact pagination exceeded the safety limit") + + +def download_t803_report_artifacts( + api: GitHubApi, + *, + candidate_sha: str, + ci_run_id: int, + gpu_run_id: int | None, + output_dir: Path, + scope: str = "all", +) -> tuple[Path, ...]: + specs = t803_artifact_specs( + candidate_sha, ci_run_id, gpu_run_id, scope=scope + ) + if output_dir.is_symlink() or (output_dir.exists() and not output_dir.is_dir()): + raise VerificationError("T.803 artifact output must be a real directory") + output_dir.mkdir(parents=True, exist_ok=True) + if any(output_dir.iterdir()): + raise VerificationError("T.803 artifact output directory must be empty") + + artifacts_by_run = { + run_id: fetch_run_artifacts(api, run_id) + for run_id in sorted({spec.run_id for spec in specs}) + } + reports: list[Path] = [] + for spec in specs: + matches = [ + artifact + for artifact in artifacts_by_run[spec.run_id] + if artifact.name == spec.artifact_name + ] + if len(matches) != 1: + raise VerificationError( + f"run {spec.run_id} must contain exactly one {spec.artifact_name} artifact" + ) + artifact = matches[0] + if artifact.size_in_bytes > MAX_T803_ARTIFACT_ARCHIVE_BYTES: + raise VerificationError( + f"workflow artifact {artifact.name} exceeds the configured size limit" + ) + archive = api.download_bytes( + f"/actions/artifacts/{artifact.artifact_id}/zip", + maximum_bytes=MAX_T803_ARTIFACT_ARCHIVE_BYTES, + ) + artifact_dir = output_dir / spec.artifact_name + reports.append( + extract_t803_report_archive( + archive, + report_stem=spec.report_stem, + output_dir=artifact_dir, + ) + ) + return tuple(reports) + + +def extract_t803_report_archive( + archive_bytes: bytes, *, report_stem: str, output_dir: Path +) -> Path: + if not archive_bytes or len(archive_bytes) > MAX_T803_ARTIFACT_ARCHIVE_BYTES: + raise VerificationError("T.803 artifact archive has an invalid size") + expected_names = {f"{report_stem}.json", f"{report_stem}.md"} + try: + with zipfile.ZipFile(io.BytesIO(archive_bytes)) as archive: + members = archive.infolist() + names = [member.filename for member in members] + if len(names) != len(set(names)): + raise VerificationError("T.803 artifact contains duplicate paths") + for member in members: + path = PurePosixPath(member.filename) + file_type = (member.external_attr >> 16) & 0o170000 + if ( + path.is_absolute() + or len(path.parts) != 1 + or ".." in path.parts + or "\\" in member.filename + or member.is_dir() + or file_type == stat.S_IFLNK + ): + raise VerificationError("T.803 artifact contains an unsafe path") + if member.flag_bits & 0x1: + raise VerificationError("T.803 artifact contains an encrypted file") + if member.file_size <= 0 or member.file_size > MAX_T803_REPORT_BYTES: + raise VerificationError("T.803 artifact report has an invalid size") + if set(names) != expected_names: + raise VerificationError( + "T.803 artifact must contain only its canonical JSON and Markdown reports" + ) + contents = { + member.filename: archive.read(member) + for member in members + } + except VerificationError: + raise + except (OSError, RuntimeError, zipfile.BadZipFile, zipfile.LargeZipFile): + raise VerificationError("T.803 artifact is not a valid bounded ZIP archive") from None + + if output_dir.exists(): + raise VerificationError("T.803 artifact extraction directory already exists") + output_dir.mkdir() + for name in sorted(contents): + (output_dir / name).write_bytes(contents[name]) + return output_dir / f"{report_stem}.json" + + def verify_workflow_run( api: GitHubApi, workflow: str, @@ -570,7 +895,8 @@ def verify_release_evidence( cuda_job: str, metal_job: str, ci_branch: str, -) -> tuple[int, int]: + t803_scope: str = "all", +) -> tuple[int, int | None]: verify_repository_origin(origin_url, server_url, repository) require_github_release_absent(api, tag) expected_sha = normalize_sha(candidate_sha, "candidate SHA") @@ -588,6 +914,7 @@ def verify_release_evidence( cuda_job=cuda_job, metal_job=metal_job, ci_branch=ci_branch, + t803_scope=t803_scope, ) @@ -601,7 +928,8 @@ def verify_candidate_evidence( cuda_job: str, metal_job: str, ci_branch: str, -) -> tuple[int, int]: + t803_scope: str = "all", +) -> tuple[int, int | None]: """Verify all post-freeze release evidence for one exact commit SHA.""" require_private_vulnerability_reporting(api) @@ -614,13 +942,21 @@ def verify_candidate_evidence( required_event="push", required_head_branch=ci_branch, ) - gpu_run = verify_workflow_run( - api, - gpu_workflow, - expected_sha, - [cuda_job, metal_job], - required_event="workflow_dispatch", - ) + scope = normalize_t803_scope(t803_scope) + required_gpu_jobs = [] + if scope in ("cuda", "all"): + required_gpu_jobs.append(cuda_job) + if scope in ("metal", "all"): + required_gpu_jobs.append(metal_job) + gpu_run = None + if required_gpu_jobs: + gpu_run = verify_workflow_run( + api, + gpu_workflow, + expected_sha, + required_gpu_jobs, + required_event="workflow_dispatch", + ) return ci_run, gpu_run @@ -668,6 +1004,10 @@ def build_parser() -> argparse.ArgumentParser: release_parser.add_argument("--cuda-job", default=CUDA_JOB) release_parser.add_argument("--metal-job", default=METAL_JOB) release_parser.add_argument("--ci-branch", default="main") + release_parser.add_argument( + "--t803-scope", choices=T803_SCOPES, default="all" + ) + release_parser.add_argument("--t803-out-dir") candidate_parser = subparsers.add_parser( "verify-candidate", @@ -681,6 +1021,10 @@ def build_parser() -> argparse.ArgumentParser: candidate_parser.add_argument("--cuda-job", default=CUDA_JOB) candidate_parser.add_argument("--metal-job", default=METAL_JOB) candidate_parser.add_argument("--ci-branch", default="main") + candidate_parser.add_argument( + "--t803-scope", choices=T803_SCOPES, default="all" + ) + candidate_parser.add_argument("--t803-out-dir") return parser @@ -739,13 +1083,30 @@ def run_command(args: argparse.Namespace) -> None: cuda_job=args.cuda_job, metal_job=args.metal_job, ci_branch=args.ci_branch, + t803_scope=args.t803_scope, ) + reports: tuple[Path, ...] = () + if args.t803_out_dir: + reports = download_t803_report_artifacts( + api, + candidate_sha=args.candidate_sha, + ci_run_id=ci_run, + gpu_run_id=gpu_run, + output_dir=Path(args.t803_out_dir), + scope=args.t803_scope, + ) print( "verified origin, private vulnerability reporting, absent GitHub Release, " f"annotated tag {args.tag}, " "and exact-SHA evidence " - f"(CI run {ci_run}, GPU run {gpu_run})" + + ( + f"(CI run {ci_run})" + if gpu_run is None + else f"(CI run {ci_run}, GPU run {gpu_run})" + ) ) + for report in reports: + print(f"downloaded T.803 report {report}") return if args.command == "verify-candidate": ci_run, gpu_run = verify_candidate_evidence( @@ -757,12 +1118,29 @@ def run_command(args: argparse.Namespace) -> None: cuda_job=args.cuda_job, metal_job=args.metal_job, ci_branch=args.ci_branch, + t803_scope=args.t803_scope, ) + reports = () + if args.t803_out_dir: + reports = download_t803_report_artifacts( + api, + candidate_sha=args.candidate_sha, + ci_run_id=ci_run, + gpu_run_id=gpu_run, + output_dir=Path(args.t803_out_dir), + scope=args.t803_scope, + ) print( "verified private vulnerability reporting and exact-SHA release candidate " f"{args.candidate_sha.lower()} " - f"(CI run {ci_run}, GPU run {gpu_run})" + + ( + f"(CI run {ci_run})" + if gpu_run is None + else f"(CI run {ci_run}, GPU run {gpu_run})" + ) ) + for report in reports: + print(f"downloaded T.803 report {report}") return raise VerificationError(f"unsupported command {args.command}") diff --git a/scripts/tests/test_github_actions_verify.py b/scripts/tests/test_github_actions_verify.py index 39eb6a00..8101c0bf 100644 --- a/scripts/tests/test_github_actions_verify.py +++ b/scripts/tests/test_github_actions_verify.py @@ -2,8 +2,13 @@ from __future__ import annotations +import io +from pathlib import Path +import tempfile import urllib.error +import urllib.request import unittest +import zipfile from typing import Any, Mapping from scripts import github_actions_verify as verifier @@ -17,6 +22,7 @@ class FakeApi: def __init__(self) -> None: self.responses: dict[tuple[str, tuple[tuple[str, str], ...]], Any] = {} + self.downloads: dict[str, bytes] = {} self.calls: list[tuple[str, tuple[tuple[str, str], ...]]] = [] @staticmethod @@ -51,6 +57,15 @@ def get_optional_json( response = self.get_json(path, params) return verifier.OptionalJsonResponse(found=response is not None, payload=response) + def add_download(self, path: str, payload: bytes) -> None: + self.downloads[path] = payload + + def download_bytes(self, path: str, *, maximum_bytes: int) -> bytes: + del maximum_bytes + if path not in self.downloads: + raise AssertionError(f"unexpected fake API download: {path}") + return self.downloads[path] + def workflow_metadata(api: FakeApi, filename: str, workflow_id: int) -> None: api.add( @@ -104,6 +119,14 @@ def add_jobs(api: FakeApi, run_id: int, jobs: list[dict[str, Any]]) -> None: ) +def report_archive(stem: str) -> bytes: + output = io.BytesIO() + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(f"{stem}.json", "{}\n") + archive.writestr(f"{stem}.md", "# evidence\n") + return output.getvalue() + + class PullRequestPolicyTests(unittest.TestCase): def test_pull_request_files_are_paginated_and_renames_use_both_paths(self) -> None: api = FakeApi() @@ -146,6 +169,22 @@ def test_non_gpu_paths_require_no_hardware_jobs(self) -> None: self.assertEqual(decision.required_jobs, ()) self.assertEqual(decision.changed_gpu_paths, ()) + def test_shared_conformance_and_routing_paths_require_both_hardware_lanes(self) -> None: + paths = [ + "crates/j2k-t803/src/runner.rs", + "corpus/j2k-conformance/t803-v3.toml", + "xtask/src/t803.rs", + "xtask/src/auto_routing/tests.rs", + ] + + decision = verifier.classify_gpu_paths(paths) + + self.assertEqual( + decision.required_jobs, + (verifier.CUDA_QUICK_JOB, verifier.METAL_QUICK_JOB), + ) + self.assertEqual(decision.changed_gpu_paths, tuple(sorted(paths))) + class ParserTests(unittest.TestCase): def test_verify_candidate_parser_smoke(self) -> None: @@ -156,12 +195,45 @@ def test_verify_candidate_parser_smoke(self) -> None: "frames-sg/j2k", "--candidate-sha", SHA, + "--t803-out-dir", + "target/t803/release-evidence", ] ) self.assertEqual(args.command, "verify-candidate") self.assertEqual(args.aggregate_job, verifier.RELEASE_CANDIDATE_JOB) self.assertEqual(args.cuda_job, verifier.CUDA_JOB) self.assertEqual(args.metal_job, verifier.METAL_JOB) + self.assertEqual(args.t803_out_dir, "target/t803/release-evidence") + + +class RedirectSecurityTests(unittest.TestCase): + def test_cross_origin_redirect_strips_every_credential_header(self) -> None: + request = urllib.request.Request( + "https://api.github.com/repos/frames-sg/j2k/actions/artifacts/1/zip", + headers={ + "Authorization": "Bearer secret", + "Cookie": "session=secret", + "Proxy-Authorization": "Basic secret", + "X-Trace": "keep-me", + }, + ) + + redirected = verifier._CredentialSafeRedirectHandler().redirect_request( + request, + None, + 302, + "Found", + {}, + "https://objects.githubusercontent.com/artifact.zip", + ) + + self.assertIsNotNone(redirected) + assert redirected is not None + headers = {name.lower(): value for name, value in redirected.header_items()} + self.assertNotIn("authorization", headers) + self.assertNotIn("cookie", headers) + self.assertNotIn("proxy-authorization", headers) + self.assertEqual(headers["x-trace"], "keep-me") def test_verify_release_parser_requires_origin_context(self) -> None: args = verifier.build_parser().parse_args( @@ -177,10 +249,13 @@ def test_verify_release_parser_requires_origin_context(self) -> None: "v0.7.0", "--candidate-sha", SHA, + "--t803-out-dir", + "target/t803/release-evidence", ] ) self.assertEqual(args.command, "verify-release") self.assertEqual(args.origin_url, "https://github.com/frames-sg/j2k.git") + self.assertEqual(args.t803_out_dir, "target/t803/release-evidence") class WorkflowVerificationTests(unittest.TestCase): @@ -361,6 +436,74 @@ def test_exact_numeric_workflow_id_is_supported(self) -> None: class ReleaseVerificationTests(unittest.TestCase): + def test_cpu_t803_scope_does_not_require_a_gpu_run(self) -> None: + specs = verifier.t803_artifact_specs(SHA, 10, None, scope="cpu") + + self.assertEqual(len(specs), 3) + self.assertTrue(all(spec.run_id == 10 for spec in specs)) + self.assertTrue(all(spec.report_stem == "cpu" for spec in specs)) + + def test_exact_run_t803_artifacts_are_downloaded_and_extracted(self) -> None: + api = FakeApi() + ci_run = 10 + gpu_run = 20 + specs = verifier.t803_artifact_specs(SHA, ci_run, gpu_run) + for run_id in (ci_run, gpu_run): + run_specs = [spec for spec in specs if spec.run_id == run_id] + api.add( + f"/actions/runs/{run_id}/artifacts", + { + "artifacts": [ + { + "id": index + run_id * 10, + "name": spec.artifact_name, + "expired": False, + "size_in_bytes": 512, + "workflow_run": {"id": run_id}, + } + for index, spec in enumerate(run_specs) + ] + }, + {"per_page": 100, "page": 1}, + ) + for index, spec in enumerate(run_specs): + artifact_id = index + run_id * 10 + api.add_download( + f"/actions/artifacts/{artifact_id}/zip", + report_archive(spec.report_stem), + ) + + with tempfile.TemporaryDirectory() as directory: + reports = verifier.download_t803_report_artifacts( + api, # type: ignore[arg-type] + candidate_sha=SHA, + ci_run_id=ci_run, + gpu_run_id=gpu_run, + output_dir=Path(directory), + ) + + self.assertEqual(len(reports), 5) + self.assertTrue(all(path.is_file() for path in reports)) + self.assertEqual( + {path.name for path in reports}, + {"cpu.json", "cuda.json", "metal.json"}, + ) + + def test_t803_artifact_extraction_rejects_path_traversal(self) -> None: + output = io.BytesIO() + with zipfile.ZipFile(output, "w") as archive: + archive.writestr("../cpu.json", "{}\n") + archive.writestr("cpu.md", "# evidence\n") + + with tempfile.TemporaryDirectory() as directory, self.assertRaisesRegex( + verifier.VerificationError, "unsafe" + ): + verifier.extract_t803_report_archive( + output.getvalue(), + report_stem="cpu", + output_dir=Path(directory), + ) + def test_post_freeze_candidate_verifies_ci_and_gpu_without_a_tag(self) -> None: api = FakeApi() api.add("/private-vulnerability-reporting", {"enabled": True}) @@ -406,6 +549,85 @@ def test_post_freeze_candidate_verifies_ci_and_gpu_without_a_tag(self) -> None: "post-freeze candidate status must not require a release tag", ) + def test_cpu_candidate_scope_does_not_query_the_gpu_workflow(self) -> None: + api = FakeApi() + api.add("/private-vulnerability-reporting", {"enabled": True}) + workflow_metadata(api, "ci.yml", 88) + api.add( + "/actions/workflows/88/runs", + { + "workflow_runs": [ + workflow_run( + 10, + workflow_id=88, + path=".github/workflows/ci.yml", + event="push", + ) + ] + }, + {"head_sha": SHA, "per_page": 100, "page": 1}, + ) + add_jobs(api, 10, [workflow_job(verifier.RELEASE_CANDIDATE_JOB)]) + + result = verifier.verify_candidate_evidence( + api, # type: ignore[arg-type] + candidate_sha=SHA, + ci_workflow="ci.yml", + aggregate_job=verifier.RELEASE_CANDIDATE_JOB, + gpu_workflow="gpu-validation.yml", + cuda_job=verifier.CUDA_JOB, + metal_job=verifier.METAL_JOB, + ci_branch="main", + t803_scope="cpu", + ) + + self.assertEqual(result, (10, None)) + self.assertFalse( + any("gpu-validation.yml" in path for path, _params in api.calls), + "CPU evidence must remain independent of accelerator availability", + ) + + def test_cuda_candidate_scope_requires_only_the_cuda_job(self) -> None: + api = FakeApi() + api.add("/private-vulnerability-reporting", {"enabled": True}) + workflow_metadata(api, "ci.yml", 88) + api.add( + "/actions/workflows/88/runs", + { + "workflow_runs": [ + workflow_run( + 10, + workflow_id=88, + path=".github/workflows/ci.yml", + event="push", + ) + ] + }, + {"head_sha": SHA, "per_page": 100, "page": 1}, + ) + add_jobs(api, 10, [workflow_job(verifier.RELEASE_CANDIDATE_JOB)]) + workflow_metadata(api, "gpu-validation.yml", 77) + add_runs(api, 77, [workflow_run(20)]) + add_jobs(api, 20, [workflow_job(verifier.CUDA_JOB)]) + + result = verifier.verify_candidate_evidence( + api, # type: ignore[arg-type] + candidate_sha=SHA, + ci_workflow="ci.yml", + aggregate_job=verifier.RELEASE_CANDIDATE_JOB, + gpu_workflow="gpu-validation.yml", + cuda_job=verifier.CUDA_JOB, + metal_job=verifier.METAL_JOB, + ci_branch="main", + t803_scope="cuda", + ) + + self.assertEqual(result, (10, 20)) + specs = verifier.t803_artifact_specs(SHA, 10, 20, scope="cuda") + self.assertEqual( + [(spec.run_id, spec.report_stem) for spec in specs], [(20, "cuda")] + ) + def test_post_freeze_candidate_requires_private_vulnerability_reporting(self) -> None: cases = ( ({"enabled": False}, "not enabled"), diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index b57409dc..c942ad14 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,6 +23,7 @@ proc-macro2 = { workspace = true, features = ["span-locations"] } serde = { workspace = true } serde_json = { workspace = true } serde_yaml_ng = { workspace = true } +sha2 = { workspace = true } syn = { workspace = true, features = ["full", "visit"] } tar = { workspace = true } toml = { workspace = true } diff --git a/xtask/src/auto_routing.rs b/xtask/src/auto_routing.rs new file mode 100644 index 00000000..4e7611d8 --- /dev/null +++ b/xtask/src/auto_routing.rs @@ -0,0 +1,645 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Component as PathComponent, Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::perf_guard::{discover_estimates, BenchEstimate}; + +const REQUIRED_OPERATIONS: [Operation; 6] = [ + Operation::FullDecode, + Operation::RoiDecode, + Operation::ScaledDecode, + Operation::BatchDecode, + Operation::LosslessEncode, + Operation::LossyEncode, +]; +const MAX_EVIDENCE_BYTES: u64 = 4 * 1024 * 1024; +const MAX_ESTIMATE_BYTES: u64 = 1024 * 1024; +const MAX_CELLS: usize = 4_096; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Evidence { + schema_version: u32, + candidate_sha: String, + backend: Backend, + platform: Platform, + external_manifest_sha256: String, + external_case_count: usize, + cells: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExternalManifest { + schema_version: u32, + corpus: String, + source_url: String, + cases: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ExternalCase { + id: String, + path: String, + kind: WorkloadKind, + pixel_format: WorkloadPixelFormat, + sha256: String, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +enum WorkloadKind { + Decode, + Encode, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +enum WorkloadPixelFormat { + Gray8, + Rgb8, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +enum Backend { + Cuda, + Metal, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Platform { + os: String, + arch: String, + hardware: String, + driver: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Cell { + id: String, + operation: Operation, + source: String, + workload: String, + cpu: Route, + hybrid: Route, + strict_device_supported: bool, + strict_device: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "kebab-case")] +enum Operation { + FullDecode, + RoiDecode, + ScaledDecode, + BatchDecode, + LosslessEncode, + LossyEncode, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Route { + criterion_id: String, + execution: Execution, + output_sha256: String, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "kebab-case")] +enum Execution { + Cpu, + Hybrid, + DeviceNative, +} + +#[derive(Debug)] +struct VerifiedEvidence { + cell_count: usize, + artifact_sha256: String, + report_json: String, +} + +pub(crate) fn auto_routing(mut args: impl Iterator) -> Result<(), String> { + let Some(command) = args.next() else { + return Err(usage()); + }; + if command != "verify" { + return Err(format!( + "unknown auto-routing command `{command}`\n{}", + usage() + )); + } + let mut evidence = None; + let mut external_manifest = None; + let mut criterion_root = None; + let mut output = None; + while let Some(argument) = args.next() { + let destination = match argument.as_str() { + "--evidence" => &mut evidence, + "--external-manifest" => &mut external_manifest, + "--criterion-root" => &mut criterion_root, + "--out" => &mut output, + _ => { + return Err(format!( + "unknown auto-routing argument `{argument}`\n{}", + usage() + )) + } + }; + let value = args + .next() + .ok_or_else(|| format!("{argument} requires a path"))?; + *destination = Some(PathBuf::from(value)); + } + let evidence = evidence.ok_or_else(usage)?; + let external_manifest = external_manifest.ok_or_else(usage)?; + let criterion_root = criterion_root.ok_or_else(usage)?; + let output = output.ok_or_else(usage)?; + let verified = verify_evidence(&evidence, &external_manifest, &criterion_root)?; + if let Some(parent) = output.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("create Auto-routing report directory: {error}"))?; + } + fs::write(&output, &verified.report_json) + .map_err(|error| format!("write Auto-routing report {}: {error}", output.display()))?; + eprintln!( + "verified {} Auto-routing workload cells; artifact SHA-256 {}; wrote {}", + verified.cell_count, + verified.artifact_sha256, + output.display() + ); + Ok(()) +} + +fn verify_evidence( + evidence_path: &Path, + external_manifest_path: &Path, + criterion_root: &Path, +) -> Result { + let evidence = read_evidence(evidence_path)?; + validate_header(&evidence)?; + let external_cases = validate_external_manifest(external_manifest_path, &evidence)?; + let estimates = discover_estimates(criterion_root)?; + let estimates = estimates + .into_iter() + .map(|estimate| (estimate.id.clone(), estimate)) + .collect::>(); + + let mut cell_ids = BTreeSet::new(); + let mut criterion_ids = BTreeSet::new(); + let mut operations = BTreeSet::new(); + let mut used_external_cases = BTreeSet::new(); + for cell in &evidence.cells { + let expected_kind = match cell.operation { + Operation::FullDecode + | Operation::RoiDecode + | Operation::ScaledDecode + | Operation::BatchDecode => WorkloadKind::Decode, + Operation::LosslessEncode | Operation::LossyEncode => WorkloadKind::Encode, + }; + if cell.id.is_empty() + || cell.workload.is_empty() + || cell.source != "external" + || external_cases.get(&cell.workload) != Some(&expected_kind) + || !cell_ids.insert(cell.id.as_str()) + { + return Err(format!( + "Auto-routing cell {:?} must have a unique id and a typed workload from the external manifest", + cell.id + )); + } + used_external_cases.insert(cell.workload.as_str()); + operations.insert(cell.operation); + validate_route( + &cell.id, + "CPU", + &cell.cpu, + Execution::Cpu, + criterion_root, + &estimates, + &mut criterion_ids, + )?; + validate_route( + &cell.id, + "hybrid", + &cell.hybrid, + Execution::Hybrid, + criterion_root, + &estimates, + &mut criterion_ids, + )?; + match (cell.strict_device_supported, &cell.strict_device) { + (true, Some(route)) => validate_route( + &cell.id, + "strict-device", + route, + Execution::DeviceNative, + criterion_root, + &estimates, + &mut criterion_ids, + )?, + (false, None) => {} + _ => { + return Err(format!( + "Auto-routing cell {} has inconsistent strict-device support", + cell.id + )); + } + } + validate_cell_metrics(cell)?; + } + let required = REQUIRED_OPERATIONS.into_iter().collect::>(); + if operations != required { + return Err("Auto-routing evidence must cover full, ROI, scaled, batch, lossless encode, and lossy encode workloads".to_string()); + } + if used_external_cases.len() != external_cases.len() { + return Err("Auto-routing evidence must exercise every external manifest case".to_string()); + } + let artifact_sha256 = artifact_sha256( + evidence_path, + external_manifest_path, + criterion_root, + &criterion_ids, + )?; + let report_json = verification_report(&evidence, &estimates, &artifact_sha256)?; + Ok(VerifiedEvidence { + cell_count: evidence.cells.len(), + artifact_sha256, + report_json, + }) +} + +fn validate_external_manifest( + path: &Path, + evidence: &Evidence, +) -> Result, String> { + let bytes = read_bounded_regular_file(path, MAX_EVIDENCE_BYTES, "external manifest")?; + let actual_sha256 = format!("{:x}", Sha256::digest(&bytes)); + if actual_sha256 != evidence.external_manifest_sha256 { + return Err(format!( + "external manifest SHA-256 mismatch: expected {}, found {actual_sha256}", + evidence.external_manifest_sha256 + )); + } + let manifest: ExternalManifest = serde_json::from_slice(&bytes) + .map_err(|error| format!("parse external manifest {}: {error}", path.display()))?; + if manifest.schema_version != 1 + || manifest.corpus.is_empty() + || !manifest.source_url.starts_with("https://") + || manifest.source_url["https://".len()..].is_empty() + || manifest + .source_url + .bytes() + .any(|byte| byte.is_ascii_whitespace()) + || manifest.cases.is_empty() + || manifest.cases.len() > MAX_CELLS + || manifest.cases.len() != evidence.external_case_count + { + return Err("external manifest identity or case inventory is invalid".to_string()); + } + let mut cases = BTreeMap::new(); + for case in manifest.cases { + if !is_safe_case_id(&case.id) + || !is_safe_relative_path(&case.path) + || !is_lower_hex(&case.sha256, 64) + || cases.insert(case.id, case.kind).is_some() + { + return Err("external manifest cases must have unique ids, safe relative paths, typed pixel formats, and lowercase SHA-256 hashes".to_string()); + } + let _ = case.pixel_format; + } + Ok(cases) +} + +fn read_evidence(path: &Path) -> Result { + let metadata = fs::metadata(path) + .map_err(|error| format!("read Auto-routing evidence {}: {error}", path.display()))?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > MAX_EVIDENCE_BYTES { + return Err("Auto-routing evidence must be a non-empty bounded regular file".to_string()); + } + let text = fs::read_to_string(path) + .map_err(|error| format!("read Auto-routing evidence {}: {error}", path.display()))?; + serde_json::from_str(&text) + .map_err(|error| format!("parse Auto-routing evidence {}: {error}", path.display())) +} + +fn validate_header(evidence: &Evidence) -> Result<(), String> { + if evidence.schema_version != 1 + || !is_hex(&evidence.candidate_sha, 40) + || !is_hex(&evidence.external_manifest_sha256, 64) + || evidence.external_case_count == 0 + || evidence.cells.is_empty() + || evidence.cells.len() > MAX_CELLS + { + return Err( + "Auto-routing evidence header or external corpus inventory is invalid".to_string(), + ); + } + if [ + &evidence.platform.os, + &evidence.platform.arch, + &evidence.platform.hardware, + &evidence.platform.driver, + ] + .into_iter() + .any(String::is_empty) + { + return Err("Auto-routing platform identity must be complete".to_string()); + } + let platform_matches = match evidence.backend { + Backend::Cuda => evidence.platform.os == "linux" && evidence.platform.arch == "x86_64", + Backend::Metal => evidence.platform.os == "macos" && evidence.platform.arch == "aarch64", + }; + if !platform_matches { + return Err("Auto-routing backend and platform identity do not match".to_string()); + } + Ok(()) +} + +fn validate_route( + cell_id: &str, + label: &str, + route: &Route, + expected_execution: Execution, + criterion_root: &Path, + estimates: &BTreeMap, + criterion_ids: &mut BTreeSet, +) -> Result<(), String> { + if route.execution != expected_execution + || !is_hex(&route.output_sha256, 64) + || !is_safe_criterion_id(&route.criterion_id) + || !criterion_ids.insert(route.criterion_id.clone()) + { + return Err(format!( + "Auto-routing cell {cell_id} has invalid or duplicate {label} route evidence" + )); + } + let estimate = estimates.get(&route.criterion_id).ok_or_else(|| { + format!( + "Auto-routing cell {cell_id} is missing Criterion estimate {}", + route.criterion_id + ) + })?; + validate_estimate(cell_id, label, estimate)?; + validate_confidence_level(criterion_root, &route.criterion_id)?; + Ok(()) +} + +fn validate_estimate(cell_id: &str, label: &str, estimate: &BenchEstimate) -> Result<(), String> { + if !estimate.median_ns.is_finite() + || !estimate.median_lower_ns.is_finite() + || !estimate.median_upper_ns.is_finite() + || estimate.median_lower_ns <= 0.0 + || estimate.median_lower_ns > estimate.median_ns + || estimate.median_ns > estimate.median_upper_ns + { + return Err(format!( + "Auto-routing cell {cell_id} has an invalid {label} Criterion median interval" + )); + } + Ok(()) +} + +fn validate_confidence_level(root: &Path, criterion_id: &str) -> Result<(), String> { + let path = root.join(criterion_id).join("new/estimates.json"); + let value: serde_json::Value = serde_json::from_slice(&read_bounded_estimate(&path)?) + .map_err(|error| format!("parse Criterion estimate {}: {error}", path.display()))?; + let level = value + .pointer("/median/confidence_interval/confidence_level") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| { + format!( + "Criterion estimate {} is missing its confidence level", + path.display() + ) + })?; + if (level - 0.95).abs() > f64::EPSILON { + return Err(format!( + "Criterion estimate {} does not use a 95% confidence interval", + path.display() + )); + } + Ok(()) +} + +fn validate_cell_metrics(cell: &Cell) -> Result<(), String> { + validate_output_parity(cell, &cell.cpu, &cell.hybrid)?; + if let Some(strict) = &cell.strict_device { + validate_output_parity(cell, &cell.cpu, strict)?; + } + Ok(()) +} + +fn validate_output_parity(cell: &Cell, expected: &Route, actual: &Route) -> Result<(), String> { + if expected.output_sha256 != actual.output_sha256 { + return Err(format!( + "Auto-routing cell {} routes do not produce identical outputs", + cell.id + )); + } + Ok(()) +} + +fn is_qualifying_win(hybrid: &BenchEstimate, competitor: &BenchEstimate) -> bool { + hybrid.median_ns <= competitor.median_ns * 0.9 + && hybrid.median_upper_ns < competitor.median_lower_ns +} + +fn is_safe_criterion_id(value: &str) -> bool { + if value.is_empty() || value.contains('\\') { + return false; + } + Path::new(value) + .components() + .all(|component| matches!(component, PathComponent::Normal(segment) if !segment.is_empty())) +} + +fn is_safe_case_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) +} + +fn is_safe_relative_path(value: &str) -> bool { + !value.is_empty() + && !value.contains('\\') + && !Path::new(value).is_absolute() + && Path::new(value).components().all( + |component| matches!(component, PathComponent::Normal(segment) if !segment.is_empty()), + ) +} + +fn is_hex(value: &str, length: usize) -> bool { + value.len() == length && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn is_lower_hex(value: &str, length: usize) -> bool { + value.len() == length + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) +} + +fn artifact_sha256( + evidence_path: &Path, + external_manifest_path: &Path, + criterion_root: &Path, + criterion_ids: &BTreeSet, +) -> Result { + let mut hasher = Sha256::new(); + hasher.update(b"j2k-auto-routing-evidence-v1\0"); + hash_artifact_file( + &mut hasher, + "evidence.json", + &read_bounded_regular_file(evidence_path, MAX_EVIDENCE_BYTES, "Auto-routing evidence")?, + )?; + hash_artifact_file( + &mut hasher, + "external-manifest.json", + &read_bounded_regular_file( + external_manifest_path, + MAX_EVIDENCE_BYTES, + "external manifest", + )?, + )?; + for criterion_id in criterion_ids { + let path = criterion_root.join(criterion_id).join("new/estimates.json"); + hash_artifact_file(&mut hasher, criterion_id, &read_bounded_estimate(&path)?)?; + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn hash_artifact_file(hasher: &mut Sha256, name: &str, bytes: &[u8]) -> Result<(), String> { + let name_len = u64::try_from(name.len()) + .map_err(|_| format!("Auto-routing artifact name is too long: {name}"))?; + let byte_len = u64::try_from(bytes.len()) + .map_err(|_| format!("Auto-routing artifact is too large: {name}"))?; + hasher.update(name_len.to_le_bytes()); + hasher.update(name.as_bytes()); + hasher.update(byte_len.to_le_bytes()); + hasher.update(bytes); + Ok(()) +} + +fn read_bounded_estimate(path: &Path) -> Result, String> { + read_bounded_regular_file(path, MAX_ESTIMATE_BYTES, "Criterion estimate") +} + +fn read_bounded_regular_file(path: &Path, limit: u64, label: &str) -> Result, String> { + let metadata = + fs::metadata(path).map_err(|error| format!("read {label} {}: {error}", path.display()))?; + if !metadata.is_file() || metadata.len() == 0 || metadata.len() > limit { + return Err(format!( + "{label} {} must be a non-empty bounded regular file", + path.display() + )); + } + fs::read(path).map_err(|error| format!("read {label} {}: {error}", path.display())) +} + +fn verification_report( + evidence: &Evidence, + estimates: &BTreeMap, + artifact_sha256: &str, +) -> Result { + let cells = evidence + .cells + .iter() + .map(|cell| { + let cpu = &estimates[&cell.cpu.criterion_id]; + let hybrid = &estimates[&cell.hybrid.criterion_id]; + let promotes_hybrid = is_qualifying_win(hybrid, cpu) + && cell + .strict_device + .as_ref() + .is_none_or(|route| is_qualifying_win(hybrid, &estimates[&route.criterion_id])); + let strict = cell.strict_device.as_ref().map(|route| { + let estimate = &estimates[&route.criterion_id]; + serde_json::json!({ + "criterion_id": route.criterion_id, + "median_ns": estimate.median_ns, + "median_lower_ns": estimate.median_lower_ns, + "median_upper_ns": estimate.median_upper_ns, + "hybrid_speedup_percent": speedup_percent(hybrid, estimate), + }) + }); + serde_json::json!({ + "id": cell.id, + "operation": cell.operation, + "source": cell.source, + "workload": cell.workload, + "decision": if promotes_hybrid { "promote-hybrid" } else { "retain-current" }, + "output_sha256": cell.hybrid.output_sha256, + "cpu": estimate_json(&cell.cpu.criterion_id, cpu), + "hybrid": estimate_json(&cell.hybrid.criterion_id, hybrid), + "hybrid_speedup_vs_cpu_percent": speedup_percent(hybrid, cpu), + "strict_device": strict, + "status": if promotes_hybrid { "promoted" } else { "retained" }, + }) + }) + .collect::>(); + let promoted_cell_count = evidence + .cells + .iter() + .filter(|cell| { + let hybrid = &estimates[&cell.hybrid.criterion_id]; + is_qualifying_win(hybrid, &estimates[&cell.cpu.criterion_id]) + && cell + .strict_device + .as_ref() + .is_none_or(|route| is_qualifying_win(hybrid, &estimates[&route.criterion_id])) + }) + .count(); + let report = serde_json::json!({ + "schema_version": 1, + "candidate_sha": evidence.candidate_sha, + "backend": evidence.backend, + "platform": evidence.platform, + "external_manifest_sha256": evidence.external_manifest_sha256, + "external_case_count": evidence.external_case_count, + "criterion_confidence_level": 0.95, + "minimum_hybrid_speedup_percent": 10.0, + "artifact_sha256": artifact_sha256, + "promoted_cell_count": promoted_cell_count, + "cells": cells, + "status": "pass", + }); + let mut json = serde_json::to_string_pretty(&report) + .map_err(|error| format!("serialize Auto-routing verification report: {error}"))?; + json.push('\n'); + Ok(json) +} + +fn estimate_json(criterion_id: &str, estimate: &BenchEstimate) -> serde_json::Value { + serde_json::json!({ + "criterion_id": criterion_id, + "median_ns": estimate.median_ns, + "median_lower_ns": estimate.median_lower_ns, + "median_upper_ns": estimate.median_upper_ns, + }) +} + +fn speedup_percent(hybrid: &BenchEstimate, competitor: &BenchEstimate) -> f64 { + (1.0 - hybrid.median_ns / competitor.median_ns) * 100.0 +} + +fn usage() -> String { + "usage: cargo xtask auto-routing verify --evidence FILE --external-manifest FILE --criterion-root DIR --out FILE".to_string() +} + +#[cfg(test)] +mod tests; diff --git a/xtask/src/auto_routing/tests.rs b/xtask/src/auto_routing/tests.rs new file mode 100644 index 00000000..3f0e4ec0 --- /dev/null +++ b/xtask/src/auto_routing/tests.rs @@ -0,0 +1,399 @@ +use std::{fs, path::Path, time::SystemTime}; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::{auto_routing, verify_evidence}; + +const OPERATIONS: [&str; 6] = [ + "full-decode", + "roi-decode", + "scaled-decode", + "batch-decode", + "lossless-encode", + "lossy-encode", +]; + +#[test] +fn qualifying_external_hybrid_cells_pass() { + let fixture = write_fixture("qualifying", false, false); + + let verified = fixture.verify().expect("qualifying evidence"); + + assert_eq!(verified.cell_count, OPERATIONS.len()); + assert_eq!(verified.artifact_sha256.len(), 64); + let report: Value = serde_json::from_str(&verified.report_json).expect("verified report JSON"); + assert_eq!(report["status"], "pass"); + assert_eq!(report["promoted_cell_count"], OPERATIONS.len()); + assert_eq!( + report["cells"].as_array().map(Vec::len), + Some(OPERATIONS.len()) + ); +} + +#[test] +fn overlapping_intervals_or_less_than_ten_percent_retain_the_current_route() { + for (label, overlap, slow_hybrid) in [ + ("overlap", true, false), + ("insufficient-speedup", false, true), + ] { + let fixture = write_fixture(label, overlap, slow_hybrid); + + let verified = fixture.verify().expect("valid nonqualifying evidence"); + let report: Value = + serde_json::from_str(&verified.report_json).expect("verified report JSON"); + assert_eq!(report["promoted_cell_count"], 0, "{label}"); + } +} + +#[test] +fn a_nonqualifying_measurement_can_retain_the_current_route() { + let fixture = write_fixture("retained-current", false, true); + + let verified = fixture.verify().expect("retained routing evidence"); + let report: Value = serde_json::from_str(&verified.report_json).expect("report JSON"); + + assert_eq!(report["promoted_cell_count"], 0); + assert!(report["cells"] + .as_array() + .expect("report cells") + .iter() + .all(|cell| cell["decision"] == "retain-current")); +} + +#[test] +fn routing_decisions_are_derived_from_measurements() { + let fixture = write_fixture("derived-decision", false, false); + let verified = fixture.verify().expect("qualifying routing evidence"); + let report: Value = serde_json::from_str(&verified.report_json).expect("report JSON"); + assert_eq!(report["promoted_cell_count"], OPERATIONS.len()); + + let mut evidence = read_json(&fixture.evidence); + evidence["cells"][0]["decision"] = json!("retain-current"); + write_json(&fixture.evidence, &evidence); + + let error = fixture + .verify() + .expect_err("raw evidence cannot override the measured decision"); + + assert!(error.contains("unknown field `decision`"), "{error}"); +} + +#[test] +fn altered_external_manifest_fails_closed() { + let fixture = write_fixture("altered-manifest", false, false); + fs::write(&fixture.manifest, b"{}\n").expect("alter manifest"); + + let error = fixture.verify().expect_err("must fail closed"); + + assert!(error.contains("external manifest SHA-256"), "{error}"); +} + +#[test] +fn external_manifest_requires_safe_typed_workload_paths() { + let fixture = write_fixture("unsafe-manifest-path", false, false); + let mut manifest = read_json(&fixture.manifest); + manifest["cases"][0]["path"] = json!("../outside.j2k"); + write_json(&fixture.manifest, &manifest); + rewrite_manifest_hash(&fixture); + + let error = fixture + .verify() + .expect_err("unsafe workload path must fail"); + + assert!(error.contains("safe relative paths"), "{error}"); +} + +#[test] +fn routes_require_identical_outputs_and_honest_execution_labels() { + for (label, field, value, expected) in [ + ( + "output-mismatch", + "output_sha256", + Value::String("e".repeat(64)), + "identical outputs", + ), + ( + "execution-mismatch", + "execution", + Value::String("cpu".to_string()), + "invalid or duplicate hybrid route evidence", + ), + ] { + let fixture = write_fixture(label, false, false); + let mut evidence = read_json(&fixture.evidence); + evidence["cells"][0]["hybrid"][field] = value; + write_json(&fixture.evidence, &evidence); + + let error = fixture.verify().expect_err("must fail closed"); + + assert!(error.contains(expected), "unexpected error: {error}"); + } +} + +#[test] +fn every_workload_class_and_exact_confidence_level_are_required() { + let missing = write_fixture("missing-operation", false, false); + let mut evidence = read_json(&missing.evidence); + evidence["cells"].as_array_mut().expect("cells").pop(); + write_json(&missing.evidence, &evidence); + let error = missing.verify().expect_err("missing operation must fail"); + assert!( + error.contains("must cover full, ROI, scaled, batch"), + "{error}" + ); + + let confidence = write_fixture("wrong-confidence", false, false); + let estimate_path = confidence + .criterion + .join("auto-routing/full-decode/decode-case/cpu/new/estimates.json"); + let mut estimate = read_json(&estimate_path); + estimate["median"]["confidence_interval"]["confidence_level"] = json!(0.90); + write_json(&estimate_path, &estimate); + let error = confidence.verify().expect_err("wrong confidence must fail"); + assert!( + error.contains("does not use a 95% confidence interval"), + "{error}" + ); +} + +#[test] +fn criterion_ids_cannot_escape_the_artifact_root() { + let fixture = write_fixture("unsafe-path", false, false); + let mut evidence = read_json(&fixture.evidence); + evidence["cells"][0]["cpu"]["criterion_id"] = json!("../outside"); + write_json(&fixture.evidence, &evidence); + + let error = fixture.verify().expect_err("unsafe path must fail"); + + assert!( + error.contains("invalid or duplicate CPU route evidence"), + "{error}" + ); +} + +#[test] +fn unsupported_strict_device_cells_are_allowed_but_must_be_explicit() { + let fixture = write_fixture("unsupported-strict-device", false, false); + let mut evidence = read_json(&fixture.evidence); + evidence["cells"][0]["strict_device_supported"] = json!(false); + evidence["cells"][0]["strict_device"] = Value::Null; + write_json(&fixture.evidence, &evidence); + + fixture + .verify() + .expect("explicit unsupported strict-device route"); +} + +#[test] +fn artifact_hash_covers_exact_criterion_estimates() { + let fixture = write_fixture("artifact-hash", false, false); + let before = fixture.verify().expect("initial evidence").artifact_sha256; + let estimate_path = fixture + .criterion + .join("auto-routing/full-decode/decode-case/cpu/new/estimates.json"); + let mut estimate = read_json(&estimate_path); + estimate["median"]["standard_error"] = json!(2.0); + write_json(&estimate_path, &estimate); + + let after = fixture + .verify() + .expect("altered but valid evidence") + .artifact_sha256; + + assert_ne!(before, after); +} + +#[test] +fn command_writes_the_verified_report_and_requires_an_output() { + let fixture = write_fixture("command", false, false); + let output = fixture.root.join("report/verified.json"); + let args = vec![ + "verify".to_string(), + "--evidence".to_string(), + fixture.evidence.display().to_string(), + "--external-manifest".to_string(), + fixture.manifest.display().to_string(), + "--criterion-root".to_string(), + fixture.criterion.display().to_string(), + "--out".to_string(), + output.display().to_string(), + ]; + + auto_routing(args.into_iter()).expect("verify command"); + + let report = read_json(&output); + assert_eq!(report["status"], "pass"); + let error = auto_routing( + [ + "verify", + "--evidence", + fixture.evidence.to_str().expect("UTF-8 path"), + ] + .into_iter() + .map(str::to_string), + ) + .expect_err("missing output must fail"); + assert!(error.contains("--out FILE"), "{error}"); +} + +struct Fixture { + root: std::path::PathBuf, + evidence: std::path::PathBuf, + manifest: std::path::PathBuf, + criterion: std::path::PathBuf, +} + +impl Fixture { + fn verify(&self) -> Result { + verify_evidence(&self.evidence, &self.manifest, &self.criterion) + } +} + +fn write_fixture(label: &str, overlap: bool, slow_hybrid: bool) -> Fixture { + let root = temp_dir(label); + let criterion = root.join("criterion"); + let evidence_path = root.join("evidence.json"); + let manifest_path = root.join("external-manifest.json"); + let manifest = json!({ + "schema_version": 1, + "corpus": "external-test-corpus", + "source_url": "https://example.invalid/j2k-routing-corpus", + "cases": [ + { + "id": "decode-case", + "path": "decode/decode-case.j2k", + "kind": "decode", + "pixel_format": "rgb8", + "sha256": "d".repeat(64) + }, + { + "id": "encode-case", + "path": "encode/encode-case.ppm", + "kind": "encode", + "pixel_format": "rgb8", + "sha256": "e".repeat(64) + } + ] + }); + let manifest_bytes = format!( + "{}\n", + serde_json::to_string_pretty(&manifest).expect("serialize manifest fixture") + ) + .into_bytes(); + fs::create_dir_all(&root).expect("create fixture root"); + fs::write(&manifest_path, &manifest_bytes).expect("write manifest fixture"); + let manifest_sha256 = format!("{:x}", Sha256::digest(&manifest_bytes)); + + let mut cells = Vec::new(); + for operation in OPERATIONS { + let workload = if operation.ends_with("encode") { + "encode-case" + } else { + "decode-case" + }; + let base = format!("auto-routing/{operation}/{workload}"); + let cpu_id = format!("{base}/cpu"); + let hybrid_id = format!("{base}/hybrid"); + let strict_id = format!("{base}/strict-device"); + write_estimate(&criterion, &cpu_id, 100.0, 98.0, 102.0); + let (median, lower, upper) = if slow_hybrid { + (91.0, 89.0, 93.0) + } else if overlap { + (80.0, 78.0, 99.0) + } else { + (80.0, 78.0, 82.0) + }; + write_estimate(&criterion, &hybrid_id, median, lower, upper); + write_estimate(&criterion, &strict_id, 110.0, 108.0, 112.0); + cells.push(json!({ + "id": format!("{operation}-{workload}"), + "operation": operation, + "source": "external", + "workload": workload, + "cpu": route(&cpu_id, "cpu"), + "hybrid": route(&hybrid_id, "hybrid"), + "strict_device_supported": true, + "strict_device": route(&strict_id, "device-native") + })); + } + let evidence = json!({ + "schema_version": 1, + "candidate_sha": "a".repeat(40), + "backend": "metal", + "platform": { + "os": "macos", + "arch": "aarch64", + "hardware": "Apple M4 Pro", + "driver": "macOS Metal" + }, + "external_manifest_sha256": manifest_sha256, + "external_case_count": 2, + "cells": cells + }); + write_json(&evidence_path, &evidence); + Fixture { + root, + evidence: evidence_path, + manifest: manifest_path, + criterion, + } +} + +fn rewrite_manifest_hash(fixture: &Fixture) { + let manifest_bytes = fs::read(&fixture.manifest).expect("read altered manifest"); + let mut evidence = read_json(&fixture.evidence); + evidence["external_manifest_sha256"] = json!(format!("{:x}", Sha256::digest(&manifest_bytes))); + write_json(&fixture.evidence, &evidence); +} + +fn route(criterion_id: &str, execution: &str) -> Value { + json!({ + "criterion_id": criterion_id, + "execution": execution, + "output_sha256": "c".repeat(64) + }) +} + +fn write_estimate(root: &Path, id: &str, median: f64, lower: f64, upper: f64) { + let path = root.join(id).join("new/estimates.json"); + fs::create_dir_all(path.parent().expect("estimate parent")).expect("create estimate path"); + let estimate = json!({ + "median": { + "confidence_interval": { + "confidence_level": 0.95, + "lower_bound": lower, + "upper_bound": upper + }, + "point_estimate": median, + "standard_error": 1.0 + } + }); + write_json(&path, &estimate); +} + +fn read_json(path: &Path) -> Value { + serde_json::from_slice(&fs::read(path).expect("read JSON fixture")).expect("parse JSON fixture") +} + +fn write_json(path: &Path, value: &Value) { + fs::write( + path, + format!( + "{}\n", + serde_json::to_string_pretty(value).expect("serialize JSON fixture") + ), + ) + .expect("write JSON fixture"); +} + +fn temp_dir(label: &str) -> std::path::PathBuf { + let nonce = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("system clock") + .as_nanos(); + std::env::temp_dir().join(format!( + "j2k-auto-routing-{label}-{}-{nonce}", + std::process::id() + )) +} diff --git a/xtask/src/benchmark_commands/tests.rs b/xtask/src/benchmark_commands/tests.rs index 524d5dc0..2adcc6c0 100644 --- a/xtask/src/benchmark_commands/tests.rs +++ b/xtask/src/benchmark_commands/tests.rs @@ -47,6 +47,10 @@ fn benchmark_build_and_signoff_execute_the_complete_fake_cargo_plan() { let log = recording.log(); assert!(log.contains("bench -p j2k --bench public_api --no-run|")); + assert!( + log.contains("bench -p j2k-cuda --bench auto_routing --features cuda-runtime --no-run|") + ); + assert!(log.contains("bench -p j2k-metal --bench auto_routing --no-run|")); assert!(log.contains( "bench -p j2k-transcode-metal --bench dct97 --features bench-internals --no-run|" )); @@ -57,7 +61,7 @@ fn benchmark_build_and_signoff_execute_the_complete_fake_cargo_plan() { assert!(log.contains("bench -p j2k-ml --bench batch_decode_cuda --features cpu,cuda --no-run|")); assert!(log.contains("test -p j2k-compare --test in_process_parity -- --nocapture|")); assert!(log.contains("test -p j2k-jpeg --features bench-libjpeg-turbo --test libjpeg_turbo_compare -- --nocapture|")); - assert_eq!(log.lines().count(), 21); + assert_eq!(log.lines().count(), 23); } #[cfg(unix)] @@ -90,13 +94,13 @@ fn benchmark_build_lanes_never_compile_the_other_accelerator() { ), ( "cuda", - 5, + 6, "j2k-ml --bench batch_decode_cuda --features cpu,cuda", "j2k-metal", ), ( "metal", - 3, + 4, "j2k-ml --bench batch_decode_metal --features cpu,metal", "j2k-cuda", ), diff --git a/xtask/src/benchmark_registry.rs b/xtask/src/benchmark_registry.rs index 2256227b..35459e31 100644 --- a/xtask/src/benchmark_registry.rs +++ b/xtask/src/benchmark_registry.rs @@ -162,6 +162,13 @@ pub(crate) const COMPILE_BENCHMARKS: &[CompileBenchmark] = &[ BenchmarkLane::Cuda, CUDA_BENCH_ENV, ), + compile( + "j2k-cuda", + Some("auto_routing"), + Some("cuda-runtime"), + BenchmarkLane::Cuda, + CUDA_BENCH_ENV, + ), compile( "j2k-cuda", Some("encode_stages"), @@ -190,6 +197,13 @@ pub(crate) const COMPILE_BENCHMARKS: &[CompileBenchmark] = &[ BenchmarkLane::Cuda, CUDA_BENCH_ENV, ), + compile( + "j2k-metal", + Some("auto_routing"), + None, + BenchmarkLane::Metal, + METAL_BENCH_ENV, + ), compile( "j2k-jpeg-metal", None, @@ -340,4 +354,42 @@ mod tests { assert_eq!(performance_benchmark.lane, BenchmarkLane::Metal); assert_eq!(performance_benchmark.env, &[("PERFORMANCE_ENV", "1")]); } + + #[test] + fn auto_routing_benchmarks_are_compiled_on_their_hardware_lanes() { + let cuda = super::COMPILE_BENCHMARKS + .iter() + .find(|benchmark| { + benchmark.package == "j2k-cuda" && benchmark.bench == Some("auto_routing") + }) + .expect("CUDA Auto-routing benchmark registry entry"); + assert_eq!(cuda.features, Some("cuda-runtime")); + assert_eq!(cuda.lane, BenchmarkLane::Cuda); + assert_eq!(cuda.runtime_env, super::CUDA_BENCH_ENV); + + let metal = super::COMPILE_BENCHMARKS + .iter() + .find(|benchmark| { + benchmark.package == "j2k-metal" && benchmark.bench == Some("auto_routing") + }) + .expect("Metal Auto-routing benchmark registry entry"); + assert_eq!(metal.features, None); + assert_eq!(metal.lane, BenchmarkLane::Metal); + assert_eq!(metal.runtime_env, super::METAL_BENCH_ENV); + } + + #[test] + fn gpu_benchmark_workflow_verifies_both_auto_routing_lanes() { + let workflow = include_str!("../../.github/workflows/gpu-benchmarks.yml"); + assert!(workflow.contains("options: [smoke, criterion, profile, adoption, routing]")); + assert!(workflow.contains( + "cargo bench --profile release-bench -p j2k-cuda --bench auto_routing --features cuda-runtime" + )); + assert!(workflow + .contains("cargo bench --profile release-bench -p j2k-metal --bench auto_routing")); + assert_eq!( + workflow.matches("cargo xtask auto-routing verify").count(), + 2 + ); + } } diff --git a/xtask/src/cuda.rs b/xtask/src/cuda.rs index 92c1edaa..9935dc26 100644 --- a/xtask/src/cuda.rs +++ b/xtask/src/cuda.rs @@ -54,7 +54,7 @@ const TRANSCODE_PARITY_TESTS: &[&str] = &[ const ML_CUDA_TESTS: &[&str] = &[ "staged_upload_session_reuses_events_and_codec_memory_for_one_thousand_batches", - "cuda_burn_batch_continues_after_one_group_submit_failure", + "cuda_burn_batch_decodes_classic_roi_and_ht_groups_together", "cuda_burn_decoder_construction_is_infallible_and_lazy", "cuda_burn_regroups_prepared_images_and_keeps_settings_failures_indexed_without_cuda", "staged_cuda_batch_writes_exact_u8_pixels_and_reuses_the_session", @@ -197,7 +197,12 @@ fn run_release_cuda(os: &str, arch: &str, mode: ValidationMode) -> Result<(), St ValidationMode::Full => { for suite in CUDA_CLIPPY_SUITES { let args = clippy_suite_args(suite); - let label = format!("{} CUDA Clippy", suite.package); + let label = format!("{} CUDA library Clippy", suite.package); + let output = run_cargo_captured(&args, CUDA_RELEASE_ENV, &label)?; + reject_cuda_skip_markers(&output, &label)?; + + let args = clippy_non_library_suite_args(suite); + let label = format!("{} CUDA test and benchmark Clippy", suite.package); let output = run_cargo_captured(&args, CUDA_RELEASE_ENV, &label)?; reject_cuda_skip_markers(&output, &label)?; } @@ -290,7 +295,24 @@ fn validate_cuda_device_probe(output: &str) -> Result<(), String> { fn clippy_suite_args(suite: &CudaClippySuite) -> Vec<&'static str> { vec![ "clippy", - "--all-targets", + "--lib", + "-p", + suite.package, + "--features", + suite.features, + "--", + "-D", + "warnings", + ] +} + +fn clippy_non_library_suite_args(suite: &CudaClippySuite) -> Vec<&'static str> { + vec![ + "clippy", + "--bins", + "--examples", + "--tests", + "--benches", "-p", suite.package, "--features", @@ -298,6 +320,10 @@ fn clippy_suite_args(suite: &CudaClippySuite) -> Vec<&'static str> { "--", "-D", "warnings", + "-A", + "clippy::disallowed_methods", + "-A", + "clippy::disallowed_macros", ] } diff --git a/xtask/src/cuda/tests.rs b/xtask/src/cuda/tests.rs index b9ae124f..cb7caa27 100644 --- a/xtask/src/cuda/tests.rs +++ b/xtask/src/cuda/tests.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::{ - clippy_suite_args, exact_suite_args, listed_rust_tests, passed_rust_tests, - reject_cuda_skip_markers, require_cuda_host, runtime_suite_args, successful_test_summaries, - validate_complete_test_run, validate_cuda_device_probe, validate_exact_named_run, - CUDA_CLIPPY_SUITES, CUDA_RUNTIME_SUITES, EXACT_CUDA_SUITES, HTJ2K_ENCODE_PARITY_TESTS, - ML_CUDA_TESTS, TRANSCODE_PARITY_TESTS, + clippy_non_library_suite_args, clippy_suite_args, exact_suite_args, listed_rust_tests, + passed_rust_tests, reject_cuda_skip_markers, require_cuda_host, runtime_suite_args, + successful_test_summaries, validate_complete_test_run, validate_cuda_device_probe, + validate_exact_named_run, CUDA_CLIPPY_SUITES, CUDA_RUNTIME_SUITES, EXACT_CUDA_SUITES, + HTJ2K_ENCODE_PARITY_TESTS, ML_CUDA_TESTS, TRANSCODE_PARITY_TESTS, }; use crate::gpu_validation::ValidationMode; @@ -70,13 +70,31 @@ fn release_commands_name_packages_features_and_non_benchmark_test_targets() { } for suite in CUDA_CLIPPY_SUITES { - let args = clippy_suite_args(suite); - assert!(args.windows(2).any(|pair| pair == ["-p", suite.package])); - assert!(args + let library_args = clippy_suite_args(suite); + assert!(library_args + .windows(2) + .any(|pair| pair == ["-p", suite.package])); + assert!(library_args .windows(2) .any(|pair| pair == ["--features", suite.features])); - assert!(args.contains(&"--all-targets")); - assert!(args.ends_with(&["--", "-D", "warnings"])); + assert!(library_args.contains(&"--lib")); + assert!(!library_args.contains(&"--all-targets")); + assert!(library_args.ends_with(&["--", "-D", "warnings"])); + + let non_library_args = clippy_non_library_suite_args(suite); + for target in ["--bins", "--examples", "--tests", "--benches"] { + assert!(non_library_args.contains(&target)); + } + assert!(!non_library_args.contains(&"--all-targets")); + assert!(non_library_args.ends_with(&[ + "--", + "-D", + "warnings", + "-A", + "clippy::disallowed_methods", + "-A", + "clippy::disallowed_macros", + ])); } } @@ -245,7 +263,10 @@ fn cuda_release_executes_the_complete_hermetic_command_plan() { .expect("platform-independent CUDA release plan"); let log = cargo.log(); - assert_eq!(log.lines().count(), expected_runs * 19); + let commands_per_run = + CUDA_CLIPPY_SUITES.len() * 2 + CUDA_RUNTIME_SUITES.len() + EXACT_CUDA_SUITES.len() * 2; + assert_eq!(commands_per_run, 26); + assert_eq!(log.lines().count(), expected_runs * commands_per_run); assert!(log.lines().all(|line| line.contains("RUST_TEST_THREADS=1"))); assert_eq!(device.log().lines().count(), expected_runs); } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index fbb66e3d..0ae5110a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -15,6 +15,7 @@ mod adoption_manifest; mod adoption_materialize; #[cfg(feature = "adoption")] mod adoption_report; +mod auto_routing; mod benchmark_commands; mod benchmark_registry; mod clone_audit; @@ -38,6 +39,7 @@ mod release_status; mod semver; mod source_audit; mod stable_api; +mod t803; #[cfg(all(test, unix))] mod test_command; @@ -54,7 +56,8 @@ use quality_commands::{ test, verify_unsafe_audit, }; use release_commands::{ - j2k_ml_package_smoke, package, published_library_packages, release_cpu, release_integrity, + j2k_ml_package_smoke, package, package_consumer_smoke, published_library_packages, release_cpu, + release_integrity, }; use stable_api::CARGO_PUBLIC_API_VERSION; @@ -102,6 +105,7 @@ fn run() -> Result<(), String> { "public-support" => public_support::public_support(env::args().skip(2)), "j2k-bench-signoff" => j2k_bench_signoff(), "j2k-perf-guard" => perf_guard::j2k_perf_guard(env::args().skip(2)), + "auto-routing" => auto_routing::auto_routing(env::args().skip(2)), "codec-math-codegen" => codec_math_codegen(env::args().skip(2)), "fuzz-build" => fuzz_build(), "fuzz-run" => fuzz_run(), @@ -118,6 +122,7 @@ fn run() -> Result<(), String> { "no-std" => no_std(), "unsafe-audit" => verify_unsafe_audit(), "repo-lint" => repo_lint(env::args().skip(2)), + "t803" => t803::t803(env::args().skip(2)), "release-integrity" => release_integrity(env::args().skip(2)), "release-status" => release_status::release_status(env::args().skip(2)), "release-cpu" => release_cpu(), @@ -126,6 +131,7 @@ fn run() -> Result<(), String> { "release-metal" => metal::release_metal(env::args().skip(2)), "coverage" => coverage::coverage(env::args().skip(2)), "j2k-ml-package-smoke" => j2k_ml_package_smoke(), + "package-consumer-smoke" => package_consumer_smoke(env::args().skip(2)), "package" => package(), "ci" => ci(), "help" | "-h" | "--help" => { @@ -157,6 +163,7 @@ fn print_help() { public-support verify the public J2K/HTJ2K support matrix and publication gates [--final]\n\ j2k-bench-signoff run required OpenJPEG/Grok parity and J2K compare bench compile gates\n\ j2k-perf-guard compare one strict host/CUDA/Metal Criterion lane against a baseline git ref\n\ + auto-routing verify Criterion-backed hybrid Auto-routing promotion evidence\n\ codec-math-codegen check generated codec-math Rust and Metal fragments\n\ fuzz-build compile fuzz harnesses\n\ fuzz-run run scheduled fuzz targets with J2K_FUZZ_RUNS\n\ @@ -169,14 +176,16 @@ fn print_help() { no-std check no_std-compatible codec crates\n\ unsafe-audit verify docs/unsafe-audit.md lists unsafe Rust sources\n\ repo-lint run repository policy checks owned by xtask\n\ + t803 fetch, run, or verify pinned T.803 v3 conformance evidence\n\ release-integrity validate offline release metadata; --publish requires final dated/signoff state\n\ - release-status verify one frozen SHA's CI aggregate and both GPU jobs [--sha SHA] [--repository owner/name]\n\ + release-status verify one frozen SHA's scoped T.803 evidence [--sha SHA] [--repository owner/name] [--scope cpu|cuda|metal|all]\n\ release-cpu run release-mode CPU codec tests\n\ release-cuda run fail-closed CUDA validation on Linux x86_64 [--mode quick|full]\n\ metal-compile compile all Metal targets and run default/pure tests on hosted macOS\n\ release-metal run fail-closed Metal hardware validation on macOS [--mode quick|full]\n\ coverage enforce >=80% host-wide or accelerator critical-path coverage [host|metal|cuda] [--base REV]\n\ j2k-ml-package-smoke compile j2k-ml from an external consumer without third-party path patches\n\ + package-consumer-smoke compile packaged j2k/CUDA/Metal archives from clean external consumers [--target ...] [--cuda-runtime]\n\ package construct all staged packages from a clean worktree and publish-dry-run registry-independent crates" ); } diff --git a/xtask/src/metal.rs b/xtask/src/metal.rs index c724d2ed..20213e2d 100644 --- a/xtask/src/metal.rs +++ b/xtask/src/metal.rs @@ -25,8 +25,10 @@ const METAL_COMPILE_PACKAGES: &[&str] = &[ ]; const J2K_METAL_REQUIRED_IGNORED_TESTS: &[&str] = &[ + "compute::tests::classic::irreversible_hybrid_cpu_tier1_matches_native_decode_exactly", "compute::tests::classic::prepared_classic_direct_plan_groups_cleanup_subbands_before_idwt", "compute::tests::classic::prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload", + "compute::tests::classic::prepared_irreversible_classic_sub_band_records_midpoint_reconstruction", "compute::tests::grouping::distinct_prepared_ht_direct_plans_support_stacked_component_batch", "compute::tests::grouping::grouped_ht_direct_plan_uses_one_group_coded_arena", "compute::tests::grouping::prepared_ht_direct_plan_encodes_full_decode_in_one_compute_encoder", diff --git a/xtask/src/metal/tests.rs b/xtask/src/metal/tests.rs index 713e7632..8b93bef9 100644 --- a/xtask/src/metal/tests.rs +++ b/xtask/src/metal/tests.rs @@ -168,7 +168,7 @@ fn ignored_inventory_is_unique_and_has_expected_size() { .iter() .copied() .collect::>(); - assert_eq!(required.len(), 19); + assert_eq!(required.len(), 21); assert_eq!(optional.len(), 1); assert_eq!(required.len(), J2K_METAL_REQUIRED_IGNORED_TESTS.len()); assert_eq!(optional.len(), METAL_OPTIONAL_IGNORED_TESTS.len()); diff --git a/xtask/src/public_support.rs b/xtask/src/public_support.rs index 28acc931..ef956e53 100644 --- a/xtask/src/public_support.rs +++ b/xtask/src/public_support.rs @@ -1,7 +1,7 @@ use std::fs; const SUPPORT_DOC: &str = "docs/public-support.md"; -const CONFORMANCE_MANIFEST: &str = "corpus/j2k-conformance/manifest.tsv"; +const SUPPORT_INVENTORY: &str = "corpus/j2k-conformance/support-inventory.tsv"; const REQUIRED_COLUMNS: &[&str] = &[ "ID", @@ -257,7 +257,7 @@ pub(crate) fn public_support(args: impl IntoIterator) -> Result<( } let doc = read(SUPPORT_DOC)?; - let manifest = read(CONFORMANCE_MANIFEST)?; + let inventory = read(SUPPORT_INVENTORY)?; let mut failures = Vec::new(); require_contains_all(&doc, REQUIRED_COLUMNS, SUPPORT_DOC, &mut failures); @@ -270,16 +270,16 @@ pub(crate) fn public_support(args: impl IntoIterator) -> Result<( &mut failures, ); require_contains_all( - &manifest, + &inventory, REQUIRED_MANIFEST_IDS, - CONFORMANCE_MANIFEST, + SUPPORT_INVENTORY, &mut failures, ); - for id in manifest_ids(&manifest) { + for id in inventory_ids(&inventory) { if !doc.contains(id) && !id.starts_with('#') { failures.push(format!( - "{SUPPORT_DOC} does not mention conformance manifest row `{id}`" + "{SUPPORT_DOC} does not mention support inventory row `{id}`" )); } } @@ -320,8 +320,8 @@ fn require_contains_all(haystack: &str, needles: &[&str], path: &str, failures: } } -fn manifest_ids(manifest: &str) -> impl Iterator { - manifest.lines().filter_map(|line| { +fn inventory_ids(inventory: &str) -> impl Iterator { + inventory.lines().filter_map(|line| { let line = line.trim(); if line.is_empty() || line.starts_with('#') { return None; @@ -360,11 +360,12 @@ fn support_row_status<'a>(doc: &'a str, id: &str) -> Option<&'a str> { #[cfg(test)] mod tests { - use super::{manifest_ids, support_row_status}; + use super::{inventory_ids, support_row_status}; #[test] - fn manifest_ids_skips_header_comments() { - let ids = manifest_ids("# id\tpath\nrow_a\ta\n\nrow_b\tb").collect::>(); + fn inventory_ids_skips_header_comments() { + let ids = inventory_ids("# id\tstatus\nrow_a\timplemented\n\nrow_b\tout-of-scope") + .collect::>(); assert_eq!(ids, ["row_a", "row_b"]); } diff --git a/xtask/src/quality_commands.rs b/xtask/src/quality_commands.rs index 2ae7877d..29680b5a 100644 --- a/xtask/src/quality_commands.rs +++ b/xtask/src/quality_commands.rs @@ -289,6 +289,11 @@ pub(super) fn fuzz_build() -> Result<(), String> { "check", "--manifest-path", "crates/j2k-transcode/fuzz/Cargo.toml", + ])?; + run_cargo(&[ + "check", + "--manifest-path", + "crates/j2k-t803/fuzz/Cargo.toml", ]) } @@ -296,6 +301,7 @@ const FUZZ_TARGETS: &[(&str, &str)] = &[ ("crates/j2k", "decode_fuzz"), ("crates/j2k", "jp2_box_fuzz"), ("crates/j2k", "jp2_metadata_fuzz"), + ("crates/j2k", "srgb8_fuzz"), ("crates/j2k", "parse_fuzz"), ("crates/j2k", "region_scaled_fuzz"), ("crates/j2k-jpeg", "decode_fuzz"), @@ -304,6 +310,8 @@ const FUZZ_TARGETS: &[(&str, &str)] = &[ ("crates/j2k-jpeg", "row_stream_fuzz"), ("crates/j2k-tilecodec", "decompress_fuzz"), ("crates/j2k-transcode", "jpeg_to_htj2k_fuzz"), + ("crates/j2k-t803", "pgx_fuzz"), + ("crates/j2k-t803", "archive_fuzz"), ]; pub(super) fn fuzz_run() -> Result<(), String> { diff --git a/xtask/src/release_commands.rs b/xtask/src/release_commands.rs index ca47ba85..b7916e8d 100644 --- a/xtask/src/release_commands.rs +++ b/xtask/src/release_commands.rs @@ -343,6 +343,14 @@ fn validate_publish_workflow_source( "--ci-workflow full-validation.yml", "--cuda-job \"CUDA full release validation\"", "--metal-job \"Metal full release validation\"", + "--t803-out-dir target/t803/release-evidence", + "--t803-scope all", + "cargo xtask t803 verify --scope all --candidate-sha", + "j2k-t803-cpu-linux-x86_64-${candidate_sha}/cpu.json", + "j2k-t803-cpu-macos-aarch64-${candidate_sha}/cpu.json", + "j2k-t803-cpu-windows-x86_64-${candidate_sha}/cpu.json", + "j2k-t803-cuda-linux-x86_64-${candidate_sha}/cuda.json", + "j2k-t803-metal-macos-aarch64-${candidate_sha}/metal.json", "cargo xtask release-integrity --publish", "scripts/publish-crate.sh --preflight-all", "python3 scripts/publish_release.py preflight", @@ -578,6 +586,40 @@ pub(super) fn package() -> Result<(), String> { package_gate::run(&metadata, &release_manifest) } +pub(super) fn package_consumer_smoke(mut args: impl Iterator) -> Result<(), String> { + let mut target = "all".to_string(); + let mut cuda_runtime = false; + while let Some(argument) = args.next() { + match argument.as_str() { + "--target" => { + target = args + .next() + .ok_or_else(|| "--target requires core, cuda, metal, or all".to_string())?; + } + "--cuda-runtime" => cuda_runtime = true, + _ => return Err(package_consumer_usage()), + } + } + let consumers = match target.as_str() { + "core" => &["j2k"][..], + "cuda" => &["j2k", "j2k-cuda"][..], + "metal" => &["j2k", "j2k-metal"][..], + "all" => &["j2k", "j2k-cuda", "j2k-metal"][..], + _ => return Err(package_consumer_usage()), + }; + if cuda_runtime && !consumers.contains(&"j2k-cuda") { + return Err("--cuda-runtime requires --target cuda or all".to_string()); + } + let metadata = cargo_metadata()?; + let release_manifest = release_manifest_contract()?; + package_gate::run_core_package_smoke(&metadata, &release_manifest, consumers, cuda_runtime) +} + +fn package_consumer_usage() -> String { + "usage: cargo xtask package-consumer-smoke [--target core|cuda|metal|all] [--cuda-runtime]" + .to_string() +} + pub(super) fn j2k_ml_package_smoke() -> Result<(), String> { let metadata = cargo_metadata()?; let release_manifest = release_manifest_contract()?; diff --git a/xtask/src/release_commands/package_gate.rs b/xtask/src/release_commands/package_gate.rs index 5e6b4826..58ec0b11 100644 --- a/xtask/src/release_commands/package_gate.rs +++ b/xtask/src/release_commands/package_gate.rs @@ -8,7 +8,10 @@ use std::path::Path; use crate::command_support::run_cargo; use crate::process::{cargo, run_command_owned, CommandContext}; -use consumer::{package_archive_path, run_j2k_ml_consumer_gate}; +use consumer::{ + package_archive_path, run_core_package_consumer_gates, run_j2k_ml_consumer_gate, + CORE_PACKAGE_CONSUMERS, +}; use super::release_manifest::{ registry_independent_packages, release_dependencies_by_package, @@ -149,17 +152,48 @@ pub(super) fn run( metadata: &serde_json::Value, manifest: &ReleaseManifestContract, ) -> Result<(), String> { - for step in package_gate_plan(metadata, manifest)? { + let plan = package_gate_plan(metadata, manifest)?; + for step in &plan { if step.registry_independent { run_cargo(&["publish", "-p", step.package.as_str(), "--dry-run"])?; - } else { - run_staged_package(&step, false)?; } + // `cargo publish --dry-run` removes its package archive after verification. + // Stage every crate so the clean-consumer checks below inspect this run's + // exact package contents instead of relying on a stale archive. + run_staged_package(step, false)?; if step.package == "j2k-ml" { - run_j2k_ml_consumer_gate(&step, &package_archive_path(metadata, &step)?)?; + run_j2k_ml_consumer_gate(step, &package_archive_path(metadata, step)?)?; } } - Ok(()) + run_core_package_consumer_gates(metadata, &plan, &CORE_PACKAGE_CONSUMERS, false) +} + +pub(super) fn run_core_package_smoke( + metadata: &serde_json::Value, + manifest: &ReleaseManifestContract, + consumers: &[&str], + cuda_runtime: bool, +) -> Result<(), String> { + let plan = package_gate_plan(metadata, manifest)?; + let mut required = BTreeSet::new(); + for package in consumers { + let step = plan + .iter() + .find(|step| step.package == *package) + .ok_or_else(|| format!("package gate plan omitted `{package}`"))?; + required.insert(step.package.as_str()); + required.extend( + step.patches + .iter() + .map(|(dependency, _)| dependency.as_str()), + ); + } + for step in &plan { + if required.contains(step.package.as_str()) { + run_staged_package(step, true)?; + } + } + run_core_package_consumer_gates(metadata, &plan, consumers, cuda_runtime) } #[cfg(test)] diff --git a/xtask/src/release_commands/package_gate/consumer.rs b/xtask/src/release_commands/package_gate/consumer.rs index db3584c8..3866b723 100644 --- a/xtask/src/release_commands/package_gate/consumer.rs +++ b/xtask/src/release_commands/package_gate/consumer.rs @@ -1,5 +1,6 @@ //! Clean consumer validation for the packaged `j2k-ml` source archive. +use std::collections::{BTreeMap, BTreeSet}; use std::env; use std::fmt::Write as _; use std::fs::{self, File}; @@ -12,6 +13,8 @@ use crate::process::{cargo, run_command_owned, CommandContext}; use super::{append_patch_config_args, PackageGateStep}; +pub(super) const CORE_PACKAGE_CONSUMERS: [&str; 3] = ["j2k", "j2k-cuda", "j2k-metal"]; + pub(super) fn j2k_ml_consumer_checks(target_os: &str) -> &'static [&'static str] { match target_os { "linux" => &["cpu", "cuda", "cpu,cuda"], @@ -48,6 +51,58 @@ pub(super) fn j2k_ml_consumer_manifest( Ok(manifest) } +pub(super) fn package_consumer_manifest( + step: &PackageGateStep, + packaged_crates: &BTreeMap, +) -> Result { + let current = packaged_crates.get(&step.package).ok_or_else(|| { + format!( + "clean consumer is missing packaged source for `{}`", + step.package + ) + })?; + let features = if step.package == "j2k-cuda" { + "[features]\ndefault = []\ncuda-runtime = [\"j2k-cuda/cuda-runtime\"]\n\n" + } else { + "" + }; + let mut manifest = format!( + "[package]\nname = \"{}-package-consumer\"\nversion = \"0.0.0\"\nedition = \"2021\"\npublish = false\n\n\ + {}\ + [dependencies]\n{} = \"={}\"\n\n\ + [patch.crates-io]\n{} = {{ path = {} }}\n", + step.package, + features, + step.package, + step.version, + step.package, + toml_string(¤t.to_string_lossy())?, + ); + for (dependency, _) in &step.patches { + let path = packaged_crates.get(dependency).ok_or_else(|| { + format!("clean consumer is missing packaged source for `{dependency}`") + })?; + writeln!( + &mut manifest, + "{dependency} = {{ path = {} }}", + toml_string(&path.to_string_lossy())? + ) + .unwrap(); + } + Ok(manifest) +} + +pub(super) fn package_consumer_source(package: &str) -> Result<&'static str, String> { + match package { + "j2k" => Ok("fn main() { let _ = j2k::J2kDecoder::new(&[]); }\n"), + "j2k-cuda" => Ok("fn main() { let _ = j2k_cuda::J2kDecoder::new(&[]); }\n"), + "j2k-metal" => Ok("fn main() { let _ = j2k_metal::J2kDecoder::new(&[]); }\n"), + _ => Err(format!( + "no clean package consumer source is defined for `{package}`" + )), + } +} + pub(super) const CONSUMER_SOURCE: &str = r#"use j2k::BatchDecodeOptions; #[cfg(feature = "cpu")] @@ -227,3 +282,114 @@ pub(super) fn run_j2k_ml_consumer_gate( }); result.and(cleanup) } + +pub(super) fn run_core_package_consumer_gates( + metadata: &serde_json::Value, + plan: &[PackageGateStep], + consumers: &[&str], + cuda_runtime: bool, +) -> Result<(), String> { + let consumer = fresh_core_consumer_dir()?; + let result = (|| { + let targets = consumers + .iter() + .map(|package| { + plan.iter() + .find(|step| step.package == *package) + .ok_or_else(|| format!("package gate plan omitted `{package}`")) + }) + .collect::, _>>()?; + let required = targets + .iter() + .flat_map(|step| { + std::iter::once(step.package.as_str()).chain( + step.patches + .iter() + .map(|(dependency, _)| dependency.as_str()), + ) + }) + .collect::>(); + let mut packaged_crates = BTreeMap::new(); + for package in required { + let step = plan + .iter() + .find(|step| step.package == package) + .ok_or_else(|| format!("package gate plan omitted dependency `{package}`"))?; + let extracted = extract_packaged_crate( + &package_archive_path(metadata, step)?, + &consumer.join("packaged"), + step, + )?; + packaged_crates.insert(package.to_string(), extracted); + } + + let target_dir = consumer.join("target"); + for step in targets { + let project = consumer.join(&step.package); + fs::create_dir_all(project.join("src")).map_err(|error| { + format!( + "failed to create clean {} consumer at {}: {error}", + step.package, + project.display() + ) + })?; + fs::write( + project.join("Cargo.toml"), + package_consumer_manifest(step, &packaged_crates)?, + ) + .map_err(|error| { + format!( + "failed to write clean {} consumer manifest: {error}", + step.package + ) + })?; + fs::write( + project.join("src/main.rs"), + package_consumer_source(&step.package)?, + ) + .map_err(|error| { + format!( + "failed to write clean {} consumer source: {error}", + step.package + ) + })?; + let mut args = vec!["check".to_string()]; + if cuda_runtime && step.package == "j2k-cuda" { + args.extend(["--features".to_string(), "cuda-runtime".to_string()]); + } + run_command_owned( + cargo(), + &args, + CommandContext::new() + .current_dir(&project) + .target_dir(&target_dir), + )?; + } + Ok(()) + })(); + let cleanup = fs::remove_dir_all(&consumer).map_err(|error| { + format!( + "failed to remove clean core/GPU consumers {}: {error}", + consumer.display() + ) + }); + result.and(cleanup) +} + +fn fresh_core_consumer_dir() -> Result { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|error| format!("system clock precedes Unix epoch: {error}"))? + .as_nanos(); + let path = env::temp_dir().join(format!( + "j2k-core-gpu-package-consumers-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&path).map_err(|error| { + format!( + "failed to create clean core/GPU consumer root at {}: {error}", + path.display() + ) + })?; + Ok(path) +} diff --git a/xtask/src/release_commands/package_gate/tests.rs b/xtask/src/release_commands/package_gate/tests.rs index df88d163..fc45ecd3 100644 --- a/xtask/src/release_commands/package_gate/tests.rs +++ b/xtask/src/release_commands/package_gate/tests.rs @@ -7,7 +7,8 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use super::consumer::{ - extract_packaged_crate, j2k_ml_consumer_checks, j2k_ml_consumer_manifest, CONSUMER_SOURCE, + extract_packaged_crate, j2k_ml_consumer_checks, j2k_ml_consumer_manifest, + package_consumer_manifest, package_consumer_source, CONSUMER_SOURCE, }; use super::{package_gate_plan, PackageGateStep}; use crate::release_commands::release_manifest::{ @@ -84,10 +85,17 @@ fn test_package_gate_plan(metadata: &serde_json::Value) -> Result>(); + + for package in ["j2k", "j2k-cuda", "j2k-metal"] { + let step = plan + .iter() + .find(|step| step.package == package) + .expect("consumer package step"); + let manifest = package_consumer_manifest(step, &packaged).expect("clean consumer manifest"); + + assert!(manifest.contains(&format!( + "{package} = {{ path = \"/packaged/{package}-0.7.5\" }}" + ))); + assert!(!manifest.contains("/workspace/")); + if package == "j2k-cuda" { + assert!(manifest.contains("cuda-runtime = [\"j2k-cuda/cuda-runtime\"]")); + } + for (dependency, _) in &step.patches { + assert!( + manifest.contains(&format!( + "{dependency} = {{ path = \"/packaged/{dependency}-0.7.5\" }}" + )), + "missing packaged dependency {dependency} in {manifest}" + ); + } + assert!(package_consumer_source(package) + .expect("consumer source") + .contains(&package.replace('-', "_"))); + } +} + #[test] fn j2k_ml_consumer_manifest_patches_only_workspace_crates() { let metadata = workspace_metadata(&[ @@ -413,7 +467,16 @@ fn package_gate_executes_registry_and_staged_steps_with_dependency_patches() { let package_root = test_root("j2k-ml-package-gate-test"); let package_dir = package_root.join("package"); fs::create_dir_all(&package_dir).expect("create package gate target"); - write_packaged_fixture(&package_dir.join("j2k-ml-0.7.5.crate")); + for package in [ + "j2k-core", + "j2k-native", + "j2k", + "j2k-cuda", + "j2k-metal", + "j2k-ml", + ] { + write_packaged_fixture(&package_dir.join(format!("{package}-0.7.5.crate"))); + } metadata["target_directory"] = serde_json::Value::String(package_root.to_string_lossy().into_owned()); let recording = RecordingProgram::new("package-gate-command-test", ""); @@ -426,12 +489,20 @@ fn package_gate_executes_registry_and_staged_steps_with_dependency_patches() { let log = recording.log(); let lines = log.lines().collect::>(); let consumer_checks = j2k_ml_consumer_checks(std::env::consts::OS); + let registry_independent = ["j2k-core", "j2k-profile", "j2k-types", "j2k-codec-math"]; assert_eq!( lines.len(), - manifest.ordered_crates().len() + consumer_checks.len() + 2 + manifest.ordered_crates().len() + registry_independent.len() + consumer_checks.len() + 5 ); assert!(lines[0].starts_with("publish -p j2k-core --dry-run|")); - assert!(lines[3].starts_with("publish -p j2k-codec-math --dry-run|")); + for package in registry_independent { + assert!(lines + .iter() + .any(|line| line.starts_with(&format!("publish -p {package} --dry-run|")))); + assert!(lines + .iter() + .any(|line| line.starts_with(&format!("package -p {package} --no-verify|")))); + } let native = lines .iter() .find(|line| line.starts_with("package -p j2k-native --no-verify")) @@ -458,4 +529,11 @@ fn package_gate_executes_registry_and_staged_steps_with_dependency_patches() { assert!(lines .iter() .any(|line| { line.contains("check --examples --no-default-features --features") })); + assert_eq!( + lines + .iter() + .filter(|line| line.starts_with("check|")) + .count(), + 3 + ); } diff --git a/xtask/src/release_commands/tests.rs b/xtask/src/release_commands/tests.rs index 8745ce4e..e650c099 100644 --- a/xtask/src/release_commands/tests.rs +++ b/xtask/src/release_commands/tests.rs @@ -4,11 +4,37 @@ use std::collections::BTreeSet; use super::release_manifest::{crates_io_publishable, release_manifest_contract}; use super::{ - has_docs_rs_metadata, has_lib_target, package_name, release_cpu, release_integrity, - validate_publish_script_source, validate_publish_workflow_source, validate_release_docs_source, - validate_unpublished_dependencies, workspace_package_records, + has_docs_rs_metadata, has_lib_target, package_consumer_smoke, package_name, release_cpu, + release_integrity, validate_publish_script_source, validate_publish_workflow_source, + validate_release_docs_source, validate_unpublished_dependencies, workspace_package_records, }; +#[test] +fn package_consumer_smoke_rejects_invalid_routes_before_packaging() { + for (args, expected) in [ + ( + vec!["--target", "unknown"], + "usage: cargo xtask package-consumer-smoke", + ), + ( + vec!["--target", "metal", "--cuda-runtime"], + "--cuda-runtime requires --target cuda or all", + ), + ] { + let error = package_consumer_smoke(args.into_iter().map(str::to_string)) + .expect_err("invalid package consumer request"); + assert!(error.contains(expected), "unexpected error: {error}"); + } +} + +#[test] +fn full_gpu_workflows_compile_the_packaged_adapter_archives() { + let workflow = include_str!("../../../.github/workflows/gpu-validation.yml"); + + assert!(workflow.contains("cargo xtask package-consumer-smoke --target cuda --cuda-runtime")); + assert!(workflow.contains("cargo xtask package-consumer-smoke --target metal")); +} + #[cfg(unix)] mod file_boundaries; #[cfg(unix)] @@ -168,6 +194,42 @@ fn checked_in_publish_workflow_script_and_docs_agree_with_the_manifest() { assert!(errors.is_empty(), "release contract drift: {errors:#?}"); } +#[test] +fn publish_workflow_requires_all_exact_sha_t803_lanes() { + let workflow = include_str!("../../../.github/workflows/publish.yml") + .replace("--t803-scope all", "--t803-scope cpu") + .replace("t803 verify --scope all", "t803 verify --scope cpu") + .replace( + " --report \"target/t803/release-evidence/j2k-t803-cuda-linux-x86_64-${candidate_sha}/cuda.json\" \\\n", + "", + ) + .replace( + " --report \"target/t803/release-evidence/j2k-t803-metal-macos-aarch64-${candidate_sha}/metal.json\"\n", + "", + ); + let mut errors = Vec::new(); + validate_publish_workflow_source(&workflow, &mut errors).expect("parse publish workflow"); + + assert!( + errors + .iter() + .any(|error| error.contains("--t803-scope all")), + "CPU-only publication evidence was accepted: {errors:#?}" + ); + assert!( + errors + .iter() + .any(|error| error.contains("j2k-t803-cuda-linux-x86_64")), + "missing CUDA report was accepted: {errors:#?}" + ); + assert!( + errors + .iter() + .any(|error| error.contains("j2k-t803-metal-macos-aarch64")), + "missing Metal report was accepted: {errors:#?}" + ); +} + #[test] fn publish_workflow_rejects_checkout_that_can_peel_an_annotated_tag() { let workflow = include_str!("../../../.github/workflows/publish.yml").replacen( diff --git a/xtask/src/release_commands/tests/orchestration.rs b/xtask/src/release_commands/tests/orchestration.rs index 0d07511d..ecc2ca66 100644 --- a/xtask/src/release_commands/tests/orchestration.rs +++ b/xtask/src/release_commands/tests/orchestration.rs @@ -86,12 +86,12 @@ fn release_integrity_publish_mode_accepts_hermetic_final_metadata() { } std::fs::write( release_root.join("Cargo.toml"), - "[workspace.package]\nversion = \"0.8.0\"\n\n[patch.crates-io]\nblock = { path = \"third_party/block-0.1.6-patched\" }\n", + "[workspace.package]\nversion = \"0.8.1\"\n\n[patch.crates-io]\nblock = { path = \"third_party/block-0.1.6-patched\" }\n", ) .expect("write workspace manifest fixture"); std::fs::write( release_root.join("CHANGELOG.md"), - "# Changelog\n\n## [0.8.0] - 2026-07-27\n", + "# Changelog\n\n## [0.8.1] - 2026-08-06\n", ) .expect("write finalized changelog fixture"); std::fs::write( @@ -150,18 +150,24 @@ fn package_command_executes_list_and_dependency_aware_gates_hermetically() { .ordered_crates() .len(); let consumer_commands = match std::env::consts::OS { - "linux" | "macos" => 5, - _ => 3, + "linux" | "macos" => 8, + _ => 6, }; + let registry_independent = ["j2k-core", "j2k-profile", "j2k-types", "j2k-codec-math"]; assert_eq!( commands.len(), - 1 + 2 * publishable_count + consumer_commands + 1 + 2 * publishable_count + registry_independent.len() + consumer_commands ); assert!(commands[0].starts_with("metadata --locked --no-deps --format-version 1|")); assert!(commands[1].starts_with("package -p j2k-core --list|")); - assert!(commands - .iter() - .any(|line| line.starts_with("publish -p j2k-core --dry-run|"))); + for package in registry_independent { + assert!(commands + .iter() + .any(|line| line.starts_with(&format!("publish -p {package} --dry-run|")))); + assert!(commands + .iter() + .any(|line| line.starts_with(&format!("package -p {package} --no-verify|")))); + } assert!(commands .iter() .any(|line| line.starts_with("package -p j2k-cli --no-verify"))); diff --git a/xtask/src/release_commands/tests/package_fixture.rs b/xtask/src/release_commands/tests/package_fixture.rs index 0b74b353..16ec52d9 100644 --- a/xtask/src/release_commands/tests/package_fixture.rs +++ b/xtask/src/release_commands/tests/package_fixture.rs @@ -6,26 +6,34 @@ use super::integrity::complete_publishable_metadata; pub(super) fn packaged_metadata() -> (serde_json::Value, PathBuf) { let mut metadata = complete_publishable_metadata(); - let version = metadata["packages"] + let packages = metadata["packages"] .as_array() .expect("package records") .iter() - .find(|package| package["name"] == "j2k-ml") - .and_then(|package| package["version"].as_str()) - .expect("j2k-ml package version") - .to_string(); + .filter_map(|package| { + Some(( + package["name"].as_str()?.to_string(), + package["version"].as_str()?.to_string(), + )) + }) + .collect::>(); let target = std::env::temp_dir().join(format!("j2k-release-package-target-{}", std::process::id())); metadata["target_directory"] = serde_json::Value::String(target.to_string_lossy().into_owned()); - let archive_path = target - .join("package") - .join(format!("j2k-ml-{version}.crate")); - std::fs::create_dir_all(archive_path.parent().expect("package fixture parent")) - .expect("create package fixture directory"); + let package_dir = target.join("package"); + std::fs::create_dir_all(&package_dir).expect("create package fixture directory"); + for (package, version) in packages { + write_package_archive(&package_dir, &package, &version); + } + (metadata, target) +} + +fn write_package_archive(package_dir: &std::path::Path, package: &str, version: &str) { + let archive_path = package_dir.join(format!("{package}-{version}.crate")); let file = std::fs::File::create(archive_path).expect("create package fixture"); let encoder = flate2::write::GzEncoder::new(file, flate2::Compression::default()); let mut archive = tar::Builder::new(encoder); - let contents = format!("[package]\nname = \"j2k-ml\"\nversion = \"{version}\"\n"); + let contents = format!("[package]\nname = \"{package}\"\nversion = \"{version}\"\n"); let mut header = tar::Header::new_gnu(); header.set_size(contents.len() as u64); header.set_mode(0o644); @@ -33,10 +41,9 @@ pub(super) fn packaged_metadata() -> (serde_json::Value, PathBuf) { archive .append_data( &mut header, - format!("j2k-ml-{version}/Cargo.toml"), + format!("{package}-{version}/Cargo.toml"), Cursor::new(contents), ) .expect("append package fixture"); archive.finish().expect("finish package fixture"); - (metadata, target) } diff --git a/xtask/src/release_status.rs b/xtask/src/release_status.rs index 645ba231..8b674b74 100644 --- a/xtask/src/release_status.rs +++ b/xtask/src/release_status.rs @@ -17,6 +17,7 @@ const METAL_JOB: &str = "Metal full release validation"; struct Options { sha: String, repository: Option, + scope: String, } pub(crate) fn release_status(args: impl Iterator) -> Result<(), String> { @@ -46,13 +47,20 @@ pub(crate) fn release_status(args: impl Iterator) -> Result<(), S )); } + let evidence_dir = root.join("target").join("t803").join(format!( + "release-status-{}-{}", + options.sha.clone(), + std::process::id() + )); + let evidence_dir_text = evidence_dir.to_string_lossy().into_owned(); + let command_args = vec![ "scripts/github_actions_verify.py".to_string(), "verify-candidate".to_string(), "--repository".to_string(), repository, "--candidate-sha".to_string(), - options.sha, + options.sha.clone(), "--token-env".to_string(), token_env.to_string(), "--ci-workflow".to_string(), @@ -67,22 +75,90 @@ pub(crate) fn release_status(args: impl Iterator) -> Result<(), S CUDA_JOB.to_string(), "--metal-job".to_string(), METAL_JOB.to_string(), + "--t803-scope".to_string(), + options.scope.clone(), + "--t803-out-dir".to_string(), + evidence_dir_text, ]; process::run_command_owned( OsString::from("python3"), &command_args, CommandContext::new().current_dir(&root), + )?; + verify_t803_reports(&root, &evidence_dir, &options.sha, &options.scope) +} + +fn verify_t803_reports( + root: &Path, + evidence_dir: &Path, + candidate_sha: &str, + scope: &str, +) -> Result<(), String> { + let mut args = [ + "run", + "--quiet", + "-p", + "j2k-t803", + "--features", + "runner", + "--bin", + "j2k-t803-runner", + "--", + "verify", + "--scope", + scope, + "--candidate-sha", + candidate_sha, + ] + .into_iter() + .map(str::to_string) + .collect::>(); + for report in t803_report_paths(evidence_dir, candidate_sha, scope) { + args.push("--report".to_string()); + args.push(report.to_string_lossy().into_owned()); + } + process::run_command_owned( + process::cargo(), + &args, + CommandContext::new().current_dir(root), ) } +fn t803_report_paths(evidence_dir: &Path, candidate_sha: &str, scope: &str) -> Vec { + let mut lanes = Vec::new(); + if matches!(scope, "cpu" | "all") { + lanes.extend([ + ("cpu-linux-x86_64", "cpu"), + ("cpu-macos-aarch64", "cpu"), + ("cpu-windows-x86_64", "cpu"), + ]); + } + if matches!(scope, "cuda" | "all") { + lanes.push(("cuda-linux-x86_64", "cuda")); + } + if matches!(scope, "metal" | "all") { + lanes.push(("metal-macos-aarch64", "metal")); + } + lanes + .into_iter() + .map(|(lane, stem)| { + evidence_dir + .join(format!("j2k-t803-{lane}-{candidate_sha}")) + .join(format!("{stem}.json")) + }) + .collect() +} + fn parse_options(args: impl Iterator) -> Result { let mut sha = None; let mut repository = None; + let mut scope = None; let mut args = args; while let Some(arg) = args.next() { let slot = match arg.as_str() { "--sha" => &mut sha, "--repository" => &mut repository, + "--scope" => &mut scope, "-h" | "--help" => return Err(usage()), other => { return Err(format!( @@ -104,7 +180,19 @@ fn parse_options(args: impl Iterator) -> Result let sha = normalize_sha(&sha.ok_or_else(|| format!("--sha is required\n{}", usage()))?)?; let repository = repository.as_deref().map(validate_repository).transpose()?; - Ok(Options { sha, repository }) + let scope = validate_scope(scope.as_deref().unwrap_or("all"))?; + Ok(Options { + sha, + repository, + scope, + }) +} + +fn validate_scope(value: &str) -> Result { + match value { + "cpu" | "cuda" | "metal" | "all" => Ok(value.to_string()), + _ => Err("--scope must be one of cpu, cuda, metal, or all".to_string()), + } } fn normalize_sha(value: &str) -> Result { @@ -225,7 +313,7 @@ fn select_token_env( } fn usage() -> String { - "usage: cargo xtask release-status --sha <40-hex-commit> [--repository owner/name]".to_string() + "usage: cargo xtask release-status --sha <40-hex-commit> [--repository owner/name] [--scope cpu|cuda|metal|all]".to_string() } #[cfg(test)] diff --git a/xtask/src/release_status/tests.rs b/xtask/src/release_status/tests.rs index 1865dc72..d8aa4031 100644 --- a/xtask/src/release_status/tests.rs +++ b/xtask/src/release_status/tests.rs @@ -19,13 +19,29 @@ const WORKSPACE_CHILD_ENV: &str = "XTASK_TEST_RELEASE_STATUS_WORKSPACE_CHILD"; #[test] fn options_require_and_normalize_an_exact_sha() { let options = parse_options( - ["--sha", &"A".repeat(40), "--repository", "frames-sg/j2k"] - .into_iter() - .map(str::to_string), + [ + "--sha", + &"A".repeat(40), + "--repository", + "frames-sg/j2k", + "--scope", + "cpu", + ] + .into_iter() + .map(str::to_string), ) .unwrap(); assert_eq!(options.sha, "a".repeat(40)); assert_eq!(options.repository.as_deref(), Some("frames-sg/j2k")); + assert_eq!(options.scope, "cpu"); + + let default_scope = parse_options( + ["--sha", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"] + .into_iter() + .map(str::to_string), + ) + .unwrap(); + assert_eq!(default_scope.scope, "all"); for invalid in ["abc", &"g".repeat(40), &"a".repeat(41)] { assert!(parse_options(["--sha", invalid].into_iter().map(str::to_string)).is_err()); @@ -38,6 +54,7 @@ fn options_reject_missing_values_duplicates_help_and_unknown_arguments() { (Vec::new(), "--sha is required"), (vec!["--sha"], "--sha` requires a value"), (vec!["--repository"], "--repository` requires a value"), + (vec!["--scope"], "--scope` requires a value"), (vec!["--help"], "usage: cargo xtask release-status"), (vec!["--unknown"], "unknown release-status argument"), ( @@ -62,6 +79,19 @@ fn options_reject_missing_values_duplicates_help_and_unknown_arguments() { ) .expect_err("malformed repository must reject"); assert!(error.contains("owner/name")); + + let error = parse_options( + [ + "--sha", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "--scope", + "gpu", + ] + .into_iter() + .map(str::to_string), + ) + .expect_err("unknown evidence scope must reject"); + assert!(error.contains("cpu, cuda, metal, or all")); } #[test] @@ -145,9 +175,14 @@ fn release_status_derives_remote_and_executes_exact_verifier_contract() { if std::env::var_os(WORKSPACE_CHILD_ENV).is_some() { release_status( - ["--sha", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"] - .into_iter() - .map(str::to_string), + [ + "--sha", + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "--scope", + "cpu", + ] + .into_iter() + .map(str::to_string), ) .expect("hermetic release-status command"); return; @@ -158,6 +193,7 @@ fn release_status_derives_remote_and_executes_exact_verifier_contract() { r#"case "${0##*/}" in git) printf '%s\n' 'git@example.invalid:frames-sg/j2k.git' ;; python3) exit 0 ;; + cargo) exit 0 ;; *) exit 90 ;; esac"#, ); @@ -167,6 +203,7 @@ esac"#, .expect("recording program parent"); symlink(recording.program(), program_dir.join("git")).expect("fake git symlink"); symlink(recording.program(), program_dir.join("python3")).expect("fake python3 symlink"); + symlink(recording.program(), program_dir.join("cargo")).expect("fake cargo symlink"); let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("xtask workspace root"); @@ -177,6 +214,7 @@ esac"#, .current_dir(workspace) .env(WORKSPACE_CHILD_ENV, "1") .env("GH_TOKEN", "test-token-placeholder") + .env("CARGO", program_dir.join("cargo")) .env_remove("GITHUB_TOKEN") .env_remove("GITHUB_REPOSITORY") .env("PATH", program_dir) @@ -191,13 +229,15 @@ esac"#, let log = recording.log(); let lines = log.lines().collect::>(); - assert_eq!(lines.len(), 2, "unexpected command log: {log}"); + assert_eq!(lines.len(), 3, "unexpected command log: {log}"); assert!(lines[0].starts_with("config --get remote.origin.url|")); assert!(lines[1].contains("verify-candidate --repository frames-sg/j2k")); assert!(lines[1] .contains("--candidate-sha aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa --token-env GH_TOKEN")); assert!(lines[1].contains("--aggregate-job Release candidate aggregate")); assert!(lines[1].contains("--ci-workflow full-validation.yml")); - assert!(lines[1].contains("--cuda-job CUDA full release validation")); - assert!(lines[1].contains("--metal-job Metal full release validation")); + assert!(lines[1].contains("--t803-scope cpu")); + assert!(lines[1].contains("--t803-out-dir")); + assert!(lines[2].contains("j2k-t803-runner -- verify --scope cpu --candidate-sha")); + assert_eq!(lines[2].matches("--report").count(), 3); } diff --git a/xtask/src/release_status/tests/boundary_errors.rs b/xtask/src/release_status/tests/boundary_errors.rs index 5557d016..8d76f8b2 100644 --- a/xtask/src/release_status/tests/boundary_errors.rs +++ b/xtask/src/release_status/tests/boundary_errors.rs @@ -28,6 +28,7 @@ fn run_child( .expect("recording program parent"); symlink(recording.program(), program_dir.join("git")).expect("fake git symlink"); symlink(recording.program(), program_dir.join("python3")).expect("fake python3 symlink"); + symlink(recording.program(), program_dir.join("cargo")).expect("fake cargo symlink"); let workspace = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("xtask workspace root"); @@ -39,6 +40,7 @@ fn run_child( .current_dir(workspace) .env(CASE_ENV, case) .env("GITHUB_TOKEN", "test-token-placeholder") + .env("CARGO", program_dir.join("cargo")) .env_remove("GH_TOKEN") .env("PATH", program_dir); if let Some(repository) = repository { @@ -66,14 +68,15 @@ fn repository_environment_empty_and_present_paths_execute_exact_contracts() { } for (case, repository, expected_repository, expected_commands) in [ - ("present", "environment/repo", "environment/repo", 1_usize), - ("empty", "", "remote/repo", 2), + ("present", "environment/repo", "environment/repo", 2_usize), + ("empty", "", "remote/repo", 3), ] { let recording = RecordingProgram::new( "release-status-environment-boundary", r#"case "${0##*/}" in git) printf '%s\n' 'git@example.invalid:remote/repo.git' ;; python3) exit 0 ;; + cargo) exit 0 ;; *) exit 90 ;; esac"#, ); @@ -88,7 +91,10 @@ esac"#, let log = recording.log(); let lines = log.lines().collect::>(); assert_eq!(lines.len(), expected_commands, "unexpected log: {log}"); - let verifier = lines.last().expect("verifier command"); + let verifier = lines + .iter() + .find(|line| line.contains("verify-candidate")) + .expect("verifier command"); assert!(verifier.contains(&format!("--repository {expected_repository}"))); assert!(verifier.contains("--token-env GITHUB_TOKEN")); } diff --git a/xtask/src/semver.rs b/xtask/src/semver.rs index f6f553b4..dc4655c7 100644 --- a/xtask/src/semver.rs +++ b/xtask/src/semver.rs @@ -25,11 +25,11 @@ use compatibility::{semver_check_args, semver_check_release_type}; const CARGO_SEMVER_CHECKS_VERSION: &str = "0.48.0"; const SEMVER_TOOLCHAIN: &str = "1.96"; -const SEMVER_BASELINE_VERSION: &str = "0.7.5"; -const SEMVER_BASELINE_TAG: &str = "v0.7.5"; -const SEMVER_BASELINE_COMMIT: &str = "a89abb6e7eba469c44b3735740712c2a85be0499"; -const API_DIFF_REPORT: &str = "engineering/reviewed-public-api-diff-0.8.0.md"; -const API_REVIEW_CONFIG: &str = "engineering/public-api-review-0.8.0.yml"; +const SEMVER_BASELINE_VERSION: &str = "0.8.0"; +const SEMVER_BASELINE_TAG: &str = "v0.8.0"; +const SEMVER_BASELINE_COMMIT: &str = "53e0ad3d4f75f492af55413e0dab5a5834bd09c6"; +const API_DIFF_REPORT: &str = "engineering/reviewed-public-api-diff-0.8.1.md"; +const API_REVIEW_CONFIG: &str = "engineering/public-api-review-0.8.1.yml"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct BaselineTransition<'a> { @@ -38,12 +38,7 @@ struct BaselineTransition<'a> { required_next_baseline_tag: &'a str, } -const INTENTIONAL_BREAK_TRANSITION: Option> = - Some(BaselineTransition { - candidate_version: "0.8.0", - required_next_baseline_version: "0.8.0", - required_next_baseline_tag: "v0.8.0", - }); +const INTENTIONAL_BREAK_TRANSITION: Option> = None; const SEMVER_BASELINE_PACKAGES: &[&str] = &[ "j2k", @@ -206,7 +201,7 @@ pub(crate) fn semver( let baseline_snapshot = baseline_api_snapshot(cargo_public_api_version)?; let baseline_apis = parse_api_snapshot(&baseline_snapshot)?; validate_snapshot_scope( - "published 0.7.5 ordinary snapshot", + "published baseline ordinary snapshot", SEMVER_BASELINE_PACKAGES, &baseline_apis, )?; diff --git a/xtask/src/semver/tests.rs b/xtask/src/semver/tests.rs index 5411ee00..76ae3956 100644 --- a/xtask/src/semver/tests.rs +++ b/xtask/src/semver/tests.rs @@ -206,7 +206,7 @@ fn published_candidate_uses_its_computed_release_type() { } #[test] -fn report_has_one_published_details_section_and_hidden_evidence() { +fn report_has_one_published_details_section_and_rotated_baseline() { let diff = PackageApiDiff { package: "alpha".to_string(), candidate_version: "0.7.0".to_string(), @@ -221,16 +221,17 @@ fn report_has_one_published_details_section_and_hidden_evidence() { assert_eq!(report.matches("## Published-package details").count(), 1); assert!(report.contains("Rustdoc-hidden candidate items: 1")); assert!(report.contains("Full hidden-inventory fingerprint: `fnv1a64:")); - assert!(report.contains("Required next semver baseline: `v0.8.0`")); + assert!(report.contains("Baseline registry version: `0.8.0`")); + assert!(!report.contains("Active intentional-break transition")); } #[test] fn parses_review_config_and_rejects_unknown_fields() { let source = "\ version: 3 -baseline_tag: v0.7.5 -baseline_version: 0.7.5 -candidate_version: 0.8.0 +baseline_tag: v0.8.0 +baseline_version: 0.8.0 +candidate_version: 0.8.1 break_ledger: - id: strict-decode-default kind: behavior @@ -257,7 +258,7 @@ reviews: "; let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(source).unwrap(); let parsed = parse_review_config(&value).unwrap(); - assert_eq!(parsed.candidate_version, "0.8.0"); + assert_eq!(parsed.candidate_version, "0.8.1"); assert_eq!(parsed.break_ledger.len(), 2); assert_eq!(parsed.break_ledger[0].kind, BreakKind::Behavior); assert_eq!(parsed.break_ledger[1].kind, BreakKind::Source); diff --git a/xtask/src/semver/tests/api_planning.rs b/xtask/src/semver/tests/api_planning.rs index 556a8096..395b359d 100644 --- a/xtask/src/semver/tests/api_planning.rs +++ b/xtask/src/semver/tests/api_planning.rs @@ -89,7 +89,7 @@ fn package_diff_planning_distinguishes_published_and_new_packages() { assert!(diffs[1].removed.is_empty()); let report = render_report("0.7.4", &diffs, "0.52.0"); - assert!(report.contains("## New packages without a 0.7.5 registry baseline")); + assert!(report.contains("## New packages without a 0.8.0 registry baseline")); assert!(report.contains("- `j2k-future` `0.7.4`: 1 ordinary public API items")); assert!(report.contains("### `j2k`")); assert!(report.contains("```text\nremoved\n```")); diff --git a/xtask/src/semver/tests/command_boundaries.rs b/xtask/src/semver/tests/command_boundaries.rs index 0b6abff5..78ca0500 100644 --- a/xtask/src/semver/tests/command_boundaries.rs +++ b/xtask/src/semver/tests/command_boundaries.rs @@ -74,7 +74,7 @@ fn committed_candidate_semver_inputs_match_the_pinned_workspace_contract() { assert!(hidden.starts_with("# J2K 1.0 Rustdoc-Hidden Public API Snapshot")); let versions = workspace_package_versions().expect("workspace package versions"); - assert_eq!(versions.get("j2k").map(String::as_str), Some("0.8.0")); + assert_eq!(versions.get("j2k").map(String::as_str), Some("0.8.1")); assert!(versions.keys().collect::>().len() > 10); } @@ -114,8 +114,8 @@ fn report_verification_is_workspace_anchored_and_empty_checks_are_a_noop() { fn semver_check_command_uses_the_computed_candidate_release_type() { let diff = PackageApiDiff { package: "j2k-core".to_string(), - candidate_version: "0.8.0".to_string(), - release_type: Some(ReleaseType::Major), + candidate_version: "0.8.1".to_string(), + release_type: Some(ReleaseType::Minor), baseline_count: 1, candidate_count: 0, added: BTreeSet::new(), @@ -136,9 +136,9 @@ fn semver_check_command_uses_the_computed_candidate_release_type() { "--package", "j2k-core", "--baseline-version", - "0.7.5", + "0.8.0", "--release-type", - "major", + "minor", "--color", "never", ] diff --git a/xtask/src/t803.rs b/xtask/src/t803.rs new file mode 100644 index 00000000..c7f0ee46 --- /dev/null +++ b/xtask/src/t803.rs @@ -0,0 +1,53 @@ +use crate::command_support::run_cargo; + +pub(super) fn t803(args: impl IntoIterator) -> Result<(), String> { + let args = args.into_iter().collect::>(); + let features = runner_features(&args); + let mut cargo_args = [ + "run", + "--quiet", + "-p", + "j2k-t803", + "--features", + features, + "--bin", + "j2k-t803-runner", + "--", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + cargo_args.extend(args); + let cargo_args = cargo_args.iter().map(String::as_str).collect::>(); + run_cargo(&cargo_args) +} + +fn runner_features(args: &[impl AsRef]) -> &'static str { + match args + .windows(2) + .find_map(|pair| (pair[0].as_ref() == "--iut").then(|| pair[1].as_ref())) + { + Some("cuda") => "runner,cuda-runner", + Some("metal") => "runner,metal-runner", + _ => "runner", + } +} + +#[cfg(test)] +mod tests { + use super::runner_features; + + #[test] + fn t803_runner_features_follow_the_selected_adapter_iut() { + assert_eq!(runner_features(&["run", "--iut", "cpu"]), "runner"); + assert_eq!( + runner_features(&["run", "--iut", "cuda"]), + "runner,cuda-runner" + ); + assert_eq!( + runner_features(&["run", "--iut", "metal"]), + "runner,metal-runner" + ); + assert_eq!(runner_features(&["verify"]), "runner"); + } +} diff --git a/xtask/tests/command_orchestration.rs b/xtask/tests/command_orchestration.rs index 4a75511a..7e4c5c94 100644 --- a/xtask/tests/command_orchestration.rs +++ b/xtask/tests/command_orchestration.rs @@ -24,6 +24,8 @@ fn release_status_executes_exact_sha_verification_without_exposing_tokens() { &explicit_sha, "--repository", "frames-sg/j2k", + "--scope", + "cpu", ], &[("GH_TOKEN", "present")], ), @@ -43,7 +45,7 @@ fn release_status_executes_exact_sha_verification_without_exposing_tokens() { "help must preserve task error handling" ); assert!(String::from_utf8_lossy(&help.stderr).contains( - "usage: cargo xtask release-status --sha <40-hex-commit> [--repository owner/name]" + "usage: cargo xtask release-status --sha <40-hex-commit> [--repository owner/name] [--scope cpu|cuda|metal|all]" )); let log = harness.log(); @@ -117,8 +119,8 @@ fn release_critical_orchestrators_run_from_the_workspace_without_real_cargo() { assert!(log.contains("package -p j2k-cli --no-verify")); #[cfg(target_os = "macos")] { - assert!(log.contains("git rev-parse v0.7.5^{commit}")); - assert!(log.contains("git show v0.7.5:docs/stable-api-1.0.public-api.txt")); + assert!(log.contains("git rev-parse v0.8.0^{commit}")); + assert!(log.contains("git show v0.8.0:docs/stable-api-1.0.public-api.txt")); } } diff --git a/xtask/tests/command_orchestration/support.rs b/xtask/tests/command_orchestration/support.rs index ba3b47ef..93a67fe2 100644 --- a/xtask/tests/command_orchestration/support.rs +++ b/xtask/tests/command_orchestration/support.rs @@ -53,14 +53,14 @@ impl Harness { .expect("write fake cargo-machete"); make_executable(&cargo_machete, "fake cargo-machete"); let git = root.join("git"); - let baseline_snapshot = root.join("stable-api-0.7.5.public-api.txt"); + let baseline_snapshot = root.join("stable-api-baseline.public-api.txt"); fs::write(&baseline_snapshot, synthetic_baseline_snapshot()) .expect("write synthetic baseline API snapshot"); let real_git = find_program("git"); fs::write( &git, format!( - "#!/bin/sh\nprintf 'git %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = status ]; then exit 0; fi\nif [ \"$1\" = config ] && [ \"$2\" = --get ] && [ \"$3\" = remote.origin.url ]; then printf '%s\\n' 'git@example.invalid:frames-sg/j2k.git'; exit 0; fi\nif [ \"$1\" = rev-parse ] && [ \"$2\" = 'v0.7.5^{{commit}}' ]; then printf '%s\\n' 'a89abb6e7eba469c44b3735740712c2a85be0499'; exit 0; fi\nif [ \"$1\" = show ] && [ \"$2\" = 'v0.7.5:docs/stable-api-1.0.public-api.txt' ]; then exec cat '{}'; fi\nif [ \"$1\" = rev-parse ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release revision: %s\\n' \"$2\" >&2; exit 97; fi\nif [ \"$1\" = show ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release object: %s\\n' \"$2\" >&2; exit 97; fi\nexec \"{}\" \"$@\"\n", + "#!/bin/sh\nprintf 'git %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = status ]; then exit 0; fi\nif [ \"$1\" = config ] && [ \"$2\" = --get ] && [ \"$3\" = remote.origin.url ]; then printf '%s\\n' 'git@example.invalid:frames-sg/j2k.git'; exit 0; fi\nif [ \"$1\" = rev-parse ] && [ \"$2\" = 'v0.8.0^{{commit}}' ]; then printf '%s\\n' '53e0ad3d4f75f492af55413e0dab5a5834bd09c6'; exit 0; fi\nif [ \"$1\" = show ] && [ \"$2\" = 'v0.8.0:docs/stable-api-1.0.public-api.txt' ]; then exec cat '{}'; fi\nif [ \"$1\" = rev-parse ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release revision: %s\\n' \"$2\" >&2; exit 97; fi\nif [ \"$1\" = show ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release object: %s\\n' \"$2\" >&2; exit 97; fi\nexec \"{}\" \"$@\"\n", log.display(), baseline_snapshot.display(), real_git.display() @@ -202,18 +202,37 @@ fn prepare_metadata_fixture(root: &Path) -> PathBuf { ); let mut metadata = serde_json::from_slice::(&output.stdout).expect("parse cargo metadata"); - let version = metadata + let workspace_members = metadata + .get("workspace_members") + .and_then(serde_json::Value::as_array) + .expect("metadata includes workspace members"); + let packaged_members = metadata .get("packages") .and_then(serde_json::Value::as_array) - .and_then(|packages| { - packages.iter().find_map(|package| { - (package.get("name").and_then(serde_json::Value::as_str) == Some("j2k-ml")) - .then(|| package.get("version").and_then(serde_json::Value::as_str)) - .flatten() - }) + .expect("metadata includes packages") + .iter() + .filter(|package| { + package + .get("id") + .is_some_and(|id| workspace_members.contains(id)) + }) + .filter(|package| match package.get("publish") { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::Array(registries)) => !registries.is_empty(), + Some(_) => panic!("workspace package has malformed publish metadata"), + }) + .map(|package| { + let name = package + .get("name") + .and_then(serde_json::Value::as_str) + .expect("workspace package has a name"); + let version = package + .get("version") + .and_then(serde_json::Value::as_str) + .expect("workspace package has a version"); + (name.to_string(), version.to_string()) }) - .expect("metadata includes j2k-ml") - .to_string(); + .collect::>(); let target = root.join("target"); metadata["target_directory"] = serde_json::Value::String(target.to_string_lossy().into_owned()); let metadata_path = root.join("metadata.json"); @@ -222,16 +241,19 @@ fn prepare_metadata_fixture(root: &Path) -> PathBuf { serde_json::to_vec(&metadata).expect("serialize metadata fixture"), ) .expect("write metadata fixture"); - write_packaged_fixture( - &target - .join("package") - .join(format!("j2k-ml-{version}.crate")), - &version, - ); + for (package, version) in packaged_members { + write_packaged_fixture( + &target + .join("package") + .join(format!("{package}-{version}.crate")), + &package, + &version, + ); + } metadata_path } -fn write_packaged_fixture(path: &Path, version: &str) { +fn write_packaged_fixture(path: &Path, package: &str, version: &str) { fs::create_dir_all(path.parent().expect("package fixture has parent")) .expect("create package fixture directory"); let encoder = GzEncoder::new( @@ -239,7 +261,7 @@ fn write_packaged_fixture(path: &Path, version: &str) { Compression::default(), ); let mut archive = tar::Builder::new(encoder); - let manifest = format!("[package]\nname = \"j2k-ml\"\nversion = \"{version}\"\n"); + let manifest = format!("[package]\nname = \"{package}\"\nversion = \"{version}\"\n"); let mut header = tar::Header::new_gnu(); header.set_mode(0o644); header.set_size(u64::try_from(manifest.len()).expect("package manifest fixture fits in u64")); @@ -247,7 +269,7 @@ fn write_packaged_fixture(path: &Path, version: &str) { archive .append_data( &mut header, - format!("j2k-ml-{version}/Cargo.toml"), + format!("{package}-{version}/Cargo.toml"), Cursor::new(manifest), ) .expect("append package manifest fixture"); diff --git a/xtask/tests/repo_lint_support/public_docs_policy.rs b/xtask/tests/repo_lint_support/public_docs_policy.rs index 3447f5a2..f6aa273f 100644 --- a/xtask/tests/repo_lint_support/public_docs_policy.rs +++ b/xtask/tests/repo_lint_support/public_docs_policy.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +mod conformance_claims; mod environment; mod metal_safety; mod navigation_packaging; diff --git a/xtask/tests/repo_lint_support/public_docs_policy/conformance_claims.rs b/xtask/tests/repo_lint_support/public_docs_policy/conformance_claims.rs new file mode 100644 index 00000000..953a5157 --- /dev/null +++ b/xtask/tests/repo_lint_support/public_docs_policy/conformance_claims.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::repo_lint_support::{assert_file_pattern_checks, repo_root, FilePatternCheck}; + +#[test] +fn t803_claims_remain_exact_and_release_scoped() { + let root = repo_root(); + assert!( + !root.join("corpus/j2k-conformance/manifest.tsv").exists(), + "the optional decode-smoke manifest must stay retired" + ); + assert!( + !root.join("crates/j2k/tests/iso_conformance.rs").exists(), + "the environment-gated decode-smoke test must stay retired" + ); + assert_file_pattern_checks( + root, + &[ + FilePatternCheck::new("README.md") + .required(&[ + "docs/t803-conformance.md", + "Profile-1 Cclass-1 compliant", + "Profile-1 Cclass-1HF compliant", + "Annex G JP2 reader compliant", + "CUDA: 0/90 device-native, 48/90 hybrid, 42/90 CPU-routed", + "Metal: 0/90 device-native, 48/90 hybrid, 42/90 CPU-routed", + ]) + .forbidden(&[ + "candidate/pending", + "full JPEG 2000 Part 1 codestream support", + ]), + FilePatternCheck::new("docs/public-support.md") + .required(&["docs/t803-conformance.md", "support-inventory.tsv"]) + .forbidden(&["full JPEG 2000 Part 1"]), + FilePatternCheck::new("docs/t803-conformance.md") + .required(&[ + "ISO/IEC 15444-4:2024 / ITU-T T.803 v3", + "Status: **0.8.1 release-scoped**", + "Formal decoder claim:", + "Profile-1 Cclass-1 compliant", + "Profile-1 Cclass-1HF compliant", + "Annex G JP2 reader compliant", + "0/90 device-native, 48/90 hybrid, and 42/90 CPU-routed", + "zero skips", + "c1-c0p0-13", + "adapter IUT", + "informative", + "T.803 does not establish robustness, security, adoption, or performance", + ]) + .forbidden(&["candidate/pending", "Formal claim: **not made**"]), + FilePatternCheck::new("corpus/j2k-conformance/README.md").required(&[ + "t803-v3.toml", + "encoder-matrix-v1.toml", + "support-inventory.tsv", + "copyrighted electronic attachment", + "must not be committed", + ]), + ], + ); +} diff --git a/xtask/tests/t803_command.rs b/xtask/tests/t803_command.rs new file mode 100644 index 00000000..e7d3156a --- /dev/null +++ b/xtask/tests/t803_command.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{fs, process::Command}; + +#[test] +fn t803_delegates_to_the_fail_closed_runner() { + let cache = std::env::temp_dir().join(format!("j2k-xtask-t803-{}", std::process::id())); + if cache.exists() { + fs::remove_dir_all(&cache).expect("remove stale test cache"); + } + fs::create_dir(&cache).expect("create test cache"); + + let output = Command::new(env!("CARGO_BIN_EXE_xtask")) + .args([ + "t803", + "run", + "--iut", + "cpu", + "--development", + "--cache-dir", + ]) + .arg(&cache) + .output() + .expect("run xtask T.803 command"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("archive is absent"), + "unexpected stderr: {stderr}" + ); + fs::remove_dir_all(cache).expect("remove test cache"); +}