From 5f8afc5c91098c017d858aaae71d1a9a36323245 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 22:13:24 +0000 Subject: [PATCH 1/2] =?UTF-8?q?simd:=20U64x8::mul=5Flo32=20=E2=80=94=20wid?= =?UTF-8?q?ening=20lo32=C3=97lo32=E2=86=92u64=20on=20every=20backend=20(ar?= =?UTF-8?q?gon2=20BlaMka)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lane-wise lo32(self) x lo32(rhs) as an exact u64; the high 32 bits of each input lane are ignored and the product cannot overflow. This is the multiply in argon2's BlaMka G function (a + b + 2*lo32(a)*lo32(b)) and the limb multiply of radix-2^26 Poly1305 / curve25519 arithmetic. AVX-512 _mm512_mul_epu32 (VPMULUDQ zmm). Intrinsic, not portable source: in BlaMka's add chain LLVM lowered the portable form to VPMULLQ. AVX2 _mm256_mul_epu32 on each 256-bit half. NEON vmovn_u64 + vmull_u32 (UMULL) per quad; LLVM left the portable form scalar inside BlaMka on aarch64. wasm i32x4_shuffle::<0,2,0,2> + u64x2_extmul_low_u32x4 per v128. scalar the per-lane reference loop. nightly (a & 0xFFFFFFFF) * (b & 0xFFFFFFFF) on core::simd::u64x8. Tests: simd::tests::u64x8_mul_lo32_matches_scalar checks every lane against the scalar definition with operands whose high halves are all non-zero and chosen so a full 64-bit multiply differs in every lane (asserted), plus commutativity and argon2's fBlaMka (RFC 9106 3.5) built from mul_lo32. The NEON (qemu) and wasm (node) parity harnesses gain the same check as 0x30F. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HnekoM1EidTwQLS3oFVFm --- crates/neon-simd-parity/src/main.rs | 7 ++++ crates/wasm-simd-parity/src/lib.rs | 7 ++++ src/simd.rs | 52 +++++++++++++++++++++++++++++ src/simd_avx2.rs | 14 ++++++++ src/simd_avx512.rs | 21 ++++++++++++ src/simd_neon.rs | 13 ++++++++ src/simd_nightly/u_word_types.rs | 10 ++++++ src/simd_scalar.rs | 15 +++++++++ src/simd_wasm.rs | 16 +++++++++ 9 files changed, 155 insertions(+) diff --git a/crates/neon-simd-parity/src/main.rs b/crates/neon-simd-parity/src/main.rs index 66aba37a..65aa1726 100644 --- a/crates/neon-simd-parity/src/main.rs +++ b/crates/neon-simd-parity/src/main.rs @@ -373,6 +373,13 @@ mod checks { if !(a == U64x8::from_array(a_arr)) || a == b { return Err(0x30E); } + // mul_lo32: lo32(a) x lo32(b) as an exact u64 (argon2's BlaMka multiply). + let m = a.mul_lo32(b).to_array(); + for i in 0..8 { + if m[i] != (a_arr[i] & 0xFFFF_FFFF) * (b_arr[i] & 0xFFFF_FFFF) { + return Err(0x30F); + } + } Ok(()) } diff --git a/crates/wasm-simd-parity/src/lib.rs b/crates/wasm-simd-parity/src/lib.rs index b84c4a6c..c89cc63a 100644 --- a/crates/wasm-simd-parity/src/lib.rs +++ b/crates/wasm-simd-parity/src/lib.rs @@ -354,6 +354,13 @@ fn check_u64x8_algebra() -> Result<(), u32> { if !(a == U64x8::from_array(a_arr)) || a == b { return Err(0x30E); } + // mul_lo32: lo32(a) x lo32(b) as an exact u64 (argon2's BlaMka multiply). + let m = a.mul_lo32(b).to_array(); + for i in 0..8 { + if m[i] != (a_arr[i] & 0xFFFF_FFFF) * (b_arr[i] & 0xFFFF_FFFF) { + return Err(0x30F); + } + } Ok(()) } diff --git a/src/simd.rs b/src/simd.rs index f7354a3f..118ef4bf 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -992,6 +992,58 @@ mod tests { } } + /// `U64x8::mul_lo32` against its scalar definition, on whichever tier this + /// build compiled (AVX-512 `VPMULUDQ`, the AVX2 halves, or scalar). + /// + /// Every operand lane carries non-zero HIGH 32 bits, and every lane is + /// chosen so a full 64-bit multiply (`VPMULLQ`, or a plain `*`) gives a + /// DIFFERENT answer — asserted below, so the fixture cannot silently lose + /// that property. Five lanes also produce products wider than 32 bits, so + /// a multiply that truncates to 32 bits (`VPMULLD`) fails too. + /// + /// The second half checks the consumer shape: argon2's BlaMka + /// `a + b + 2·lo32(a)·lo32(b)` built from `mul_lo32` against the RFC 9106 + /// definition (`fBlaMka`) computed in scalar `u64` wrapping arithmetic. + #[test] + fn u64x8_mul_lo32_matches_scalar() { + use super::U64x8; + + let a_arr: [u64; 8] = [ + 0xFFFF_FFFF_FFFF_FFFF, 0xDEAD_BEEF_0000_0000, 0x0000_0001_0000_0001, 0x1234_5678_9ABC_DEF0, + 0x8000_0000_8000_0000, 0xFFFF_FFFF_0000_0001, 0x0123_4567_89AB_CDEF, 0xAAAA_AAAA_5555_5555, + ]; + let b_arr: [u64; 8] = [ + 0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_FFFF_FFFF, 0xFFFF_FFFF_FFFF_FFFF, 0x0FED_CBA9_8765_4321, + 0x8000_0001_8000_0000, 0x7777_7777_C0DE_CAFE, 0xFEDC_BA98_7654_3210, 0x5555_5555_AAAA_AAAA, + ]; + let (a, b) = (U64x8::from_array(a_arr), U64x8::from_array(b_arr)); + + let got = a.mul_lo32(b).to_array(); + for i in 0..8 { + let want = (a_arr[i] & 0xFFFF_FFFF) * (b_arr[i] & 0xFFFF_FFFF); + assert_eq!(got[i], want, "lane {i}: mul_lo32"); + assert_ne!( + want, + a_arr[i].wrapping_mul(b_arr[i]), + "fixture lane {i} no longer distinguishes mul_lo32 from a full 64-bit multiply" + ); + } + // mul_lo32 is commutative; a backend that read one operand's high + // half by mistake would break this before it broke `want`. + assert_eq!(b.mul_lo32(a).to_array(), got, "mul_lo32 is not commutative"); + + // BlaMka, RFC 9106 §3.5: fBlaMka(x, y) = x + y + 2 * trunc(x) * trunc(y). + let m = a.mul_lo32(b); + let blamka = (a + b + m + m).to_array(); + for i in 0..8 { + let t = (a_arr[i] & 0xFFFF_FFFF) * (b_arr[i] & 0xFFFF_FFFF); + let want = a_arr[i] + .wrapping_add(b_arr[i]) + .wrapping_add(t.wrapping_mul(2)); + assert_eq!(blamka[i], want, "lane {i}: BlaMka"); + } + } + /// The BLAKE3 shuffle surface on `U32x16`, checked against the REAL x86 /// intrinsics it reproduces — applied to each 256-bit half. /// diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 71f46ea7..494dcdb9 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1674,6 +1674,20 @@ impl U64x8 { let (lo, hi) = self.avx2_halves(); Self::from_avx2_halves(Self::rotl_half(lo, 64 - n), Self::rotl_half(hi, 64 - n)) } + + /// Lane-wise `lo32(self) × lo32(rhs)` as an exact `u64` — the widening + /// 32×32→64 multiply (`VPMULUDQ`, one per 256-bit half). The high 32 + /// bits of every input lane are ignored; the product cannot overflow. + /// argon2's BlaMka multiply; see the AVX-512 backend for the contract. + #[inline(always)] + pub fn mul_lo32(self, rhs: Self) -> Self { + let (a_lo, a_hi) = self.avx2_halves(); + let (b_lo, b_hi) = rhs.avx2_halves(); + // SAFETY: same obligation as `avx2_halves` — AVX2 is present on any + // host this arm runs on; `_mm256_mul_epu32` operates on register + // values only. + unsafe { Self::from_avx2_halves(_mm256_mul_epu32(a_lo, b_lo), _mm256_mul_epu32(a_hi, b_hi)) } + } } /// Lane-wise variable shifts for the mask family's word ops (the Morton hex diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 986017b9..f33595e3 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -1784,6 +1784,27 @@ impl U64x8 { Self(unsafe { _mm512_rorv_epi64(self.0, _mm512_set1_epi64(n as i64)) }) } + /// Lane-wise `lo32(self) × lo32(rhs)` as an exact `u64` — the widening + /// 32×32→64 multiply (`VPMULUDQ`). The high 32 bits of every input lane + /// are ignored; the product cannot overflow, since `(2³²−1)² < 2⁶⁴`. + /// + /// This is the multiply in argon2's BlaMka G function, + /// `a + b + 2·lo32(a)·lo32(b)`, and the limb multiply of radix-2²⁶ + /// Poly1305 and curve25519 field arithmetic. + /// + /// Written as the intrinsic rather than portable source on this tier: in + /// BlaMka's add chain LLVM lowered the portable `(a as u32 as u64) * + /// (b as u32 as u64)` to `VPMULLQ` (the full 64-bit multiply) instead of + /// `VPMULUDQ` on x86-64-v4, although both operands are provably + /// zero-extended. + #[inline(always)] + pub fn mul_lo32(self, rhs: Self) -> Self { + // SAFETY: `Self` is a native `__m512i`; this arm is compiled only + // under the avx512f dispatch. `_mm512_mul_epu32` reads the low 32 bits + // of each 64-bit lane and writes the full 64-bit product. + Self(unsafe { _mm512_mul_epu32(self.0, rhs.0) }) + } + #[inline(always)] pub fn splat(v: u64) -> Self { Self(unsafe { _mm512_set1_epi64(v as i64) }) diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 487b8688..ccc77062 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -2824,6 +2824,19 @@ impl U64x8 { self.rotate_left(64 - n) } + /// Lane-wise `lo32(self) × lo32(rhs)` as an exact `u64` — the widening + /// 32×32→64 multiply. Per quad: `vmovn_u64` keeps the low 32 bits of each + /// lane, then `vmull_u32` (`UMULL`) widens the product back to 64 bits. + /// The high 32 bits of every input lane are ignored; the product cannot + /// overflow. argon2's BlaMka multiply — written explicitly because LLVM + /// left the portable form as scalar `madd` inside BlaMka's add chain on + /// aarch64. + #[inline(always)] + pub fn mul_lo32(self, rhs: Self) -> Self { + // SAFETY: NEON baseline; pure register ops on uint64x2_t / uint32x2_t. + Self(core::array::from_fn(|p| unsafe { U64x2(vmull_u32(vmovn_u64(self.0[p].0), vmovn_u64(rhs.0[p].0))) })) + } + /// Lane-wise population count: `vcntq_u8` on the bytes, then the /// `vpaddlq_u8 → vpaddlq_u16 → vpaddlq_u32` widening-add ladder back to /// one count per u64 lane (0..=64). diff --git a/src/simd_nightly/u_word_types.rs b/src/simd_nightly/u_word_types.rs index 9c8dd1c9..b87eb797 100644 --- a/src/simd_nightly/u_word_types.rs +++ b/src/simd_nightly/u_word_types.rs @@ -62,6 +62,16 @@ impl U64x8 { Self::from_array(o) } + /// Lane-wise `lo32(self) × lo32(rhs)` as an exact `u64` — the widening + /// 32×32→64 multiply (argon2's BlaMka). Masking both operands to their + /// low 32 bits makes the 64-bit `core::simd` multiply exact: the product + /// of two values below 2³² cannot overflow a `u64`. + #[inline(always)] + pub fn mul_lo32(self, rhs: Self) -> Self { + let lo = u64x8::splat(0xFFFF_FFFF); + Self((self.0 & lo) * (rhs.0 & lo)) + } + #[inline(always)] pub fn splat(v: u64) -> Self { Self(u64x8::splat(v)) diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 6ed3efd6..cc4d3375 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -572,6 +572,21 @@ impl U64x8 { } Self::from_array(o) } + + /// Lane-wise `lo32(self) × lo32(rhs)` as an exact `u64` — the widening + /// 32×32→64 multiply. The high 32 bits of every input lane are ignored; + /// the product cannot overflow, since `(2³²−1)² < 2⁶⁴`. argon2's BlaMka + /// multiply. This loop is the reference the native arms are tested + /// against. + #[inline(always)] + pub fn mul_lo32(self, rhs: Self) -> Self { + let (a, b) = (self.to_array(), rhs.to_array()); + let mut o = [0u64; 8]; + for i in 0..8 { + o[i] = (a[i] as u32 as u64) * (b[i] as u32 as u64); + } + Self::from_array(o) + } } // I8/I16 SIMD types (scalar fallback) diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index dff29d18..8a61a35c 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -1859,6 +1859,22 @@ pub mod wasm32_simd { self.rotate_left(64 - n) } + /// Lane-wise `lo32(self) × lo32(rhs)` as an exact `u64` — the + /// widening 32×32→64 multiply. Per `v128`: the low 32 bits of the two + /// u64 lanes are u32 lanes 0 and 2 (little-endian), gathered into + /// lanes 0 and 1 by `i32x4_shuffle::<0, 2, 0, 2>`, then + /// `u64x2_extmul_low_u32x4` widens their product. The high 32 bits + /// of every input lane are ignored; the product cannot overflow. + /// argon2's BlaMka multiply. + #[inline(always)] + pub fn mul_lo32(self, rhs: Self) -> Self { + Self(core::array::from_fn(|p| { + let a = i32x4_shuffle::<0, 2, 0, 2>(self.0[p].0, self.0[p].0); + let b = i32x4_shuffle::<0, 2, 0, 2>(rhs.0[p].0, rhs.0[p].0); + U64x2(u64x2_extmul_low_u32x4(a, b)) + })) + } + /// Lane-wise population count: `i8x16_popcnt`, then the pairwise /// widening adds up to 32-bit halves, then the two halves of each u64 /// summed (`u64x2_shr` 32 + masked add). From 2cd424642481dd750c88b5a12f8184ed936109b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 22:22:40 +0000 Subject: [PATCH 2/2] blackboard: U64x8::mul_lo32 landed; clippy --config placement trap Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019HnekoM1EidTwQLS3oFVFm --- .claude/blackboard.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 1bcff2e2..6e547797 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,15 @@ +## 2026-09-24 (2) — U64x8::mul_lo32: the widening lo32×lo32→u64 multiply (argon2 BlaMka) + +Added on all six realizations: AVX-512 `_mm512_mul_epu32`; AVX2 `_mm256_mul_epu32` per half; NEON `vmovn_u64` + `vmull_u32`; wasm `i32x4_shuffle::<0,2,0,2>` + `u64x2_extmul_low_u32x4`; scalar reference loop; nightly masked `core::simd` multiply. Unblocks argon2's BlaMka (`a + b + 2·lo32(a)·lo32(b)`), which `ogar-encryption` → a2ui sessions and ogar-auth logins run; also the limb multiply radix-2²⁶ Poly1305 / curve25519 need. + +Why intrinsics, not portable source, on AVX-512 and NEON: the LLVM probe (see the 2026-09-24 inventory entry) showed the portable form in BlaMka's add chain lowered to `VPMULLQ` on x86-64-v4 and stayed scalar `madd` on aarch64. + +**Evidence:** `simd::tests::u64x8_mul_lo32_matches_scalar` (every operand lane has non-zero high bits and is asserted to differ under a full 64-bit multiply; 5 lanes exceed 32-bit products; commutativity; RFC 9106 fBlaMka) passes native (AVX-512) and pinned v3; `neon-parity.sh` (qemu) and `wasm-parity.sh` (node) pass with the new `0x30F` check. **Disable runs, all red:** AVX-512 → `_mm512_mullo_epi64` fails lane 0; NEON high-half `vshrn` → rc 783 (0x30F); wasm `<1,3,1,3>` shuffle → rc 783. Full `cargo test --lib` native: 2478 passed. clippy `-D warnings` clean native and v3. Not run: the nightly-simd arm (no nightly toolchain here; CI's nightly rows cover it). + +**Trap found:** `cargo --config X clippy` does NOT apply `X` — `clippy` is an external subcommand and cargo does not forward `--config` given before it. Measured: rustc ran with `target-cpu=native` only. `cargo clippy --config X` works (`native` then `x86-64-v3`, last wins). `cargo --config X test` is fine (built-in subcommand). A "v3 clippy" run the first way silently re-checks the native tier. + +**Next:** vendor `argon2` and give `Block::compress` a vertical `U64x8` lane (8 independent G's per register; rotates already exist); RFC 9106 vectors as the oracle. + ## 2026-09-24 — per-CPU SIMD inventory generated from LLVM's TableGen source `tools/gen_llvm_inventory.py` reads llvm-project's `X86.td`, `AArch64Processors.td`, `AArch64Features.td` (plus the instruction/predicate `.td` files) at the `llvmorg-*` tag matching `rustc -vV` (22.1.8 under the pinned 1.98.1), resolves every processor's features, and writes `tools/llvm-inventory/inventory.json` + `.claude/knowledge/llvm-cpu-inventory.md`. Same pattern as `gen_ternlog_bodies.py`: committed output, generator as provenance; default mode exits 1 if the output is stale.