From 619a39e10db196c63c9d05f21fde9241a53a81c2 Mon Sep 17 00:00:00 2001 From: Patrik Wenger Date: Fri, 17 Jul 2026 12:41:50 +0200 Subject: [PATCH 1/5] salsa20: add SSE2/AVX2 intrinsics backends Add SIMD-accelerated Salsa20 backends using x86 intrinsics with runtime CPU detection via cpufeatures (no_std compatible, same pattern as chacha20). Both backends use diagonal SIMD layout: each register holds one diagonal of the 4x4 state matrix, with shuffles rotating between column and row round alignment. SSE2: 4 parallel blocks, 4 __m128i registers per block. AVX2: 4 parallel blocks via 2 __m256i sets, each packing 2 blocks in its 128-bit lanes. Dispatch order: AVX2 > SSE2 > scalar fallback. Measured 3.1x throughput on Salsa20/20 (543 -> 1673 MB/s) on x86_64 with AVX2. --- Cargo.lock | 313 +++++++++++++++++++++++++++++++++++ salsa20/CHANGELOG.md | 5 + salsa20/Cargo.toml | 4 + salsa20/README.md | 4 + salsa20/examples/ctbench.rs | 144 ++++++++++++++++ salsa20/src/backends.rs | 7 + salsa20/src/backends/avx2.rs | 228 +++++++++++++++++++++++++ salsa20/src/backends/soft.rs | 2 +- salsa20/src/backends/sse2.rs | 169 +++++++++++++++++++ salsa20/src/lib.rs | 49 +++++- salsa20/src/xsalsa.rs | 13 +- 11 files changed, 927 insertions(+), 11 deletions(-) create mode 100644 salsa20/examples/ctbench.rs create mode 100644 salsa20/src/backends/avx2.rs create mode 100644 salsa20/src/backends/sse2.rs diff --git a/Cargo.lock b/Cargo.lock index c49e7b75..b6b73500 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,38 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "blobby" version = "0.4.0" @@ -18,12 +50,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.1" @@ -36,6 +83,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + [[package]] name = "cipher" version = "0.5.2" @@ -49,6 +107,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -67,6 +140,53 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "dudect-bencher" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6174adfb35811a6845001876823965032406784c36e795edb84a74353455e1" +dependencies = [ + "clap", + "ctrlc", + "rand", + "rand_chacha", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + [[package]] name = "hc-256" version = "0.6.0" @@ -75,6 +195,15 @@ dependencies = [ "hex-literal", ] +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex-literal" version = "1.1.0" @@ -106,6 +235,66 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rabbit" version = "0.5.0" @@ -114,6 +303,27 @@ dependencies = [ "hex-literal", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.10.1" @@ -134,15 +344,118 @@ version = "0.11.0" dependencies = [ "cfg-if", "cipher", + "cpufeatures", + "dudect-bencher", "hex-literal", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zeroize" version = "1.9.0" diff --git a/salsa20/CHANGELOG.md b/salsa20/CHANGELOG.md index efc51c61..410fdf34 100644 --- a/salsa20/CHANGELOG.md +++ b/salsa20/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- SSE2 and AVX2 intrinsics backends with runtime detection via `cpufeatures` +- `dudect` constant-time timing tests (`examples/ctbench.rs`) + ## 0.11.0 (2026-03-30) ### Added - SSE2 backend ([#333]) diff --git a/salsa20/Cargo.toml b/salsa20/Cargo.toml index 3f3e549a..06ba2618 100644 --- a/salsa20/Cargo.toml +++ b/salsa20/Cargo.toml @@ -16,8 +16,12 @@ description = "Pure Rust implementation of the Salsa20 stream cipher" cfg-if = "1" cipher = { version = "0.5", features = ["stream-wrapper"] } +[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'.dependencies] +cpufeatures = "0.3" + [dev-dependencies] cipher = { version = "0.5", features = ["dev"] } +dudect-bencher = "0.7" hex-literal = "1" [features] diff --git a/salsa20/README.md b/salsa20/README.md index 9b7b38be..903e2a7f 100644 --- a/salsa20/README.md +++ b/salsa20/README.md @@ -11,6 +11,9 @@ Implementation of the [Salsa] family of stream ciphers, including the [XSalsa] variants with an extended 192-bit (24-byte) nonce. +SIMD-accelerated on x86/x86\_64 with SSE2 and AVX2 backends, selected +automatically at runtime via [cpufeatures]. + ## ⚠️ Security Warning: [Hazmat!][hazmat-link] This crate does not ensure ciphertexts are authentic (i.e. by using a MAC to @@ -96,3 +99,4 @@ dual licensed as above, without any additional terms or conditions. [Salsa]: https://en.wikipedia.org/wiki/Salsa20 [XSalsa]: https://cr.yp.to/snuffle/xsalsa-20081128.pdf +[cpufeatures]: https://crates.io/crates/cpufeatures diff --git a/salsa20/examples/ctbench.rs b/salsa20/examples/ctbench.rs new file mode 100644 index 00000000..b8da4f4f --- /dev/null +++ b/salsa20/examples/ctbench.rs @@ -0,0 +1,144 @@ +//! Constant-time verification using dudect statistical timing tests. +//! +//! Run with: cargo run --release --example ctbench + +use dudect_bencher::rand::{Rng, RngExt}; +use dudect_bencher::{BenchRng, Class, CtRunner, ctbench_main}; +use salsa20::cipher::{KeyIvInit, StreamCipher}; + +fn hsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng) { + use salsa20::hsalsa; + + let input = [0x13u8; 16]; + let fixed_key = [0u8; 32]; + + let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(100_000); + for _ in 0..100_000 { + if rng.random::() { + inputs.push((fixed_key, Class::Left)); + } else { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + inputs.push((key, Class::Right)); + } + } + + for (key, class) in &inputs { + let key: salsa20::Key = (*key).into(); + let input: salsa20::cipher::array::Array = input.into(); + runner.run_one(*class, || { + let _ = hsalsa::(&key, &input); + }); + } +} + +fn xsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { + let nonce = [0x13u8; 24]; + let fixed_key = [0u8; 32]; + + let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(n); + for _ in 0..n { + if rng.random::() { + inputs.push((fixed_key, Class::Left)); + } else { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + inputs.push((key, Class::Right)); + } + } + + let mut buf = vec![0u8; size]; + for (key, class) in &inputs { + buf.fill(0); + let key = *key; + let buf_ptr = buf.as_mut_ptr(); + let buf_len = buf.len(); + runner.run_one(*class, || { + let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; + let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); + cipher.apply_keystream(buf); + }); + } +} + +fn xsalsa20_data(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { + let key = [0x42u8; 32]; + let nonce = [0x13u8; 24]; + + let mut inputs: Vec<(Vec, Class)> = Vec::with_capacity(n); + for _ in 0..n { + if rng.random::() { + inputs.push((vec![0u8; size], Class::Left)); + } else { + let mut data = vec![0u8; size]; + rng.fill_bytes(&mut data); + inputs.push((data, Class::Right)); + } + } + + let mut buf = vec![0u8; size]; + for (data, class) in &inputs { + buf.copy_from_slice(data); + let buf_ptr = buf.as_mut_ptr(); + let buf_len = buf.len(); + runner.run_one(*class, || { + let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; + let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); + cipher.apply_keystream(buf); + }); + } +} + +fn xsalsa20_key_64(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 64, 100_000); +} +fn xsalsa20_key_256(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 256, 50_000); +} +fn xsalsa20_key_1024(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 1024, 10_000); +} +fn xsalsa20_key_4096(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 4096, 5_000); +} +fn xsalsa20_key_16384(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 16384, 2_000); +} +fn xsalsa20_key_65536(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 65536, 1_000); +} + +fn xsalsa20_data_64(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 64, 100_000); +} +fn xsalsa20_data_256(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 256, 50_000); +} +fn xsalsa20_data_1024(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 1024, 10_000); +} +fn xsalsa20_data_4096(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 4096, 5_000); +} +fn xsalsa20_data_16384(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 16384, 2_000); +} +fn xsalsa20_data_65536(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 65536, 1_000); +} + +ctbench_main!( + hsalsa20_key, + xsalsa20_key_64, + xsalsa20_key_256, + xsalsa20_key_1024, + xsalsa20_key_4096, + xsalsa20_key_16384, + xsalsa20_key_65536, + xsalsa20_data_64, + xsalsa20_data_256, + xsalsa20_data_1024, + xsalsa20_data_4096, + xsalsa20_data_16384, + xsalsa20_data_65536 +); diff --git a/salsa20/src/backends.rs b/salsa20/src/backends.rs index fbc9393c..a1ddcb6a 100644 --- a/salsa20/src/backends.rs +++ b/salsa20/src/backends.rs @@ -1 +1,8 @@ pub(crate) mod soft; + +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + pub(crate) mod avx2; + pub(crate) mod sse2; + } +} diff --git a/salsa20/src/backends/avx2.rs b/salsa20/src/backends/avx2.rs new file mode 100644 index 00000000..5ede5393 --- /dev/null +++ b/salsa20/src/backends/avx2.rs @@ -0,0 +1,228 @@ +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(clippy::cast_possible_wrap)] + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::{STATE_WORDS, Unsigned}; +use cipher::{ + BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure, + consts::{U4, U64}, +}; +use core::marker::PhantomData; + +/// Number of blocks processed in parallel. +const PAR_BLOCKS: usize = 4; +/// Number of `__m256i` sets (each holds 2 blocks). +const N: usize = PAR_BLOCKS / 2; + +/// Diagonal indices for the Salsa20 state matrix. +const IDX: [[usize; 4]; 4] = [[0, 5, 10, 15], [4, 9, 14, 3], [8, 13, 2, 7], [12, 1, 6, 11]]; + +macro_rules! rotl_epi32 { + ($v:expr, $n:literal) => {{ + let v = $v; + _mm256_or_si256(_mm256_slli_epi32(v, $n), _mm256_srli_epi32(v, 32 - $n)) + }}; +} + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamCipherClosure, +{ + f.call(&mut Backend:: { + state, + _pd: PhantomData, + }); +} + +struct Backend<'a, R: Unsigned> { + state: &'a mut [u32; STATE_WORDS], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend<'_, R> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend<'_, R> { + type ParBlocksSize = U4; +} + +impl StreamCipherBackend for Backend<'_, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut cipher::Block) { + unsafe { + let res = rounds::(self.state); + let lo = extract_lo(&res[0]); + store_block(&lo, block); + let counter = get_counter(self.state); + set_counter(self.state, counter + 1); + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + unsafe { + let res = rounds::(self.state); + for i in 0..N { + let lo = extract_lo(&res[i]); + let hi = extract_hi(&res[i]); + store_block(&lo, &mut blocks[2 * i]); + store_block(&hi, &mut blocks[2 * i + 1]); + } + let counter = get_counter(self.state); + set_counter(self.state, counter + PAR_BLOCKS as u64); + } + } +} + +/// Load two blocks' diagonal values into `__m256i` registers. +/// +/// Each `__m256i` holds two blocks: the lower 128 bits contain block 0's +/// diagonal and the upper 128 bits contain block 1's diagonal. +#[inline(always)] +unsafe fn load_diag_pair(state: &[u32; STATE_WORDS], ctr0: u64, ctr1: u64) -> [__m256i; 4] { + let s = state; + [ + // a: [s0, s5, s10, s15] (same for both blocks) + _mm256_set_epi32( + s[15] as i32, + s[10] as i32, + s[5] as i32, + s[0] as i32, + s[15] as i32, + s[10] as i32, + s[5] as i32, + s[0] as i32, + ), + // b: [s4, ctr_hi, s14, s3] (ctr_hi differs) + _mm256_set_epi32( + s[3] as i32, + s[14] as i32, + (ctr1 >> 32) as i32, + s[4] as i32, + s[3] as i32, + s[14] as i32, + (ctr0 >> 32) as i32, + s[4] as i32, + ), + // c: [ctr_lo, s13, s2, s7] (ctr_lo differs) + _mm256_set_epi32( + s[7] as i32, + s[2] as i32, + s[13] as i32, + ctr1 as i32, + s[7] as i32, + s[2] as i32, + s[13] as i32, + ctr0 as i32, + ), + // d: [s12, s1, s6, s11] (same for both blocks) + _mm256_set_epi32( + s[11] as i32, + s[6] as i32, + s[1] as i32, + s[12] as i32, + s[11] as i32, + s[6] as i32, + s[1] as i32, + s[12] as i32, + ), + ] +} + +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn rounds(state: &[u32; STATE_WORDS]) -> [[__m256i; 4]; N] { + let counter = get_counter(state); + let mut res = [[_mm256_setzero_si256(); 4]; N]; + for (i, block) in res.iter_mut().enumerate() { + let base = counter + (i * 2) as u64; + *block = load_diag_pair(state, base, base + 1); + } + let initial = res; + + for _ in 0..R::USIZE * 2 { + for block in res.iter_mut() { + quarter_round(block); + shuffle(block); + } + } + + for i in 0..N { + for j in 0..4 { + res[i][j] = _mm256_add_epi32(res[i][j], initial[i][j]); + } + } + + res +} + +#[inline(always)] +unsafe fn quarter_round(block: &mut [__m256i; 4]) { + let [a, b, c, d] = block; + *b = _mm256_xor_si256(*b, rotl_epi32!(_mm256_add_epi32(*a, *d), 7)); + *c = _mm256_xor_si256(*c, rotl_epi32!(_mm256_add_epi32(*b, *a), 9)); + *d = _mm256_xor_si256(*d, rotl_epi32!(_mm256_add_epi32(*c, *b), 13)); + *a = _mm256_xor_si256(*a, rotl_epi32!(_mm256_add_epi32(*d, *c), 18)); +} + +#[inline(always)] +unsafe fn shuffle(block: &mut [__m256i; 4]) { + let [_, b, c, d] = block; + let tmp = *b; + *b = _mm256_shuffle_epi32(*d, 0b_00_11_10_01); + *c = _mm256_shuffle_epi32(*c, 0b_01_00_11_10); + *d = _mm256_shuffle_epi32(tmp, 0b_10_01_00_11); +} + +#[inline(always)] +unsafe fn extract_lo(set: &[__m256i; 4]) -> [__m128i; 4] { + [ + _mm256_castsi256_si128(set[0]), + _mm256_castsi256_si128(set[1]), + _mm256_castsi256_si128(set[2]), + _mm256_castsi256_si128(set[3]), + ] +} + +#[inline(always)] +unsafe fn extract_hi(set: &[__m256i; 4]) -> [__m128i; 4] { + [ + _mm256_extracti128_si256(set[0], 1), + _mm256_extracti128_si256(set[1], 1), + _mm256_extracti128_si256(set[2], 1), + _mm256_extracti128_si256(set[3], 1), + ] +} + +/// Convert diagonal SIMD layout back to sequential little-endian bytes. +#[inline(always)] +unsafe fn store_block(diag: &[__m128i; 4], block: &mut [u8]) { + let mut vals = [[0u32; 4]; 4]; + for (i, v) in diag.iter().enumerate() { + _mm_storeu_si128(vals[i].as_mut_ptr().cast(), *v); + } + for (reg, idxs) in vals.iter().zip(IDX.iter()) { + for (&val, &idx) in reg.iter().zip(idxs.iter()) { + block[idx * 4..idx * 4 + 4].copy_from_slice(&val.to_le_bytes()); + } + } +} + +#[inline(always)] +fn get_counter(state: &[u32; STATE_WORDS]) -> u64 { + (state[8] as u64) | ((state[9] as u64) << 32) +} + +#[inline(always)] +fn set_counter(state: &mut [u32; STATE_WORDS], pos: u64) { + state[8] = pos as u32; + state[9] = (pos >> 32) as u32; +} diff --git a/salsa20/src/backends/soft.rs b/salsa20/src/backends/soft.rs index caf2693f..66b52b36 100644 --- a/salsa20/src/backends/soft.rs +++ b/salsa20/src/backends/soft.rs @@ -46,7 +46,7 @@ pub(crate) fn quarter_round( } #[inline(always)] -fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { +pub(crate) fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { let mut res = *state; for _ in 0..R::USIZE { diff --git a/salsa20/src/backends/sse2.rs b/salsa20/src/backends/sse2.rs new file mode 100644 index 00000000..2ea9df98 --- /dev/null +++ b/salsa20/src/backends/sse2.rs @@ -0,0 +1,169 @@ +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(clippy::cast_possible_wrap)] + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::{STATE_WORDS, Unsigned}; +use cipher::{ + BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure, + consts::{U4, U64}, +}; +use core::marker::PhantomData; + +const PAR_BLOCKS: usize = 4; + +/// Diagonal indices for the Salsa20 state matrix. +/// +/// The 4x4 state is packed into 4 SSE registers so that each register +/// holds one diagonal. All four column quarter-rounds then execute +/// as a single SIMD operation, and shuffles rotate between +/// column and row round alignment. +const IDX: [[usize; 4]; 4] = [[0, 5, 10, 15], [4, 9, 14, 3], [8, 13, 2, 7], [12, 1, 6, 11]]; + +macro_rules! rotl_epi32 { + ($v:expr, $n:literal) => {{ + let v = $v; + _mm_or_si128(_mm_slli_epi32(v, $n), _mm_srli_epi32(v, 32 - $n)) + }}; +} + +#[inline] +#[target_feature(enable = "sse2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamCipherClosure, +{ + f.call(&mut Backend:: { + state, + _pd: PhantomData, + }); +} + +struct Backend<'a, R: Unsigned> { + state: &'a mut [u32; STATE_WORDS], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend<'_, R> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend<'_, R> { + type ParBlocksSize = U4; +} + +impl StreamCipherBackend for Backend<'_, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut cipher::Block) { + unsafe { + let res = rounds::(self.state); + store_block(&res[0], block); + let counter = get_counter(self.state); + set_counter(self.state, counter + 1); + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + unsafe { + let res = rounds::(self.state); + for i in 0..PAR_BLOCKS { + store_block(&res[i], &mut blocks[i]); + } + let counter = get_counter(self.state); + set_counter(self.state, counter + PAR_BLOCKS as u64); + } + } +} + +#[inline(always)] +unsafe fn load_diag(state: &[u32; STATE_WORDS]) -> [__m128i; 4] { + let s = |i: usize| state[i] as i32; + [ + _mm_set_epi32(s(IDX[0][3]), s(IDX[0][2]), s(IDX[0][1]), s(IDX[0][0])), + _mm_set_epi32(s(IDX[1][3]), s(IDX[1][2]), s(IDX[1][1]), s(IDX[1][0])), + _mm_set_epi32(s(IDX[2][3]), s(IDX[2][2]), s(IDX[2][1]), s(IDX[2][0])), + _mm_set_epi32(s(IDX[3][3]), s(IDX[3][2]), s(IDX[3][1]), s(IDX[3][0])), + ] +} + +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn rounds(state: &[u32; STATE_WORDS]) -> [[__m128i; 4]; PAR_BLOCKS] { + let counter = get_counter(state); + let mut tmp = *state; + let mut res = [[_mm_setzero_si128(); 4]; PAR_BLOCKS]; + for (i, block) in res.iter_mut().enumerate() { + let ctr = counter + i as u64; + tmp[8] = ctr as u32; + tmp[9] = (ctr >> 32) as u32; + *block = load_diag(&tmp); + } + let initial = res; + + for _ in 0..R::USIZE * 2 { + for block in res.iter_mut() { + quarter_round(block); + shuffle(block); + } + } + + for i in 0..PAR_BLOCKS { + for j in 0..4 { + res[i][j] = _mm_add_epi32(res[i][j], initial[i][j]); + } + } + + res +} + +#[inline(always)] +unsafe fn quarter_round(block: &mut [__m128i; 4]) { + let [a, b, c, d] = block; + *b = _mm_xor_si128(*b, rotl_epi32!(_mm_add_epi32(*a, *d), 7)); + *c = _mm_xor_si128(*c, rotl_epi32!(_mm_add_epi32(*b, *a), 9)); + *d = _mm_xor_si128(*d, rotl_epi32!(_mm_add_epi32(*c, *b), 13)); + *a = _mm_xor_si128(*a, rotl_epi32!(_mm_add_epi32(*d, *c), 18)); +} + +/// Rotate diagonal alignment between column and row rounds. +/// +/// The same shuffle+swap sequence converts column layout to row layout +/// and vice versa, so it is applied identically after every quarter round. +#[inline(always)] +unsafe fn shuffle(block: &mut [__m128i; 4]) { + let [_, b, c, d] = block; + let tmp = *b; + *b = _mm_shuffle_epi32(*d, 0b_00_11_10_01); + *c = _mm_shuffle_epi32(*c, 0b_01_00_11_10); + *d = _mm_shuffle_epi32(tmp, 0b_10_01_00_11); +} + +/// Convert diagonal SIMD layout back to sequential little-endian bytes. +#[inline(always)] +unsafe fn store_block(diag: &[__m128i; 4], block: &mut [u8]) { + let mut vals = [[0u32; 4]; 4]; + for (i, v) in diag.iter().enumerate() { + _mm_storeu_si128(vals[i].as_mut_ptr().cast(), *v); + } + for (reg, idxs) in vals.iter().zip(IDX.iter()) { + for (&val, &idx) in reg.iter().zip(idxs.iter()) { + block[idx * 4..idx * 4 + 4].copy_from_slice(&val.to_le_bytes()); + } + } +} + +#[inline(always)] +fn get_counter(state: &[u32; STATE_WORDS]) -> u64 { + (state[8] as u64) | ((state[9] as u64) << 32) +} + +#[inline(always)] +fn set_counter(state: &mut [u32; STATE_WORDS], pos: u64) { + state[8] = pos as u32; + state[9] = (pos >> 32) as u32; +} diff --git a/salsa20/src/lib.rs b/salsa20/src/lib.rs index e794b593..e6e966e2 100644 --- a/salsa20/src/lib.rs +++ b/salsa20/src/lib.rs @@ -23,6 +23,16 @@ use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; mod backends; mod xsalsa; +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cpufeatures::new!(avx2_cpuid, "avx2"); + cpufeatures::new!(sse2_cpuid, "sse2"); + type Tokens = (avx2_cpuid::InitToken, sse2_cpuid::InitToken); + } else { + type Tokens = (); + } +} + pub use xsalsa::{XSalsa8, XSalsa12, XSalsa20, XSalsaCore, hsalsa}; /// Salsa20/8 stream cipher @@ -56,6 +66,9 @@ const CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574] pub struct SalsaCore { /// Internal state of the core function state: [u32; STATE_WORDS], + /// CPU target feature tokens + #[allow(dead_code)] + tokens: Tokens, /// Number of rounds to perform rounds: PhantomData, } @@ -66,8 +79,16 @@ impl SalsaCore { /// This method is mainly intended for the `scrypt` crate. /// Other users generally should not use this method. pub fn from_raw_state(state: [u32; STATE_WORDS]) -> Self { + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); + } else { + let tokens = (); + } + } Self { state, + tokens, rounds: PhantomData, } } @@ -110,8 +131,17 @@ impl KeyIvInit for SalsaCore { state[15] = CONSTANTS[3]; + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); + } else { + let tokens = (); + } + } + Self { state, + tokens, rounds: PhantomData, } } @@ -124,7 +154,24 @@ impl StreamCipherCore for SalsaCore { rem.try_into().ok() } fn process_with_backend(&mut self, f: impl StreamCipherClosure) { - f.call(&mut backends::soft::Backend(self)); + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + let (avx2_token, sse2_token) = self.tokens; + if avx2_token.get() { + unsafe { + backends::avx2::inner::(&mut self.state, f); + } + } else if sse2_token.get() { + unsafe { + backends::sse2::inner::(&mut self.state, f); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } } } diff --git a/salsa20/src/xsalsa.rs b/salsa20/src/xsalsa.rs index 9f84cef5..ae1482fc 100644 --- a/salsa20/src/xsalsa.rs +++ b/salsa20/src/xsalsa.rs @@ -8,8 +8,6 @@ use cipher::{ consts::{U4, U6, U10, U16, U24, U32, U64}, }; -use crate::backends::soft::quarter_round; - #[cfg(feature = "zeroize")] use cipher::zeroize::ZeroizeOnDrop; @@ -113,27 +111,24 @@ pub fn hsalsa(key: &Key, input: &Array) -> Array .for_each(|(v, chunk)| *v = to_u32(chunk)); state[15] = CONSTANTS[3]; - // 20 rounds consisting of 10 column rounds and 10 diagonal rounds + use crate::backends::soft::quarter_round; for _ in 0..R::USIZE { - // column rounds quarter_round(0, 4, 8, 12, &mut state); quarter_round(5, 9, 13, 1, &mut state); quarter_round(10, 14, 2, 6, &mut state); quarter_round(15, 3, 7, 11, &mut state); - - // diagonal rounds quarter_round(0, 1, 2, 3, &mut state); quarter_round(5, 6, 7, 4, &mut state); quarter_round(10, 11, 8, 9, &mut state); quarter_round(15, 12, 13, 14, &mut state); } - let mut output = Array::default(); let key_idx: [usize; 8] = [0, 5, 10, 15, 6, 7, 8, 9]; + let words: [u32; 8] = core::array::from_fn(|i| state[key_idx[i]]); + let mut output = Array::default(); for (i, chunk) in output.chunks_exact_mut(4).enumerate() { - chunk.copy_from_slice(&state[key_idx[i]].to_le_bytes()); + chunk.copy_from_slice(&words[i].to_le_bytes()); } - output } From c14c65e358027d533308ecd6c1c962d5b0cb338b Mon Sep 17 00:00:00 2001 From: Patrik Wenger Date: Fri, 17 Jul 2026 16:03:18 +0200 Subject: [PATCH 2/5] salsa20: remove dudect constant-time bench example Remove `examples/ctbench.rs` and the `dudect-bencher` dev-dependency as requested during review. Criterion-based benchmarks will be added in the repo-level `benches/` crate instead. --- Cargo.lock | 312 ------------------------------------ salsa20/Cargo.toml | 1 - salsa20/examples/ctbench.rs | 144 ----------------- 3 files changed, 457 deletions(-) delete mode 100644 salsa20/examples/ctbench.rs diff --git a/Cargo.lock b/Cargo.lock index b6b73500..402cd83b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,38 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "ansi_term" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" -dependencies = [ - "winapi", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" - [[package]] name = "blobby" version = "0.4.0" @@ -50,27 +18,12 @@ dependencies = [ "zeroize", ] -[[package]] -name = "block2" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" -dependencies = [ - "objc2", -] - [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - [[package]] name = "chacha20" version = "0.10.1" @@ -83,17 +36,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures", - "rand_core", -] - [[package]] name = "cipher" version = "0.5.2" @@ -107,21 +49,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "clap" -version = "2.34.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" -dependencies = [ - "ansi_term", - "atty", - "bitflags 1.3.2", - "strsim", - "textwrap", - "unicode-width", - "vec_map", -] - [[package]] name = "cpufeatures" version = "0.3.0" @@ -140,53 +67,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "ctrlc" -version = "3.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" -dependencies = [ - "dispatch2", - "nix", - "windows-sys", -] - -[[package]] -name = "dispatch2" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" -dependencies = [ - "bitflags 2.13.1", - "block2", - "libc", - "objc2", -] - -[[package]] -name = "dudect-bencher" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6174adfb35811a6845001876823965032406784c36e795edb84a74353455e1" -dependencies = [ - "clap", - "ctrlc", - "rand", - "rand_chacha", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "rand_core", -] - [[package]] name = "hc-256" version = "0.6.0" @@ -195,15 +75,6 @@ dependencies = [ "hex-literal", ] -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hex-literal" version = "1.1.0" @@ -235,66 +106,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.1", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "objc2" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" -dependencies = [ - "objc2-encode", -] - -[[package]] -name = "objc2-encode" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "rabbit" version = "0.5.0" @@ -303,27 +114,6 @@ dependencies = [ "hex-literal", ] -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "getrandom", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - [[package]] name = "rand_core" version = "0.10.1" @@ -345,117 +135,15 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures", - "dudect-bencher", "hex-literal", ] -[[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "textwrap" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -dependencies = [ - "unicode-width", -] - [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "zerocopy" -version = "0.8.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zeroize" version = "1.9.0" diff --git a/salsa20/Cargo.toml b/salsa20/Cargo.toml index 06ba2618..75d8af20 100644 --- a/salsa20/Cargo.toml +++ b/salsa20/Cargo.toml @@ -21,7 +21,6 @@ cpufeatures = "0.3" [dev-dependencies] cipher = { version = "0.5", features = ["dev"] } -dudect-bencher = "0.7" hex-literal = "1" [features] diff --git a/salsa20/examples/ctbench.rs b/salsa20/examples/ctbench.rs deleted file mode 100644 index b8da4f4f..00000000 --- a/salsa20/examples/ctbench.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! Constant-time verification using dudect statistical timing tests. -//! -//! Run with: cargo run --release --example ctbench - -use dudect_bencher::rand::{Rng, RngExt}; -use dudect_bencher::{BenchRng, Class, CtRunner, ctbench_main}; -use salsa20::cipher::{KeyIvInit, StreamCipher}; - -fn hsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng) { - use salsa20::hsalsa; - - let input = [0x13u8; 16]; - let fixed_key = [0u8; 32]; - - let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(100_000); - for _ in 0..100_000 { - if rng.random::() { - inputs.push((fixed_key, Class::Left)); - } else { - let mut key = [0u8; 32]; - rng.fill_bytes(&mut key); - inputs.push((key, Class::Right)); - } - } - - for (key, class) in &inputs { - let key: salsa20::Key = (*key).into(); - let input: salsa20::cipher::array::Array = input.into(); - runner.run_one(*class, || { - let _ = hsalsa::(&key, &input); - }); - } -} - -fn xsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { - let nonce = [0x13u8; 24]; - let fixed_key = [0u8; 32]; - - let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(n); - for _ in 0..n { - if rng.random::() { - inputs.push((fixed_key, Class::Left)); - } else { - let mut key = [0u8; 32]; - rng.fill_bytes(&mut key); - inputs.push((key, Class::Right)); - } - } - - let mut buf = vec![0u8; size]; - for (key, class) in &inputs { - buf.fill(0); - let key = *key; - let buf_ptr = buf.as_mut_ptr(); - let buf_len = buf.len(); - runner.run_one(*class, || { - let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; - let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); - cipher.apply_keystream(buf); - }); - } -} - -fn xsalsa20_data(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { - let key = [0x42u8; 32]; - let nonce = [0x13u8; 24]; - - let mut inputs: Vec<(Vec, Class)> = Vec::with_capacity(n); - for _ in 0..n { - if rng.random::() { - inputs.push((vec![0u8; size], Class::Left)); - } else { - let mut data = vec![0u8; size]; - rng.fill_bytes(&mut data); - inputs.push((data, Class::Right)); - } - } - - let mut buf = vec![0u8; size]; - for (data, class) in &inputs { - buf.copy_from_slice(data); - let buf_ptr = buf.as_mut_ptr(); - let buf_len = buf.len(); - runner.run_one(*class, || { - let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; - let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); - cipher.apply_keystream(buf); - }); - } -} - -fn xsalsa20_key_64(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_key(r, rng, 64, 100_000); -} -fn xsalsa20_key_256(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_key(r, rng, 256, 50_000); -} -fn xsalsa20_key_1024(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_key(r, rng, 1024, 10_000); -} -fn xsalsa20_key_4096(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_key(r, rng, 4096, 5_000); -} -fn xsalsa20_key_16384(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_key(r, rng, 16384, 2_000); -} -fn xsalsa20_key_65536(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_key(r, rng, 65536, 1_000); -} - -fn xsalsa20_data_64(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_data(r, rng, 64, 100_000); -} -fn xsalsa20_data_256(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_data(r, rng, 256, 50_000); -} -fn xsalsa20_data_1024(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_data(r, rng, 1024, 10_000); -} -fn xsalsa20_data_4096(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_data(r, rng, 4096, 5_000); -} -fn xsalsa20_data_16384(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_data(r, rng, 16384, 2_000); -} -fn xsalsa20_data_65536(r: &mut CtRunner, rng: &mut BenchRng) { - xsalsa20_data(r, rng, 65536, 1_000); -} - -ctbench_main!( - hsalsa20_key, - xsalsa20_key_64, - xsalsa20_key_256, - xsalsa20_key_1024, - xsalsa20_key_4096, - xsalsa20_key_16384, - xsalsa20_key_65536, - xsalsa20_data_64, - xsalsa20_data_256, - xsalsa20_data_1024, - xsalsa20_data_4096, - xsalsa20_data_16384, - xsalsa20_data_65536 -); From be58b6e917fc2267eadf9b0e5667751f706ba84d Mon Sep 17 00:00:00 2001 From: Patrik Wenger Date: Fri, 17 Jul 2026 16:03:31 +0200 Subject: [PATCH 3/5] salsa20: address review feedback - Add `reason` to `#[allow(dead_code)]` on the `tokens` field. - Extract `init_tokens()` helper to deduplicate the `cfg_if!` token initialization in `from_raw_state` and `KeyIvInit::new`. (`Default::default()` as suggested doesn't compile because `cpufeatures::InitToken` doesn't impl `Default`.) - Add PR reference `([#576])` to CHANGELOG entry. --- salsa20/CHANGELOG.md | 4 ++-- salsa20/src/lib.rs | 30 ++++++++++++------------------ 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/salsa20/CHANGELOG.md b/salsa20/CHANGELOG.md index 410fdf34..43e80312 100644 --- a/salsa20/CHANGELOG.md +++ b/salsa20/CHANGELOG.md @@ -7,8 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added -- SSE2 and AVX2 intrinsics backends with runtime detection via `cpufeatures` -- `dudect` constant-time timing tests (`examples/ctbench.rs`) +- SSE2 and AVX2 intrinsics backends with runtime detection via `cpufeatures` ([#576]) ## 0.11.0 (2026-03-30) ### Added @@ -25,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#333]: https://github.com/RustCrypto/stream-ciphers/pull/333 [#338]: https://github.com/RustCrypto/stream-ciphers/pull/338 [#397]: https://github.com/RustCrypto/stream-ciphers/pull/397 +[#576]: https://github.com/RustCrypto/stream-ciphers/pull/576 ## 0.10.2 (2022-02-17) ### Added diff --git a/salsa20/src/lib.rs b/salsa20/src/lib.rs index e6e966e2..8a344a7f 100644 --- a/salsa20/src/lib.rs +++ b/salsa20/src/lib.rs @@ -28,8 +28,14 @@ cfg_if::cfg_if! { cpufeatures::new!(avx2_cpuid, "avx2"); cpufeatures::new!(sse2_cpuid, "sse2"); type Tokens = (avx2_cpuid::InitToken, sse2_cpuid::InitToken); + + fn init_tokens() -> Tokens { + (avx2_cpuid::init(), sse2_cpuid::init()) + } } else { type Tokens = (); + + fn init_tokens() -> Tokens {} } } @@ -67,7 +73,10 @@ pub struct SalsaCore { /// Internal state of the core function state: [u32; STATE_WORDS], /// CPU target feature tokens - #[allow(dead_code)] + #[allow( + dead_code, + reason = "tokens are held for their type-level proof, used by SIMD backends" + )] tokens: Tokens, /// Number of rounds to perform rounds: PhantomData, @@ -79,16 +88,9 @@ impl SalsaCore { /// This method is mainly intended for the `scrypt` crate. /// Other users generally should not use this method. pub fn from_raw_state(state: [u32; STATE_WORDS]) -> Self { - cfg_if::cfg_if! { - if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); - } else { - let tokens = (); - } - } Self { state, - tokens, + tokens: init_tokens(), rounds: PhantomData, } } @@ -131,17 +133,9 @@ impl KeyIvInit for SalsaCore { state[15] = CONSTANTS[3]; - cfg_if::cfg_if! { - if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); - } else { - let tokens = (); - } - } - Self { state, - tokens, + tokens: init_tokens(), rounds: PhantomData, } } From 55b939a7c802ec1b5cf83ea7b6359554403e96a1 Mon Sep 17 00:00:00 2001 From: Patrik Wenger Date: Fri, 17 Jul 2026 16:03:47 +0200 Subject: [PATCH 4/5] benches: add salsa20 criterion benchmark Add `salsa20` throughput benchmark to the repo-level `benches/` crate, matching the existing `chacha20` benchmark structure. --- benches/Cargo.toml | 6 ++++++ benches/src/salsa20.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 benches/src/salsa20.rs diff --git a/benches/Cargo.toml b/benches/Cargo.toml index 47e3700c..2d9f25a8 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -12,6 +12,7 @@ publish = false [dependencies] criterion = "0.5" chacha20 = { path = "../chacha20/", features = ["rng", "zeroize"] } +salsa20 = { path = "../salsa20/" } [target.'cfg(any(target_arch = "x86_64", target_arch = "x86", all(target_arch = "aarch64", target_os = "linux")))'.dependencies] criterion-cycles-per-byte = "0.6.0" @@ -19,4 +20,9 @@ criterion-cycles-per-byte = "0.6.0" [[bench]] name = "chacha20" path = "src/chacha20.rs" +harness = false + +[[bench]] +name = "salsa20" +path = "src/salsa20.rs" harness = false \ No newline at end of file diff --git a/benches/src/salsa20.rs b/benches/src/salsa20.rs new file mode 100644 index 00000000..2b8717bb --- /dev/null +++ b/benches/src/salsa20.rs @@ -0,0 +1,32 @@ +//! Salsa20 benchmark +use benches::{criterion_group_bench, Benchmarker}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; + +use salsa20::{ + cipher::{KeyIvInit, StreamCipher}, + Salsa20, +}; + +const KB: usize = 1024; +fn bench(c: &mut Benchmarker) { + let mut group = c.benchmark_group("stream-cipher"); + + for size in &[KB, 2 * KB, 4 * KB, 8 * KB, 16 * KB] { + let mut buf = vec![0u8; *size]; + + group.throughput(Throughput::Bytes(*size as u64)); + + group.bench_function(BenchmarkId::new("apply_keystream", size), |b| { + let key = Default::default(); + let nonce = Default::default(); + let mut cipher = Salsa20::new(&key, &nonce); + b.iter(|| cipher.apply_keystream(&mut buf)); + }); + } + + group.finish(); +} + +criterion_group_bench!(benches, bench); + +criterion_main!(benches); From f91ec0ab02caa9c53fcbb5646011f8d1118ddb1a Mon Sep 17 00:00:00 2001 From: Patrik Wenger Date: Sat, 18 Jul 2026 03:43:07 +0200 Subject: [PATCH 5/5] salsa20: drop SSE2 backend, keep AVX2 only Per review feedback, start with AVX2 and follow up on SSE2 later once there is user demand to guide tuning. --- salsa20/CHANGELOG.md | 2 +- salsa20/README.md | 2 +- salsa20/src/backends.rs | 1 - salsa20/src/backends/sse2.rs | 169 ----------------------------------- salsa20/src/lib.rs | 12 +-- 5 files changed, 5 insertions(+), 181 deletions(-) delete mode 100644 salsa20/src/backends/sse2.rs diff --git a/salsa20/CHANGELOG.md b/salsa20/CHANGELOG.md index 43e80312..6ec3c5f2 100644 --- a/salsa20/CHANGELOG.md +++ b/salsa20/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased ### Added -- SSE2 and AVX2 intrinsics backends with runtime detection via `cpufeatures` ([#576]) +- AVX2 intrinsics backend with runtime detection via `cpufeatures` ([#576]) ## 0.11.0 (2026-03-30) ### Added diff --git a/salsa20/README.md b/salsa20/README.md index 903e2a7f..22f7c91b 100644 --- a/salsa20/README.md +++ b/salsa20/README.md @@ -11,7 +11,7 @@ Implementation of the [Salsa] family of stream ciphers, including the [XSalsa] variants with an extended 192-bit (24-byte) nonce. -SIMD-accelerated on x86/x86\_64 with SSE2 and AVX2 backends, selected +SIMD-accelerated on x86/x86\_64 with an AVX2 backend, selected automatically at runtime via [cpufeatures]. ## ⚠️ Security Warning: [Hazmat!][hazmat-link] diff --git a/salsa20/src/backends.rs b/salsa20/src/backends.rs index a1ddcb6a..f0ffda7c 100644 --- a/salsa20/src/backends.rs +++ b/salsa20/src/backends.rs @@ -3,6 +3,5 @@ pub(crate) mod soft; cfg_if::cfg_if! { if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { pub(crate) mod avx2; - pub(crate) mod sse2; } } diff --git a/salsa20/src/backends/sse2.rs b/salsa20/src/backends/sse2.rs deleted file mode 100644 index 2ea9df98..00000000 --- a/salsa20/src/backends/sse2.rs +++ /dev/null @@ -1,169 +0,0 @@ -#![allow(unsafe_op_in_unsafe_fn)] -#![allow(clippy::cast_possible_wrap)] - -#[cfg(target_arch = "x86")] -use core::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64::*; - -use crate::{STATE_WORDS, Unsigned}; -use cipher::{ - BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure, - consts::{U4, U64}, -}; -use core::marker::PhantomData; - -const PAR_BLOCKS: usize = 4; - -/// Diagonal indices for the Salsa20 state matrix. -/// -/// The 4x4 state is packed into 4 SSE registers so that each register -/// holds one diagonal. All four column quarter-rounds then execute -/// as a single SIMD operation, and shuffles rotate between -/// column and row round alignment. -const IDX: [[usize; 4]; 4] = [[0, 5, 10, 15], [4, 9, 14, 3], [8, 13, 2, 7], [12, 1, 6, 11]]; - -macro_rules! rotl_epi32 { - ($v:expr, $n:literal) => {{ - let v = $v; - _mm_or_si128(_mm_slli_epi32(v, $n), _mm_srli_epi32(v, 32 - $n)) - }}; -} - -#[inline] -#[target_feature(enable = "sse2")] -pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) -where - R: Unsigned, - F: StreamCipherClosure, -{ - f.call(&mut Backend:: { - state, - _pd: PhantomData, - }); -} - -struct Backend<'a, R: Unsigned> { - state: &'a mut [u32; STATE_WORDS], - _pd: PhantomData, -} - -impl BlockSizeUser for Backend<'_, R> { - type BlockSize = U64; -} - -impl ParBlocksSizeUser for Backend<'_, R> { - type ParBlocksSize = U4; -} - -impl StreamCipherBackend for Backend<'_, R> { - #[inline(always)] - fn gen_ks_block(&mut self, block: &mut cipher::Block) { - unsafe { - let res = rounds::(self.state); - store_block(&res[0], block); - let counter = get_counter(self.state); - set_counter(self.state, counter + 1); - } - } - - #[inline(always)] - fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { - unsafe { - let res = rounds::(self.state); - for i in 0..PAR_BLOCKS { - store_block(&res[i], &mut blocks[i]); - } - let counter = get_counter(self.state); - set_counter(self.state, counter + PAR_BLOCKS as u64); - } - } -} - -#[inline(always)] -unsafe fn load_diag(state: &[u32; STATE_WORDS]) -> [__m128i; 4] { - let s = |i: usize| state[i] as i32; - [ - _mm_set_epi32(s(IDX[0][3]), s(IDX[0][2]), s(IDX[0][1]), s(IDX[0][0])), - _mm_set_epi32(s(IDX[1][3]), s(IDX[1][2]), s(IDX[1][1]), s(IDX[1][0])), - _mm_set_epi32(s(IDX[2][3]), s(IDX[2][2]), s(IDX[2][1]), s(IDX[2][0])), - _mm_set_epi32(s(IDX[3][3]), s(IDX[3][2]), s(IDX[3][1]), s(IDX[3][0])), - ] -} - -#[inline] -#[target_feature(enable = "sse2")] -unsafe fn rounds(state: &[u32; STATE_WORDS]) -> [[__m128i; 4]; PAR_BLOCKS] { - let counter = get_counter(state); - let mut tmp = *state; - let mut res = [[_mm_setzero_si128(); 4]; PAR_BLOCKS]; - for (i, block) in res.iter_mut().enumerate() { - let ctr = counter + i as u64; - tmp[8] = ctr as u32; - tmp[9] = (ctr >> 32) as u32; - *block = load_diag(&tmp); - } - let initial = res; - - for _ in 0..R::USIZE * 2 { - for block in res.iter_mut() { - quarter_round(block); - shuffle(block); - } - } - - for i in 0..PAR_BLOCKS { - for j in 0..4 { - res[i][j] = _mm_add_epi32(res[i][j], initial[i][j]); - } - } - - res -} - -#[inline(always)] -unsafe fn quarter_round(block: &mut [__m128i; 4]) { - let [a, b, c, d] = block; - *b = _mm_xor_si128(*b, rotl_epi32!(_mm_add_epi32(*a, *d), 7)); - *c = _mm_xor_si128(*c, rotl_epi32!(_mm_add_epi32(*b, *a), 9)); - *d = _mm_xor_si128(*d, rotl_epi32!(_mm_add_epi32(*c, *b), 13)); - *a = _mm_xor_si128(*a, rotl_epi32!(_mm_add_epi32(*d, *c), 18)); -} - -/// Rotate diagonal alignment between column and row rounds. -/// -/// The same shuffle+swap sequence converts column layout to row layout -/// and vice versa, so it is applied identically after every quarter round. -#[inline(always)] -unsafe fn shuffle(block: &mut [__m128i; 4]) { - let [_, b, c, d] = block; - let tmp = *b; - *b = _mm_shuffle_epi32(*d, 0b_00_11_10_01); - *c = _mm_shuffle_epi32(*c, 0b_01_00_11_10); - *d = _mm_shuffle_epi32(tmp, 0b_10_01_00_11); -} - -/// Convert diagonal SIMD layout back to sequential little-endian bytes. -#[inline(always)] -unsafe fn store_block(diag: &[__m128i; 4], block: &mut [u8]) { - let mut vals = [[0u32; 4]; 4]; - for (i, v) in diag.iter().enumerate() { - _mm_storeu_si128(vals[i].as_mut_ptr().cast(), *v); - } - for (reg, idxs) in vals.iter().zip(IDX.iter()) { - for (&val, &idx) in reg.iter().zip(idxs.iter()) { - block[idx * 4..idx * 4 + 4].copy_from_slice(&val.to_le_bytes()); - } - } -} - -#[inline(always)] -fn get_counter(state: &[u32; STATE_WORDS]) -> u64 { - (state[8] as u64) | ((state[9] as u64) << 32) -} - -#[inline(always)] -fn set_counter(state: &mut [u32; STATE_WORDS], pos: u64) { - state[8] = pos as u32; - state[9] = (pos >> 32) as u32; -} diff --git a/salsa20/src/lib.rs b/salsa20/src/lib.rs index 8a344a7f..3bf1b556 100644 --- a/salsa20/src/lib.rs +++ b/salsa20/src/lib.rs @@ -26,11 +26,10 @@ mod xsalsa; cfg_if::cfg_if! { if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { cpufeatures::new!(avx2_cpuid, "avx2"); - cpufeatures::new!(sse2_cpuid, "sse2"); - type Tokens = (avx2_cpuid::InitToken, sse2_cpuid::InitToken); + type Tokens = avx2_cpuid::InitToken; fn init_tokens() -> Tokens { - (avx2_cpuid::init(), sse2_cpuid::init()) + avx2_cpuid::init() } } else { type Tokens = (); @@ -150,15 +149,10 @@ impl StreamCipherCore for SalsaCore { fn process_with_backend(&mut self, f: impl StreamCipherClosure) { cfg_if::cfg_if! { if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { - let (avx2_token, sse2_token) = self.tokens; - if avx2_token.get() { + if self.tokens.get() { unsafe { backends::avx2::inner::(&mut self.state, f); } - } else if sse2_token.get() { - unsafe { - backends::sse2::inner::(&mut self.state, f); - } } else { f.call(&mut backends::soft::Backend(self)); }