diff --git a/HashLib.Benchmark/Delphi/HashLib.BenchmarkConsole.dpr b/HashLib.Benchmark/Delphi/HashLib.BenchmarkConsole.dpr index 56afdffc..d9151d4d 100644 --- a/HashLib.Benchmark/Delphi/HashLib.BenchmarkConsole.dpr +++ b/HashLib.Benchmark/Delphi/HashLib.BenchmarkConsole.dpr @@ -20,12 +20,14 @@ uses HlpHMACNotBuildInAdapter in '..\..\HashLib\src\Base\HlpHMACNotBuildInAdapter.pas', HlpMultipleTransformNonBlock in '..\..\HashLib\src\Base\HlpMultipleTransformNonBlock.pas', HlpAdler32 in '..\..\HashLib\src\Checksum\HlpAdler32.pas', - HlpAdler32Dispatch in '..\..\HashLib\src\Checksum\HlpAdler32Dispatch.pas', + HlpAdler32Simd in '..\..\HashLib\src\Simd\Facade\HlpAdler32Simd.pas', + HlpAdler32X86Backend in '..\..\HashLib\src\Simd\Backend\HlpAdler32X86Backend.pas', + HlpAdler32ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpAdler32ArmBackend.pas', HlpCRC in '..\..\HashLib\src\Checksum\HlpCRC.pas', - HlpCRCDispatch in '..\..\HashLib\src\Checksum\HlpCRCDispatch.pas', - HlpCRCFoldConstants in '..\..\HashLib\src\Checksum\HlpCRCFoldConstants.pas', - HlpCRCStandard in '..\..\HashLib\src\Checksum\HlpCRCStandard.pas', - HlpCRC32Fast in '..\..\HashLib\src\Checksum\HlpCRC32Fast.pas', + HlpCRCCore in '..\..\HashLib\src\Checksum\HlpCRCCore.pas', + HlpCRCSimd in '..\..\HashLib\src\Simd\Facade\HlpCRCSimd.pas', + HlpCRCX86Backend in '..\..\HashLib\src\Simd\Backend\HlpCRCX86Backend.pas', + HlpCRCArmBackend in '..\..\HashLib\src\Simd\Backend\HlpCRCArmBackend.pas', HlpGost in '..\..\HashLib\src\Crypto\HlpGost.pas', HlpGrindahl256 in '..\..\HashLib\src\Crypto\HlpGrindahl256.pas', HlpGrindahl512 in '..\..\HashLib\src\Crypto\HlpGrindahl512.pas', @@ -45,27 +47,39 @@ uses HlpRIPEMD320 in '..\..\HashLib\src\Crypto\HlpRIPEMD320.pas', HlpSHA0 in '..\..\HashLib\src\Crypto\HlpSHA0.pas', HlpSHA1 in '..\..\HashLib\src\Crypto\HlpSHA1.pas', - HlpSHA1Dispatch in '..\..\HashLib\src\Crypto\HlpSHA1Dispatch.pas', + HlpSHA1Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA1Simd.pas', + HlpSHA1X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA1X86Backend.pas', + HlpSHA1ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA1ArmBackend.pas', HlpSHA2_224 in '..\..\HashLib\src\Crypto\HlpSHA2_224.pas', HlpSHA2_256 in '..\..\HashLib\src\Crypto\HlpSHA2_256.pas', - HlpSHA2_256Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_256Dispatch.pas', - HlpSHA2_512Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_512Dispatch.pas', + HlpSHA2_256Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_256Simd.pas', + HlpSHA2_256X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256X86Backend.pas', + HlpSHA2_256ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256ArmBackend.pas', + HlpSHA2_512Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_512Simd.pas', + HlpSHA2_512X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512X86Backend.pas', + HlpSHA2_512ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512ArmBackend.pas', HlpSHA2_256Base in '..\..\HashLib\src\Crypto\HlpSHA2_256Base.pas', HlpSHA2_384 in '..\..\HashLib\src\Crypto\HlpSHA2_384.pas', HlpSHA2_512 in '..\..\HashLib\src\Crypto\HlpSHA2_512.pas', HlpSHA2_512_224 in '..\..\HashLib\src\Crypto\HlpSHA2_512_224.pas', HlpSHA2_512_256 in '..\..\HashLib\src\Crypto\HlpSHA2_512_256.pas', HlpSHA2_512Base in '..\..\HashLib\src\Crypto\HlpSHA2_512Base.pas', - HlpSHA3Dispatch in '..\..\HashLib\src\Crypto\HlpSHA3Dispatch.pas', + HlpSHA3Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA3Simd.pas', + HlpSHA3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA3X86Backend.pas', + HlpSHA3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA3ArmBackend.pas', HlpSHA3 in '..\..\HashLib\src\Crypto\HlpSHA3.pas', HlpSnefru in '..\..\HashLib\src\Crypto\HlpSnefru.pas', HlpTiger in '..\..\HashLib\src\Crypto\HlpTiger.pas', HlpTiger2 in '..\..\HashLib\src\Crypto\HlpTiger2.pas', HlpWhirlPool in '..\..\HashLib\src\Crypto\HlpWhirlPool.pas', HlpGOST3411_2012 in '..\..\HashLib\src\Crypto\HlpGOST3411_2012.pas', - HlpBlake2BDispatch in '..\..\HashLib\src\Crypto\HlpBlake2BDispatch.pas', + HlpBlake2BSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2BSimd.pas', + HlpBlake2BX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BX86Backend.pas', + HlpBlake2BArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BArmBackend.pas', HlpBlake2B in '..\..\HashLib\src\Crypto\HlpBlake2B.pas', - HlpBlake2SDispatch in '..\..\HashLib\src\Crypto\HlpBlake2SDispatch.pas', + HlpBlake2SSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2SSimd.pas', + HlpBlake2SX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SX86Backend.pas', + HlpBlake2SArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SArmBackend.pas', HlpBlake2S in '..\..\HashLib\src\Crypto\HlpBlake2S.pas', HlpBlake2BParams in '..\..\HashLib\src\Crypto\Blake2BParams\HlpBlake2BParams.pas', HlpBlake2SParams in '..\..\HashLib\src\Crypto\Blake2SParams\HlpBlake2SParams.pas', @@ -96,7 +110,9 @@ uses HlpMurmur2_64 in '..\..\HashLib\src\Hash64\HlpMurmur2_64.pas', HlpSipHash in '..\..\HashLib\src\Hash64\HlpSipHash.pas', HlpXXHash64 in '..\..\HashLib\src\Hash64\HlpXXHash64.pas', - HlpXXHash3Dispatch in '..\..\HashLib\src\Hash64\HlpXXHash3Dispatch.pas', + HlpXXHash3Simd in '..\..\HashLib\src\Simd\Facade\HlpXXHash3Simd.pas', + HlpXXHash3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3X86Backend.pas', + HlpXXHash3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3ArmBackend.pas', HlpXXHash3 in '..\..\HashLib\src\Hash64\HlpXXHash3.pas', HlpMurmurHash3_x86_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x86_128.pas', HlpMurmurHash3_x64_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x64_128.pas', @@ -111,14 +127,20 @@ uses HlpIBlake2SParams in '..\..\HashLib\src\Interfaces\IBlake2SParams\HlpIBlake2SParams.pas', HlpBlake2BP in '..\..\HashLib\src\Crypto\HlpBlake2BP.pas', HlpBlake2SP in '..\..\HashLib\src\Crypto\HlpBlake2SP.pas', - HlpBlake3Dispatch in '..\..\HashLib\src\Crypto\HlpBlake3Dispatch.pas', + HlpBlake3Simd in '..\..\HashLib\src\Simd\Facade\HlpBlake3Simd.pas', + HlpBlake3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake3X86Backend.pas', + HlpBlake3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake3ArmBackend.pas', HlpBlake3 in '..\..\HashLib\src\Crypto\HlpBlake3.pas', HlpPBKDF2_HMACNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF2_HMACNotBuildInAdapter.pas', HlpPBKDF_Argon2NotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_Argon2NotBuildInAdapter.pas', - HlpArgon2Dispatch in '..\..\HashLib\src\KDF\HlpArgon2Dispatch.pas', + HlpArgon2Simd in '..\..\HashLib\src\Simd\Facade\HlpArgon2Simd.pas', + HlpArgon2X86Backend in '..\..\HashLib\src\Simd\Backend\HlpArgon2X86Backend.pas', + HlpArgon2ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpArgon2ArmBackend.pas', HlpArgon2TypeAndVersion in '..\..\HashLib\src\KDF\HlpArgon2TypeAndVersion.pas', HlpPBKDF_ScryptNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_ScryptNotBuildInAdapter.pas', - HlpScryptDispatch in '..\..\HashLib\src\KDF\HlpScryptDispatch.pas', + HlpScryptSimd in '..\..\HashLib\src\Simd\Facade\HlpScryptSimd.pas', + HlpScryptX86Backend in '..\..\HashLib\src\Simd\Backend\HlpScryptX86Backend.pas', + HlpScryptArmBackend in '..\..\HashLib\src\Simd\Backend\HlpScryptArmBackend.pas', HlpConverters in '..\..\HashLib\src\Utils\HlpConverters.pas', HlpBitOperations in '..\..\HashLib\src\Utils\HlpBitOperations.pas', HlpBinaryPrimitives in '..\..\HashLib\src\Utils\HlpBinaryPrimitives.pas', diff --git a/HashLib.Benchmark/Delphi/HashLib.BenchmarkFMX.dpr b/HashLib.Benchmark/Delphi/HashLib.BenchmarkFMX.dpr index 66c0bfc2..3b4c4866 100644 --- a/HashLib.Benchmark/Delphi/HashLib.BenchmarkFMX.dpr +++ b/HashLib.Benchmark/Delphi/HashLib.BenchmarkFMX.dpr @@ -19,12 +19,14 @@ uses HlpHMACNotBuildInAdapter in '..\..\HashLib\src\Base\HlpHMACNotBuildInAdapter.pas', HlpMultipleTransformNonBlock in '..\..\HashLib\src\Base\HlpMultipleTransformNonBlock.pas', HlpAdler32 in '..\..\HashLib\src\Checksum\HlpAdler32.pas', - HlpAdler32Dispatch in '..\..\HashLib\src\Checksum\HlpAdler32Dispatch.pas', + HlpAdler32Simd in '..\..\HashLib\src\Simd\Facade\HlpAdler32Simd.pas', + HlpAdler32X86Backend in '..\..\HashLib\src\Simd\Backend\HlpAdler32X86Backend.pas', + HlpAdler32ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpAdler32ArmBackend.pas', HlpCRC in '..\..\HashLib\src\Checksum\HlpCRC.pas', - HlpCRCDispatch in '..\..\HashLib\src\Checksum\HlpCRCDispatch.pas', - HlpCRCFoldConstants in '..\..\HashLib\src\Checksum\HlpCRCFoldConstants.pas', - HlpCRCStandard in '..\..\HashLib\src\Checksum\HlpCRCStandard.pas', - HlpCRC32Fast in '..\..\HashLib\src\Checksum\HlpCRC32Fast.pas', + HlpCRCCore in '..\..\HashLib\src\Checksum\HlpCRCCore.pas', + HlpCRCSimd in '..\..\HashLib\src\Simd\Facade\HlpCRCSimd.pas', + HlpCRCX86Backend in '..\..\HashLib\src\Simd\Backend\HlpCRCX86Backend.pas', + HlpCRCArmBackend in '..\..\HashLib\src\Simd\Backend\HlpCRCArmBackend.pas', HlpGost in '..\..\HashLib\src\Crypto\HlpGost.pas', HlpGrindahl256 in '..\..\HashLib\src\Crypto\HlpGrindahl256.pas', HlpGrindahl512 in '..\..\HashLib\src\Crypto\HlpGrindahl512.pas', @@ -44,27 +46,39 @@ uses HlpRIPEMD320 in '..\..\HashLib\src\Crypto\HlpRIPEMD320.pas', HlpSHA0 in '..\..\HashLib\src\Crypto\HlpSHA0.pas', HlpSHA1 in '..\..\HashLib\src\Crypto\HlpSHA1.pas', - HlpSHA1Dispatch in '..\..\HashLib\src\Crypto\HlpSHA1Dispatch.pas', + HlpSHA1Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA1Simd.pas', + HlpSHA1X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA1X86Backend.pas', + HlpSHA1ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA1ArmBackend.pas', HlpSHA2_224 in '..\..\HashLib\src\Crypto\HlpSHA2_224.pas', HlpSHA2_256 in '..\..\HashLib\src\Crypto\HlpSHA2_256.pas', - HlpSHA2_256Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_256Dispatch.pas', - HlpSHA2_512Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_512Dispatch.pas', + HlpSHA2_256Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_256Simd.pas', + HlpSHA2_256X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256X86Backend.pas', + HlpSHA2_256ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256ArmBackend.pas', + HlpSHA2_512Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_512Simd.pas', + HlpSHA2_512X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512X86Backend.pas', + HlpSHA2_512ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512ArmBackend.pas', HlpSHA2_256Base in '..\..\HashLib\src\Crypto\HlpSHA2_256Base.pas', HlpSHA2_384 in '..\..\HashLib\src\Crypto\HlpSHA2_384.pas', HlpSHA2_512 in '..\..\HashLib\src\Crypto\HlpSHA2_512.pas', HlpSHA2_512_224 in '..\..\HashLib\src\Crypto\HlpSHA2_512_224.pas', HlpSHA2_512_256 in '..\..\HashLib\src\Crypto\HlpSHA2_512_256.pas', HlpSHA2_512Base in '..\..\HashLib\src\Crypto\HlpSHA2_512Base.pas', - HlpSHA3Dispatch in '..\..\HashLib\src\Crypto\HlpSHA3Dispatch.pas', + HlpSHA3Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA3Simd.pas', + HlpSHA3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA3X86Backend.pas', + HlpSHA3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA3ArmBackend.pas', HlpSHA3 in '..\..\HashLib\src\Crypto\HlpSHA3.pas', HlpSnefru in '..\..\HashLib\src\Crypto\HlpSnefru.pas', HlpTiger in '..\..\HashLib\src\Crypto\HlpTiger.pas', HlpTiger2 in '..\..\HashLib\src\Crypto\HlpTiger2.pas', HlpWhirlPool in '..\..\HashLib\src\Crypto\HlpWhirlPool.pas', HlpGOST3411_2012 in '..\..\HashLib\src\Crypto\HlpGOST3411_2012.pas', - HlpBlake2BDispatch in '..\..\HashLib\src\Crypto\HlpBlake2BDispatch.pas', + HlpBlake2BSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2BSimd.pas', + HlpBlake2BX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BX86Backend.pas', + HlpBlake2BArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BArmBackend.pas', HlpBlake2B in '..\..\HashLib\src\Crypto\HlpBlake2B.pas', - HlpBlake2SDispatch in '..\..\HashLib\src\Crypto\HlpBlake2SDispatch.pas', + HlpBlake2SSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2SSimd.pas', + HlpBlake2SX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SX86Backend.pas', + HlpBlake2SArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SArmBackend.pas', HlpBlake2S in '..\..\HashLib\src\Crypto\HlpBlake2S.pas', HlpBlake2BParams in '..\..\HashLib\src\Crypto\Blake2BParams\HlpBlake2BParams.pas', HlpBlake2SParams in '..\..\HashLib\src\Crypto\Blake2SParams\HlpBlake2SParams.pas', @@ -95,7 +109,9 @@ uses HlpMurmur2_64 in '..\..\HashLib\src\Hash64\HlpMurmur2_64.pas', HlpSipHash in '..\..\HashLib\src\Hash64\HlpSipHash.pas', HlpXXHash64 in '..\..\HashLib\src\Hash64\HlpXXHash64.pas', - HlpXXHash3Dispatch in '..\..\HashLib\src\Hash64\HlpXXHash3Dispatch.pas', + HlpXXHash3Simd in '..\..\HashLib\src\Simd\Facade\HlpXXHash3Simd.pas', + HlpXXHash3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3X86Backend.pas', + HlpXXHash3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3ArmBackend.pas', HlpXXHash3 in '..\..\HashLib\src\Hash64\HlpXXHash3.pas', HlpMurmurHash3_x86_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x86_128.pas', HlpMurmurHash3_x64_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x64_128.pas', @@ -110,14 +126,20 @@ uses HlpIBlake2SParams in '..\..\HashLib\src\Interfaces\IBlake2SParams\HlpIBlake2SParams.pas', HlpBlake2BP in '..\..\HashLib\src\Crypto\HlpBlake2BP.pas', HlpBlake2SP in '..\..\HashLib\src\Crypto\HlpBlake2SP.pas', - HlpBlake3Dispatch in '..\..\HashLib\src\Crypto\HlpBlake3Dispatch.pas', + HlpBlake3Simd in '..\..\HashLib\src\Simd\Facade\HlpBlake3Simd.pas', + HlpBlake3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake3X86Backend.pas', + HlpBlake3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake3ArmBackend.pas', HlpBlake3 in '..\..\HashLib\src\Crypto\HlpBlake3.pas', HlpPBKDF2_HMACNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF2_HMACNotBuildInAdapter.pas', HlpPBKDF_Argon2NotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_Argon2NotBuildInAdapter.pas', - HlpArgon2Dispatch in '..\..\HashLib\src\KDF\HlpArgon2Dispatch.pas', + HlpArgon2Simd in '..\..\HashLib\src\Simd\Facade\HlpArgon2Simd.pas', + HlpArgon2X86Backend in '..\..\HashLib\src\Simd\Backend\HlpArgon2X86Backend.pas', + HlpArgon2ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpArgon2ArmBackend.pas', HlpArgon2TypeAndVersion in '..\..\HashLib\src\KDF\HlpArgon2TypeAndVersion.pas', HlpPBKDF_ScryptNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_ScryptNotBuildInAdapter.pas', - HlpScryptDispatch in '..\..\HashLib\src\KDF\HlpScryptDispatch.pas', + HlpScryptSimd in '..\..\HashLib\src\Simd\Facade\HlpScryptSimd.pas', + HlpScryptX86Backend in '..\..\HashLib\src\Simd\Backend\HlpScryptX86Backend.pas', + HlpScryptArmBackend in '..\..\HashLib\src\Simd\Backend\HlpScryptArmBackend.pas', HlpConverters in '..\..\HashLib\src\Utils\HlpConverters.pas', HlpBitOperations in '..\..\HashLib\src\Utils\HlpBitOperations.pas', HlpBinaryPrimitives in '..\..\HashLib\src\Utils\HlpBinaryPrimitives.pas', diff --git a/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dpr b/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dpr index 6a548b6b..122693ca 100644 --- a/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dpr +++ b/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dpr @@ -21,12 +21,14 @@ uses HlpHMACNotBuildInAdapter in '..\..\HashLib\src\Base\HlpHMACNotBuildInAdapter.pas', HlpMultipleTransformNonBlock in '..\..\HashLib\src\Base\HlpMultipleTransformNonBlock.pas', HlpAdler32 in '..\..\HashLib\src\Checksum\HlpAdler32.pas', - HlpAdler32Dispatch in '..\..\HashLib\src\Checksum\HlpAdler32Dispatch.pas', + HlpAdler32Simd in '..\..\HashLib\src\Simd\Facade\HlpAdler32Simd.pas', + HlpAdler32X86Backend in '..\..\HashLib\src\Simd\Backend\HlpAdler32X86Backend.pas', + HlpAdler32ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpAdler32ArmBackend.pas', HlpCRC in '..\..\HashLib\src\Checksum\HlpCRC.pas', - HlpCRCDispatch in '..\..\HashLib\src\Checksum\HlpCRCDispatch.pas', - HlpCRCFoldConstants in '..\..\HashLib\src\Checksum\HlpCRCFoldConstants.pas', - HlpCRCStandard in '..\..\HashLib\src\Checksum\HlpCRCStandard.pas', - HlpCRC32Fast in '..\..\HashLib\src\Checksum\HlpCRC32Fast.pas', + HlpCRCCore in '..\..\HashLib\src\Checksum\HlpCRCCore.pas', + HlpCRCSimd in '..\..\HashLib\src\Simd\Facade\HlpCRCSimd.pas', + HlpCRCX86Backend in '..\..\HashLib\src\Simd\Backend\HlpCRCX86Backend.pas', + HlpCRCArmBackend in '..\..\HashLib\src\Simd\Backend\HlpCRCArmBackend.pas', HlpGost in '..\..\HashLib\src\Crypto\HlpGost.pas', HlpGrindahl256 in '..\..\HashLib\src\Crypto\HlpGrindahl256.pas', HlpGrindahl512 in '..\..\HashLib\src\Crypto\HlpGrindahl512.pas', @@ -46,27 +48,39 @@ uses HlpRIPEMD320 in '..\..\HashLib\src\Crypto\HlpRIPEMD320.pas', HlpSHA0 in '..\..\HashLib\src\Crypto\HlpSHA0.pas', HlpSHA1 in '..\..\HashLib\src\Crypto\HlpSHA1.pas', - HlpSHA1Dispatch in '..\..\HashLib\src\Crypto\HlpSHA1Dispatch.pas', + HlpSHA1Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA1Simd.pas', + HlpSHA1X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA1X86Backend.pas', + HlpSHA1ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA1ArmBackend.pas', HlpSHA2_224 in '..\..\HashLib\src\Crypto\HlpSHA2_224.pas', HlpSHA2_256 in '..\..\HashLib\src\Crypto\HlpSHA2_256.pas', - HlpSHA2_256Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_256Dispatch.pas', - HlpSHA2_512Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_512Dispatch.pas', + HlpSHA2_256Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_256Simd.pas', + HlpSHA2_256X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256X86Backend.pas', + HlpSHA2_256ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256ArmBackend.pas', + HlpSHA2_512Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_512Simd.pas', + HlpSHA2_512X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512X86Backend.pas', + HlpSHA2_512ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512ArmBackend.pas', HlpSHA2_256Base in '..\..\HashLib\src\Crypto\HlpSHA2_256Base.pas', HlpSHA2_384 in '..\..\HashLib\src\Crypto\HlpSHA2_384.pas', HlpSHA2_512 in '..\..\HashLib\src\Crypto\HlpSHA2_512.pas', HlpSHA2_512_224 in '..\..\HashLib\src\Crypto\HlpSHA2_512_224.pas', HlpSHA2_512_256 in '..\..\HashLib\src\Crypto\HlpSHA2_512_256.pas', HlpSHA2_512Base in '..\..\HashLib\src\Crypto\HlpSHA2_512Base.pas', - HlpSHA3Dispatch in '..\..\HashLib\src\Crypto\HlpSHA3Dispatch.pas', + HlpSHA3Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA3Simd.pas', + HlpSHA3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA3X86Backend.pas', + HlpSHA3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA3ArmBackend.pas', HlpSHA3 in '..\..\HashLib\src\Crypto\HlpSHA3.pas', HlpSnefru in '..\..\HashLib\src\Crypto\HlpSnefru.pas', HlpTiger in '..\..\HashLib\src\Crypto\HlpTiger.pas', HlpTiger2 in '..\..\HashLib\src\Crypto\HlpTiger2.pas', HlpWhirlPool in '..\..\HashLib\src\Crypto\HlpWhirlPool.pas', HlpGOST3411_2012 in '..\..\HashLib\src\Crypto\HlpGOST3411_2012.pas', - HlpBlake2BDispatch in '..\..\HashLib\src\Crypto\HlpBlake2BDispatch.pas', + HlpBlake2BSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2BSimd.pas', + HlpBlake2BX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BX86Backend.pas', + HlpBlake2BArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BArmBackend.pas', HlpBlake2B in '..\..\HashLib\src\Crypto\HlpBlake2B.pas', - HlpBlake2SDispatch in '..\..\HashLib\src\Crypto\HlpBlake2SDispatch.pas', + HlpBlake2SSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2SSimd.pas', + HlpBlake2SX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SX86Backend.pas', + HlpBlake2SArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SArmBackend.pas', HlpBlake2S in '..\..\HashLib\src\Crypto\HlpBlake2S.pas', HlpBlake2BParams in '..\..\HashLib\src\Crypto\Blake2BParams\HlpBlake2BParams.pas', HlpBlake2SParams in '..\..\HashLib\src\Crypto\Blake2SParams\HlpBlake2SParams.pas', @@ -97,7 +111,9 @@ uses HlpMurmur2_64 in '..\..\HashLib\src\Hash64\HlpMurmur2_64.pas', HlpSipHash in '..\..\HashLib\src\Hash64\HlpSipHash.pas', HlpXXHash64 in '..\..\HashLib\src\Hash64\HlpXXHash64.pas', - HlpXXHash3Dispatch in '..\..\HashLib\src\Hash64\HlpXXHash3Dispatch.pas', + HlpXXHash3Simd in '..\..\HashLib\src\Simd\Facade\HlpXXHash3Simd.pas', + HlpXXHash3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3X86Backend.pas', + HlpXXHash3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3ArmBackend.pas', HlpXXHash3 in '..\..\HashLib\src\Hash64\HlpXXHash3.pas', HlpMurmurHash3_x86_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x86_128.pas', HlpMurmurHash3_x64_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x64_128.pas', @@ -112,14 +128,20 @@ uses HlpIBlake2SParams in '..\..\HashLib\src\Interfaces\IBlake2SParams\HlpIBlake2SParams.pas', HlpBlake2BP in '..\..\HashLib\src\Crypto\HlpBlake2BP.pas', HlpBlake2SP in '..\..\HashLib\src\Crypto\HlpBlake2SP.pas', - HlpBlake3Dispatch in '..\..\HashLib\src\Crypto\HlpBlake3Dispatch.pas', + HlpBlake3Simd in '..\..\HashLib\src\Simd\Facade\HlpBlake3Simd.pas', + HlpBlake3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake3X86Backend.pas', + HlpBlake3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake3ArmBackend.pas', HlpBlake3 in '..\..\HashLib\src\Crypto\HlpBlake3.pas', HlpPBKDF2_HMACNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF2_HMACNotBuildInAdapter.pas', HlpPBKDF_Argon2NotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_Argon2NotBuildInAdapter.pas', - HlpArgon2Dispatch in '..\..\HashLib\src\KDF\HlpArgon2Dispatch.pas', + HlpArgon2Simd in '..\..\HashLib\src\Simd\Facade\HlpArgon2Simd.pas', + HlpArgon2X86Backend in '..\..\HashLib\src\Simd\Backend\HlpArgon2X86Backend.pas', + HlpArgon2ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpArgon2ArmBackend.pas', HlpArgon2TypeAndVersion in '..\..\HashLib\src\KDF\HlpArgon2TypeAndVersion.pas', HlpPBKDF_ScryptNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_ScryptNotBuildInAdapter.pas', - HlpScryptDispatch in '..\..\HashLib\src\KDF\HlpScryptDispatch.pas', + HlpScryptSimd in '..\..\HashLib\src\Simd\Facade\HlpScryptSimd.pas', + HlpScryptX86Backend in '..\..\HashLib\src\Simd\Backend\HlpScryptX86Backend.pas', + HlpScryptArmBackend in '..\..\HashLib\src\Simd\Backend\HlpScryptArmBackend.pas', HlpConverters in '..\..\HashLib\src\Utils\HlpConverters.pas', HlpBitOperations in '..\..\HashLib\src\Utils\HlpBitOperations.pas', HlpBinaryPrimitives in '..\..\HashLib\src\Utils\HlpBinaryPrimitives.pas', diff --git a/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dproj b/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dproj index 9de9700b..302935a4 100644 --- a/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dproj +++ b/HashLib.Tests/Delphi.Tests/HashLib.Tests.Mobile.dproj @@ -361,12 +361,14 @@ - + + + - - - - + + + + @@ -386,27 +388,39 @@ - + + + - - + + + + + + - + + + - + + + - + + + @@ -437,7 +451,9 @@ - + + + @@ -452,14 +468,20 @@ - + + + - + + + - + + + diff --git a/HashLib.Tests/Delphi.Tests/HashLib.Tests.dpr b/HashLib.Tests/Delphi.Tests/HashLib.Tests.dpr index a1792736..f6f545b3 100644 --- a/HashLib.Tests/Delphi.Tests/HashLib.Tests.dpr +++ b/HashLib.Tests/Delphi.Tests/HashLib.Tests.dpr @@ -38,12 +38,14 @@ uses HlpHMACNotBuildInAdapter in '..\..\HashLib\src\Base\HlpHMACNotBuildInAdapter.pas', HlpMultipleTransformNonBlock in '..\..\HashLib\src\Base\HlpMultipleTransformNonBlock.pas', HlpAdler32 in '..\..\HashLib\src\Checksum\HlpAdler32.pas', - HlpAdler32Dispatch in '..\..\HashLib\src\Checksum\HlpAdler32Dispatch.pas', + HlpAdler32Simd in '..\..\HashLib\src\Simd\Facade\HlpAdler32Simd.pas', + HlpAdler32X86Backend in '..\..\HashLib\src\Simd\Backend\HlpAdler32X86Backend.pas', + HlpAdler32ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpAdler32ArmBackend.pas', HlpCRC in '..\..\HashLib\src\Checksum\HlpCRC.pas', - HlpCRCDispatch in '..\..\HashLib\src\Checksum\HlpCRCDispatch.pas', - HlpCRCFoldConstants in '..\..\HashLib\src\Checksum\HlpCRCFoldConstants.pas', - HlpCRCStandard in '..\..\HashLib\src\Checksum\HlpCRCStandard.pas', - HlpCRC32Fast in '..\..\HashLib\src\Checksum\HlpCRC32Fast.pas', + HlpCRCCore in '..\..\HashLib\src\Checksum\HlpCRCCore.pas', + HlpCRCSimd in '..\..\HashLib\src\Simd\Facade\HlpCRCSimd.pas', + HlpCRCX86Backend in '..\..\HashLib\src\Simd\Backend\HlpCRCX86Backend.pas', + HlpCRCArmBackend in '..\..\HashLib\src\Simd\Backend\HlpCRCArmBackend.pas', HlpGost in '..\..\HashLib\src\Crypto\HlpGost.pas', HlpGrindahl256 in '..\..\HashLib\src\Crypto\HlpGrindahl256.pas', HlpGrindahl512 in '..\..\HashLib\src\Crypto\HlpGrindahl512.pas', @@ -63,27 +65,39 @@ uses HlpRIPEMD320 in '..\..\HashLib\src\Crypto\HlpRIPEMD320.pas', HlpSHA0 in '..\..\HashLib\src\Crypto\HlpSHA0.pas', HlpSHA1 in '..\..\HashLib\src\Crypto\HlpSHA1.pas', - HlpSHA1Dispatch in '..\..\HashLib\src\Crypto\HlpSHA1Dispatch.pas', + HlpSHA1Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA1Simd.pas', + HlpSHA1X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA1X86Backend.pas', + HlpSHA1ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA1ArmBackend.pas', HlpSHA2_224 in '..\..\HashLib\src\Crypto\HlpSHA2_224.pas', HlpSHA2_256 in '..\..\HashLib\src\Crypto\HlpSHA2_256.pas', - HlpSHA2_256Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_256Dispatch.pas', - HlpSHA2_512Dispatch in '..\..\HashLib\src\Crypto\HlpSHA2_512Dispatch.pas', + HlpSHA2_256Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_256Simd.pas', + HlpSHA2_256X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256X86Backend.pas', + HlpSHA2_256ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_256ArmBackend.pas', + HlpSHA2_512Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA2_512Simd.pas', + HlpSHA2_512X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512X86Backend.pas', + HlpSHA2_512ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA2_512ArmBackend.pas', HlpSHA2_256Base in '..\..\HashLib\src\Crypto\HlpSHA2_256Base.pas', HlpSHA2_384 in '..\..\HashLib\src\Crypto\HlpSHA2_384.pas', HlpSHA2_512 in '..\..\HashLib\src\Crypto\HlpSHA2_512.pas', HlpSHA2_512_224 in '..\..\HashLib\src\Crypto\HlpSHA2_512_224.pas', HlpSHA2_512_256 in '..\..\HashLib\src\Crypto\HlpSHA2_512_256.pas', HlpSHA2_512Base in '..\..\HashLib\src\Crypto\HlpSHA2_512Base.pas', - HlpSHA3Dispatch in '..\..\HashLib\src\Crypto\HlpSHA3Dispatch.pas', + HlpSHA3Simd in '..\..\HashLib\src\Simd\Facade\HlpSHA3Simd.pas', + HlpSHA3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpSHA3X86Backend.pas', + HlpSHA3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpSHA3ArmBackend.pas', HlpSHA3 in '..\..\HashLib\src\Crypto\HlpSHA3.pas', HlpSnefru in '..\..\HashLib\src\Crypto\HlpSnefru.pas', HlpTiger in '..\..\HashLib\src\Crypto\HlpTiger.pas', HlpTiger2 in '..\..\HashLib\src\Crypto\HlpTiger2.pas', HlpWhirlPool in '..\..\HashLib\src\Crypto\HlpWhirlPool.pas', HlpGOST3411_2012 in '..\..\HashLib\src\Crypto\HlpGOST3411_2012.pas', - HlpBlake2BDispatch in '..\..\HashLib\src\Crypto\HlpBlake2BDispatch.pas', + HlpBlake2BSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2BSimd.pas', + HlpBlake2BX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BX86Backend.pas', + HlpBlake2BArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2BArmBackend.pas', HlpBlake2B in '..\..\HashLib\src\Crypto\HlpBlake2B.pas', - HlpBlake2SDispatch in '..\..\HashLib\src\Crypto\HlpBlake2SDispatch.pas', + HlpBlake2SSimd in '..\..\HashLib\src\Simd\Facade\HlpBlake2SSimd.pas', + HlpBlake2SX86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SX86Backend.pas', + HlpBlake2SArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake2SArmBackend.pas', HlpBlake2S in '..\..\HashLib\src\Crypto\HlpBlake2S.pas', HlpBlake2BParams in '..\..\HashLib\src\Crypto\Blake2BParams\HlpBlake2BParams.pas', HlpBlake2SParams in '..\..\HashLib\src\Crypto\Blake2SParams\HlpBlake2SParams.pas', @@ -114,7 +128,9 @@ uses HlpMurmur2_64 in '..\..\HashLib\src\Hash64\HlpMurmur2_64.pas', HlpSipHash in '..\..\HashLib\src\Hash64\HlpSipHash.pas', HlpXXHash64 in '..\..\HashLib\src\Hash64\HlpXXHash64.pas', - HlpXXHash3Dispatch in '..\..\HashLib\src\Hash64\HlpXXHash3Dispatch.pas', + HlpXXHash3Simd in '..\..\HashLib\src\Simd\Facade\HlpXXHash3Simd.pas', + HlpXXHash3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3X86Backend.pas', + HlpXXHash3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpXXHash3ArmBackend.pas', HlpXXHash3 in '..\..\HashLib\src\Hash64\HlpXXHash3.pas', HlpMurmurHash3_x86_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x86_128.pas', HlpMurmurHash3_x64_128 in '..\..\HashLib\src\Hash128\HlpMurmurHash3_x64_128.pas', @@ -129,14 +145,20 @@ uses HlpIBlake2SParams in '..\..\HashLib\src\Interfaces\IBlake2SParams\HlpIBlake2SParams.pas', HlpBlake2BP in '..\..\HashLib\src\Crypto\HlpBlake2BP.pas', HlpBlake2SP in '..\..\HashLib\src\Crypto\HlpBlake2SP.pas', - HlpBlake3Dispatch in '..\..\HashLib\src\Crypto\HlpBlake3Dispatch.pas', + HlpBlake3Simd in '..\..\HashLib\src\Simd\Facade\HlpBlake3Simd.pas', + HlpBlake3X86Backend in '..\..\HashLib\src\Simd\Backend\HlpBlake3X86Backend.pas', + HlpBlake3ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpBlake3ArmBackend.pas', HlpBlake3 in '..\..\HashLib\src\Crypto\HlpBlake3.pas', HlpPBKDF2_HMACNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF2_HMACNotBuildInAdapter.pas', HlpPBKDF_Argon2NotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_Argon2NotBuildInAdapter.pas', - HlpArgon2Dispatch in '..\..\HashLib\src\KDF\HlpArgon2Dispatch.pas', + HlpArgon2Simd in '..\..\HashLib\src\Simd\Facade\HlpArgon2Simd.pas', + HlpArgon2X86Backend in '..\..\HashLib\src\Simd\Backend\HlpArgon2X86Backend.pas', + HlpArgon2ArmBackend in '..\..\HashLib\src\Simd\Backend\HlpArgon2ArmBackend.pas', HlpArgon2TypeAndVersion in '..\..\HashLib\src\KDF\HlpArgon2TypeAndVersion.pas', HlpPBKDF_ScryptNotBuildInAdapter in '..\..\HashLib\src\KDF\HlpPBKDF_ScryptNotBuildInAdapter.pas', - HlpScryptDispatch in '..\..\HashLib\src\KDF\HlpScryptDispatch.pas', + HlpScryptSimd in '..\..\HashLib\src\Simd\Facade\HlpScryptSimd.pas', + HlpScryptX86Backend in '..\..\HashLib\src\Simd\Backend\HlpScryptX86Backend.pas', + HlpScryptArmBackend in '..\..\HashLib\src\Simd\Backend\HlpScryptArmBackend.pas', HlpConverters in '..\..\HashLib\src\Utils\HlpConverters.pas', HlpBitOperations in '..\..\HashLib\src\Utils\HlpBitOperations.pas', HlpBinaryPrimitives in '..\..\HashLib\src\Utils\HlpBinaryPrimitives.pas', diff --git a/HashLib.Tests/src/CRCTests.pas b/HashLib.Tests/src/CRCTests.pas index ac82308e..b445acf8 100644 --- a/HashLib.Tests/src/CRCTests.pas +++ b/HashLib.Tests/src/CRCTests.pas @@ -30,6 +30,7 @@ TTestCRCModel = class(THashLibAlgorithmTestCase) procedure TestCheckValue; procedure TestCheckValueWithIncrementalHash; procedure TestAnotherChunkedDataIncrementalHash; + procedure TestFoldBoundariesMatchByteTable; procedure TestHashCloneIsCorrect; procedure TestHashCloneIsUnique; @@ -171,6 +172,54 @@ procedure TTestCRCModel.TestCheckValueWithIncrementalHash; end; +procedure TTestCRCModel.TestFoldBoundariesMatchByteTable; +const + // Lengths straddling the SIMD fold-path boundaries: fold-by-1 entry (16B), + // fold-by-4 entry (64B), fold-by-8 entry (128B), their loop repeats and + // reduce/tail edges. + Lengths: array [0 .. 21] of Int32 = (16, 17, 31, 32, 33, 63, 64, 65, 79, 95, + 96, 127, 128, 129, 143, 191, 192, 255, 256, 257, 512, 640); +var + LIdx: TCRCStandard; + LBuffer: TBytes; + LI, LJ, LLen, LPos, LChunk: Int32; + LOneShot, LChunked: IHash; +begin + System.SetLength(LBuffer, 640); + for LI := 0 to System.High(LBuffer) do + LBuffer[LI] := Byte((LI * 131) + 17); + + for LIdx := System.Low(TCRCStandard) to System.High(TCRCStandard) do + begin + for LJ := 0 to System.High(Lengths) do + begin + LLen := Lengths[LJ]; + + LOneShot := THashFactory.TChecksum.TCRC.CreateCRC(LIdx); + ActualString := LOneShot.ComputeBytes(System.Copy(LBuffer, 0, LLen)) + .ToString(); + + // Reference: sub-16-byte chunks keep everything on the byte-table path. + LChunked := THashFactory.TChecksum.TCRC.CreateCRC(LIdx); + LChunked.Initialize; + LPos := 0; + while LPos < LLen do + begin + LChunk := 15; + if LLen - LPos < LChunk then + LChunk := LLen - LPos; + LChunked.TransformBytes(LBuffer, LPos, LChunk); + System.Inc(LPos, LChunk); + end; + ExpectedString := LChunked.TransformFinal().ToString(); + + CheckEquals(ExpectedString, ActualString, + Format('Expected %s but got %s at length %d. %s', + [ExpectedString, ActualString, LLen, LOneShot.Name])); + end; + end; +end; + procedure TTestCRCModel.TestHashCloneIsCorrect; var LOriginal, LCopy: IHash; diff --git a/HashLib.Tests/src/Hash64Tests.pas b/HashLib.Tests/src/Hash64Tests.pas index 265998ce..0fe379a0 100644 --- a/HashLib.Tests/src/Hash64Tests.pas +++ b/HashLib.Tests/src/Hash64Tests.pas @@ -3,6 +3,7 @@ interface uses + SysUtils, {$IFDEF FPC} fpcunit, testregistry, @@ -10,6 +11,7 @@ interface TestFramework, {$ENDIF FPC} HashLibTestBase, + HlpIHashInfo, HlpHashFactory; // Hash64 @@ -70,6 +72,18 @@ TTestXXHash3 = class(THashWithUInt64AsKeyAlgorithmTestCase) procedure SetUp; override; procedure TearDown; override; + published + // Known-answer tests for the long-input paths (values cross-checked + // against the official xxHash implementation): the self-consistency + // chunked tests alone cannot catch a wrong SIMD kernel because both + // sides run the same code. + procedure TestChunkedDataKnownAnswer; // 299 bytes: accumulate512 + procedure TestTwoKiBDataKnownAnswer; // 2048 bytes: scrambleAcc + procedure TestChunkedDataWithMaxUInt64AsKeyKnownAnswer; // seeded: initSecret + // An asymmetric key (lo32 <> hi32) - a symmetric one like MaxUInt64 + // cannot catch seed dword swaps or wrong hi/lo pickup in the kernels. + procedure TestChunkedDataWithAsymmetricKeyKnownAnswer; + end; implementation @@ -188,6 +202,69 @@ procedure TTestXXHash3.TearDown; inherited; end; +procedure TTestXXHash3.TestChunkedDataKnownAnswer; +begin + ExpectedString := '3D3D7245B763ACE8'; + ActualString := HashInstance.ComputeString(ChunkedData, TEncoding.UTF8) + .ToString(); + CheckEquals(ExpectedString, ActualString, Format('Expected %s but got %s.', + [ExpectedString, ActualString])); +end; + +procedure TTestXXHash3.TestTwoKiBDataKnownAnswer; +var + LData: TBytes; + LIdx: Int32; +begin + System.SetLength(LData, 2048); + for LIdx := 0 to 2047 do + begin + LData[LIdx] := Byte(LIdx and $FF); + end; + ExpectedString := 'DD420471FF96BD00'; + ActualString := HashInstance.ComputeBytes(LData).ToString(); + CheckEquals(ExpectedString, ActualString, Format('Expected %s but got %s.', + [ExpectedString, ActualString])); +end; + +procedure TTestXXHash3.TestChunkedDataWithAsymmetricKeyKnownAnswer; +var + LIHashWithKey: IHashWithKey; + LKey: TBytes; +begin + // key = $0123456789ABCDEF (little-endian bytes below) + ExpectedString := 'FBF9CD92890CC82A'; + CheckTrue(Supports(HashInstance, IHashWithKey, LIHashWithKey), + 'HashInstance must support IHashWithKey'); + LKey := TBytes.Create($EF, $CD, $AB, $89, $67, $45, $23, $01); + LIHashWithKey.Key := LKey; + ActualString := LIHashWithKey.ComputeString(ChunkedData, TEncoding.UTF8) + .ToString(); + CheckEquals(ExpectedString, ActualString, Format('Expected %s but got %s.', + [ExpectedString, ActualString])); +end; + +procedure TTestXXHash3.TestChunkedDataWithMaxUInt64AsKeyKnownAnswer; +var + LIHashWithKey: IHashWithKey; + LKey: TBytes; + LIdx: Int32; +begin + ExpectedString := '7DEFEE70FC854900'; + CheckTrue(Supports(HashInstance, IHashWithKey, LIHashWithKey), + 'HashInstance must support IHashWithKey'); + System.SetLength(LKey, System.SizeOf(UInt64)); + for LIdx := 0 to System.High(LKey) do + begin + LKey[LIdx] := $FF; + end; + LIHashWithKey.Key := LKey; + ActualString := LIHashWithKey.ComputeString(ChunkedData, TEncoding.UTF8) + .ToString(); + CheckEquals(ExpectedString, ActualString, Format('Expected %s but got %s.', + [ExpectedString, ActualString])); +end; + initialization // Register any test cases with the test runner diff --git a/HashLib.Tests/src/HashLibTestBase.pas b/HashLib.Tests/src/HashLibTestBase.pas index cdd1792c..d544364e 100644 --- a/HashLib.Tests/src/HashLibTestBase.pas +++ b/HashLib.Tests/src/HashLibTestBase.pas @@ -164,8 +164,13 @@ THashLibAlgorithmTestCase = class abstract(THashLibTestCase) EmptyData: String = ''; DefaultData = 'HashLib4Pascal'; OneToNine = '123456789'; + // 299 bytes (13 x 23) so one-shot hashing crosses the SIMD wide-fold + // thresholds (>= 128 bytes) that the chunked passes then cross-validate. ChunkedData = - 'HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678'; + 'HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678' + + 'HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678' + + 'HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678HashLib4Pascal012345678' + + 'HashLib4Pascal012345678'; EEAABEEF = 'EEAABEEF'; ZeroToThreeInHex = '00010203'; ZeroToFifteenInHex = '000102030405060708090A0B0C0D0E0F'; diff --git a/HashLib/src/Base/HlpHashFactory.pas b/HashLib/src/Base/HlpHashFactory.pas index 9a1e7e1a..3cacc5ce 100644 --- a/HashLib/src/Base/HlpHashFactory.pas +++ b/HashLib/src/Base/HlpHashFactory.pas @@ -16,8 +16,6 @@ interface HlpAdler32, // CRC Units // HlpCRC, - HlpCRCStandard, - HlpCRC32Fast, // Hash32 Units // HlpAP, HlpBernstein, @@ -696,44 +694,44 @@ class function THashFactory.TChecksum.TCRC.CreateCRC16(APolynomial, AInitialValue: UInt64; AReflectIn, AReflectOut: Boolean; AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray): IHash; begin - Result := TCRC16.Create(APolynomial, AInitialValue, AReflectIn, AReflectOut, - AOutputXor, ACheckValue, ANames); + Result := HlpCRC.TCRC.Create(16, APolynomial, AInitialValue, AReflectIn, + AReflectOut, AOutputXor, ACheckValue, ANames); end; class function THashFactory.TChecksum.TCRC.CreateCRC16_BUYPASS: IHash; begin - Result := TCRC16_BUYPASS.Create(); + Result := HlpCRC.TCRC.CreateCRCObject(TCRCStandard.CRC16_BUYPASS); end; class function THashFactory.TChecksum.TCRC.CreateCRC32(APolynomial, AInitialValue: UInt64; AReflectIn, AReflectOut: Boolean; AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray): IHash; begin - Result := TCRC32.Create(APolynomial, AInitialValue, AReflectIn, AReflectOut, - AOutputXor, ACheckValue, ANames); + Result := HlpCRC.TCRC.Create(32, APolynomial, AInitialValue, AReflectIn, + AReflectOut, AOutputXor, ACheckValue, ANames); end; class function THashFactory.TChecksum.TCRC.CreateCRC32_CASTAGNOLI: IHash; begin - Result := HlpCRC32Fast.TCRC32_CASTAGNOLI.Create(); + Result := HlpCRC.TCRC.CreateCRCObject(TCRCStandard.CRC32C); end; class function THashFactory.TChecksum.TCRC.CreateCRC32_PKZIP: IHash; begin - Result := HlpCRC32Fast.TCRC32_PKZIP.Create(); + Result := HlpCRC.TCRC.CreateCRCObject(TCRCStandard.CRC32); end; class function THashFactory.TChecksum.TCRC.CreateCRC64(APolynomial, AInitialValue: UInt64; AReflectIn, AReflectOut: Boolean; AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray): IHash; begin - Result := TCRC64.Create(APolynomial, AInitialValue, AReflectIn, AReflectOut, - AOutputXor, ACheckValue, ANames); + Result := HlpCRC.TCRC.Create(64, APolynomial, AInitialValue, AReflectIn, + AReflectOut, AOutputXor, ACheckValue, ANames); end; class function THashFactory.TChecksum.TCRC.CreateCRC64_ECMA_182: IHash; begin - Result := TCRC64_ECMA_182.Create(); + Result := HlpCRC.TCRC.CreateCRCObject(TCRCStandard.CRC64); end; { THashFactory.TChecksum } diff --git a/HashLib/src/Checksum/HlpAdler32.pas b/HashLib/src/Checksum/HlpAdler32.pas index b7b6a15d..fc78f244 100644 --- a/HashLib/src/Checksum/HlpAdler32.pas +++ b/HashLib/src/Checksum/HlpAdler32.pas @@ -13,6 +13,19 @@ interface HlpHashResult, HlpIHashResult; +type + TAdler32UpdateProc = procedure(AData: PByte; ALength: UInt32; ASums: Pointer); + TAdler32ProcessBlocksProc = procedure(AData: PByte; ANumBlocks: UInt32; + ASums, AConstants: Pointer); + +var + Adler32_Update: TAdler32UpdateProc; + +// Shared SIMD chunking driver, exposed so each backend's Update wrapper can feed +// it the backend's block-processing kernel. +procedure Adler32_Update_Simd(AData: PByte; ALength: UInt32; ASums: Pointer; + AProcessBlocks: TAdler32ProcessBlocksProc); + type TAdler32 = class sealed(THash, IChecksum, IHash32, ITransformBlock) @@ -33,7 +46,94 @@ TAdler32 = class sealed(THash, IChecksum, IHash32, ITransformBlock) implementation uses - HlpAdler32Dispatch; + HlpAdler32Simd; + +const + ModAdler = UInt32(65521); + NMAX = UInt32(5552); + // MAX_BLOCKS_PER_CHUNK = NMAX div UInt32(32); // 173 + + Adler32Constants: array [0 .. 63] of Byte = ( + // Offset 0..31: weights [32,31,...,1] + // SSE2/SSSE3 use as two 16-byte halves; AVX2 uses full 32 bytes. + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + // Offset 32..63: ones_16 (16-bit value 1 in little-endian, repeated) + // SSSE3 uses first 16 bytes; AVX2 uses all 32 bytes. + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 + ); + +// ============================================================================= +// Scalar fallback implementation +// ============================================================================= + +procedure Adler32_Update_Scalar(AData: PByte; ALength: UInt32; ASums: Pointer); +var + LChunkLen: UInt32; + LPSumA, LPSumB: PUInt32; +begin + LPSumA := PUInt32(ASums); + LPSumB := PUInt32(PByte(ASums) + SizeOf(UInt32)); + + while ALength > 0 do + begin + LChunkLen := ALength; + if LChunkLen > NMAX then + LChunkLen := NMAX; + Dec(ALength, LChunkLen); + + while LChunkLen > 0 do + begin + LPSumA^ := LPSumA^ + AData^; + LPSumB^ := LPSumB^ + LPSumA^; + Inc(AData); + Dec(LChunkLen); + end; + + LPSumA^ := LPSumA^ mod ModAdler; + LPSumB^ := LPSumB^ mod ModAdler; + end; +end; + +procedure Adler32_Update_Simd(AData: PByte; ALength: UInt32; ASums: Pointer; + AProcessBlocks: TAdler32ProcessBlocksProc); +const + BLOCK_SIZE = UInt32(32); +var + LChunkLen, LBlocks: UInt32; + LPSumA, LPSumB: PUInt32; +begin + LPSumA := PUInt32(ASums); + LPSumB := PUInt32(PByte(ASums) + SizeOf(UInt32)); + + while ALength > 0 do + begin + LChunkLen := ALength; + if LChunkLen > NMAX then + LChunkLen := NMAX; + Dec(ALength, LChunkLen); + + LBlocks := LChunkLen div BLOCK_SIZE; + if LBlocks > 0 then + begin + AProcessBlocks(AData, LBlocks, ASums, @Adler32Constants[0]); + Inc(AData, LBlocks * BLOCK_SIZE); + Dec(LChunkLen, LBlocks * BLOCK_SIZE); + end; + + while LChunkLen > 0 do + begin + LPSumA^ := LPSumA^ + AData^; + LPSumB^ := LPSumB^ + LPSumA^; + Inc(AData); + Dec(LChunkLen); + end; + + LPSumA^ := LPSumA^ mod ModAdler; + LPSumB^ := LPSumB^ mod ModAdler; + end; +end; { TAdler32 } @@ -87,4 +187,7 @@ function TAdler32.TransformFinal: IHashResult; Initialize(); end; +initialization + Adler32_Update := TAdler32Simd.Select(@Adler32_Update_Scalar); + end. diff --git a/HashLib/src/Checksum/HlpAdler32Dispatch.pas b/HashLib/src/Checksum/HlpAdler32Dispatch.pas deleted file mode 100644 index ad92822c..00000000 --- a/HashLib/src/Checksum/HlpAdler32Dispatch.pas +++ /dev/null @@ -1,255 +0,0 @@ -unit HlpAdler32Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TAdler32UpdateProc = procedure(AData: PByte; ALength: UInt32; ASums: Pointer); - -var - Adler32_Update: TAdler32UpdateProc; - -implementation - -uses - HlpCpuFeatures, - HlpSimdLevels; - -const - ModAdler = UInt32(65521); - NMAX = UInt32(5552); - // MAX_BLOCKS_PER_CHUNK = NMAX div UInt32(32); // 173 - - Adler32Constants: array [0 .. 63] of Byte = ( - // Offset 0..31: weights [32,31,...,1] - // SSE2/SSSE3 use as two 16-byte halves; AVX2 uses full 32 bytes. - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, - 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, - // Offset 32..63: ones_16 (16-bit value 1 in little-endian, repeated) - // SSSE3 uses first 16 bytes; AVX2 uses all 32 bytes. - 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0 - ); - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure Adler32_Update_Scalar(AData: PByte; ALength: UInt32; ASums: Pointer); -var - LChunkLen: UInt32; - LPSumA, LPSumB: PUInt32; -begin - LPSumA := PUInt32(ASums); - LPSumB := PUInt32(PByte(ASums) + SizeOf(UInt32)); - - while ALength > 0 do - begin - LChunkLen := ALength; - if LChunkLen > NMAX then - LChunkLen := NMAX; - Dec(ALength, LChunkLen); - - while LChunkLen > 0 do - begin - LPSumA^ := LPSumA^ + AData^; - LPSumB^ := LPSumB^ + LPSumA^; - Inc(AData); - Dec(LChunkLen); - end; - - LPSumA^ := LPSumA^ mod ModAdler; - LPSumB^ := LPSumB^ mod ModAdler; - end; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2, SSSE3 -// x86_64: AVX2, SSSE3, SSE2 -// aarch64: NEON -// ============================================================================= - -{$IF DEFINED(HASHLIB_X86_SIMD) OR DEFINED(HASHLIB_ARM_SIMD)} - -type - TProcessBlocksProc = procedure(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - -procedure Adler32_Update_Simd(AData: PByte; ALength: UInt32; ASums: Pointer; - AProcessBlocks: TProcessBlocksProc); -const - BLOCK_SIZE = UInt32(32); -var - LChunkLen, LBlocks: UInt32; - LPSumA, LPSumB: PUInt32; -begin - LPSumA := PUInt32(ASums); - LPSumB := PUInt32(PByte(ASums) + SizeOf(UInt32)); - - while ALength > 0 do - begin - LChunkLen := ALength; - if LChunkLen > NMAX then - LChunkLen := NMAX; - Dec(ALength, LChunkLen); - - LBlocks := LChunkLen div BLOCK_SIZE; - if LBlocks > 0 then - begin - AProcessBlocks(AData, LBlocks, ASums, @Adler32Constants[0]); - Inc(AData, LBlocks * BLOCK_SIZE); - Dec(LChunkLen, LBlocks * BLOCK_SIZE); - end; - - while LChunkLen > 0 do - begin - LPSumA^ := LPSumA^ + AData^; - LPSumB^ := LPSumB^ + LPSumA^; - Inc(AData); - Dec(LChunkLen); - end; - - LPSumA^ := LPSumA^ mod ModAdler; - LPSumB^ := LPSumB^ mod ModAdler; - end; -end; - -{$IFEND} - -{$IFDEF HASHLIB_X86_SIMD} - -{$IFDEF HASHLIB_I386_ASM} - -procedure Adler32_ProcessBlocks_Sse2(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\Adler32\Adler32BlocksSse2_i386.inc} -end; - -procedure Adler32_ProcessBlocks_Ssse3(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\Adler32\Adler32BlocksSsse3_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure Adler32_ProcessBlocks_Sse2(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Adler32\Adler32BlocksSse2_x86_64.inc} -end; - -procedure Adler32_ProcessBlocks_Ssse3(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Adler32\Adler32BlocksSsse3_x86_64.inc} -end; - -procedure Adler32_ProcessBlocks_Avx2(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Adler32\Adler32BlocksAvx2_x86_64.inc} -end; - -procedure Adler32_Update_Ssse3(AData: PByte; ALength: UInt32; ASums: Pointer); -begin - Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Ssse3); -end; - -procedure Adler32_Update_Avx2(AData: PByte; ALength: UInt32; ASums: Pointer); -begin - Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Avx2); -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -procedure Adler32_Update_Sse2(AData: PByte; ALength: UInt32; ASums: Pointer); -begin - Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Sse2); -end; - -{$IFDEF HASHLIB_I386_ASM} - -procedure Adler32_Update_Ssse3(AData: PByte; ALength: UInt32; ASums: Pointer); -begin - Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Ssse3); -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$ENDIF HASHLIB_X86_SIMD} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure Adler32_ProcessBlocks_Neon(AData: PByte; ANumBlocks: UInt32; - ASums, AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\Adler32\Adler32BlocksNeon_aarch64.inc} -end; - -{$IFDEF HASHLIB_ARM_SIMD} - -procedure Adler32_Update_Neon(AData: PByte; ALength: UInt32; ASums: Pointer); -begin - Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Neon); -end; - -{$ENDIF HASHLIB_ARM_SIMD} - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - Adler32_Update := @Adler32_Update_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSSE3: - begin - Adler32_Update := @Adler32_Update_Ssse3; - end; - TX86SimdLevel.SSE2: - begin - Adler32_Update := @Adler32_Update_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - Adler32_Update := @Adler32_Update_Avx2; - end; - TX86SimdLevel.SSSE3: - begin - Adler32_Update := @Adler32_Update_Ssse3; - end; - TX86SimdLevel.SSE2: - begin - Adler32_Update := @Adler32_Update_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - TArmSimdLevel.NEON: - begin - Adler32_Update := @Adler32_Update_Neon; - end; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Checksum/HlpCRC.pas b/HashLib/src/Checksum/HlpCRC.pas index 1a697e90..9149fefa 100644 --- a/HashLib/src/Checksum/HlpCRC.pas +++ b/HashLib/src/Checksum/HlpCRC.pas @@ -1,7 +1,15 @@ unit HlpCRC; -// A vast majority if not all of the parameters for these CRC standards -// were gotten from http://reveng.sourceforge.net/crc-catalogue/. +{ + Generic CRC engine for any width 3..64 and any polynomial, plus the catalogue + of named standards (parameters from http://reveng.sourceforge.net/crc-catalogue/). + + One engine path serves every width: a slicing-by-16 table plus the SIMD / + scalar fold selected in HlpCRCCore. Widths below 8 with MSB-first input are + computed in a left-aligned domain (polynomial and state shifted up by + 8 - Width, un-shifted at finalization); LSB-first (reflected) CRCs are + width-agnostic and run as-is. +} {$I ..\Include\HashLib.inc} @@ -9,7 +17,7 @@ interface uses SysUtils, - TypInfo, + SyncObjs, Generics.Collections, HlpHashLibTypes, HlpHash, @@ -18,628 +26,71 @@ interface HlpHashResult, HlpIHashResult, HlpICRC, - HlpCRCDispatch; + HlpCRCCore; resourcestring - SUnSupportedCRCType = 'UnSupported CRC Type: "%s"'; SWidthOutOfRange = 'Width Must be Between 3 and 64. "%d"'; type /// - /// Enum of all defined and implemented CRC standards. + /// All defined and implemented CRC standards. /// - TCRCStandard = ( - - /// - /// CRC standard named "CRC3_GSM". - /// - CRC3_GSM, - - /// - /// CRC standard named "CRC3_ROHC". - /// - CRC3_ROHC, - - /// - /// CRC standard named "CRC4_INTERLAKEN". - /// - CRC4_INTERLAKEN, - - /// - /// CRC standard named "CRC4_ITU". - /// - CRC4_ITU, - - /// - /// CRC standard named "CRC5_EPC". - /// - CRC5_EPC, - - /// - /// CRC standard named "CRC5_ITU". - /// - CRC5_ITU, - - /// - /// CRC standard named "CRC5_USB". - /// - CRC5_USB, - - /// - /// CRC standard named "CRC6_CDMA2000A". - /// - CRC6_CDMA2000A, - - /// - /// CRC standard named "CRC6_CDMA2000B". - /// - CRC6_CDMA2000B, - - /// - /// CRC standard named "CRC6_DARC". - /// - CRC6_DARC, - - /// - /// CRC standard named "CRC6_GSM". - /// - CRC6_GSM, - - /// - /// CRC standard named "CRC6_ITU". - /// - CRC6_ITU, - - /// - /// CRC standard named "CRC7". - /// - CRC7, - - /// - /// CRC standard named "CRC7_ROHC". - /// - CRC7_ROHC, - - /// - /// CRC standard named "CRC7_UMTS". - /// - CRC7_UMTS, - - /// - /// CRC standard named "CRC8". - /// - CRC8, - - /// - /// CRC standard named "CRC8_AUTOSAR". - /// - CRC8_AUTOSAR, - - /// - /// CRC standard named "CRC8_BLUETOOTH". - /// - CRC8_BLUETOOTH, - - /// - /// CRC standard named "CRC8_CDMA2000". - /// - CRC8_CDMA2000, - - /// - /// CRC standard named "CRC8_DARC". - /// - CRC8_DARC, - - /// - /// CRC standard named "CRC8_DVBS2". - /// - CRC8_DVBS2, - - /// - /// CRC standard named "CRC8_EBU". - /// - CRC8_EBU, - - /// - /// CRC standard named "CRC8_GSMA". - /// - CRC8_GSMA, - - /// - /// CRC standard named "CRC8_GSMB". - /// - CRC8_GSMB, - - /// - /// CRC standard named "CRC8_ICODE". - /// - CRC8_ICODE, - - /// - /// CRC standard named "CRC8_ITU". - /// - CRC8_ITU, - - /// - /// CRC standard named "CRC8_LTE". - /// - CRC8_LTE, - - /// - /// CRC standard named "CRC8_MAXIM". - /// - CRC8_MAXIM, - - /// - /// CRC standard named "CRC8_OPENSAFETY". - /// - CRC8_OPENSAFETY, - - /// - /// CRC standard named "CRC8_ROHC". - /// - CRC8_ROHC, - - /// - /// CRC standard named "CRC8_SAEJ1850". - /// - CRC8_SAEJ1850, - - /// - /// CRC standard named "CRC8_WCDMA". - /// - CRC8_WCDMA, - - /// - /// CRC standard named "CRC8_MIFAREMAD". - /// - CRC8_MIFAREMAD, - - /// - /// CRC standard named "CRC8_NRSC5". - /// - CRC8_NRSC5, - - /// - /// CRC standard named "CRC10". - /// - CRC10, - - /// - /// CRC standard named "CRC10_CDMA2000". - /// - CRC10_CDMA2000, - - /// - /// CRC standard named "CRC10_GSM". - /// - CRC10_GSM, - - /// - /// CRC standard named "CRC11". - /// - CRC11, - - /// - /// CRC standard named "CRC11_UMTS". - /// - CRC11_UMTS, - - /// - /// CRC standard named "CRC12_CDMA2000". - /// - CRC12_CDMA2000, - - /// - /// CRC standard named "CRC12_DECT". - /// - CRC12_DECT, - - /// - /// CRC standard named "CRC12_GSM". - /// - CRC12_GSM, - - /// - /// CRC standard named "CRC12_UMTS". - /// - CRC12_UMTS, - - /// - /// CRC standard named "CRC13_BBC". - /// - CRC13_BBC, - - /// - /// CRC standard named "CRC14_DARC". - /// - CRC14_DARC, - - /// - /// CRC standard named "CRC14_GSM". - /// - CRC14_GSM, - - /// - /// CRC standard named "CRC15". - /// - CRC15, - - /// - /// CRC standard named "CRC15_MPT1327". - /// - CRC15_MPT1327, - - /// - /// CRC standard named "ARC". - /// - ARC, - - /// - /// CRC standard named "CRC16_AUGCCITT". - /// - CRC16_AUGCCITT, - - /// - /// CRC standard named "CRC16_BUYPASS". - /// - CRC16_BUYPASS, - - /// - /// CRC standard named "CRC16_CCITTFALSE". - /// - CRC16_CCITTFALSE, - - /// - /// CRC standard named "CRC16_CDMA2000". - /// - CRC16_CDMA2000, - - /// - /// CRC standard named "CRC16_CMS". - /// - CRC16_CMS, - - /// - /// CRC standard named "CRC16_DDS110". - /// - CRC16_DDS110, - - /// - /// CRC standard named "CRC16_DECTR". - /// - CRC16_DECTR, - - /// - /// CRC standard named "CRC16_DECTX". - /// - CRC16_DECTX, - - /// - /// CRC standard named "CRC16_DNP". - /// - CRC16_DNP, - - /// - /// CRC standard named "CRC16_EN13757". - /// - CRC16_EN13757, - - /// - /// CRC standard named "CRC16_GENIBUS". - /// - CRC16_GENIBUS, - - /// - /// CRC standard named "CRC16_GSM". - /// - CRC16_GSM, - - /// - /// CRC standard named "CRC16_LJ1200". - /// - CRC16_LJ1200, - - /// - /// CRC standard named "CRC16_MAXIM". - /// - CRC16_MAXIM, - - /// - /// CRC standard named "CRC16_MCRF4XX". - /// - CRC16_MCRF4XX, - - /// - /// CRC standard named "CRC16_OPENSAFETYA". - /// - CRC16_OPENSAFETYA, - - /// - /// CRC standard named "CRC16_OPENSAFETYB". - /// - CRC16_OPENSAFETYB, - - /// - /// CRC standard named "CRC16_PROFIBUS". - /// - CRC16_PROFIBUS, - - /// - /// CRC standard named "CRC16_RIELLO". - /// - CRC16_RIELLO, - - /// - /// CRC standard named "CRC16_T10DIF". - /// - CRC16_T10DIF, - - /// - /// CRC standard named "CRC16_TELEDISK". - /// - CRC16_TELEDISK, - - /// - /// CRC standard named "CRC16_TMS37157". - /// - CRC16_TMS37157, - - /// - /// CRC standard named "CRC16_USB". - /// - CRC16_USB, - - /// - /// CRC standard named "CRCA". - /// - CRCA, - - /// - /// CRC standard named "KERMIT". - /// - KERMIT, - - /// - /// CRC standard named "MODBUS". - /// - MODBUS, - - /// - /// CRC standard named "X25". - /// - X25, - - /// - /// CRC standard named "XMODEM". - /// - XMODEM, - - /// - /// CRC standard named "CRC16_NRSC5". - /// - CRC16_NRSC5, - - /// - /// CRC standard named "CRC17_CANFD". - /// - CRC17_CANFD, - - /// - /// CRC standard named "CRC21_CANFD". - /// - CRC21_CANFD, - - /// - /// CRC standard named "CRC24". - /// - CRC24, - - /// - /// CRC standard named "CRC24_BLE". - /// - CRC24_BLE, - - /// - /// CRC standard named "CRC24_FLEXRAYA". - /// - CRC24_FLEXRAYA, - - /// - /// CRC standard named "CRC24_FLEXRAYB". - /// - CRC24_FLEXRAYB, - - /// - /// CRC standard named "CRC24_INTERLAKEN". - /// - CRC24_INTERLAKEN, - - /// - /// CRC standard named "CRC24_LTEA". - /// - CRC24_LTEA, - - /// - /// CRC standard named "CRC24_LTEB". - /// - CRC24_LTEB, - - /// - /// CRC standard named "CRC24_OS9". - /// - CRC24_OS9, - - /// - /// CRC standard named "CRC30_CDMA". - /// - CRC30_CDMA, - - /// - /// CRC standard named "CRC31_PHILIPS". - /// - CRC31_PHILIPS, - - /// - /// CRC standard named "CRC32". - /// - CRC32, - - /// - /// CRC standard named "CRC32_AUTOSAR". - /// - CRC32_AUTOSAR, - - /// - /// CRC standard named "CRC32_BZIP2". - /// - CRC32_BZIP2, - - /// - /// CRC standard named "CRC32C". - /// - CRC32C, - - /// - /// CRC standard named "CRC32D". - /// - CRC32D, - - /// - /// CRC standard named "CRC32_MPEG2". - /// - CRC32_MPEG2, - - /// - /// CRC standard named "CRC32_POSIX". - /// - CRC32_POSIX, - - /// - /// CRC standard named "CRC32Q". - /// - CRC32Q, - - /// - /// CRC standard named "JAMCRC". - /// - JAMCRC, - - /// - /// CRC standard named "XFER". - /// - XFER, - - /// - /// CRC standard named "CRC32_CDROMEDC". - /// - CRC32_CDROMEDC, - - /// - /// CRC standard named "CRC40_GSM". - /// - CRC40_GSM, - - /// - /// CRC standard named "CRC64". - /// - CRC64, - - /// - /// CRC standard named "CRC64_GOISO". - /// - CRC64_GOISO, - - /// - /// CRC standard named "CRC64_WE". - /// - CRC64_WE, - - /// - /// CRC standard named "CRC64_XZ". - /// - CRC64_XZ, - - /// - /// CRC standard named "CRC64_1B". - /// - CRC64_1B, - - /// - /// CRC standard named "CRC64_Jones". - /// - CRC64_Jones); + TCRCStandard = (CRC3_GSM, CRC3_ROHC, CRC4_INTERLAKEN, CRC4_ITU, CRC5_EPC, + CRC5_ITU, CRC5_USB, CRC6_CDMA2000A, CRC6_CDMA2000B, CRC6_DARC, CRC6_GSM, + CRC6_ITU, CRC7, CRC7_ROHC, CRC7_UMTS, CRC8, CRC8_AUTOSAR, CRC8_BLUETOOTH, + CRC8_CDMA2000, CRC8_DARC, CRC8_DVBS2, CRC8_EBU, CRC8_GSMA, CRC8_GSMB, + CRC8_ICODE, CRC8_ITU, CRC8_LTE, CRC8_MAXIM, CRC8_OPENSAFETY, CRC8_ROHC, + CRC8_SAEJ1850, CRC8_WCDMA, CRC8_MIFAREMAD, CRC8_NRSC5, CRC10, + CRC10_CDMA2000, CRC10_GSM, CRC11, CRC11_UMTS, CRC12_CDMA2000, CRC12_DECT, + CRC12_GSM, CRC12_UMTS, CRC13_BBC, CRC14_DARC, CRC14_GSM, CRC15, + CRC15_MPT1327, ARC, CRC16_AUGCCITT, CRC16_BUYPASS, CRC16_CCITTFALSE, + CRC16_CDMA2000, CRC16_CMS, CRC16_DDS110, CRC16_DECTR, CRC16_DECTX, + CRC16_DNP, CRC16_EN13757, CRC16_GENIBUS, CRC16_GSM, CRC16_LJ1200, + CRC16_MAXIM, CRC16_MCRF4XX, CRC16_OPENSAFETYA, CRC16_OPENSAFETYB, + CRC16_PROFIBUS, CRC16_RIELLO, CRC16_T10DIF, CRC16_TELEDISK, CRC16_TMS37157, + CRC16_USB, CRCA, KERMIT, MODBUS, X25, XMODEM, CRC16_NRSC5, CRC17_CANFD, + CRC21_CANFD, CRC24, CRC24_BLE, CRC24_FLEXRAYA, CRC24_FLEXRAYB, + CRC24_INTERLAKEN, CRC24_LTEA, CRC24_LTEB, CRC24_OS9, CRC30_CDMA, + CRC31_PHILIPS, CRC32, CRC32_AUTOSAR, CRC32_BZIP2, CRC32C, CRC32D, + CRC32_MPEG2, CRC32_POSIX, CRC32Q, JAMCRC, XFER, CRC32_CDROMEDC, CRC40_GSM, + CRC64, CRC64_GOISO, CRC64_WE, CRC64_XZ, CRC64_1B, CRC64_Jones); type TCRC = class sealed(THash, IChecksum, ICRC, ITransformBlock) strict private - type - TCRCCacheValue = record - Table: THashLibMatrixUInt64Array; - FoldRuntime: TCRCFoldRuntimeCtx; - end; - class var FCache: TDictionary; + FCacheLock: TCriticalSection; var FNames: THashLibStringArray; - FWidth: Int32; + FWidth, FEngineWidth, FEngineShift: Int32; FPolynomial, FInitialValue, FOutputXor, FCheckValue, FCRCMask, - FCRCHighBitMask, FHash: UInt64; + FHash: UInt64; FIsInputReflected, FIsOutputReflected: Boolean; FCacheEntry: TCRCCacheValue; - const - MinTableWidth = Int32(7); - class constructor CreateCRCCache; class destructor DestroyCRCCache; - class function MakeCacheKey(APoly: UInt64; AWidth: Int32; - AReflected: Boolean): String; static; - class function GetOrCreateCacheEntry(APoly: UInt64; AWidth: Int32; - AReflected: Boolean): TCRCCacheValue; static; class function GenerateCRCTable(APoly: UInt64; AWidth: Int32; AReflected: Boolean): THashLibMatrixUInt64Array; static; + class function GetOrCreateCacheEntry(APoly: UInt64; AWidth: Int32; + AReflected: Boolean): TCRCCacheValue; static; - function GetNames: THashLibStringArray; inline; - procedure SetNames(const AValue: THashLibStringArray); inline; - function GetWidth: Int32; inline; - procedure SetWidth(AValue: Int32); inline; - function GetPolynomial: UInt64; inline; - procedure SetPolynomial(AValue: UInt64); inline; - function GetInitialValue: UInt64; inline; - procedure SetInitialValue(AValue: UInt64); inline; - function GetIsInputReflected: Boolean; inline; - procedure SetIsInputReflected(AValue: Boolean); inline; - function GetIsOutputReflected: Boolean; inline; - procedure SetIsOutputReflected(AValue: Boolean); inline; - function GetOutputXor: UInt64; inline; - procedure SetOutputXor(AValue: UInt64); inline; - function GetCheckValue: UInt64; inline; - procedure SetCheckValue(AValue: UInt64); inline; + function GetNames: THashLibStringArray; + function GetWidth: Int32; + function GetPolynomial: UInt64; + function GetInitialValue: UInt64; + function GetIsInputReflected: Boolean; + function GetIsOutputReflected: Boolean; + function GetOutputXor: UInt64; + function GetCheckValue: UInt64; - // Table-driven byte path: length < MinSimdBytes or tail after fold (no 16-byte block). + // Table-driven byte path: length < MinSimdBytes or tail after fold. procedure UpdateCRCViaByteTable(AData: PByte; ADataLength, AIndex: Int32); - // Bit-serial update without table (width <= MinTableWidth). - procedure UpdateCRCViaBitSerial(AData: PByte; ADataLength, AIndex: Int32); - - // reflects the lower 'width' LBits of 'value' - class function Reflect(AValue: UInt64; AWidth: Int32): UInt64; static; - - property Names: THashLibStringArray read GetNames write SetNames; - property Width: Int32 read GetWidth write SetWidth; - property Polynomial: UInt64 read GetPolynomial write SetPolynomial; - property InitialValue: UInt64 read GetInitialValue write SetInitialValue; - property IsInputReflected: Boolean read GetIsInputReflected - write SetIsInputReflected; - property IsOutputReflected: Boolean read GetIsOutputReflected - write SetIsOutputReflected; - property OutputXor: UInt64 read GetOutputXor write SetOutputXor; - property CheckValue: UInt64 read GetCheckValue write SetCheckValue; strict protected function GetName: String; override; @@ -663,91 +114,465 @@ TCRCCacheValue = record implementation -{ TCRC } +type + TCRCStandardDef = record + Width: Int32; + Poly, Init: UInt64; + RefIn, RefOut: Boolean; + XorOut, Check: UInt64; + Names: String; // comma-separated aliases; first one is the primary name + end; -function TCRC.GetCheckValue: UInt64; +const + CRCStandardDefs: array [TCRCStandard] of TCRCStandardDef = ( + (Width: 3; Poly: $3; Init: $0; RefIn: False; RefOut: False; XorOut: $7; + Check: $4; Names: 'CRC-3/GSM'), + (Width: 3; Poly: $3; Init: $7; RefIn: True; RefOut: True; XorOut: $0; + Check: $6; Names: 'CRC-3/ROHC'), + (Width: 4; Poly: $3; Init: $F; RefIn: False; RefOut: False; XorOut: $F; + Check: $B; Names: 'CRC-4/INTERLAKEN'), + (Width: 4; Poly: $3; Init: $0; RefIn: True; RefOut: True; XorOut: $0; + Check: $7; Names: 'CRC-4/ITU,CRC-4/G-704'), + (Width: 5; Poly: $9; Init: $9; RefIn: False; RefOut: False; XorOut: $00; + Check: $00; Names: 'CRC-5/EPC,CRC-5/EPC-C1G2'), + (Width: 5; Poly: $15; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $07; Names: 'CRC-5/ITU,CRC-5/G-704'), + (Width: 5; Poly: $05; Init: $1F; RefIn: True; RefOut: True; XorOut: $1F; + Check: $19; Names: 'CRC-5/USB'), + (Width: 6; Poly: $27; Init: $3F; RefIn: False; RefOut: False; XorOut: $00; + Check: $0D; Names: 'CRC-6/CDMA2000-A'), + (Width: 6; Poly: $07; Init: $3F; RefIn: False; RefOut: False; XorOut: $00; + Check: $3B; Names: 'CRC-6/CDMA2000-B'), + (Width: 6; Poly: $19; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $26; Names: 'CRC-6/DARC'), + (Width: 6; Poly: $2F; Init: $00; RefIn: False; RefOut: False; XorOut: $3F; + Check: $13; Names: 'CRC-6/GSM'), + (Width: 6; Poly: $03; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $06; Names: 'CRC-6/ITU,CRC-6/G-704'), + (Width: 7; Poly: $09; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $75; Names: 'CRC-7,CRC-7/MMC'), + (Width: 7; Poly: $4F; Init: $7F; RefIn: True; RefOut: True; XorOut: $00; + Check: $53; Names: 'CRC-7/ROHC'), + (Width: 7; Poly: $45; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $61; Names: 'CRC-7/UMTS'), + (Width: 8; Poly: $07; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $F4; Names: 'CRC-8,CRC-8/SMBUS'), + (Width: 8; Poly: $2F; Init: $FF; RefIn: False; RefOut: False; XorOut: $FF; + Check: $DF; Names: 'CRC-8/AUTOSAR'), + (Width: 8; Poly: $A7; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $26; Names: 'CRC-8/BLUETOOTH'), + (Width: 8; Poly: $9B; Init: $FF; RefIn: False; RefOut: False; XorOut: $00; + Check: $DA; Names: 'CRC-8/CDMA2000'), + (Width: 8; Poly: $39; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $15; Names: 'CRC-8/DARC'), + (Width: 8; Poly: $D5; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $BC; Names: 'CRC-8/DVB-S2'), + (Width: 8; Poly: $1D; Init: $FF; RefIn: True; RefOut: True; XorOut: $00; + Check: $97; Names: 'CRC-8/EBU,CRC-8/AES,CRC-8/TECH-3250'), + (Width: 8; Poly: $1D; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $37; Names: 'CRC-8/GSM-A'), + (Width: 8; Poly: $49; Init: $00; RefIn: False; RefOut: False; XorOut: $FF; + Check: $94; Names: 'CRC-8/GSM-B'), + (Width: 8; Poly: $1D; Init: $FD; RefIn: False; RefOut: False; XorOut: $00; + Check: $7E; Names: 'CRC-8/I-CODE'), + (Width: 8; Poly: $07; Init: $00; RefIn: False; RefOut: False; XorOut: $55; + Check: $A1; Names: 'CRC-8/ITU,CRC-8/I-432-1'), + (Width: 8; Poly: $9B; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $EA; Names: 'CRC-8/LTE'), + (Width: 8; Poly: $31; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $A1; Names: 'CRC-8/MAXIM,DOW-CRC,CRC-8/MAXIM-DOW'), + (Width: 8; Poly: $2F; Init: $00; RefIn: False; RefOut: False; XorOut: $00; + Check: $3E; Names: 'CRC-8/OPENSAFETY'), + (Width: 8; Poly: $07; Init: $FF; RefIn: True; RefOut: True; XorOut: $00; + Check: $D0; Names: 'CRC-8/ROHC'), + (Width: 8; Poly: $1D; Init: $FF; RefIn: False; RefOut: False; XorOut: $FF; + Check: $4B; Names: 'CRC-8/SAE-J1850'), + (Width: 8; Poly: $9B; Init: $00; RefIn: True; RefOut: True; XorOut: $00; + Check: $25; Names: 'CRC-8/WCDMA'), + (Width: 8; Poly: $1D; Init: $C7; RefIn: False; RefOut: False; XorOut: $00; + Check: $99; Names: 'CRC-8/MIFARE-MAD'), + (Width: 8; Poly: $31; Init: $FF; RefIn: False; RefOut: False; XorOut: $00; + Check: $F7; Names: 'CRC-8/NRSC-5'), + (Width: 10; Poly: $233; Init: $000; RefIn: False; RefOut: False; + XorOut: $000; Check: $199; Names: 'CRC-10,CRC-10/ATM,CRC-10/I-610'), + (Width: 10; Poly: $3D9; Init: $3FF; RefIn: False; RefOut: False; + XorOut: $000; Check: $233; Names: 'CRC-10/CDMA2000'), + (Width: 10; Poly: $175; Init: $000; RefIn: False; RefOut: False; + XorOut: $3FF; Check: $12A; Names: 'CRC-10/GSM'), + (Width: 11; Poly: $385; Init: $01A; RefIn: False; RefOut: False; + XorOut: $000; Check: $5A3; Names: 'CRC-11,CRC-11/FLEXRAY'), + (Width: 11; Poly: $307; Init: $000; RefIn: False; RefOut: False; + XorOut: $000; Check: $061; Names: 'CRC-11/UMTS'), + (Width: 12; Poly: $F13; Init: $FFF; RefIn: False; RefOut: False; + XorOut: $000; Check: $D4D; Names: 'CRC-12/CDMA2000'), + (Width: 12; Poly: $80F; Init: $000; RefIn: False; RefOut: False; + XorOut: $000; Check: $F5B; Names: 'CRC-12/DECT,X-CRC-12'), + (Width: 12; Poly: $D31; Init: $000; RefIn: False; RefOut: False; + XorOut: $FFF; Check: $B34; Names: 'CRC-12/GSM'), + (Width: 12; Poly: $80F; Init: $000; RefIn: False; RefOut: True; + XorOut: $000; Check: $DAF; Names: 'CRC-12/UMTS,CRC-12/3GPP'), + (Width: 13; Poly: $1CF5; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $04FA; Names: 'CRC-13/BBC'), + (Width: 14; Poly: $0805; Init: $0000; RefIn: True; RefOut: True; + XorOut: $0000; Check: $082D; Names: 'CRC-14/DARC'), + (Width: 14; Poly: $202D; Init: $0000; RefIn: False; RefOut: False; + XorOut: $3FFF; Check: $30AE; Names: 'CRC-14/GSM'), + (Width: 15; Poly: $4599; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $059E; Names: 'CRC-15,CRC-15/CAN'), + (Width: 15; Poly: $6815; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0001; Check: $2566; Names: 'CRC-15/MPT1327'), + (Width: 16; Poly: $8005; Init: $0000; RefIn: True; RefOut: True; + XorOut: $0000; Check: $BB3D; + Names: 'CRC-16,ARC,CRC-IBM,CRC-16/ARC,CRC-16/LHA'), + (Width: 16; Poly: $1021; Init: $1D0F; RefIn: False; RefOut: False; + XorOut: $0000; Check: $E5CC; + Names: 'CRC-16/AUG-CCITT,CRC-16/SPI-FUJITSU'), + (Width: 16; Poly: $8005; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $FEE8; + Names: 'CRC-16/BUYPASS,CRC-16/VERIFONE,CRC-16/UMTS'), + (Width: 16; Poly: $1021; Init: $FFFF; RefIn: False; RefOut: False; + XorOut: $0000; Check: $29B1; + Names: 'CRC-16/CCITT-False,CRC-16/AUTOSAR,CRC-16/IBM-3740'), + (Width: 16; Poly: $C867; Init: $FFFF; RefIn: False; RefOut: False; + XorOut: $0000; Check: $4C06; Names: 'CRC-16/CDMA2000'), + (Width: 16; Poly: $8005; Init: $FFFF; RefIn: False; RefOut: False; + XorOut: $0000; Check: $AEE7; Names: 'CRC-16/CMS'), + (Width: 16; Poly: $8005; Init: $800D; RefIn: False; RefOut: False; + XorOut: $0000; Check: $9ECF; Names: 'CRC-16/DDS-110'), + (Width: 16; Poly: $0589; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0001; Check: $007E; Names: 'CRC-16/DECT-R,R-CRC-16'), + (Width: 16; Poly: $0589; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $007F; Names: 'CRC-16/DECT-X,X-CRC-16'), + (Width: 16; Poly: $3D65; Init: $0000; RefIn: True; RefOut: True; + XorOut: $FFFF; Check: $EA82; Names: 'CRC-16/DNP'), + (Width: 16; Poly: $3D65; Init: $0000; RefIn: False; RefOut: False; + XorOut: $FFFF; Check: $C2B7; Names: 'CRC-16/EN13757'), + (Width: 16; Poly: $1021; Init: $FFFF; RefIn: False; RefOut: False; + XorOut: $FFFF; Check: $D64E; + Names: 'CRC-16/GENIBUS,CRC-16/EPC,CRC-16/I-CODE,CRC-16/DARC,CRC-16/EPC-C1G2'), + (Width: 16; Poly: $1021; Init: $0000; RefIn: False; RefOut: False; + XorOut: $FFFF; Check: $CE3C; Names: 'CRC-16/GSM'), + (Width: 16; Poly: $6F63; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $BDF4; Names: 'CRC-16/LJ1200'), + (Width: 16; Poly: $8005; Init: $0000; RefIn: True; RefOut: True; + XorOut: $FFFF; Check: $44C2; Names: 'CRC-16/MAXIM,CRC-16/MAXIM-DOW'), + (Width: 16; Poly: $1021; Init: $FFFF; RefIn: True; RefOut: True; + XorOut: $0000; Check: $6F91; Names: 'CRC-16/MCRF4XX'), + (Width: 16; Poly: $5935; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $5D38; Names: 'CRC-16/OPENSAFETY-A'), + (Width: 16; Poly: $755B; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $20FE; Names: 'CRC-16/OPENSAFETY-B'), + (Width: 16; Poly: $1DCF; Init: $FFFF; RefIn: False; RefOut: False; + XorOut: $FFFF; Check: $A819; Names: 'CRC-16/PROFIBUS,CRC-16/IEC-61158-2'), + (Width: 16; Poly: $1021; Init: $B2AA; RefIn: True; RefOut: True; + XorOut: $0000; Check: $63D0; Names: 'CRC-16/RIELLO'), + (Width: 16; Poly: $8BB7; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $D0DB; Names: 'CRC-16/T10-DIF'), + (Width: 16; Poly: $A097; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $0FB3; Names: 'CRC-16/TELEDISK'), + (Width: 16; Poly: $1021; Init: $89EC; RefIn: True; RefOut: True; + XorOut: $0000; Check: $26B1; Names: 'CRC-16/TMS37157'), + (Width: 16; Poly: $8005; Init: $FFFF; RefIn: True; RefOut: True; + XorOut: $FFFF; Check: $B4C8; Names: 'CRC-16/USB'), + (Width: 16; Poly: $1021; Init: $C6C6; RefIn: True; RefOut: True; + XorOut: $0000; Check: $BF05; Names: 'CRC-A,CRC-16/ISO-IEC-14443-3-A'), + (Width: 16; Poly: $1021; Init: $0000; RefIn: True; RefOut: True; + XorOut: $0000; Check: $2189; + Names: 'KERMIT,CRC-16/CCITT,CRC-16/CCITT-True,CRC-CCITT,CRC-16/KERMIT,CRC-16/V-41-LSB'), + (Width: 16; Poly: $8005; Init: $FFFF; RefIn: True; RefOut: True; + XorOut: $0000; Check: $4B37; Names: 'MODBUS,CRC-16/MODBUS'), + (Width: 16; Poly: $1021; Init: $FFFF; RefIn: True; RefOut: True; + XorOut: $FFFF; Check: $906E; + Names: 'X-25,CRC-16/IBM-SDLC,CRC-16/ISO-HDLC,CRC-16/ISO-IEC-14443-3-B,CRC-B,CRC-16/X-25'), + (Width: 16; Poly: $1021; Init: $0000; RefIn: False; RefOut: False; + XorOut: $0000; Check: $31C3; + Names: 'XMODEM,ZMODEM,CRC-16/ACORN,CRC-16/XMODEM,CRC-16/V-41-MSB'), + (Width: 16; Poly: $080B; Init: $FFFF; RefIn: True; RefOut: True; + XorOut: $0000; Check: $A066; Names: 'CRC-16/NRSC-5'), + (Width: 17; Poly: $1685B; Init: $00000; RefIn: False; RefOut: False; + XorOut: $00000; Check: $04F03; Names: 'CRC-17/CAN-FD'), + (Width: 21; Poly: $102899; Init: $00000; RefIn: False; RefOut: False; + XorOut: $00000; Check: $0ED841; Names: 'CRC-21/CAN-FD'), + (Width: 24; Poly: $864CFB; Init: $B704CE; RefIn: False; RefOut: False; + XorOut: $000000; Check: $21CF02; Names: 'CRC-24,CRC-24/OPENPGP'), + (Width: 24; Poly: $00065B; Init: $555555; RefIn: True; RefOut: True; + XorOut: $000000; Check: $C25A56; Names: 'CRC-24/BLE'), + (Width: 24; Poly: $5D6DCB; Init: $FEDCBA; RefIn: False; RefOut: False; + XorOut: $000000; Check: $7979BD; Names: 'CRC-24/FLEXRAY-A'), + (Width: 24; Poly: $5D6DCB; Init: $ABCDEF; RefIn: False; RefOut: False; + XorOut: $000000; Check: $1F23B8; Names: 'CRC-24/FLEXRAY-B'), + (Width: 24; Poly: $328B63; Init: $FFFFFF; RefIn: False; RefOut: False; + XorOut: $FFFFFF; Check: $B4F3E6; Names: 'CRC-24/INTERLAKEN'), + (Width: 24; Poly: $864CFB; Init: $000000; RefIn: False; RefOut: False; + XorOut: $000000; Check: $CDE703; Names: 'CRC-24/LTE-A'), + (Width: 24; Poly: $800063; Init: $000000; RefIn: False; RefOut: False; + XorOut: $000000; Check: $23EF52; Names: 'CRC-24/LTE-B'), + (Width: 24; Poly: $800063; Init: $FFFFFF; RefIn: False; RefOut: False; + XorOut: $FFFFFF; Check: $200FA5; Names: 'CRC-24/OS-9'), + (Width: 30; Poly: $2030B9C7; Init: $3FFFFFFF; RefIn: False; RefOut: False; + XorOut: $3FFFFFFF; Check: $04C34ABF; Names: 'CRC-30/CDMA'), + (Width: 31; Poly: $04C11DB7; Init: $7FFFFFFF; RefIn: False; RefOut: False; + XorOut: $7FFFFFFF; Check: $0CE9E46C; Names: 'CRC-31/PHILLIPS'), + (Width: 32; Poly: $04C11DB7; Init: $FFFFFFFF; RefIn: True; RefOut: True; + XorOut: $FFFFFFFF; Check: $CBF43926; + Names: 'CRC-32,CRC-32/ADCCP,CRC-32/V-42,CRC-32/XZ,PKZIP,CRC-32/ISO-HDLC'), + (Width: 32; Poly: $F4ACFB13; Init: $FFFFFFFF; RefIn: True; RefOut: True; + XorOut: $FFFFFFFF; Check: $1697D06A; Names: 'CRC-32/AUTOSAR'), + (Width: 32; Poly: $04C11DB7; Init: $FFFFFFFF; RefIn: False; RefOut: False; + XorOut: $FFFFFFFF; Check: $FC891918; + Names: 'CRC-32/BZIP2,CRC-32/AAL5,CRC-32/DECT-B,B-CRC-32'), + (Width: 32; Poly: $1EDC6F41; Init: $FFFFFFFF; RefIn: True; RefOut: True; + XorOut: $FFFFFFFF; Check: $E3069283; + Names: 'CRC-32C,CRC-32/BASE91-C,CRC-32/CASTAGNOLI,CRC-32/INTERLAKEN,CRC-32/ISCSI'), + (Width: 32; Poly: $A833982B; Init: $FFFFFFFF; RefIn: True; RefOut: True; + XorOut: $FFFFFFFF; Check: $87315576; Names: 'CRC-32D,CRC-32/BASE91-D'), + (Width: 32; Poly: $04C11DB7; Init: $FFFFFFFF; RefIn: False; RefOut: False; + XorOut: $00000000; Check: $0376E6E7; Names: 'CRC-32/MPEG-2'), + (Width: 32; Poly: $04C11DB7; Init: $FFFFFFFF; RefIn: False; RefOut: False; + XorOut: $00000000; Check: $0376E6E7; Names: 'CRC-32/POSIX,CKSUM'), + (Width: 32; Poly: $814141AB; Init: $00000000; RefIn: False; RefOut: False; + XorOut: $00000000; Check: $3010BF7F; Names: 'CRC-32Q,CRC-32/AIXM'), + (Width: 32; Poly: $04C11DB7; Init: $FFFFFFFF; RefIn: True; RefOut: True; + XorOut: $00000000; Check: $340BC6D9; Names: 'JAMCRC,CRC-32/JAMCRC'), + (Width: 32; Poly: $000000AF; Init: $00000000; RefIn: False; RefOut: False; + XorOut: $00000000; Check: $BD0BE338; Names: 'XFER,CRC-32/XFER'), + (Width: 32; Poly: $8001801B; Init: $00000000; RefIn: True; RefOut: True; + XorOut: $00000000; Check: $6EC2EDC4; Names: 'CRC-32/CD-ROM-EDC'), + (Width: 40; Poly: $0004820009; Init: $0000000000; RefIn: False; + RefOut: False; XorOut: $FFFFFFFFFF; Check: $D4164FC646; + Names: 'CRC-40/GSM'), + (Width: 64; Poly: $42F0E1EBA9EA3693; Init: $0000000000000000; + RefIn: False; RefOut: False; XorOut: $0000000000000000; + Check: $6C40DF5F0B497347; Names: 'CRC-64,CRC-64/ECMA-182'), + (Width: 64; Poly: $000000000000001B; Init: UInt64($FFFFFFFFFFFFFFFF); + RefIn: True; RefOut: True; XorOut: UInt64($FFFFFFFFFFFFFFFF); + Check: UInt64($B90956C775A41001); Names: 'CRC-64/GO-ISO'), + (Width: 64; Poly: $42F0E1EBA9EA3693; Init: UInt64($FFFFFFFFFFFFFFFF); + RefIn: False; RefOut: False; XorOut: UInt64($FFFFFFFFFFFFFFFF); + Check: $62EC59E3F1A4F00A; Names: 'CRC-64/WE'), + (Width: 64; Poly: $42F0E1EBA9EA3693; Init: UInt64($FFFFFFFFFFFFFFFF); + RefIn: True; RefOut: True; XorOut: UInt64($FFFFFFFFFFFFFFFF); + Check: UInt64($995DC9BBDF1939FA); Names: 'CRC-64/XZ,CRC-64/GO-ECMA'), + (Width: 64; Poly: $000000000000001B; Init: $0000000000000000; + RefIn: True; RefOut: True; XorOut: $0000000000000000; + Check: $46A5A9388A5BEFFE; Names: 'CRC-64/1B'), + (Width: 64; Poly: UInt64($AD93D23594C935A9); + Init: UInt64($FFFFFFFFFFFFFFFF); RefIn: True; RefOut: True; + XorOut: $0000000000000000; Check: UInt64($CAA717168609F281); + Names: 'CRC-64/Jones')); + +function SplitNames(const ACsv: String): THashLibStringArray; +var + LCount, LPos, LStart, LIdx: Int32; begin - Result := FCheckValue; + LCount := 1; + for LPos := 1 to System.Length(ACsv) do + if ACsv[LPos] = ',' then + System.Inc(LCount); + System.SetLength(Result, LCount); + LIdx := 0; + LStart := 1; + for LPos := 1 to System.Length(ACsv) do + if ACsv[LPos] = ',' then + begin + Result[LIdx] := System.Copy(ACsv, LStart, LPos - LStart); + System.Inc(LIdx); + LStart := LPos + 1; + end; + Result[LIdx] := System.Copy(ACsv, LStart, + System.Length(ACsv) - LStart + 1); end; -function TCRC.GetInitialValue: UInt64; +{ TCRC } + +class constructor TCRC.CreateCRCCache; begin - Result := FInitialValue; + FCache := TDictionary.Create; + FCacheLock := TCriticalSection.Create; end; -function TCRC.GetNames: THashLibStringArray; +class destructor TCRC.DestroyCRCCache; begin - Result := FNames; + FCache.Free; + FCacheLock.Free; end; -function TCRC.GetPolynomial: UInt64; +class function TCRC.GenerateCRCTable(APoly: UInt64; AWidth: Int32; + AReflected: Boolean): THashLibMatrixUInt64Array; +var + LCRC: UInt64; + LIdx, LRow, LBit: Int32; + LReflectedPoly, LHighBitMask, LMask: UInt64; begin - Result := FPolynomial; + System.SetLength(Result, 16); + for LIdx := 0 to 15 do + System.SetLength(Result[LIdx], 256); + + if AReflected then + begin + LReflectedPoly := TGF2.BitReverse(APoly, AWidth); + for LIdx := 0 to 255 do + begin + LCRC := UInt64(LIdx); + for LRow := 0 to 15 do + begin + for LBit := 0 to 7 do + LCRC := (LCRC shr 1) xor (UInt64(-Int64(LCRC and 1)) and LReflectedPoly); + Result[LRow][LIdx] := LCRC; + end; + end; + end + else + begin + LHighBitMask := UInt64(1) shl (AWidth - 1); + LMask := ((LHighBitMask - 1) shl 1) or 1; + for LIdx := 0 to 255 do + begin + LCRC := UInt64(LIdx) shl (AWidth - 8); + for LRow := 0 to 15 do + begin + for LBit := 0 to 7 do + begin + if (LCRC and LHighBitMask) <> 0 then + LCRC := ((LCRC shl 1) xor APoly) and LMask + else + LCRC := (LCRC shl 1) and LMask; + end; + Result[LRow][LIdx] := LCRC; + end; + end; + end; end; -function TCRC.GetIsInputReflected: Boolean; +class function TCRC.GetOrCreateCacheEntry(APoly: UInt64; AWidth: Int32; + AReflected: Boolean): TCRCCacheValue; +var + LKey: String; begin - Result := FIsInputReflected; + LKey := Format('Poly-%x:Width-%d:Reflected-%s', + [APoly, AWidth, BoolToStr(AReflected, True)]); + FCacheLock.Acquire; + try + if not FCache.TryGetValue(LKey, Result) then + begin + Result.Table := GenerateCRCTable(APoly, AWidth, AReflected); + CRC_InitFoldRuntimeCtx(Result.Table, APoly, AWidth, AReflected, + Result.FoldRuntime); + FCache.Add(LKey, Result); + end; + finally + FCacheLock.Release; + end; end; -function TCRC.GetIsOutputReflected: Boolean; +constructor TCRC.Create(AWidth: Int32; APolynomial, AInitial: UInt64; + AIsInputReflected, AIsOutputReflected: Boolean; + AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); begin - Result := FIsOutputReflected; + if not(AWidth in [3 .. 64]) then + begin + raise EArgumentOutOfRangeHashLibException.CreateResFmt(@SWidthOutOfRange, + [AWidth]); + end; + + inherited Create(-1, -1); // Dummy State + + case AWidth of + 3 .. 7: + Self.HashSize := 1; + 8 .. 16: + Self.HashSize := 2; + 17 .. 39: + Self.HashSize := 4; + else + Self.HashSize := 8; + end; + Self.BlockSize := 1; + + FNames := ANames; + FWidth := AWidth; + FPolynomial := APolynomial; + FInitialValue := AInitial; + FIsInputReflected := AIsInputReflected; + FIsOutputReflected := AIsOutputReflected; + FOutputXor := AOutputXor; + FCheckValue := ACheckValue; + + // MSB-first CRCs narrower than a byte run left-aligned at width 8 + // (polynomial and state shifted up by FEngineShift, un-shifted at + // finalization). LSB-first CRCs are width-agnostic and run as-is. + if (AWidth < 8) and not AIsInputReflected then + FEngineShift := 8 - AWidth + else + FEngineShift := 0; + FEngineWidth := AWidth + FEngineShift; + + FCRCMask := (((UInt64(1) shl (FEngineWidth - 1)) - 1) shl 1) or 1; + + FCacheEntry := GetOrCreateCacheEntry(APolynomial shl FEngineShift, + FEngineWidth, AIsInputReflected); end; -function TCRC.GetWidth: Int32; +class function TCRC.CreateCRCObject(AValue: TCRCStandard): ICRC; +var + LDef: TCRCStandardDef; begin - Result := FWidth; + LDef := CRCStandardDefs[AValue]; + Result := TCRC.Create(LDef.Width, LDef.Poly, LDef.Init, LDef.RefIn, + LDef.RefOut, LDef.XorOut, LDef.Check, SplitNames(LDef.Names)); end; -function TCRC.GetOutputXor: UInt64; +function TCRC.GetNames: THashLibStringArray; begin - Result := FOutputXor; + Result := FNames; end; -procedure TCRC.SetCheckValue(AValue: UInt64); +function TCRC.GetWidth: Int32; begin - FCheckValue := AValue; + Result := FWidth; end; -procedure TCRC.SetInitialValue(AValue: UInt64); +function TCRC.GetPolynomial: UInt64; begin - FInitialValue := AValue; + Result := FPolynomial; end; -procedure TCRC.SetNames(const AValue: THashLibStringArray); +function TCRC.GetInitialValue: UInt64; begin - FNames := AValue; + Result := FInitialValue; end; -procedure TCRC.SetPolynomial(AValue: UInt64); +function TCRC.GetIsInputReflected: Boolean; begin - FPolynomial := AValue; + Result := FIsInputReflected; end; -procedure TCRC.SetIsInputReflected(AValue: Boolean); +function TCRC.GetIsOutputReflected: Boolean; begin - FIsInputReflected := AValue; + Result := FIsOutputReflected; end; -procedure TCRC.SetIsOutputReflected(AValue: Boolean); +function TCRC.GetOutputXor: UInt64; begin - FIsOutputReflected := AValue; + Result := FOutputXor; end; -procedure TCRC.SetWidth(AValue: Int32); +function TCRC.GetCheckValue: UInt64; begin - FWidth := AValue; + Result := FCheckValue; end; -procedure TCRC.SetOutputXor(AValue: UInt64); +function TCRC.GetName: String; begin - FOutputXor := AValue; + Result := Format('T%s', [FNames[0]]); end; -function TCRC.GetName: String; +procedure TCRC.Initialize; begin - Result := Format('T%s', [Names[0]]); + if FIsInputReflected then + FHash := TGF2.BitReverse(FInitialValue, FWidth) + else + FHash := FInitialValue shl FEngineShift; end; procedure TCRC.UpdateCRCViaByteTable(AData: PByte; ADataLength, AIndex: Int32); @@ -762,7 +587,7 @@ procedure TCRC.UpdateCRCViaByteTable(AData: PByte; ADataLength, AIndex: Int32); LTemp := FHash; LCRCTable := FCacheEntry.Table; - if IsInputReflected then + if FIsInputReflected then begin while LLength > 0 do begin @@ -776,7 +601,7 @@ procedure TCRC.UpdateCRCViaByteTable(AData: PByte; ADataLength, AIndex: Int32); while LLength > 0 do begin LTemp := (LTemp shl 8) xor LCRCTable[0] - [Byte((LTemp shr (Width - 8)) xor LPtrData^)]; + [Byte((LTemp shr (FEngineWidth - 8)) xor LPtrData^)]; System.Inc(LPtrData); System.Dec(LLength); end; @@ -785,652 +610,13 @@ procedure TCRC.UpdateCRCViaByteTable(AData: PByte; ADataLength, AIndex: Int32); FHash := LTemp; end; -procedure TCRC.UpdateCRCViaBitSerial(AData: PByte; ADataLength, AIndex: Int32); -begin - CRC_UpdateViaBitSerial(AData, ADataLength, AIndex, FHash, Polynomial, Width, - IsInputReflected, FCRCHighBitMask); -end; - -function TCRC.Clone(): IHash; +procedure TCRC.TransformBytes(const AData: THashLibByteArray; + AIndex, ALength: Int32); var - LHashInstance: TCRC; -begin - LHashInstance := TCRC.Create(Width, Polynomial, InitialValue, - IsInputReflected, IsOutputReflected, OutputXor, CheckValue, - System.Copy(Names)); - LHashInstance.FHash := FHash; - Result := LHashInstance; - Result.BufferSize := BufferSize; -end; - -constructor TCRC.Create(AWidth: Int32; APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); -begin - - if not(AWidth in [3 .. 64]) then - begin - raise EArgumentOutOfRangeHashLibException.CreateResFmt(@SWidthOutOfRange, - [AWidth]); - end; - - inherited Create(-1, -1); // Dummy State - - case AWidth of - 0 .. 7: - begin - Self.HashSize := 1; - Self.BlockSize := 1; - end; - - 8 .. 16: - begin - Self.HashSize := 2; - Self.BlockSize := 1; - end; - - 17 .. 39: - begin - Self.HashSize := 4; - Self.BlockSize := 1; - end; - - else - begin - Self.HashSize := 8; - Self.BlockSize := 1; - end; - - end; - - Names := ANames; - Width := AWidth; - Polynomial := APolynomial; - InitialValue := AInitial; - IsInputReflected := AIsInputReflected; - IsOutputReflected := AIsOutputReflected; - OutputXor := AOutputXor; - CheckValue := ACheckValue; - - FCRCHighBitMask := UInt64(1) shl (Width - 1); - FCRCMask := ((FCRCHighBitMask - 1) shl 1) or 1; - - if Width > MinTableWidth then - FCacheEntry := GetOrCreateCacheEntry(Polynomial, Width, IsInputReflected); -end; - -class function TCRC.CreateCRCObject(AValue: TCRCStandard): ICRC; -begin - case AValue of - - TCRCStandard.CRC3_GSM: - Result := TCRC.Create(3, $3, $0, False, False, $7, $4, - THashLibStringArray.Create('CRC-3/GSM')); - - TCRCStandard.CRC3_ROHC: - Result := TCRC.Create(3, $3, $7, True, True, $0, $6, - THashLibStringArray.Create('CRC-3/ROHC')); - - TCRCStandard.CRC4_INTERLAKEN: - Result := TCRC.Create(4, $3, $F, False, False, $F, $B, - THashLibStringArray.Create('CRC-4/INTERLAKEN')); - - TCRCStandard.CRC4_ITU: - Result := TCRC.Create(4, $3, $0, True, True, $0, $7, - THashLibStringArray.Create('CRC-4/ITU', 'CRC-4/G-704')); - - TCRCStandard.CRC5_EPC: - Result := TCRC.Create(5, $9, $9, False, False, $00, $00, - THashLibStringArray.Create('CRC-5/EPC', 'CRC-5/EPC-C1G2')); - - TCRCStandard.CRC5_ITU: - Result := TCRC.Create(5, $15, $00, True, True, $00, $07, - THashLibStringArray.Create('CRC-5/ITU', 'CRC-5/G-704')); - - TCRCStandard.CRC5_USB: - Result := TCRC.Create(5, $05, $1F, True, True, $1F, $19, - THashLibStringArray.Create('CRC-5/USB')); - - TCRCStandard.CRC6_CDMA2000A: - Result := TCRC.Create(6, $27, $3F, False, False, $00, $0D, - THashLibStringArray.Create('CRC-6/CDMA2000-A')); - - TCRCStandard.CRC6_CDMA2000B: - Result := TCRC.Create(6, $07, $3F, False, False, $00, $3B, - THashLibStringArray.Create('CRC-6/CDMA2000-B')); - - TCRCStandard.CRC6_DARC: - Result := TCRC.Create(6, $19, $00, True, True, $00, $26, - THashLibStringArray.Create('CRC-6/DARC')); - - TCRCStandard.CRC6_GSM: - Result := TCRC.Create(6, $2F, $00, False, False, $3F, $13, - THashLibStringArray.Create('CRC-6/GSM')); - - TCRCStandard.CRC6_ITU: - Result := TCRC.Create(6, $03, $00, True, True, $00, $06, - THashLibStringArray.Create('CRC-6/ITU', 'CRC-6/G-704')); - - TCRCStandard.CRC7: - Result := TCRC.Create(7, $09, $00, False, False, $00, $75, - THashLibStringArray.Create('CRC-7', 'CRC-7/MMC')); - - TCRCStandard.CRC7_ROHC: - Result := TCRC.Create(7, $4F, $7F, True, True, $00, $53, - THashLibStringArray.Create('CRC-7/ROHC')); - - TCRCStandard.CRC7_UMTS: - Result := TCRC.Create(7, $45, $00, False, False, $00, $61, - THashLibStringArray.Create('CRC-7/UMTS')); - - TCRCStandard.CRC8: - Result := TCRC.Create(8, $07, $00, False, False, $00, $F4, - THashLibStringArray.Create('CRC-8', 'CRC-8/SMBUS')); - - TCRCStandard.CRC8_AUTOSAR: - Result := TCRC.Create(8, $2F, $FF, False, False, $FF, $DF, - THashLibStringArray.Create('CRC-8/AUTOSAR')); - - TCRCStandard.CRC8_BLUETOOTH: - Result := TCRC.Create(8, $A7, $00, True, True, $00, $26, - THashLibStringArray.Create('CRC-8/BLUETOOTH')); - - TCRCStandard.CRC8_CDMA2000: - Result := TCRC.Create(8, $9B, $FF, False, False, $00, $DA, - THashLibStringArray.Create('CRC-8/CDMA2000')); - - TCRCStandard.CRC8_DARC: - Result := TCRC.Create(8, $39, $00, True, True, $00, $15, - THashLibStringArray.Create('CRC-8/DARC')); - - TCRCStandard.CRC8_DVBS2: - Result := TCRC.Create(8, $D5, $00, False, False, $00, $BC, - THashLibStringArray.Create('CRC-8/DVB-S2')); - - TCRCStandard.CRC8_EBU: - Result := TCRC.Create(8, $1D, $FF, True, True, $00, $97, - THashLibStringArray.Create('CRC-8/EBU', 'CRC-8/AES', - 'CRC-8/TECH-3250')); - - TCRCStandard.CRC8_GSMA: - Result := TCRC.Create(8, $1D, $00, False, False, $00, $37, - THashLibStringArray.Create('CRC-8/GSM-A')); - - TCRCStandard.CRC8_GSMB: - Result := TCRC.Create(8, $49, $00, False, False, $FF, $94, - THashLibStringArray.Create('CRC-8/GSM-B')); - - TCRCStandard.CRC8_ICODE: - Result := TCRC.Create(8, $1D, $FD, False, False, $00, $7E, - THashLibStringArray.Create('CRC-8/I-CODE')); - - TCRCStandard.CRC8_ITU: - Result := TCRC.Create(8, $07, $00, False, False, $55, $A1, - THashLibStringArray.Create('CRC-8/ITU', 'CRC-8/I-432-1')); - - TCRCStandard.CRC8_LTE: - Result := TCRC.Create(8, $9B, $00, False, False, $00, $EA, - THashLibStringArray.Create('CRC-8/LTE')); - - TCRCStandard.CRC8_MAXIM: - Result := TCRC.Create(8, $31, $00, True, True, $00, $A1, - THashLibStringArray.Create('CRC-8/MAXIM', 'DOW-CRC', - 'CRC-8/MAXIM-DOW')); - - TCRCStandard.CRC8_OPENSAFETY: - Result := TCRC.Create(8, $2F, $00, False, False, $00, $3E, - THashLibStringArray.Create('CRC-8/OPENSAFETY')); - - TCRCStandard.CRC8_ROHC: - Result := TCRC.Create(8, $07, $FF, True, True, $00, $D0, - THashLibStringArray.Create('CRC-8/ROHC')); - - TCRCStandard.CRC8_SAEJ1850: - Result := TCRC.Create(8, $1D, $FF, False, False, $FF, $4B, - THashLibStringArray.Create('CRC-8/SAE-J1850')); - - TCRCStandard.CRC8_WCDMA: - Result := TCRC.Create(8, $9B, $00, True, True, $00, $25, - THashLibStringArray.Create('CRC-8/WCDMA')); - - TCRCStandard.CRC8_MIFAREMAD: - Result := TCRC.Create(8, $1D, $C7, False, False, $00, $99, - THashLibStringArray.Create('CRC-8/MIFARE-MAD')); - - TCRCStandard.CRC8_NRSC5: - Result := TCRC.Create(8, $31, $FF, False, False, $00, $F7, - THashLibStringArray.Create('CRC-8/NRSC-5')); - - TCRCStandard.CRC10: - Result := TCRC.Create(10, $233, $000, False, False, $000, $199, - THashLibStringArray.Create('CRC-10', 'CRC-10/ATM', 'CRC-10/I-610')); - - TCRCStandard.CRC10_CDMA2000: - Result := TCRC.Create(10, $3D9, $3FF, False, False, $000, $233, - THashLibStringArray.Create('CRC-10/CDMA2000')); - - TCRCStandard.CRC10_GSM: - Result := TCRC.Create(10, $175, $000, False, False, $3FF, $12A, - THashLibStringArray.Create('CRC-10/GSM')); - - TCRCStandard.CRC11: - Result := TCRC.Create(11, $385, $01A, False, False, $000, $5A3, - THashLibStringArray.Create('CRC-11', 'CRC-11/FLEXRAY')); - - TCRCStandard.CRC11_UMTS: - Result := TCRC.Create(11, $307, $000, False, False, $000, $061, - THashLibStringArray.Create('CRC-11/UMTS')); - - TCRCStandard.CRC12_CDMA2000: - Result := TCRC.Create(12, $F13, $FFF, False, False, $000, $D4D, - THashLibStringArray.Create('CRC-12/CDMA2000')); - - TCRCStandard.CRC12_DECT: - Result := TCRC.Create(12, $80F, $000, False, False, $000, $F5B, - THashLibStringArray.Create('CRC-12/DECT', 'X-CRC-12')); - - TCRCStandard.CRC12_GSM: - Result := TCRC.Create(12, $D31, $000, False, False, $FFF, $B34, - THashLibStringArray.Create('CRC-12/GSM')); - - TCRCStandard.CRC12_UMTS: - Result := TCRC.Create(12, $80F, $000, False, True, $000, $DAF, - THashLibStringArray.Create('CRC-12/UMTS', 'CRC-12/3GPP')); - - TCRCStandard.CRC13_BBC: - Result := TCRC.Create(13, $1CF5, $0000, False, False, $0000, $04FA, - THashLibStringArray.Create('CRC-13/BBC')); - - TCRCStandard.CRC14_DARC: - Result := TCRC.Create(14, $0805, $0000, True, True, $0000, $082D, - THashLibStringArray.Create('CRC-14/DARC')); - - TCRCStandard.CRC14_GSM: - Result := TCRC.Create(14, $202D, $0000, False, False, $3FFF, $30AE, - THashLibStringArray.Create('CRC-14/GSM')); - - TCRCStandard.CRC15: - Result := TCRC.Create(15, $4599, $0000, False, False, $0000, $059E, - THashLibStringArray.Create('CRC-15', 'CRC-15/CAN')); - - TCRCStandard.CRC15_MPT1327: - Result := TCRC.Create(15, $6815, $0000, False, False, $0001, $2566, - THashLibStringArray.Create('CRC-15/MPT1327')); - - TCRCStandard.ARC: - Result := TCRC.Create(16, $8005, $0000, True, True, $0000, $BB3D, - THashLibStringArray.Create('CRC-16', 'ARC', 'CRC-IBM', 'CRC-16/ARC', - 'CRC-16/LHA')); - - TCRCStandard.CRC16_AUGCCITT: - Result := TCRC.Create(16, $1021, $1D0F, False, False, $0000, $E5CC, - THashLibStringArray.Create('CRC-16/AUG-CCITT', 'CRC-16/SPI-FUJITSU')); - - TCRCStandard.CRC16_BUYPASS: - Result := TCRC.Create(16, $8005, $0000, False, False, $0000, $FEE8, - THashLibStringArray.Create('CRC-16/BUYPASS', 'CRC-16/VERIFONE', - 'CRC-16/UMTS')); - - TCRCStandard.CRC16_CCITTFALSE: - Result := TCRC.Create(16, $1021, $FFFF, False, False, $0000, $29B1, - THashLibStringArray.Create('CRC-16/CCITT-False', 'CRC-16/AUTOSAR', - 'CRC-16/IBM-3740')); - - TCRCStandard.CRC16_CDMA2000: - Result := TCRC.Create(16, $C867, $FFFF, False, False, $0000, $4C06, - THashLibStringArray.Create('CRC-16/CDMA2000')); - - TCRCStandard.CRC16_CMS: - Result := TCRC.Create(16, $8005, $FFFF, False, False, $0000, $AEE7, - THashLibStringArray.Create('CRC-16/CMS')); - - TCRCStandard.CRC16_DDS110: - Result := TCRC.Create(16, $8005, $800D, False, False, $0000, $9ECF, - THashLibStringArray.Create('CRC-16/DDS-110')); - - TCRCStandard.CRC16_DECTR: - Result := TCRC.Create(16, $0589, $0000, False, False, $0001, $007E, - THashLibStringArray.Create('CRC-16/DECT-R', 'R-CRC-16')); - - TCRCStandard.CRC16_DECTX: - Result := TCRC.Create(16, $0589, $0000, False, False, $0000, $007F, - THashLibStringArray.Create('CRC-16/DECT-X', 'X-CRC-16')); - - TCRCStandard.CRC16_DNP: - Result := TCRC.Create(16, $3D65, $0000, True, True, $FFFF, $EA82, - THashLibStringArray.Create('CRC-16/DNP')); - - TCRCStandard.CRC16_EN13757: - Result := TCRC.Create(16, $3D65, $0000, False, False, $FFFF, $C2B7, - THashLibStringArray.Create('CRC-16/EN13757')); - - TCRCStandard.CRC16_GENIBUS: - Result := TCRC.Create(16, $1021, $FFFF, False, False, $FFFF, $D64E, - THashLibStringArray.Create('CRC-16/GENIBUS', 'CRC-16/EPC', - 'CRC-16/I-CODE', 'CRC-16/DARC', 'CRC-16/EPC-C1G2')); - - TCRCStandard.CRC16_GSM: - Result := TCRC.Create(16, $1021, $0000, False, False, $FFFF, $CE3C, - THashLibStringArray.Create('CRC-16/GSM')); - - TCRCStandard.CRC16_LJ1200: - Result := TCRC.Create(16, $6F63, $0000, False, False, $0000, $BDF4, - THashLibStringArray.Create('CRC-16/LJ1200')); - - TCRCStandard.CRC16_MAXIM: - Result := TCRC.Create(16, $8005, $0000, True, True, $FFFF, $44C2, - THashLibStringArray.Create('CRC-16/MAXIM', 'CRC-16/MAXIM-DOW')); - - TCRCStandard.CRC16_MCRF4XX: - Result := TCRC.Create(16, $1021, $FFFF, True, True, $0000, $6F91, - THashLibStringArray.Create('CRC-16/MCRF4XX')); - - TCRCStandard.CRC16_OPENSAFETYA: - Result := TCRC.Create(16, $5935, $0000, False, False, $0000, $5D38, - THashLibStringArray.Create('CRC-16/OPENSAFETY-A')); - - TCRCStandard.CRC16_OPENSAFETYB: - Result := TCRC.Create(16, $755B, $0000, False, False, $0000, $20FE, - THashLibStringArray.Create('CRC-16/OPENSAFETY-B')); - - TCRCStandard.CRC16_PROFIBUS: - Result := TCRC.Create(16, $1DCF, $FFFF, False, False, $FFFF, $A819, - THashLibStringArray.Create('CRC-16/PROFIBUS', 'CRC-16/IEC-61158-2')); - - TCRCStandard.CRC16_RIELLO: - Result := TCRC.Create(16, $1021, $B2AA, True, True, $0000, $63D0, - THashLibStringArray.Create('CRC-16/RIELLO')); - - TCRCStandard.CRC16_T10DIF: - Result := TCRC.Create(16, $8BB7, $0000, False, False, $0000, $D0DB, - THashLibStringArray.Create('CRC-16/T10-DIF')); - - TCRCStandard.CRC16_TELEDISK: - Result := TCRC.Create(16, $A097, $0000, False, False, $0000, $0FB3, - THashLibStringArray.Create('CRC-16/TELEDISK')); - - TCRCStandard.CRC16_TMS37157: - Result := TCRC.Create(16, $1021, $89EC, True, True, $0000, $26B1, - THashLibStringArray.Create('CRC-16/TMS37157')); - - TCRCStandard.CRC16_USB: - Result := TCRC.Create(16, $8005, $FFFF, True, True, $FFFF, $B4C8, - THashLibStringArray.Create('CRC-16/USB')); - - TCRCStandard.CRCA: - Result := TCRC.Create(16, $1021, $C6C6, True, True, $0000, $BF05, - THashLibStringArray.Create('CRC-A', 'CRC-16/ISO-IEC-14443-3-A')); - - TCRCStandard.KERMIT: - Result := TCRC.Create(16, $1021, $0000, True, True, $0000, $2189, - THashLibStringArray.Create('KERMIT', 'CRC-16/CCITT', - 'CRC-16/CCITT-True', 'CRC-CCITT', 'CRC-16/KERMIT', 'CRC-16/V-41-LSB')); - - TCRCStandard.MODBUS: - Result := TCRC.Create(16, $8005, $FFFF, True, True, $0000, $4B37, - THashLibStringArray.Create('MODBUS', 'CRC-16/MODBUS')); - - TCRCStandard.X25: - Result := TCRC.Create(16, $1021, $FFFF, True, True, $FFFF, $906E, - THashLibStringArray.Create('X-25', 'CRC-16/IBM-SDLC', 'CRC-16/ISO-HDLC', - 'CRC-16/ISO-IEC-14443-3-B', 'CRC-B', 'CRC-16/X-25')); - - TCRCStandard.XMODEM: - Result := TCRC.Create(16, $1021, $0000, False, False, $0000, $31C3, - THashLibStringArray.Create('XMODEM', 'ZMODEM', 'CRC-16/ACORN', - 'CRC-16/XMODEM', 'CRC-16/V-41-MSB')); - - TCRCStandard.CRC16_NRSC5: - Result := TCRC.Create(16, $080B, $FFFF, True, True, $0000, $A066, - THashLibStringArray.Create('CRC-16/NRSC-5')); - - TCRCStandard.CRC17_CANFD: - Result := TCRC.Create(17, $1685B, $00000, False, False, $00000, $04F03, - THashLibStringArray.Create('CRC-17/CAN-FD')); - - TCRCStandard.CRC21_CANFD: - Result := TCRC.Create(21, $102899, $00000, False, False, $00000, $0ED841, - THashLibStringArray.Create('CRC-21/CAN-FD')); - - TCRCStandard.CRC24: - Result := TCRC.Create(24, $864CFB, $B704CE, False, False, $000000, - $21CF02, THashLibStringArray.Create('CRC-24', 'CRC-24/OPENPGP')); - - TCRCStandard.CRC24_BLE: - Result := TCRC.Create(24, $00065B, $555555, True, True, $000000, $C25A56, - THashLibStringArray.Create('CRC-24/BLE')); - - TCRCStandard.CRC24_FLEXRAYA: - Result := TCRC.Create(24, $5D6DCB, $FEDCBA, False, False, $000000, - $7979BD, THashLibStringArray.Create('CRC-24/FLEXRAY-A')); - - TCRCStandard.CRC24_FLEXRAYB: - Result := TCRC.Create(24, $5D6DCB, $ABCDEF, False, False, $000000, - $1F23B8, THashLibStringArray.Create('CRC-24/FLEXRAY-B')); - - TCRCStandard.CRC24_INTERLAKEN: - Result := TCRC.Create(24, $328B63, $FFFFFF, False, False, $FFFFFF, - $B4F3E6, THashLibStringArray.Create('CRC-24/INTERLAKEN')); - - TCRCStandard.CRC24_LTEA: - Result := TCRC.Create(24, $864CFB, $000000, False, False, $000000, - $CDE703, THashLibStringArray.Create('CRC-24/LTE-A')); - - TCRCStandard.CRC24_LTEB: - Result := TCRC.Create(24, $800063, $000000, False, False, $000000, - $23EF52, THashLibStringArray.Create('CRC-24/LTE-B')); - - TCRCStandard.CRC24_OS9: - Result := TCRC.Create(24, $800063, $FFFFFF, False, False, $FFFFFF, - $200FA5, THashLibStringArray.Create('CRC-24/OS-9')); - - TCRCStandard.CRC30_CDMA: - Result := TCRC.Create(30, $2030B9C7, $3FFFFFFF, False, False, $3FFFFFFF, - $04C34ABF, THashLibStringArray.Create('CRC-30/CDMA')); - - TCRCStandard.CRC31_PHILIPS: - Result := TCRC.Create(31, $04C11DB7, $7FFFFFFF, False, False, $7FFFFFFF, - $0CE9E46C, THashLibStringArray.Create('CRC-31/PHILLIPS')); - - TCRCStandard.CRC32: - Result := TCRC.Create(32, $04C11DB7, $FFFFFFFF, True, True, $FFFFFFFF, - $CBF43926, THashLibStringArray.Create('CRC-32', 'CRC-32/ADCCP', - 'CRC-32/V-42', 'CRC-32/XZ', 'PKZIP', 'CRC-32/ISO-HDLC')); - - TCRCStandard.CRC32_AUTOSAR: - Result := TCRC.Create(32, $F4ACFB13, $FFFFFFFF, True, True, $FFFFFFFF, - $1697D06A, THashLibStringArray.Create('CRC-32/AUTOSAR')); - - TCRCStandard.CRC32_BZIP2: - Result := TCRC.Create(32, $04C11DB7, $FFFFFFFF, False, False, $FFFFFFFF, - $FC891918, THashLibStringArray.Create('CRC-32/BZIP2', 'CRC-32/AAL5', - 'CRC-32/DECT-B', 'B-CRC-32')); - - TCRCStandard.CRC32C: - Result := TCRC.Create(32, $1EDC6F41, $FFFFFFFF, True, True, $FFFFFFFF, - $E3069283, THashLibStringArray.Create('CRC-32C', 'CRC-32/BASE91-C', - 'CRC-32/CASTAGNOLI', 'CRC-32/INTERLAKEN', 'CRC-32/ISCSI')); - - TCRCStandard.CRC32D: - Result := TCRC.Create(32, $A833982B, $FFFFFFFF, True, True, $FFFFFFFF, - $87315576, THashLibStringArray.Create('CRC-32D', 'CRC-32/BASE91-D')); - - TCRCStandard.CRC32_MPEG2: - Result := TCRC.Create(32, $04C11DB7, $FFFFFFFF, False, False, $00000000, - $0376E6E7, THashLibStringArray.Create('CRC-32/MPEG-2')); - - TCRCStandard.CRC32_POSIX: - Result := TCRC.Create(32, $04C11DB7, $FFFFFFFF, False, False, $00000000, - $0376E6E7, THashLibStringArray.Create('CRC-32/POSIX', 'CKSUM')); - - TCRCStandard.CRC32Q: - Result := TCRC.Create(32, $814141AB, $00000000, False, False, $00000000, - $3010BF7F, THashLibStringArray.Create('CRC-32Q', 'CRC-32/AIXM')); - - TCRCStandard.JAMCRC: - Result := TCRC.Create(32, $04C11DB7, $FFFFFFFF, True, True, $00000000, - $340BC6D9, THashLibStringArray.Create('JAMCRC', 'CRC-32/JAMCRC')); - - TCRCStandard.XFER: - Result := TCRC.Create(32, $000000AF, $00000000, False, False, $00000000, - $BD0BE338, THashLibStringArray.Create('XFER', 'CRC-32/XFER')); - - TCRCStandard.CRC32_CDROMEDC: - Result := TCRC.Create(32, $8001801B, $00000000, True, True, $00000000, - $6EC2EDC4, THashLibStringArray.Create('CRC-32/CD-ROM-EDC')); - - TCRCStandard.CRC40_GSM: - Result := TCRC.Create(40, $0004820009, $0000000000, False, False, - $FFFFFFFFFF, $D4164FC646, THashLibStringArray.Create('CRC-40/GSM')); - - TCRCStandard.CRC64: - Result := TCRC.Create(64, $42F0E1EBA9EA3693, $0000000000000000, False, - False, $0000000000000000, $6C40DF5F0B497347, - THashLibStringArray.Create('CRC-64', 'CRC-64/ECMA-182')); - - TCRCStandard.CRC64_GOISO: - Result := TCRC.Create(64, $000000000000001B, UInt64($FFFFFFFFFFFFFFFF), - True, True, UInt64($FFFFFFFFFFFFFFFF), UInt64($B90956C775A41001), - THashLibStringArray.Create('CRC-64/GO-ISO')); - - TCRCStandard.CRC64_WE: - Result := TCRC.Create(64, $42F0E1EBA9EA3693, UInt64($FFFFFFFFFFFFFFFF), - False, False, UInt64($FFFFFFFFFFFFFFFF), $62EC59E3F1A4F00A, - THashLibStringArray.Create('CRC-64/WE')); - - TCRCStandard.CRC64_XZ: - Result := TCRC.Create(64, $42F0E1EBA9EA3693, UInt64($FFFFFFFFFFFFFFFF), - True, True, UInt64($FFFFFFFFFFFFFFFF), UInt64($995DC9BBDF1939FA), - THashLibStringArray.Create('CRC-64/XZ', 'CRC-64/GO-ECMA')); - - TCRCStandard.CRC64_1B: - Result := TCRC.Create(64, $000000000000001B, UInt64($0000000000000000), - True, True, UInt64($0000000000000000), $46A5A9388A5BEFFE, - THashLibStringArray.Create('CRC-64/1B')); - - TCRCStandard.CRC64_Jones: - Result := TCRC.Create(64, UInt64($AD93D23594C935A9), - UInt64($FFFFFFFFFFFFFFFF), True, True, UInt64($0000000000000000), - UInt64($CAA717168609F281), THashLibStringArray.Create('CRC-64/Jones')) - - else - raise EArgumentInvalidHashLibException.CreateResFmt(@SUnSupportedCRCType, - [GetEnumName(TypeInfo(TCRCStandard), Ord(AValue))]); - - end; -end; - -class constructor TCRC.CreateCRCCache; -begin - FCache := TDictionary.Create; -end; - -class destructor TCRC.DestroyCRCCache; -begin - FCache.Free; -end; - -class function TCRC.MakeCacheKey(APoly: UInt64; AWidth: Int32; - AReflected: Boolean): String; -begin - Result := Format('Poly-%x:Width-%d:Reflected-%s', - [APoly, AWidth, BoolToStr(AReflected, True)]); -end; - -class function TCRC.GenerateCRCTable(APoly: UInt64; AWidth: Int32; - AReflected: Boolean): THashLibMatrixUInt64Array; -var - LCRC: UInt64; - LIdx, LRow, LBit: Int32; - LReflectedPoly, LHighBitMask, LMask: UInt64; -begin - System.SetLength(Result, 16); - for LIdx := 0 to 15 do - System.SetLength(Result[LIdx], 256); - - if AReflected then - begin - LReflectedPoly := Reflect(APoly, AWidth); - for LIdx := 0 to 255 do - begin - LCRC := UInt64(LIdx); - for LRow := 0 to 15 do - begin - for LBit := 0 to 7 do - LCRC := (LCRC shr 1) xor (UInt64(-Int64(LCRC and 1)) and LReflectedPoly); - Result[LRow][LIdx] := LCRC; - end; - end; - end - else - begin - LHighBitMask := UInt64(1) shl (AWidth - 1); - LMask := ((LHighBitMask - 1) shl 1) or 1; - for LIdx := 0 to 255 do - begin - LCRC := UInt64(LIdx) shl (AWidth - 8); - for LRow := 0 to 15 do - begin - for LBit := 0 to 7 do - begin - if (LCRC and LHighBitMask) <> 0 then - LCRC := ((LCRC shl 1) xor APoly) and LMask - else - LCRC := (LCRC shl 1) and LMask; - end; - Result[LRow][LIdx] := LCRC; - end; - end; - end; -end; - -class function TCRC.GetOrCreateCacheEntry(APoly: UInt64; AWidth: Int32; - AReflected: Boolean): TCRCCacheValue; -var - LKey: String; -begin - LKey := MakeCacheKey(APoly, AWidth, AReflected); - if not FCache.TryGetValue(LKey, Result) then - begin - Result.Table := GenerateCRCTable(APoly, AWidth, AReflected); - CRCDispatch_InitRuntimeCtx(Result.Table, APoly, AWidth, AReflected, - Result.FoldRuntime); - FCache.Add(LKey, Result); - end; -end; - -procedure TCRC.Initialize; -begin - FHash := InitialValue; - if (Width > MinTableWidth) and IsInputReflected then - FHash := Reflect(FHash, Width); -end; - -class function TCRC.Reflect(AValue: UInt64; AWidth: Int32): UInt64; -var - LIdx, LJdx: UInt64; -begin - LJdx := 1; - Result := 0; - LIdx := UInt64(1) shl (AWidth - 1); - while LIdx <> 0 do - begin - if ((AValue and LIdx) <> 0) then - begin - Result := Result or LJdx; - end; - LJdx := LJdx shl 1; - LIdx := LIdx shr 1; - end; -end; - -procedure TCRC.TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); -var - LPtrAData: PByte; - LFoldFunc: TCRCFoldFunc; - LState: array [0 .. 1] of UInt64; - LProcessed, LTail: Int32; + LPtrAData: PByte; + LFoldFunc: TCRCFoldFunc; + LState: UInt64; + LProcessed, LTail: Int32; begin {$IFDEF DEBUG} System.Assert(AIndex >= 0); @@ -1440,111 +626,64 @@ procedure TCRC.TransformBytes(const AData: THashLibByteArray; LPtrAData := PByte(AData); - if Width > MinTableWidth then + if ALength >= MinSimdBytes then begin - if ALength >= MinSimdBytes then + if FIsInputReflected then begin - if IsInputReflected then - begin - LFoldFunc := CRC_Fold_Reflected; - LState[0] := FHash; - end - else - begin - LFoldFunc := CRC_Fold_Forward; - if CRC_Fold_UsesCarrylessMul then - begin - if Width < 64 then - LState[0] := FHash shl (64 - Width) - else - LState[0] := FHash; - end - else - LState[0] := FHash; - end; - - LState[1] := 0; - LProcessed := ALength and (not Int32(15)); - FHash := LFoldFunc(LPtrAData + AIndex, UInt32(LProcessed), @LState[0], - @FCacheEntry.FoldRuntime) and FCRCMask; - LTail := ALength - LProcessed; - if LTail > 0 then - UpdateCRCViaByteTable(LPtrAData, LTail, AIndex + LProcessed); + LFoldFunc := CRC_Fold.Reflected; + LState := FHash; end else - UpdateCRCViaByteTable(LPtrAData, ALength, AIndex); + begin + LFoldFunc := CRC_Fold.Fwd; + if CRC_Fold.UsesCarrylessMul then + LState := FHash shl (64 - FEngineWidth) + else + LState := FHash; + end; + + LProcessed := ALength and (not Int32(15)); + FHash := LFoldFunc(LPtrAData + AIndex, UInt32(LProcessed), @LState, + @FCacheEntry.FoldRuntime) and FCRCMask; + LTail := ALength - LProcessed; + if LTail > 0 then + UpdateCRCViaByteTable(LPtrAData, LTail, AIndex + LProcessed); end else - UpdateCRCViaBitSerial(LPtrAData, ALength, AIndex); + UpdateCRCViaByteTable(LPtrAData, ALength, AIndex); end; function TCRC.TransformFinal: IHashResult; -var - LUInt64: UInt64; - LUInt32: UInt32; - LUInt16: UInt16; - LUInt8: UInt8; begin - - if Width > MinTableWidth then - begin - if (IsInputReflected xor IsOutputReflected) then - begin - FHash := Reflect(FHash, Width); - end; - end - else - begin - if (IsOutputReflected) then - begin - FHash := Reflect(FHash, Width); - end; - end; - - FHash := FHash xor OutputXor; - FHash := FHash and FCRCMask; - - if Width = 21 then // special case - begin - LUInt32 := UInt32(FHash); - - Result := THashResult.Create(LUInt32); - - Initialize(); - - Exit; - end; - - case Width shr 3 of - 0: - begin - LUInt8 := UInt8(FHash); - Result := THashResult.Create(LUInt8); - - end; - - 1 .. 2: - begin - LUInt16 := UInt16(FHash); - - Result := THashResult.Create(LUInt16); - end; - - 3 .. 4: - begin - LUInt32 := UInt32(FHash); - - Result := THashResult.Create(LUInt32); - end + FHash := FHash shr FEngineShift; + if FIsInputReflected xor FIsOutputReflected then + FHash := TGF2.BitReverse(FHash, FWidth); + FHash := (FHash xor FOutputXor) and (FCRCMask shr FEngineShift); + + case HashSize of + 1: + Result := THashResult.Create(UInt8(FHash)); + 2: + Result := THashResult.Create(UInt16(FHash)); + 4: + Result := THashResult.Create(UInt32(FHash)); else - begin - LUInt64 := (FHash); - - Result := THashResult.Create(LUInt64); - end; + Result := THashResult.Create(FHash); end; Initialize(); end; +function TCRC.Clone(): IHash; +var + LHashInstance: TCRC; +begin + LHashInstance := TCRC.Create(FWidth, FPolynomial, FInitialValue, + FIsInputReflected, FIsOutputReflected, FOutputXor, FCheckValue, + System.Copy(FNames)); + LHashInstance.FHash := FHash; + Result := LHashInstance; + Result.BufferSize := BufferSize; +end; + end. diff --git a/HashLib/src/Checksum/HlpCRC32Fast.pas b/HashLib/src/Checksum/HlpCRC32Fast.pas deleted file mode 100644 index f75a8c3b..00000000 --- a/HashLib/src/Checksum/HlpCRC32Fast.pas +++ /dev/null @@ -1,170 +0,0 @@ -unit HlpCRC32Fast; - -{$I ..\Include\HashLib.inc} - -interface - -uses - HlpBinaryPrimitives, - HlpHashLibTypes, - HlpHash, - HlpIHash, - HlpIHashInfo, - HlpHashResult, - HlpIHashResult, - HlpCRCDispatch; - -type - - TCRC32Fast = class(THash, IChecksum, IHash32, ITransformBlock) - - strict protected - var - FCurrentCRC: UInt32; - - public - - constructor Create(); - - procedure Initialize(); override; - function TransformFinal(): IHashResult; override; - - end; - - TCRC32_PKZIP = class sealed(TCRC32Fast) - - strict private - - const - Crc32PkzipPolynomial = UInt32($EDB88320); // Polynomial Reversed - Crc32PkzipMsbPoly = UInt32($04C11DB7); // MSB-first form - - class var - FCrc32PkzipTable: THashLibMatrixUInt32Array; - FCrc32PkzipFoldRuntime: TCRCFoldRuntimeCtx; - - class constructor Crc32Pkzip(); - - public - constructor Create(); - procedure TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); override; - function Clone(): IHash; override; - - end; - - TCRC32_CASTAGNOLI = class sealed(TCRC32Fast) - - strict private - - const - Crc32CastagnoliPolynomial = UInt32($82F63B78); // Polynomial Reversed - Crc32CastagnoliMsbPoly = UInt32($1EDC6F41); // MSB-first form - - class var - FCrc32CastagnoliTable: THashLibMatrixUInt32Array; - FCrc32CastagnoliFoldRuntime: TCRCFoldRuntimeCtx; - - class constructor Crc32Castagnoli(); - - public - constructor Create(); - procedure TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); override; - function Clone(): IHash; override; - - end; - -implementation - -procedure CRC32FastInitFoldTables(const AReflectedPoly, AMsbPoly: UInt32; - var ATable: THashLibMatrixUInt32Array; var ACtx: TCRCFoldRuntimeCtx); -begin - ATable := CRCDispatch_BuildSlicingTable32Reflect(AReflectedPoly); - CRCDispatch_InitRuntimeCtx(ATable, AMsbPoly, ACtx); -end; - -{ TCRC32Fast } - -constructor TCRC32Fast.Create(); -begin - inherited Create(4, 1); -end; - -procedure TCRC32Fast.Initialize; -begin - FCurrentCRC := 0; -end; - -function TCRC32Fast.TransformFinal: IHashResult; -var - LBufferBytes: THashLibByteArray; -begin - System.SetLength(LBufferBytes, HashSize); - TBinaryPrimitives.WriteUInt32BigEndian(LBufferBytes, 0, FCurrentCRC); - - Result := THashResult.Create(LBufferBytes); - Initialize(); -end; - -{ TCRC32_PKZIP } - -function TCRC32_PKZIP.Clone(): IHash; -var - LHashInstance: TCRC32_PKZIP; -begin - LHashInstance := TCRC32_PKZIP.Create(); - LHashInstance.FCurrentCRC := FCurrentCRC; - Result := LHashInstance; - Result.BufferSize := BufferSize; -end; - -constructor TCRC32_PKZIP.Create; -begin - inherited Create(); -end; - -procedure TCRC32_PKZIP.TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); -begin - CRCDispatch_UpdateReflectedCrc32(FCurrentCRC, PByte(AData) + AIndex, - UInt32(ALength), @FCrc32PkzipFoldRuntime); -end; - -class constructor TCRC32_PKZIP.Crc32Pkzip(); -begin - CRC32FastInitFoldTables(Crc32PkzipPolynomial, Crc32PkzipMsbPoly, - FCrc32PkzipTable, FCrc32PkzipFoldRuntime); -end; - -{ TCRC32_CASTAGNOLI } - -function TCRC32_CASTAGNOLI.Clone(): IHash; -var - LHashInstance: TCRC32_CASTAGNOLI; -begin - LHashInstance := TCRC32_CASTAGNOLI.Create(); - LHashInstance.FCurrentCRC := FCurrentCRC; - Result := LHashInstance; - Result.BufferSize := BufferSize; -end; - -constructor TCRC32_CASTAGNOLI.Create; -begin - inherited Create(); -end; - -procedure TCRC32_CASTAGNOLI.TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); -begin - CRCDispatch_UpdateReflectedCrc32(FCurrentCRC, PByte(AData) + AIndex, - UInt32(ALength), @FCrc32CastagnoliFoldRuntime); -end; - -class constructor TCRC32_CASTAGNOLI.Crc32Castagnoli(); -begin - CRC32FastInitFoldTables(Crc32CastagnoliPolynomial, Crc32CastagnoliMsbPoly, - FCrc32CastagnoliTable, FCrc32CastagnoliFoldRuntime); -end; - -end. diff --git a/HashLib/src/Checksum/HlpCRCFoldConstants.pas b/HashLib/src/Checksum/HlpCRCCore.pas similarity index 52% rename from HashLib/src/Checksum/HlpCRCFoldConstants.pas rename to HashLib/src/Checksum/HlpCRCCore.pas index 36e143fd..9c596be8 100644 --- a/HashLib/src/Checksum/HlpCRCFoldConstants.pas +++ b/HashLib/src/Checksum/HlpCRCCore.pas @@ -1,15 +1,40 @@ -unit HlpCRCFoldConstants; +unit HlpCRCCore; + +{ + CRC engine internals, in three parts: + + 1. TGF2 - GF(2) polynomial math (carry-less multiply, division, PowerMod) + used to generate the PCLMUL/VPCLMUL/PMULL folding and Barrett reduction + constants at runtime for ANY generator polynomial (Linux kernel + gen-crc-consts.py algorithm). One generic carry-less-multiply kernel per + arch serves every CRC standard through these computed constants. + + 2. TCRCFoldRuntimeCtx - the runtime context handed to the fold kernels. + The carry-less-multiply paths read FoldConstants (offset 0) only; the + scalar/SSE2 slicing tiers read the 16 slicing-table row pointers at + offset 96 (= SizeOf(TCRCFoldConstants)), matching the fixed + +96+I*SizeOf(Pointer) offsets in the CRCFold*.inc kernels. + + 3. Scalar slicing-by-16 fold routines - the portable fallback selected by + TCRCSimd when no SIMD backend is built/available. +} {$I ..\Include\HashLib.inc} interface +uses + HlpHashLibTypes; + +const + MinSimdBytes = Int32(16); + type TUInt128 = record Lo, Hi: UInt64; end; - // PCLMULQDQ / VPCLMULQDQ CRC folding and Barrett reduction constants. + // PCLMULQDQ / VPCLMULQDQ / PMULL CRC folding and Barrett reduction constants. // Layout must match the assembly expectations in the CRCFold*.inc files. TCRCFoldConstants = packed record Fold_4x128: array [0 .. 1] of UInt64; // offset 0: fold-by-4 constants (stride 512) @@ -21,10 +46,38 @@ TUInt128 = record BarrettShift: UInt64; // offset 88: = 64 - CrcBits (MSB psrlq alignment) end; + TCRCFoldRuntimeCtx = packed record + FoldConstants: TCRCFoldConstants; + TableRow: array [0 .. 15] of PUInt64; + end; + + PCRCFoldRuntimeCtx = ^TCRCFoldRuntimeCtx; + + // A slicing table plus its fold runtime context, cached by TCRC per + // (polynomial, width, reflection). + TCRCCacheValue = record + Table: THashLibMatrixUInt64Array; + FoldRuntime: TCRCFoldRuntimeCtx; + end; + + // AData: data pointer, ALength: byte count (>= MinSimdBytes, multiple of 16). + // AState: pointer to UInt64 (the running CRC / fold state). + // AConstants: pointer to TCRCFoldRuntimeCtx (FoldConstants at offset 0). + TCRCFoldFunc = function(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; + + // Tier-selection result: the two fold entry points plus whether they use + // carry-less multiply (PCLMUL/VPCLMUL/PMULL) rather than table slicing. + TCRCFoldSelection = record + Reflected: TCRCFoldFunc; + Fwd: TCRCFoldFunc; + UsesCarrylessMul: Boolean; + end; + TGF2 = class sealed strict private class function BitLength64(A: UInt64): Int32; static; - class function BitLength128(const A: TUInt128): Int32; static; + class function BitLength128(const A: TUInt128): Int32; static; inline; // 64 x 64 -> 128 carry-less multiply (MSB-first polynomial arithmetic). class function CLMul(A, B: UInt64): TUInt128; static; @@ -42,6 +95,12 @@ TGF2 = class sealed class procedure XorShiftedG(var AVal: TUInt128; APoly: UInt64; ABits, AShift: Int32); static; + // One (x^(AStride+64), x^AStride) fold constant pair, stored in the + // domain (bit-reflected or plain) the kernels expect. + class procedure FoldConstPair(AStride: Int32; APoly: UInt64; + ABits, AK: Int32; AReflected: Boolean; + var APair: array of UInt64); static; + public // Compute x^N mod G using repeated squaring. // APoly: generator polynomial WITHOUT the leading x^ABits term. @@ -60,8 +119,19 @@ TGF2 = class sealed AReflected: Boolean; out AConstants: TCRCFoldConstants); static; end; +procedure CRC_InitFoldRuntimeCtx(const Table: THashLibMatrixUInt64Array; + APoly: UInt64; AWidth: Int32; AReflected: Boolean; + out Ctx: TCRCFoldRuntimeCtx); + +var + CRC_Fold: TCRCFoldSelection; + implementation +uses + HlpBinaryPrimitives, + HlpCRCSimd; + { TGF2 } class function TGF2.BitLength64(A: UInt64): Int32; @@ -213,11 +283,30 @@ class function TGF2.BitReverse(AValue: UInt64; ANumBits: Int32): UInt64; end; end; +class procedure TGF2.FoldConstPair(AStride: Int32; APoly: UInt64; + ABits, AK: Int32; AReflected: Boolean; var APair: array of UInt64); +var + LHiExp, LLoExp: UInt64; +begin + LHiExp := PowerMod(AStride + 64 + AK, APoly, ABits); + LLoExp := PowerMod(AStride + AK, APoly, ABits); + if AReflected then + begin + APair[0] := BitReverse(LHiExp shl (64 - ABits), 64); + APair[1] := BitReverse(LLoExp shl (64 - ABits), 64); + end + else + begin + APair[0] := LLoExp; + APair[1] := LHiExp; + end; +end; + class procedure TGF2.GenerateFoldConstants(APoly: UInt64; ABits: Int32; AReflected: Boolean; out AConstants: TCRCFoldConstants); var LK, LPowOfX, I: Int32; - LConst0, LConst1, LBarrett0, LGMinusXn: UInt64; + LBarrett0, LGMinusXn: UInt64; LDiv128: TUInt128; begin // Following the Linux kernel gen-crc-consts.py algorithm. @@ -230,47 +319,9 @@ class procedure TGF2.GenerateFoldConstants(APoly: UInt64; ABits: Int32; else LK := 0; - // --- Fold-by-4 constants (stride = 512 bits) --- - LConst0 := PowerMod(512 + 64 + LK, APoly, ABits); - LConst1 := PowerMod(512 + LK, APoly, ABits); - if AReflected then - begin - AConstants.Fold_4x128[0] := BitReverse(LConst0 shl (64 - ABits), 64); - AConstants.Fold_4x128[1] := BitReverse(LConst1 shl (64 - ABits), 64); - end - else - begin - AConstants.Fold_4x128[0] := LConst1; - AConstants.Fold_4x128[1] := LConst0; - end; - - // --- Fold-by-1 constants (stride = 128 bits) --- - LConst0 := PowerMod(128 + 64 + LK, APoly, ABits); - LConst1 := PowerMod(128 + LK, APoly, ABits); - if AReflected then - begin - AConstants.Fold_1x128[0] := BitReverse(LConst0 shl (64 - ABits), 64); - AConstants.Fold_1x128[1] := BitReverse(LConst1 shl (64 - ABits), 64); - end - else - begin - AConstants.Fold_1x128[0] := LConst1; - AConstants.Fold_1x128[1] := LConst0; - end; - - // --- Fold-by-8 constants (stride = 1024 bits, for VPCLMULQDQ) --- - LConst0 := PowerMod(1024 + 64 + LK, APoly, ABits); - LConst1 := PowerMod(1024 + LK, APoly, ABits); - if AReflected then - begin - AConstants.Fold_8x128[0] := BitReverse(LConst0 shl (64 - ABits), 64); - AConstants.Fold_8x128[1] := BitReverse(LConst1 shl (64 - ABits), 64); - end - else - begin - AConstants.Fold_8x128[0] := LConst1; - AConstants.Fold_8x128[1] := LConst0; - end; + FoldConstPair(512, APoly, ABits, LK, AReflected, AConstants.Fold_4x128); + FoldConstPair(128, APoly, ABits, LK, AReflected, AConstants.Fold_1x128); + FoldConstPair(1024, APoly, ABits, LK, AReflected, AConstants.Fold_8x128); // --- Barrett reduction constants --- LGMinusXn := APoly; @@ -330,4 +381,130 @@ class procedure TGF2.GenerateFoldConstants(APoly: UInt64; ABits: Int32; AConstants.BarrettShift := 0; end; +// ============================================================================= +// Scalar slicing-by-16 fold routines +// ============================================================================= + +function CrcTableU64(const Row: PUInt64; B: Byte): UInt64; inline; +begin + Result := PUInt64(NativeUInt(Row) + UInt64(B) * SizeOf(UInt64))^; +end; + +function CRCMaskFromWidth(AWidth: Int32): UInt64; inline; +begin + Result := ((UInt64(1) shl (AWidth - 1)) - 1) shl 1 or 1; +end; + +procedure CRC_FoldReflected_OneSlice(Ctx: PCRCFoldRuntimeCtx; + var ATemp: UInt64; LPtr: PByte); +begin + ATemp := CrcTableU64(Ctx.TableRow[15], LPtr[0] xor Byte(ATemp)) + xor CrcTableU64(Ctx.TableRow[14], LPtr[1] xor Byte(ATemp shr 8)) + xor CrcTableU64(Ctx.TableRow[13], LPtr[2] xor Byte(ATemp shr 16)) + xor CrcTableU64(Ctx.TableRow[12], LPtr[3] xor Byte(ATemp shr 24)) + xor CrcTableU64(Ctx.TableRow[11], LPtr[4] xor Byte(ATemp shr 32)) + xor CrcTableU64(Ctx.TableRow[10], LPtr[5] xor Byte(ATemp shr 40)) + xor CrcTableU64(Ctx.TableRow[9], LPtr[6] xor Byte(ATemp shr 48)) + xor CrcTableU64(Ctx.TableRow[8], LPtr[7] xor Byte(ATemp shr 56)) + xor CrcTableU64(Ctx.TableRow[7], LPtr[8]) + xor CrcTableU64(Ctx.TableRow[6], LPtr[9]) + xor CrcTableU64(Ctx.TableRow[5], LPtr[10]) + xor CrcTableU64(Ctx.TableRow[4], LPtr[11]) + xor CrcTableU64(Ctx.TableRow[3], LPtr[12]) + xor CrcTableU64(Ctx.TableRow[2], LPtr[13]) + xor CrcTableU64(Ctx.TableRow[1], LPtr[14]) + xor CrcTableU64(Ctx.TableRow[0], LPtr[15]); +end; + +function CRC_Fold_Reflected_Scalar(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +var + Ctx: PCRCFoldRuntimeCtx; + LTemp: UInt64; + LPtr: PByte; + LLen: UInt32; +begin + Ctx := PCRCFoldRuntimeCtx(AConstants); + LPtr := AData; + LLen := ALength; + LTemp := TBinaryPrimitives.LoadUInt64(PUInt64(AState)); + + while LLen >= 16 do + begin + CRC_FoldReflected_OneSlice(Ctx, LTemp, LPtr); + System.Inc(LPtr, 16); + System.Dec(LLen, 16); + end; + + TBinaryPrimitives.StoreUInt64(PUInt64(AState), LTemp); + Result := LTemp; +end; + +function CRC_Fold_Forward_Scalar(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +var + Ctx: PCRCFoldRuntimeCtx; + LTemp, LNewTemp, LTempCopy: UInt64; + LPtr: PByte; + LLen: UInt32; + LWidth: Int32; + LCrcMask: UInt64; + LCrcBytes, LBIdx: Int32; + LByte: Byte; +begin + Ctx := PCRCFoldRuntimeCtx(AConstants); + LPtr := AData; + LLen := ALength; + LTemp := TBinaryPrimitives.LoadUInt64(PUInt64(AState)); + LWidth := Int32(Ctx.FoldConstants.CrcBits); + LCrcMask := CRCMaskFromWidth(LWidth); + LCrcBytes := (LWidth + 7) shr 3; + + while LLen >= 16 do + begin + LNewTemp := UInt64(0); + LTempCopy := LTemp; + + LBIdx := 0; + while LBIdx < LCrcBytes do + begin + LByte := LPtr[LBIdx] xor Byte(LTempCopy shr (LWidth - 8)); + LTempCopy := (LTempCopy shl 8) and LCrcMask; + LNewTemp := LNewTemp xor CrcTableU64(Ctx.TableRow[15 - LBIdx], LByte); + System.Inc(LBIdx); + end; + while LBIdx < 16 do + begin + LNewTemp := LNewTemp xor CrcTableU64(Ctx.TableRow[15 - LBIdx], LPtr[LBIdx]); + System.Inc(LBIdx); + end; + + LTemp := LNewTemp; + System.Inc(LPtr, 16); + System.Dec(LLen, 16); + end; + + TBinaryPrimitives.StoreUInt64(PUInt64(AState), LTemp); + Result := LTemp; +end; + +procedure CRC_InitFoldRuntimeCtx(const Table: THashLibMatrixUInt64Array; + APoly: UInt64; AWidth: Int32; AReflected: Boolean; + out Ctx: TCRCFoldRuntimeCtx); +var + I: Int32; +begin + TGF2.GenerateFoldConstants(APoly, AWidth, AReflected, Ctx.FoldConstants); + for I := 0 to 15 do + Ctx.TableRow[I] := @Table[I][0]; +end; + +// ============================================================================= +// Fold routine selection (once, at init) +// ============================================================================= + +initialization + CRC_Fold := TCRCSimd.Select(@CRC_Fold_Reflected_Scalar, + @CRC_Fold_Forward_Scalar); + end. diff --git a/HashLib/src/Checksum/HlpCRCDispatch.pas b/HashLib/src/Checksum/HlpCRCDispatch.pas deleted file mode 100644 index edd6e6d9..00000000 --- a/HashLib/src/Checksum/HlpCRCDispatch.pas +++ /dev/null @@ -1,529 +0,0 @@ -unit HlpCRCDispatch; - -{ - CRC fold dispatch (scalar + SIMD + PCLMUL) and fast reflected-CRC32 update. - - TCRC (HlpCRC.pas): generic widths, UInt64 table rows, FHash state — uses - CRC_Fold_Reflected/Forward + UpdateCRCViaByteTable; not the PKZIP wire inverted form. - - CRC32Fast (HlpCRC32Fast.pas): PKZIP/Castagnoli only; FCurrentCRC with not/xor - convention; uses CRCDispatch_UpdateReflectedCrc32 + TCRCFoldRuntimeCtx. - - TCRCFoldRuntimeCtx: FoldConstants + TableRow only. One record serves both the - UInt64 generic table and the UInt32 CRC32 table (TableRow stores untyped - pointers, cast at point of use). MSB fold reads CRC width from - FoldConstants.CrcBits (see TGF2.GenerateFoldConstants) and derives the state - mask the same way as TCRC.FCRCMask. -} - -{$I ..\Include\HashLib.inc} - -interface - -uses - HlpHashLibTypes, - HlpCRCFoldConstants; - -const - MinSimdBytes = Int32(16); - -type - // Unified fold runtime context. The PCLMUL/VPCLMUL (and future PMULL) paths - // read FoldConstants only (offset 0). The scalar/SSE2 slicing tiers read - // TableRow at offset 96 (= SizeOf(TCRCFoldConstants)); rows point at UInt64 - // (generic CRC) or UInt32 (CRC32 fast path) tables and are cast at point of - // use. TableRow is declared as Pointer so one record serves both table cell - // widths with identical byte layout on i386 (4-byte rows) and x86_64 - // (8-byte rows) — matching the fixed +96+I*SizeOf(Pointer) asm offsets. - TCRCFoldRuntimeCtx = packed record - FoldConstants: TCRCFoldConstants; - TableRow: array [0 .. 15] of Pointer; - end; - - PCRCFoldRuntimeCtx = ^TCRCFoldRuntimeCtx; - - // AData: data pointer, ALength: byte count (>= MinSimdBytes, multiple of 16). - // AState: pointer to 2 x UInt64 ([0]=CRC / state, [1]=0 for PCLMUL). - // AConstants: pointer to TCRCFoldRuntimeCtx (FoldConstants at offset 0). - TCRCFoldFunc = function(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - -procedure CRCDispatch_InitRuntimeCtx(const Table: THashLibMatrixUInt64Array; - APoly: UInt64; AWidth: Int32; AReflected: Boolean; - out Ctx: TCRCFoldRuntimeCtx); overload; - -procedure CRCDispatch_InitRuntimeCtx(const Table: THashLibMatrixUInt32Array; - AMsbPoly: UInt32; out Ctx: TCRCFoldRuntimeCtx); overload; - -// 16 x 256 slicing-by-16 table for reflected CRC32 (AReflectedPoly = e.g. $EDB88320). -function CRCDispatch_BuildSlicingTable32Reflect(AReflectedPoly: UInt32) - : THashLibMatrixUInt32Array; - -// PKZIP-style reflected CRC32: updates AWireCrc (e.g. FCurrentCRC) with ALength -// bytes at AData using fold dispatch + row-0 tail (same as former LocalCRCCompute). -procedure CRCDispatch_UpdateReflectedCrc32(var AWireCrc: UInt32; - AData: PByte; ALength: UInt32; ACtx: PCRCFoldRuntimeCtx); - -procedure CRC_UpdateViaBitSerial(AData: PByte; ADataLength, AIndex: Int32; - var AHash: UInt64; APolynomial: UInt64; AWidth: Int32; - AInputReflected: Boolean; AHighBitMask: UInt64); - -var - CRC_Fold_Reflected: TCRCFoldFunc; - CRC_Fold_Forward: TCRCFoldFunc; - CRC_Fold_Reflected32: TCRCFoldFunc; - CRC_Fold_UsesCarrylessMul: Boolean; - -implementation - -uses - HlpCpuFeatures, - HlpSimdLevels; - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -function CrcTableU64(const Row: Pointer; B: Byte): UInt64; inline; -begin - Result := PUInt64(NativeUInt(Row) + UInt64(B) * SizeOf(UInt64))^; -end; - -function CrcTableU32(const Row: Pointer; B: Byte): UInt32; inline; -begin - Result := PUInt32(NativeUInt(Row) + UInt64(B) * SizeOf(UInt32))^; -end; - -function CRCMaskFromWidth(AWidth: Int32): UInt64; inline; -begin - Result := ((UInt64(1) shl (AWidth - 1)) - 1) shl 1 or 1; -end; - -procedure CRC_FoldReflected_OneSlice(Ctx: PCRCFoldRuntimeCtx; - var ATemp: UInt64; LPtr: PByte); -begin - ATemp := CrcTableU64(Ctx.TableRow[15], LPtr[0] xor Byte(ATemp)) - xor CrcTableU64(Ctx.TableRow[14], LPtr[1] xor Byte(ATemp shr 8)) - xor CrcTableU64(Ctx.TableRow[13], LPtr[2] xor Byte(ATemp shr 16)) - xor CrcTableU64(Ctx.TableRow[12], LPtr[3] xor Byte(ATemp shr 24)) - xor CrcTableU64(Ctx.TableRow[11], LPtr[4] xor Byte(ATemp shr 32)) - xor CrcTableU64(Ctx.TableRow[10], LPtr[5] xor Byte(ATemp shr 40)) - xor CrcTableU64(Ctx.TableRow[9], LPtr[6] xor Byte(ATemp shr 48)) - xor CrcTableU64(Ctx.TableRow[8], LPtr[7] xor Byte(ATemp shr 56)) - xor CrcTableU64(Ctx.TableRow[7], LPtr[8]) - xor CrcTableU64(Ctx.TableRow[6], LPtr[9]) - xor CrcTableU64(Ctx.TableRow[5], LPtr[10]) - xor CrcTableU64(Ctx.TableRow[4], LPtr[11]) - xor CrcTableU64(Ctx.TableRow[3], LPtr[12]) - xor CrcTableU64(Ctx.TableRow[2], LPtr[13]) - xor CrcTableU64(Ctx.TableRow[1], LPtr[14]) - xor CrcTableU64(Ctx.TableRow[0], LPtr[15]); -end; - -function CRC_Fold_Reflected_Scalar(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; -var - Ctx: PCRCFoldRuntimeCtx; - LTemp: UInt64; - LPtr: PByte; - LLen: UInt32; -begin - Ctx := PCRCFoldRuntimeCtx(AConstants); - LPtr := AData; - LLen := ALength; - LTemp := PUInt64(AState)^; - - while LLen >= 16 do - begin - CRC_FoldReflected_OneSlice(Ctx, LTemp, LPtr); - System.Inc(LPtr, 16); - System.Dec(LLen, 16); - end; - - PUInt64(AState)^ := LTemp; - Result := LTemp; -end; - -function CRC_Fold_Forward_Scalar(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; -var - Ctx: PCRCFoldRuntimeCtx; - LTemp, LNewTemp, LTempCopy: UInt64; - LPtr: PByte; - LLen: UInt32; - LWidth: Int32; - LCrcMask: UInt64; - LCrcBytes, LBIdx: Int32; - LByte: Byte; -begin - Ctx := PCRCFoldRuntimeCtx(AConstants); - LPtr := AData; - LLen := ALength; - LTemp := PUInt64(AState)^; - LWidth := Int32(Ctx.FoldConstants.CrcBits); - LCrcMask := CRCMaskFromWidth(LWidth); - LCrcBytes := (LWidth + 7) shr 3; - - while LLen >= 16 do - begin - LNewTemp := UInt64(0); - LTempCopy := LTemp; - - LBIdx := 0; - while LBIdx < LCrcBytes do - begin - LByte := LPtr[LBIdx] xor Byte(LTempCopy shr (LWidth - 8)); - LTempCopy := (LTempCopy shl 8) and LCrcMask; - LNewTemp := LNewTemp xor CrcTableU64(Ctx.TableRow[15 - LBIdx], LByte); - System.Inc(LBIdx); - end; - while LBIdx < 16 do - begin - LNewTemp := LNewTemp xor CrcTableU64(Ctx.TableRow[15 - LBIdx], LPtr[LBIdx]); - System.Inc(LBIdx); - end; - - LTemp := LNewTemp; - System.Inc(LPtr, 16); - System.Dec(LLen, 16); - end; - - PUInt64(AState)^ := LTemp; - Result := LTemp; -end; - -procedure CRC32_FoldReflected32_OneSlice(Ctx: PCRCFoldRuntimeCtx; - var LCRC: UInt32; LPtr: PByte); -var - LTempCrc: UInt32; -begin - LTempCrc := LCRC; - LCRC := CrcTableU32(Ctx.TableRow[0], LPtr[15]) - xor CrcTableU32(Ctx.TableRow[1], LPtr[14]) - xor CrcTableU32(Ctx.TableRow[2], LPtr[13]) - xor CrcTableU32(Ctx.TableRow[3], LPtr[12]) - xor CrcTableU32(Ctx.TableRow[4], LPtr[11]) - xor CrcTableU32(Ctx.TableRow[5], LPtr[10]) - xor CrcTableU32(Ctx.TableRow[6], LPtr[9]) - xor CrcTableU32(Ctx.TableRow[7], LPtr[8]) - xor CrcTableU32(Ctx.TableRow[8], LPtr[7]) - xor CrcTableU32(Ctx.TableRow[9], LPtr[6]) - xor CrcTableU32(Ctx.TableRow[10], LPtr[5]) - xor CrcTableU32(Ctx.TableRow[11], LPtr[4]) - xor CrcTableU32(Ctx.TableRow[12], LPtr[3] xor Byte(LTempCrc shr 24)) - xor CrcTableU32(Ctx.TableRow[13], LPtr[2] xor Byte(LTempCrc shr 16)) - xor CrcTableU32(Ctx.TableRow[14], LPtr[1] xor Byte(LTempCrc shr 8)) - xor CrcTableU32(Ctx.TableRow[15], LPtr[0] xor Byte(LTempCrc)); -end; - -function CRC_Fold_Reflected32_Scalar(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; -var - Ctx: PCRCFoldRuntimeCtx; - LCRC: UInt32; - LPtr: PByte; - LLen: UInt32; -begin - Ctx := PCRCFoldRuntimeCtx(AConstants); - LPtr := AData; - LLen := ALength; - LCRC := UInt32(PUInt64(AState)^); - - while LLen >= 16 do - begin - CRC32_FoldReflected32_OneSlice(Ctx, LCRC, LPtr); - System.Inc(LPtr, 16); - System.Dec(LLen, 16); - end; - - PUInt64(AState)^ := LCRC; - Result := LCRC; -end; - -procedure CRCDispatch_UpdateReflectedCrc32(var AWireCrc: UInt32; - AData: PByte; ALength: UInt32; ACtx: PCRCFoldRuntimeCtx); -var - LInternal: UInt32; - LPtr: PByte; - LLen, LProcessed: UInt32; - LState: array [0 .. 1] of UInt64; -begin - LInternal := not AWireCrc; - LPtr := AData; - LLen := ALength; - if LLen >= UInt32(MinSimdBytes) then - begin - LProcessed := LLen and (not UInt32(15)); - LState[0] := UInt64(LInternal); - LState[1] := 0; - LInternal := UInt32(CRC_Fold_Reflected32(LPtr, LProcessed, @LState[0], ACtx)); - System.Inc(LPtr, LProcessed); - System.Dec(LLen, LProcessed); - end; - while LLen > 0 do - begin - LInternal := (LInternal shr 8) xor CrcTableU32(ACtx.TableRow[0], - Byte(LInternal and $FF) xor LPtr^); - System.Inc(LPtr); - System.Dec(LLen); - end; - AWireCrc := not LInternal; -end; - -procedure CRCDispatch_InitRuntimeCtx(const Table: THashLibMatrixUInt64Array; - APoly: UInt64; AWidth: Int32; AReflected: Boolean; - out Ctx: TCRCFoldRuntimeCtx); -var - I: Int32; -begin - TGF2.GenerateFoldConstants(APoly, AWidth, AReflected, Ctx.FoldConstants); - for I := 0 to 15 do - Ctx.TableRow[I] := @Table[I][0]; -end; - -procedure CRCDispatch_InitRuntimeCtx(const Table: THashLibMatrixUInt32Array; - AMsbPoly: UInt32; out Ctx: TCRCFoldRuntimeCtx); -var - I: Int32; -begin - TGF2.GenerateFoldConstants(UInt64(AMsbPoly), 32, True, Ctx.FoldConstants); - for I := 0 to 15 do - Ctx.TableRow[I] := @Table[I][0]; -end; - -function CRCDispatch_BuildSlicingTable32Reflect(AReflectedPoly: UInt32) - : THashLibMatrixUInt32Array; -var - LIdx, LJIdx, LKIdx: Int32; - LRes: UInt32; -begin - System.SetLength(Result, 16); - for LIdx := System.Low(Result) to System.High(Result) do - System.SetLength(Result[LIdx], 256); - for LIdx := 0 to System.Pred(256) do - begin - LRes := LIdx; - for LJIdx := 0 to System.Pred(16) do - begin - LKIdx := 0; - while LKIdx < System.Pred(9) do - begin - LRes := (LRes shr 1) xor (-Int32(LRes and 1) and AReflectedPoly); - Result[LJIdx][LIdx] := LRes; - System.Inc(LKIdx); - end; - end; - end; -end; - -procedure CRC_UpdateViaBitSerial(AData: PByte; ADataLength, AIndex: Int32; - var AHash: UInt64; APolynomial: UInt64; AWidth: Int32; - AInputReflected: Boolean; AHighBitMask: UInt64); -var - LLength, LIdx: Int32; - LTemp, LBit, LJdx, LHash: UInt64; -begin - LLength := ADataLength; - LIdx := AIndex; - while LLength > 0 do - begin - LTemp := UInt64(AData[LIdx]); - if AInputReflected then - LTemp := TGF2.BitReverse(LTemp, 8); - - LJdx := $80; - LHash := AHash; - while LJdx > 0 do - begin - LBit := LHash and AHighBitMask; - LHash := LHash shl 1; - if ((LTemp and LJdx) > 0) then - LBit := LBit xor AHighBitMask; - if (LBit > 0) then - LHash := LHash xor APolynomial; - LJdx := LJdx shr 1; - end; - AHash := LHash; - System.Inc(LIdx); - System.Dec(LLength); - end; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: VPCLMULQDQ, PCLMULQDQ, SSE2 -// aarch64: PMULL -// ============================================================================= - -{$IFDEF HASHLIB_X86_64_ASM} - -function CRC_Fold_Reflected_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflectedSse2_x86_64.inc} -end; - -function CRC_Fold_Forward_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldForwardSse2_x86_64.inc} -end; - -function CRC_Fold_Reflected32_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflected32Sse2_x86_64.inc} -end; - -{$ELSE} - -{$IFDEF HASHLIB_I386_ASM} - -function CRC_Fold_Reflected_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflectedSse2_i386.inc} -end; - -function CRC_Fold_Forward_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\CRC\CRCFoldForwardSse2_i386.inc} -end; - -function CRC_Fold_Reflected32_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflected32Sse2_i386.inc} -end; - -{$ELSE} - -function CRC_Fold_Reflected_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; -begin - Result := CRC_Fold_Reflected_Scalar(AData, ALength, AState, AConstants); -end; - -function CRC_Fold_Forward_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; -begin - Result := CRC_Fold_Forward_Scalar(AData, ALength, AState, AConstants); -end; - -function CRC_Fold_Reflected32_Sse2(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; -begin - Result := CRC_Fold_Reflected32_Scalar(AData, ALength, AState, AConstants); -end; - -{$ENDIF HASHLIB_I386_ASM} -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -function CRC_Fold_Reflected_Pclmul(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflectedPclmul_x86_64.inc} -end; - -function CRC_Fold_Reflected_Vpclmul(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflectedVpclmul_x86_64.inc} -end; - -function CRC_Fold_Forward_Pclmul(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldForwardPclmul_x86_64.inc} -end; - -function CRC_Fold_Forward_Vpclmul(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\CRC\CRCFoldForwardVpclmul_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -function CRC_Fold_Reflected_Pmull(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\CRC\CRCFoldReflectedPmull_aarch64.inc} -end; - -function CRC_Fold_Forward_Pmull(AData: PByte; ALength: UInt32; - AState: Pointer; AConstants: Pointer): UInt64; - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\CRC\CRCFoldForwardPmull_aarch64.inc} -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); - - procedure BindSse2CrcFold; - begin - CRC_Fold_Reflected := @CRC_Fold_Reflected_Sse2; - CRC_Fold_Forward := @CRC_Fold_Forward_Sse2; - CRC_Fold_Reflected32 := @CRC_Fold_Reflected32_Sse2; - end; - -begin - CRC_Fold_Reflected := @CRC_Fold_Reflected_Scalar; - CRC_Fold_Forward := @CRC_Fold_Forward_Scalar; - CRC_Fold_Reflected32 := @CRC_Fold_Reflected32_Scalar; - CRC_Fold_UsesCarrylessMul := False; - -{$IFDEF HASHLIB_AARCH64_ASM} - if TCpuFeatures.Arm.HasPMULL() then - begin - CRC_Fold_Reflected := @CRC_Fold_Reflected_Pmull; - CRC_Fold_Forward := @CRC_Fold_Forward_Pmull; - CRC_Fold_Reflected32 := @CRC_Fold_Reflected_Pmull; - CRC_Fold_UsesCarrylessMul := True; - Exit; - end; -{$ENDIF HASHLIB_AARCH64_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - if TCpuFeatures.X86.HasVPCLMULQDQ() then - begin - CRC_Fold_Reflected := @CRC_Fold_Reflected_Vpclmul; - CRC_Fold_Forward := @CRC_Fold_Forward_Vpclmul; - CRC_Fold_Reflected32 := @CRC_Fold_Reflected_Vpclmul; - CRC_Fold_UsesCarrylessMul := True; - Exit; - end; - if TCpuFeatures.X86.HasPCLMULQDQ() then - begin - CRC_Fold_Reflected := @CRC_Fold_Reflected_Pclmul; - CRC_Fold_Forward := @CRC_Fold_Forward_Pclmul; - CRC_Fold_Reflected32 := @CRC_Fold_Reflected_Pclmul; - CRC_Fold_UsesCarrylessMul := True; - Exit; - end; -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_X86_SIMD} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - BindSse2CrcFold; - end; -{$ENDIF HASHLIB_X86_SIMD} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Checksum/HlpCRCStandard.pas b/HashLib/src/Checksum/HlpCRCStandard.pas deleted file mode 100644 index 1a86a1dc..00000000 --- a/HashLib/src/Checksum/HlpCRCStandard.pas +++ /dev/null @@ -1,273 +0,0 @@ -unit HlpCRCStandard; - -{$I ..\Include\HashLib.inc} - -{ - Width-specific CRC wrappers (TCRC16 / TCRC32 / TCRC64) that expose the - IHash16 / IHash32 / IHash64 interfaces while delegating all work to the - generic engine in HlpCRC (TCRC). The high-throughput PKZIP/Castagnoli - variants live separately in HlpCRC32Fast. -} - -interface - -uses - HlpHashLibTypes, - HlpHash, - HlpIHash, - HlpICRC, - HlpIHashResult, - HlpIHashInfo, - HlpCRC; - -type - - TCRC16Polynomials = class sealed(TObject) - - private - - const - - BUYPASS = UInt16($8005); - - end; - - TCRC16 = class(THash, IChecksum, IHash16, ITransformBlock) - - strict private - var - FCRCAlgorithm: ICRC; - - public - constructor Create(APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); - - procedure Initialize(); override; - procedure TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); override; - function TransformFinal(): IHashResult; override; - - end; - - TCRC16_BUYPASS = class sealed(TCRC16) - - public - constructor Create(); - - end; - - TCRC32Polynomials = class sealed(TObject) - - private - - const - - PKZIP = UInt32($04C11DB7); - Castagnoli = UInt32($1EDC6F41); - - end; - - TCRC32 = class(THash, IChecksum, IHash32, ITransformBlock) - - strict private - var - FCRCAlgorithm: ICRC; - - public - - constructor Create(APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); - - procedure Initialize(); override; - procedure TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); override; - function TransformFinal(): IHashResult; override; - function Clone(): IHash; override; - - end; - - TCRC32_PKZIP = class sealed(TCRC32) - - public - constructor Create(); - - end; - - TCRC32_CASTAGNOLI = class sealed(TCRC32) - - public - constructor Create(); - - end; - - TCRC64Polynomials = class sealed(TObject) - - private - - const - - ECMA_182 = UInt64($42F0E1EBA9EA3693); - - end; - - TCRC64 = class(THash, IChecksum, IHash64, ITransformBlock) - - strict private - var - FCRCAlgorithm: ICRC; - - public - constructor Create(APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); - - procedure Initialize(); override; - procedure TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); override; - function TransformFinal(): IHashResult; override; - function Clone(): IHash; override; - - end; - - TCRC64_ECMA_182 = class sealed(TCRC64) - - public - constructor Create(); - - end; - -implementation - -{ TCRC16 } - -constructor TCRC16.Create(APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); -begin - inherited Create(2, 1); - FCRCAlgorithm := TCRC.Create(16, APolynomial, AInitial, AIsInputReflected, - AIsOutputReflected, AOutputXor, ACheckValue, ANames); -end; - -procedure TCRC16.Initialize; -begin - FCRCAlgorithm.Initialize; -end; - -procedure TCRC16.TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); -begin - FCRCAlgorithm.TransformBytes(AData, AIndex, ALength); -end; - -function TCRC16.TransformFinal: IHashResult; -begin - Result := FCRCAlgorithm.TransformFinal(); -end; - -{ TCRC16_BUYPASS } - -constructor TCRC16_BUYPASS.Create; -begin - inherited Create(TCRC16Polynomials.BUYPASS, $0000, False, False, $0000, $FEE8, - THashLibStringArray.Create('CRC-16/BUYPASS', 'CRC-16/VERIFONE', - 'CRC-16/UMTS')); -end; - -{ TCRC32 } - -function TCRC32.Clone(): IHash; -begin - Result := FCRCAlgorithm.Clone(); -end; - -constructor TCRC32.Create(APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); -begin - inherited Create(4, 1); - FCRCAlgorithm := TCRC.Create(32, APolynomial, AInitial, AIsInputReflected, - AIsOutputReflected, AOutputXor, ACheckValue, ANames); -end; - -procedure TCRC32.Initialize; -begin - FCRCAlgorithm.Initialize; -end; - -procedure TCRC32.TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); -begin - FCRCAlgorithm.TransformBytes(AData, AIndex, ALength); -end; - -function TCRC32.TransformFinal: IHashResult; -begin - Result := FCRCAlgorithm.TransformFinal(); -end; - -{ TCRC32_PKZIP } - -constructor TCRC32_PKZIP.Create; -begin - inherited Create(TCRC32Polynomials.PKZIP, $FFFFFFFF, True, True, $FFFFFFFF, - $CBF43926, THashLibStringArray.Create('CRC-32', 'CRC-32/ADCCP', - 'CRC-32/V-42', 'CRC-32/XZ', 'PKZIP', 'CRC-32/ISO-HDLC')); - -end; - -{ TCRC32_CASTAGNOLI } - -constructor TCRC32_CASTAGNOLI.Create; -begin - inherited Create(TCRC32Polynomials.Castagnoli, $FFFFFFFF, True, True, - $FFFFFFFF, $E3069283, THashLibStringArray.Create('CRC-32C', - 'CRC-32/BASE91-C', 'CRC-32/CASTAGNOLI', 'CRC-32/INTERLAKEN', - 'CRC-32/ISCSI')); - -end; - -{ TCRC64 } - -function TCRC64.Clone(): IHash; -begin - Result := FCRCAlgorithm.Clone(); -end; - -constructor TCRC64.Create(APolynomial, AInitial: UInt64; - AIsInputReflected, AIsOutputReflected: Boolean; - AOutputXor, ACheckValue: UInt64; const ANames: THashLibStringArray); -begin - inherited Create(8, 1); - FCRCAlgorithm := TCRC.Create(64, APolynomial, AInitial, AIsInputReflected, - AIsOutputReflected, AOutputXor, ACheckValue, ANames); -end; - -procedure TCRC64.Initialize; -begin - FCRCAlgorithm.Initialize; -end; - -procedure TCRC64.TransformBytes(const AData: THashLibByteArray; - AIndex, ALength: Int32); -begin - FCRCAlgorithm.TransformBytes(AData, AIndex, ALength); -end; - -function TCRC64.TransformFinal: IHashResult; -begin - Result := FCRCAlgorithm.TransformFinal(); -end; - -{ TCRC64_ECMA_182 } - -constructor TCRC64_ECMA_182.Create; -begin - inherited Create(TCRC64Polynomials.ECMA_182, $0000000000000000, False, False, - $0000000000000000, $6C40DF5F0B497347, THashLibStringArray.Create('CRC-64', - 'CRC-64/ECMA-182')); -end; - -end. diff --git a/HashLib/src/Crypto/HlpBlake2B.pas b/HashLib/src/Crypto/HlpBlake2B.pas index 582a36c6..7bc4d730 100644 --- a/HashLib/src/Crypto/HlpBlake2B.pas +++ b/HashLib/src/Crypto/HlpBlake2B.pas @@ -18,6 +18,20 @@ interface HlpArrayUtils, HlpHashLibTypes; +type + TBlake2BCompressProc = procedure(AState, AMsg, ACounterFlags, AIV: Pointer); + +var + Blake2B_Compress: TBlake2BCompressProc; + +const + Blake2BIV: array [0 .. 7] of UInt64 = ( + UInt64($6A09E667F3BCC908), UInt64($BB67AE8584CAA73B), + UInt64($3C6EF372FE94F82B), UInt64($A54FF53A5F1D36F1), + UInt64($510E527FADE682D1), UInt64($9B05688C2B3E6C1F), + UInt64($1F83D9ABFB41BD6B), UInt64($5BE0CD19137E2179) + ); + resourcestring SInvalidConfigLength = 'Config Length Must Be 8 Words'; SConfigNil = 'Config Cannot Be nil'; @@ -222,7 +236,85 @@ TBlake2BMACNotBuildInAdapter = class sealed(THash, IBlake2BMAC, implementation uses - HlpBlake2BDispatch; + HlpBitOperations, + HlpBlake2BSimd; + +const + Blake2BSigma: array [0 .. 11, 0 .. 15] of Int32 = ( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), + (14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3), + (11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4), + (7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8), + (9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13), + (2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9), + (12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11), + (13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10), + (6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5), + (10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0), + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), + (14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3) + ); + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure Blake2B_Compress_Scalar(AState, AMsg, ACounterFlags, AIV: Pointer); +var + LV: array [0 .. 15] of UInt64; + LPState, LPMsg, LPCounterFlags, LPIV: PByte; + LRound, I: Int32; + + procedure G(AA, AB, AC, AD, AMsgIdx0, AMsgIdx1: Int32); + begin + LV[AA] := LV[AA] + LV[AB] + TBinaryPrimitives.LoadUInt64 + (PUInt64(LPMsg + AMsgIdx0 * SizeOf(UInt64))); + LV[AD] := TBitOperations.RotateRight64(LV[AD] xor LV[AA], 32); + LV[AC] := LV[AC] + LV[AD]; + LV[AB] := TBitOperations.RotateRight64(LV[AB] xor LV[AC], 24); + LV[AA] := LV[AA] + LV[AB] + TBinaryPrimitives.LoadUInt64 + (PUInt64(LPMsg + AMsgIdx1 * SizeOf(UInt64))); + LV[AD] := TBitOperations.RotateRight64(LV[AD] xor LV[AA], 16); + LV[AC] := LV[AC] + LV[AD]; + LV[AB] := TBitOperations.RotateRight64(LV[AB] xor LV[AC], 63); + end; + +begin + LPState := PByte(AState); + LPMsg := PByte(AMsg); + LPCounterFlags := PByte(ACounterFlags); + LPIV := PByte(AIV); + + for I := 0 to 7 do + LV[I] := TBinaryPrimitives.LoadUInt64(PUInt64(LPState + I * SizeOf(UInt64))); + for I := 0 to 7 do + LV[I + 8] := TBinaryPrimitives.LoadUInt64(PUInt64(LPIV + I * SizeOf(UInt64))); + + LV[12] := LV[12] xor TBinaryPrimitives.LoadUInt64(PUInt64(LPCounterFlags)); + LV[13] := LV[13] xor TBinaryPrimitives.LoadUInt64 + (PUInt64(LPCounterFlags + SizeOf(UInt64))); + LV[14] := LV[14] xor TBinaryPrimitives.LoadUInt64 + (PUInt64(LPCounterFlags + 2 * SizeOf(UInt64))); + LV[15] := LV[15] xor TBinaryPrimitives.LoadUInt64 + (PUInt64(LPCounterFlags + 3 * SizeOf(UInt64))); + + for LRound := 0 to 11 do + begin + G(0, 4, 8, 12, Blake2BSigma[LRound, 0], Blake2BSigma[LRound, 1]); + G(1, 5, 9, 13, Blake2BSigma[LRound, 2], Blake2BSigma[LRound, 3]); + G(2, 6, 10, 14, Blake2BSigma[LRound, 4], Blake2BSigma[LRound, 5]); + G(3, 7, 11, 15, Blake2BSigma[LRound, 6], Blake2BSigma[LRound, 7]); + G(0, 5, 10, 15, Blake2BSigma[LRound, 8], Blake2BSigma[LRound, 9]); + G(1, 6, 11, 12, Blake2BSigma[LRound, 10], Blake2BSigma[LRound, 11]); + G(2, 7, 8, 13, Blake2BSigma[LRound, 12], Blake2BSigma[LRound, 13]); + G(3, 4, 9, 14, Blake2BSigma[LRound, 14], Blake2BSigma[LRound, 15]); + end; + + for I := 0 to 7 do + TBinaryPrimitives.StoreUInt64(PUInt64(LPState + I * SizeOf(UInt64)), + TBinaryPrimitives.LoadUInt64(PUInt64(LPState + I * SizeOf(UInt64))) + xor (LV[I] xor LV[I + 8])); +end; type // Self-contained Blake2XB squeeze engine. Owns the finalized root digest and @@ -394,8 +486,8 @@ procedure TBlake2B.Compress(ABlock: PByte; AStart: Int32); LCounterFlags[1] := FCounter1; LCounterFlags[2] := FFinalizationFlag0; LCounterFlags[3] := FFinalizationFlag1; - HlpBlake2BDispatch.Blake2B_Compress(@FState[0], @FM[0], - @LCounterFlags[0], @HlpBlake2BDispatch.Blake2BIV[0]); + Blake2B_Compress(@FState[0], @FM[0], + @LCounterFlags[0], @Blake2BIV[0]); end; constructor TBlake2B.Create(const AConfig: IBlake2BConfig); @@ -988,4 +1080,7 @@ function TBlake2BMACNotBuildInAdapter.TransformFinal: IHashResult; Result := FHash.TransformFinal(); end; +initialization + Blake2B_Compress := TBlake2BSimd.Select(@Blake2B_Compress_Scalar); + end. diff --git a/HashLib/src/Crypto/HlpBlake2BDispatch.pas b/HashLib/src/Crypto/HlpBlake2BDispatch.pas deleted file mode 100644 index 29c2dbc3..00000000 --- a/HashLib/src/Crypto/HlpBlake2BDispatch.pas +++ /dev/null @@ -1,179 +0,0 @@ -unit HlpBlake2BDispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TBlake2BCompressProc = procedure(AState, AMsg, ACounterFlags, AIV: Pointer); - -const - Blake2BIV: array [0 .. 7] of UInt64 = ( - UInt64($6A09E667F3BCC908), UInt64($BB67AE8584CAA73B), - UInt64($3C6EF372FE94F82B), UInt64($A54FF53A5F1D36F1), - UInt64($510E527FADE682D1), UInt64($9B05688C2B3E6C1F), - UInt64($1F83D9ABFB41BD6B), UInt64($5BE0CD19137E2179) - ); - -var - Blake2B_Compress: TBlake2BCompressProc; - -implementation - -uses - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -const - Blake2BSigma: array [0 .. 11, 0 .. 15] of Int32 = ( - (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), - (14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3), - (11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4), - (7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8), - (9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13), - (2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9), - (12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11), - (13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10), - (6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5), - (10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0), - (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), - (14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3) - ); - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure Blake2B_Compress_Scalar(AState, AMsg, ACounterFlags, AIV: Pointer); -var - LV: array [0 .. 15] of UInt64; - LPState, LPMsg, LPCounterFlags, LPIV: PByte; - LRound, I: Int32; - - procedure G(AA, AB, AC, AD, AMsgIdx0, AMsgIdx1: Int32); - begin - LV[AA] := LV[AA] + LV[AB] + PUInt64(LPMsg + AMsgIdx0 * SizeOf(UInt64))^; - LV[AD] := TBitOperations.RotateRight64(LV[AD] xor LV[AA], 32); - LV[AC] := LV[AC] + LV[AD]; - LV[AB] := TBitOperations.RotateRight64(LV[AB] xor LV[AC], 24); - LV[AA] := LV[AA] + LV[AB] + PUInt64(LPMsg + AMsgIdx1 * SizeOf(UInt64))^; - LV[AD] := TBitOperations.RotateRight64(LV[AD] xor LV[AA], 16); - LV[AC] := LV[AC] + LV[AD]; - LV[AB] := TBitOperations.RotateRight64(LV[AB] xor LV[AC], 63); - end; - -begin - LPState := PByte(AState); - LPMsg := PByte(AMsg); - LPCounterFlags := PByte(ACounterFlags); - LPIV := PByte(AIV); - - for I := 0 to 7 do - LV[I] := PUInt64(LPState + I * SizeOf(UInt64))^; - for I := 0 to 7 do - LV[I + 8] := PUInt64(LPIV + I * SizeOf(UInt64))^; - - LV[12] := LV[12] xor PUInt64(LPCounterFlags)^; - LV[13] := LV[13] xor PUInt64(LPCounterFlags + SizeOf(UInt64))^; - LV[14] := LV[14] xor PUInt64(LPCounterFlags + 2 * SizeOf(UInt64))^; - LV[15] := LV[15] xor PUInt64(LPCounterFlags + 3 * SizeOf(UInt64))^; - - for LRound := 0 to 11 do - begin - G(0, 4, 8, 12, Blake2BSigma[LRound, 0], Blake2BSigma[LRound, 1]); - G(1, 5, 9, 13, Blake2BSigma[LRound, 2], Blake2BSigma[LRound, 3]); - G(2, 6, 10, 14, Blake2BSigma[LRound, 4], Blake2BSigma[LRound, 5]); - G(3, 7, 11, 15, Blake2BSigma[LRound, 6], Blake2BSigma[LRound, 7]); - G(0, 5, 10, 15, Blake2BSigma[LRound, 8], Blake2BSigma[LRound, 9]); - G(1, 6, 11, 12, Blake2BSigma[LRound, 10], Blake2BSigma[LRound, 11]); - G(2, 7, 8, 13, Blake2BSigma[LRound, 12], Blake2BSigma[LRound, 13]); - G(3, 4, 9, 14, Blake2BSigma[LRound, 14], Blake2BSigma[LRound, 15]); - end; - - for I := 0 to 7 do - PUInt64(LPState + I * SizeOf(UInt64))^ := - PUInt64(LPState + I * SizeOf(UInt64))^ xor (LV[I] xor LV[I + 8]); -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: NEON -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure Blake2B_Compress_Sse2(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\Blake2B\Blake2BCompressSse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure Blake2B_Compress_Sse2(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Blake2B\Blake2BCompressSse2_x86_64.inc} -end; - -procedure Blake2B_Compress_Avx2(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Blake2B\Blake2BCompressAvx2_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure Blake2B_Compress_Neon(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\Blake2B\Blake2BCompressNeon_aarch64.inc} -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - Blake2B_Compress := @Blake2B_Compress_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - Blake2B_Compress := @Blake2B_Compress_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - Blake2B_Compress := @Blake2B_Compress_Avx2; - end; - TX86SimdLevel.SSE2: - begin - Blake2B_Compress := @Blake2B_Compress_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - TArmSimdLevel.NEON: - begin - Blake2B_Compress := @Blake2B_Compress_Neon; - end; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Crypto/HlpBlake2S.pas b/HashLib/src/Crypto/HlpBlake2S.pas index d9b2a04f..b5e71c9f 100644 --- a/HashLib/src/Crypto/HlpBlake2S.pas +++ b/HashLib/src/Crypto/HlpBlake2S.pas @@ -18,6 +18,20 @@ interface HlpArrayUtils, HlpHashLibTypes; +type + TBlake2SCompressProc = procedure(AState, AMsg, ACounterFlags, AIV: Pointer); + +var + Blake2S_Compress: TBlake2SCompressProc; + +const + Blake2SIV: array [0 .. 7] of UInt32 = ( + UInt32($6A09E667), UInt32($BB67AE85), + UInt32($3C6EF372), UInt32($A54FF53A), + UInt32($510E527F), UInt32($9B05688C), + UInt32($1F83D9AB), UInt32($5BE0CD19) + ); + resourcestring SInvalidConfigLength = 'Config Length Must Be 8 Words'; SConfigNil = 'Config Cannot Be nil'; @@ -220,7 +234,83 @@ TBlake2SMACNotBuildInAdapter = class sealed(THash, IBlake2SMAC, implementation uses - HlpBlake2SDispatch; + HlpBitOperations, + HlpBlake2SSimd; + +const + Blake2SSigma: array [0 .. 9, 0 .. 15] of Int32 = ( + (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), + (14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3), + (11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4), + (7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8), + (9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13), + (2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9), + (12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11), + (13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10), + (6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5), + (10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0) + ); + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure Blake2S_Compress_Scalar(AState, AMsg, ACounterFlags, AIV: Pointer); +var + LV: array [0 .. 15] of UInt32; + LPState, LPMsg, LPCounterFlags, LPIV: PByte; + LRound, I: Int32; + + procedure G(AA, AB, AC, AD, AMsgIdx0, AMsgIdx1: Int32); + begin + LV[AA] := LV[AA] + LV[AB] + TBinaryPrimitives.LoadUInt32 + (PCardinal(LPMsg + AMsgIdx0 * SizeOf(UInt32))); + LV[AD] := TBitOperations.RotateRight32(LV[AD] xor LV[AA], 16); + LV[AC] := LV[AC] + LV[AD]; + LV[AB] := TBitOperations.RotateRight32(LV[AB] xor LV[AC], 12); + LV[AA] := LV[AA] + LV[AB] + TBinaryPrimitives.LoadUInt32 + (PCardinal(LPMsg + AMsgIdx1 * SizeOf(UInt32))); + LV[AD] := TBitOperations.RotateRight32(LV[AD] xor LV[AA], 8); + LV[AC] := LV[AC] + LV[AD]; + LV[AB] := TBitOperations.RotateRight32(LV[AB] xor LV[AC], 7); + end; + +begin + LPState := PByte(AState); + LPMsg := PByte(AMsg); + LPCounterFlags := PByte(ACounterFlags); + LPIV := PByte(AIV); + + for I := 0 to 7 do + LV[I] := TBinaryPrimitives.LoadUInt32(PCardinal(LPState + I * SizeOf(UInt32))); + for I := 0 to 7 do + LV[I + 8] := TBinaryPrimitives.LoadUInt32(PCardinal(LPIV + I * SizeOf(UInt32))); + + LV[12] := LV[12] xor TBinaryPrimitives.LoadUInt32(PCardinal(LPCounterFlags)); + LV[13] := LV[13] xor TBinaryPrimitives.LoadUInt32 + (PCardinal(LPCounterFlags + SizeOf(UInt32))); + LV[14] := LV[14] xor TBinaryPrimitives.LoadUInt32 + (PCardinal(LPCounterFlags + 2 * SizeOf(UInt32))); + LV[15] := LV[15] xor TBinaryPrimitives.LoadUInt32 + (PCardinal(LPCounterFlags + 3 * SizeOf(UInt32))); + + for LRound := 0 to 9 do + begin + G(0, 4, 8, 12, Blake2SSigma[LRound, 0], Blake2SSigma[LRound, 1]); + G(1, 5, 9, 13, Blake2SSigma[LRound, 2], Blake2SSigma[LRound, 3]); + G(2, 6, 10, 14, Blake2SSigma[LRound, 4], Blake2SSigma[LRound, 5]); + G(3, 7, 11, 15, Blake2SSigma[LRound, 6], Blake2SSigma[LRound, 7]); + G(0, 5, 10, 15, Blake2SSigma[LRound, 8], Blake2SSigma[LRound, 9]); + G(1, 6, 11, 12, Blake2SSigma[LRound, 10], Blake2SSigma[LRound, 11]); + G(2, 7, 8, 13, Blake2SSigma[LRound, 12], Blake2SSigma[LRound, 13]); + G(3, 4, 9, 14, Blake2SSigma[LRound, 14], Blake2SSigma[LRound, 15]); + end; + + for I := 0 to 7 do + TBinaryPrimitives.StoreUInt32(PCardinal(LPState + I * SizeOf(UInt32)), + TBinaryPrimitives.LoadUInt32(PCardinal(LPState + I * SizeOf(UInt32))) + xor (LV[I] xor LV[I + 8])); +end; type // Self-contained Blake2XS squeeze engine. Owns the finalized root digest and @@ -392,8 +482,8 @@ procedure TBlake2S.Compress(ABlock: PByte; AStart: Int32); LCounterFlags[1] := FCounter1; LCounterFlags[2] := FFinalizationFlag0; LCounterFlags[3] := FFinalizationFlag1; - HlpBlake2SDispatch.Blake2S_Compress(@FState[0], @FM[0], - @LCounterFlags[0], @HlpBlake2SDispatch.Blake2SIV[0]); + Blake2S_Compress(@FState[0], @FM[0], + @LCounterFlags[0], @Blake2SIV[0]); end; constructor TBlake2S.Create(const AConfig: IBlake2SConfig); @@ -984,4 +1074,7 @@ function TBlake2SMACNotBuildInAdapter.TransformFinal: IHashResult; Result := FHash.TransformFinal(); end; +initialization + Blake2S_Compress := TBlake2SSimd.Select(@Blake2S_Compress_Scalar); + end. diff --git a/HashLib/src/Crypto/HlpBlake2SDispatch.pas b/HashLib/src/Crypto/HlpBlake2SDispatch.pas deleted file mode 100644 index 4afb7f89..00000000 --- a/HashLib/src/Crypto/HlpBlake2SDispatch.pas +++ /dev/null @@ -1,177 +0,0 @@ -unit HlpBlake2SDispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TBlake2SCompressProc = procedure(AState, AMsg, ACounterFlags, AIV: Pointer); - -const - Blake2SIV: array [0 .. 7] of UInt32 = ( - UInt32($6A09E667), UInt32($BB67AE85), - UInt32($3C6EF372), UInt32($A54FF53A), - UInt32($510E527F), UInt32($9B05688C), - UInt32($1F83D9AB), UInt32($5BE0CD19) - ); - -var - Blake2S_Compress: TBlake2SCompressProc; - -implementation - -uses - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -const - Blake2SSigma: array [0 .. 9, 0 .. 15] of Int32 = ( - (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), - (14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3), - (11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4), - (7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8), - (9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13), - (2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9), - (12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11), - (13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10), - (6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5), - (10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0) - ); - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure Blake2S_Compress_Scalar(AState, AMsg, ACounterFlags, AIV: Pointer); -var - LV: array [0 .. 15] of UInt32; - LPState, LPMsg, LPCounterFlags, LPIV: PByte; - LRound, I: Int32; - - procedure G(AA, AB, AC, AD, AMsgIdx0, AMsgIdx1: Int32); - begin - LV[AA] := LV[AA] + LV[AB] + PUInt32(LPMsg + AMsgIdx0 * SizeOf(UInt32))^; - LV[AD] := TBitOperations.RotateRight32(LV[AD] xor LV[AA], 16); - LV[AC] := LV[AC] + LV[AD]; - LV[AB] := TBitOperations.RotateRight32(LV[AB] xor LV[AC], 12); - LV[AA] := LV[AA] + LV[AB] + PUInt32(LPMsg + AMsgIdx1 * SizeOf(UInt32))^; - LV[AD] := TBitOperations.RotateRight32(LV[AD] xor LV[AA], 8); - LV[AC] := LV[AC] + LV[AD]; - LV[AB] := TBitOperations.RotateRight32(LV[AB] xor LV[AC], 7); - end; - -begin - LPState := PByte(AState); - LPMsg := PByte(AMsg); - LPCounterFlags := PByte(ACounterFlags); - LPIV := PByte(AIV); - - for I := 0 to 7 do - LV[I] := PUInt32(LPState + I * SizeOf(UInt32))^; - for I := 0 to 7 do - LV[I + 8] := PUInt32(LPIV + I * SizeOf(UInt32))^; - - LV[12] := LV[12] xor PUInt32(LPCounterFlags)^; - LV[13] := LV[13] xor PUInt32(LPCounterFlags + SizeOf(UInt32))^; - LV[14] := LV[14] xor PUInt32(LPCounterFlags + 2 * SizeOf(UInt32))^; - LV[15] := LV[15] xor PUInt32(LPCounterFlags + 3 * SizeOf(UInt32))^; - - for LRound := 0 to 9 do - begin - G(0, 4, 8, 12, Blake2SSigma[LRound, 0], Blake2SSigma[LRound, 1]); - G(1, 5, 9, 13, Blake2SSigma[LRound, 2], Blake2SSigma[LRound, 3]); - G(2, 6, 10, 14, Blake2SSigma[LRound, 4], Blake2SSigma[LRound, 5]); - G(3, 7, 11, 15, Blake2SSigma[LRound, 6], Blake2SSigma[LRound, 7]); - G(0, 5, 10, 15, Blake2SSigma[LRound, 8], Blake2SSigma[LRound, 9]); - G(1, 6, 11, 12, Blake2SSigma[LRound, 10], Blake2SSigma[LRound, 11]); - G(2, 7, 8, 13, Blake2SSigma[LRound, 12], Blake2SSigma[LRound, 13]); - G(3, 4, 9, 14, Blake2SSigma[LRound, 14], Blake2SSigma[LRound, 15]); - end; - - for I := 0 to 7 do - PUInt32(LPState + I * SizeOf(UInt32))^ := - PUInt32(LPState + I * SizeOf(UInt32))^ xor (LV[I] xor LV[I + 8]); -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: NEON -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure Blake2S_Compress_Sse2(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\Blake2S\Blake2SCompressSse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure Blake2S_Compress_Sse2(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Blake2S\Blake2SCompressSse2_x86_64.inc} -end; - -procedure Blake2S_Compress_Avx2(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Blake2S\Blake2SCompressAvx2_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure Blake2S_Compress_Neon(AState, AMsg, ACounterFlags, AIV: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\Blake2S\Blake2SCompressNeon_aarch64.inc} -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - Blake2S_Compress := @Blake2S_Compress_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - Blake2S_Compress := @Blake2S_Compress_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - Blake2S_Compress := @Blake2S_Compress_Avx2; - end; - TX86SimdLevel.SSE2: - begin - Blake2S_Compress := @Blake2S_Compress_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - TArmSimdLevel.NEON: - begin - Blake2S_Compress := @Blake2S_Compress_Neon; - end; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Crypto/HlpBlake3.pas b/HashLib/src/Crypto/HlpBlake3.pas index 8a85fc90..a37a8ac8 100644 --- a/HashLib/src/Crypto/HlpBlake3.pas +++ b/HashLib/src/Crypto/HlpBlake3.pas @@ -17,6 +17,21 @@ interface HlpArrayUtils, HlpHashLibTypes; +type + TBlake3CompressProc = procedure(AState, AMsg, ACV, ACounterFlags: Pointer); + TBlake3HashManyProc = procedure(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); + +var + Blake3_Compress: TBlake3CompressProc; + Blake3_HashMany: TBlake3HashManyProc; + Blake3_ParallelDegree: Int32; + +// Scalar HashMany reference, exposed so the SIMD backends can delegate their +// sub-parallel-degree remainder to it. +procedure Blake3_HashMany_Scalar(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); + resourcestring SInvalidXOFSize = 'XOFSize in Bits must be Multiples of 8 and be Greater than Zero Bytes'; @@ -250,7 +265,588 @@ TBlake3XOF = class sealed(TBlake3, IXOF, IXOFStream) implementation uses - HlpBlake3Dispatch; + HlpBitOperations, + HlpBlake3Simd; + +const + Blake3IV: array [0 .. 3] of UInt32 = ( + UInt32($6A09E667), UInt32($BB67AE85), + UInt32($3C6EF372), UInt32($A54FF53A) + ); + + FlagChunkStart = UInt32(1 shl 0); + FlagChunkEnd = UInt32(1 shl 1); + +// ============================================================================= +// Scalar fallback implementation (fully unrolled, 7 rounds, inlined G) +// ============================================================================= + +procedure Blake3_Compress_Scalar(AState, AMsg, ACV, ACounterFlags: Pointer); +var + LV0, LV1, LV2, LV3, LV4, LV5, LV6, LV7: UInt32; + LV8, LV9, LV10, LV11, LV12, LV13, LV14, LV15: UInt32; + LPMsg, LPCV, LPCounterFlags, LPState: PCardinal; +begin + LPMsg := PCardinal(AMsg); + LPCV := PCardinal(ACV); + LPCounterFlags := PCardinal(ACounterFlags); + LPState := PCardinal(AState); + + // Initialize state from chaining value + LV0 := LPCV[0]; + LV1 := LPCV[1]; + LV2 := LPCV[2]; + LV3 := LPCV[3]; + LV4 := LPCV[4]; + LV5 := LPCV[5]; + LV6 := LPCV[6]; + LV7 := LPCV[7]; + // Initialize counter half from IV and counter/flags + LV8 := Blake3IV[0]; + LV9 := Blake3IV[1]; + LV10 := Blake3IV[2]; + LV11 := Blake3IV[3]; + LV12 := LPCounterFlags[0]; + LV13 := LPCounterFlags[1]; + LV14 := LPCounterFlags[2]; + LV15 := LPCounterFlags[3]; + + // Round 0 + LV0 := LV0 + LV4 + LPMsg[0]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[1]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[2]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[3]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[4]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[5]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[6]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[7]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[8]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[9]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[10]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[11]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[12]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[13]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[14]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[15]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Round 1 + LV0 := LV0 + LV4 + LPMsg[2]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[6]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[3]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[10]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[7]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[0]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[4]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[13]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[1]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[11]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[12]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[5]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[9]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[14]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[15]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[8]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Round 2 + LV0 := LV0 + LV4 + LPMsg[3]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[4]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[10]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[12]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[13]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[2]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[7]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[14]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[6]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[5]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[9]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[0]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[11]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[15]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[8]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[1]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Round 3 + LV0 := LV0 + LV4 + LPMsg[10]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[7]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[12]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[9]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[14]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[3]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[13]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[15]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[4]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[0]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[11]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[2]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[5]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[8]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[1]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[6]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Round 4 + LV0 := LV0 + LV4 + LPMsg[12]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[13]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[9]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[11]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[15]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[10]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[14]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[8]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[7]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[2]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[5]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[3]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[0]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[1]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[6]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[4]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Round 5 + LV0 := LV0 + LV4 + LPMsg[9]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[14]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[11]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[5]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[8]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[12]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[15]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[1]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[13]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[3]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[0]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[10]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[2]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[6]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[4]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[7]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Round 6 + LV0 := LV0 + LV4 + LPMsg[11]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); + LV0 := LV0 + LV4 + LPMsg[15]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); + LV8 := LV8 + LV12; + LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); + LV1 := LV1 + LV5 + LPMsg[5]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); + LV1 := LV1 + LV5 + LPMsg[0]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); + LV9 := LV9 + LV13; + LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); + LV2 := LV2 + LV6 + LPMsg[1]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); + LV2 := LV2 + LV6 + LPMsg[9]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); + LV10 := LV10 + LV14; + LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); + LV3 := LV3 + LV7 + LPMsg[8]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); + LV3 := LV3 + LV7 + LPMsg[6]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); + LV11 := LV11 + LV15; + LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); + LV0 := LV0 + LV5 + LPMsg[14]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); + LV0 := LV0 + LV5 + LPMsg[10]; + LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); + LV10 := LV10 + LV15; + LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); + LV1 := LV1 + LV6 + LPMsg[2]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); + LV1 := LV1 + LV6 + LPMsg[12]; + LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); + LV11 := LV11 + LV12; + LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); + LV2 := LV2 + LV7 + LPMsg[3]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); + LV2 := LV2 + LV7 + LPMsg[4]; + LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); + LV8 := LV8 + LV13; + LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); + LV3 := LV3 + LV4 + LPMsg[7]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); + LV3 := LV3 + LV4 + LPMsg[13]; + LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); + LV9 := LV9 + LV14; + LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); + + // Finalization: XOR top and bottom halves + LPState[0] := LV0 xor LV8; + LPState[1] := LV1 xor LV9; + LPState[2] := LV2 xor LV10; + LPState[3] := LV3 xor LV11; + LPState[4] := LV4 xor LV12; + LPState[5] := LV5 xor LV13; + LPState[6] := LV6 xor LV14; + LPState[7] := LV7 xor LV15; + LPState[8] := LV8 xor LPCV[0]; + LPState[9] := LV9 xor LPCV[1]; + LPState[10] := LV10 xor LPCV[2]; + LPState[11] := LV11 xor LPCV[3]; + LPState[12] := LV12 xor LPCV[4]; + LPState[13] := LV13 xor LPCV[5]; + LPState[14] := LV14 xor LPCV[6]; + LPState[15] := LV15 xor LPCV[7]; +end; + +// ============================================================================= +// Scalar HashMany fallback (sequential, no parallelism) +// ============================================================================= + +procedure Blake3_HashMany_Scalar(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); +var + LChunk, LBlock: Int32; + LCV: array [0 .. 7] of UInt32; + LBlockWords: array [0 .. 15] of UInt32; + LState: array [0 .. 15] of UInt32; + LCounterFlags: array [0 .. 3] of UInt32; + LBlockFlags: UInt32; + LPInput, LPOut: PByte; +begin + LPInput := PByte(AInput); + LPOut := PByte(AOut); + + for LChunk := 0 to ANumChunks - 1 do + begin + // Initialize CV from key/IV + System.Move(AKey^, LCV[0], 8 * System.SizeOf(UInt32)); + + // Process 16 blocks per chunk + for LBlock := 0 to 15 do + begin + TBinaryPrimitives.CopyUInt32LittleEndian(LPInput, 0, @LBlockWords[0], 0, 64); + + // Set flags for this block + LBlockFlags := AFlags; + if LBlock = 0 then + LBlockFlags := LBlockFlags or FlagChunkStart; + if LBlock = 15 then + LBlockFlags := LBlockFlags or FlagChunkEnd; + + LCounterFlags[0] := UInt32(ACounter); + LCounterFlags[1] := UInt32(ACounter shr 32); + LCounterFlags[2] := 64; // BlockLen = 64 + LCounterFlags[3] := LBlockFlags; + + Blake3_Compress(@LState[0], @LBlockWords[0], @LCV[0], @LCounterFlags[0]); + + // Extract chaining value (first 8 words of output) + if LBlock < 15 then + System.Move(LState[0], LCV[0], 8 * System.SizeOf(UInt32)); + + System.Inc(LPInput, 64); + end; + + // Write final chaining value for this chunk + System.Move(LState[0], LPOut^, 8 * System.SizeOf(UInt32)); + System.Inc(LPOut, 8 * System.SizeOf(UInt32)); + System.Inc(ACounter); + end; +end; { TBlake3.TBlake3Node } @@ -910,4 +1506,9 @@ function TBlake3XOF.TransformFinal: IHashResult; Result := THashResult.Create(LBuffer); end; +initialization + Blake3_Compress := TBlake3Simd.SelectCompress(@Blake3_Compress_Scalar); + Blake3_HashMany := TBlake3Simd.SelectHashMany(@Blake3_HashMany_Scalar); + Blake3_ParallelDegree := TBlake3Simd.SelectParallelDegree(1); + end. diff --git a/HashLib/src/Crypto/HlpBlake3Dispatch.pas b/HashLib/src/Crypto/HlpBlake3Dispatch.pas deleted file mode 100644 index a065a7d4..00000000 --- a/HashLib/src/Crypto/HlpBlake3Dispatch.pas +++ /dev/null @@ -1,795 +0,0 @@ -unit HlpBlake3Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TBlake3CompressProc = procedure(AState, AMsg, ACV, ACounterFlags: Pointer); - - // Hash N complete chunks in parallel. - // AInput: pointer to N * 1024 bytes of contiguous input - // AKey: pointer to 8 x UInt32 key/IV - // AOut: pointer to N * 8 x UInt32 output chaining values - // ANumChunks: number of chunks to hash - // ACounter: starting chunk counter - // AFlags: base flags (chunk start/end handled internally) - TBlake3HashManyProc = procedure(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); - -var - Blake3_Compress: TBlake3CompressProc; - Blake3_HashMany: TBlake3HashManyProc; - Blake3_ParallelDegree: Int32; - -implementation - -uses - HlpBinaryPrimitives, - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -const - Blake3IV: array [0 .. 3] of UInt32 = ( - UInt32($6A09E667), UInt32($BB67AE85), - UInt32($3C6EF372), UInt32($A54FF53A) - ); - - FlagChunkStart = UInt32(1 shl 0); - FlagChunkEnd = UInt32(1 shl 1); - -// ============================================================================= -// Scalar fallback implementation (fully unrolled, 7 rounds, inlined G) -// ============================================================================= - -procedure Blake3_Compress_Scalar(AState, AMsg, ACV, ACounterFlags: Pointer); -var - LV0, LV1, LV2, LV3, LV4, LV5, LV6, LV7: UInt32; - LV8, LV9, LV10, LV11, LV12, LV13, LV14, LV15: UInt32; - LPMsg, LPCV, LPCounterFlags, LPState: PCardinal; -begin - LPMsg := PCardinal(AMsg); - LPCV := PCardinal(ACV); - LPCounterFlags := PCardinal(ACounterFlags); - LPState := PCardinal(AState); - - // Initialize state from chaining value - LV0 := LPCV[0]; - LV1 := LPCV[1]; - LV2 := LPCV[2]; - LV3 := LPCV[3]; - LV4 := LPCV[4]; - LV5 := LPCV[5]; - LV6 := LPCV[6]; - LV7 := LPCV[7]; - // Initialize counter half from IV and counter/flags - LV8 := Blake3IV[0]; - LV9 := Blake3IV[1]; - LV10 := Blake3IV[2]; - LV11 := Blake3IV[3]; - LV12 := LPCounterFlags[0]; - LV13 := LPCounterFlags[1]; - LV14 := LPCounterFlags[2]; - LV15 := LPCounterFlags[3]; - - // Round 0 - LV0 := LV0 + LV4 + LPMsg[0]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[1]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[2]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[3]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[4]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[5]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[6]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[7]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[8]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[9]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[10]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[11]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[12]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[13]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[14]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[15]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Round 1 - LV0 := LV0 + LV4 + LPMsg[2]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[6]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[3]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[10]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[7]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[0]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[4]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[13]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[1]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[11]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[12]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[5]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[9]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[14]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[15]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[8]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Round 2 - LV0 := LV0 + LV4 + LPMsg[3]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[4]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[10]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[12]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[13]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[2]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[7]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[14]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[6]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[5]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[9]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[0]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[11]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[15]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[8]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[1]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Round 3 - LV0 := LV0 + LV4 + LPMsg[10]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[7]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[12]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[9]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[14]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[3]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[13]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[15]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[4]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[0]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[11]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[2]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[5]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[8]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[1]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[6]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Round 4 - LV0 := LV0 + LV4 + LPMsg[12]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[13]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[9]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[11]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[15]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[10]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[14]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[8]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[7]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[2]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[5]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[3]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[0]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[1]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[6]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[4]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Round 5 - LV0 := LV0 + LV4 + LPMsg[9]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[14]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[11]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[5]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[8]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[12]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[15]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[1]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[13]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[3]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[0]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[10]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[2]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[6]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[4]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[7]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Round 6 - LV0 := LV0 + LV4 + LPMsg[11]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 16); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 12); - LV0 := LV0 + LV4 + LPMsg[15]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV0, 8); - LV8 := LV8 + LV12; - LV4 := TBitOperations.RotateRight32(LV4 xor LV8, 7); - LV1 := LV1 + LV5 + LPMsg[5]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 16); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 12); - LV1 := LV1 + LV5 + LPMsg[0]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV1, 8); - LV9 := LV9 + LV13; - LV5 := TBitOperations.RotateRight32(LV5 xor LV9, 7); - LV2 := LV2 + LV6 + LPMsg[1]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 16); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 12); - LV2 := LV2 + LV6 + LPMsg[9]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV2, 8); - LV10 := LV10 + LV14; - LV6 := TBitOperations.RotateRight32(LV6 xor LV10, 7); - LV3 := LV3 + LV7 + LPMsg[8]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 16); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 12); - LV3 := LV3 + LV7 + LPMsg[6]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV3, 8); - LV11 := LV11 + LV15; - LV7 := TBitOperations.RotateRight32(LV7 xor LV11, 7); - LV0 := LV0 + LV5 + LPMsg[14]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 16); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 12); - LV0 := LV0 + LV5 + LPMsg[10]; - LV15 := TBitOperations.RotateRight32(LV15 xor LV0, 8); - LV10 := LV10 + LV15; - LV5 := TBitOperations.RotateRight32(LV5 xor LV10, 7); - LV1 := LV1 + LV6 + LPMsg[2]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 16); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 12); - LV1 := LV1 + LV6 + LPMsg[12]; - LV12 := TBitOperations.RotateRight32(LV12 xor LV1, 8); - LV11 := LV11 + LV12; - LV6 := TBitOperations.RotateRight32(LV6 xor LV11, 7); - LV2 := LV2 + LV7 + LPMsg[3]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 16); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 12); - LV2 := LV2 + LV7 + LPMsg[4]; - LV13 := TBitOperations.RotateRight32(LV13 xor LV2, 8); - LV8 := LV8 + LV13; - LV7 := TBitOperations.RotateRight32(LV7 xor LV8, 7); - LV3 := LV3 + LV4 + LPMsg[7]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 16); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 12); - LV3 := LV3 + LV4 + LPMsg[13]; - LV14 := TBitOperations.RotateRight32(LV14 xor LV3, 8); - LV9 := LV9 + LV14; - LV4 := TBitOperations.RotateRight32(LV4 xor LV9, 7); - - // Finalization: XOR top and bottom halves - LPState[0] := LV0 xor LV8; - LPState[1] := LV1 xor LV9; - LPState[2] := LV2 xor LV10; - LPState[3] := LV3 xor LV11; - LPState[4] := LV4 xor LV12; - LPState[5] := LV5 xor LV13; - LPState[6] := LV6 xor LV14; - LPState[7] := LV7 xor LV15; - LPState[8] := LV8 xor LPCV[0]; - LPState[9] := LV9 xor LPCV[1]; - LPState[10] := LV10 xor LPCV[2]; - LPState[11] := LV11 xor LPCV[3]; - LPState[12] := LV12 xor LPCV[4]; - LPState[13] := LV13 xor LPCV[5]; - LPState[14] := LV14 xor LPCV[6]; - LPState[15] := LV15 xor LPCV[7]; -end; - -// ============================================================================= -// Scalar HashMany fallback (sequential, no parallelism) -// ============================================================================= - -procedure Blake3_HashMany_Scalar(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); -var - LChunk, LBlock: Int32; - LCV: array [0 .. 7] of UInt32; - LBlockWords: array [0 .. 15] of UInt32; - LState: array [0 .. 15] of UInt32; - LCounterFlags: array [0 .. 3] of UInt32; - LBlockFlags: UInt32; - LPInput, LPOut: PByte; -begin - LPInput := PByte(AInput); - LPOut := PByte(AOut); - - for LChunk := 0 to ANumChunks - 1 do - begin - // Initialize CV from key/IV - System.Move(AKey^, LCV[0], 8 * System.SizeOf(UInt32)); - - // Process 16 blocks per chunk - for LBlock := 0 to 15 do - begin - TBinaryPrimitives.CopyUInt32LittleEndian(LPInput, 0, @LBlockWords[0], 0, 64); - - // Set flags for this block - LBlockFlags := AFlags; - if LBlock = 0 then - LBlockFlags := LBlockFlags or FlagChunkStart; - if LBlock = 15 then - LBlockFlags := LBlockFlags or FlagChunkEnd; - - LCounterFlags[0] := UInt32(ACounter); - LCounterFlags[1] := UInt32(ACounter shr 32); - LCounterFlags[2] := 64; // BlockLen = 64 - LCounterFlags[3] := LBlockFlags; - - Blake3_Compress(@LState[0], @LBlockWords[0], @LCV[0], @LCounterFlags[0]); - - // Extract chaining value (first 8 words of output) - if LBlock < 15 then - System.Move(LState[0], LCV[0], 8 * System.SizeOf(UInt32)); - - System.Inc(LPInput, 64); - end; - - // Write final chaining value for this chunk - System.Move(LState[0], LPOut^, 8 * System.SizeOf(UInt32)); - System.Inc(LPOut, 8 * System.SizeOf(UInt32)); - System.Inc(ACounter); - end; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: NEON -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure Blake3_Compress_Sse2(AState, AMsg, ACV, ACounterFlags: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\Blake3\Blake3CompressSse2_i386.inc} -end; - -procedure Blake3_Hash4_Sse2(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); - {$I ..\Include\Simd\Common\HlpSimdProc6Begin_i386.inc} - {$I ..\Include\Simd\Blake3\Blake3Hash4Sse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure Blake3_Compress_Sse2(AState, AMsg, ACV, ACounterFlags: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Blake3\Blake3CompressSse2_x86_64.inc} -end; - -procedure Blake3_Compress_Avx2(AState, AMsg, ACV, ACounterFlags: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Blake3\Blake3CompressAvx2_x86_64.inc} -end; - -procedure Blake3_Hash4_Sse2(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); - {$I ..\Include\Simd\Common\HlpSimdProc6Begin_x86_64.inc} - {$I ..\Include\Simd\Blake3\Blake3Hash4Sse2_x86_64.inc} -end; - -procedure Blake3_Hash8_Avx2(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); - {$I ..\Include\Simd\Common\HlpSimdProc6Begin_x86_64.inc} - {$I ..\Include\Simd\Blake3\Blake3Hash8Avx2_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_X86_SIMD} - -// SSE2 hash_many: hash4 -> delegate remainder to scalar hash_many -procedure Blake3_HashMany_Sse2(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); -var - LPInput, LPOut: PByte; -begin - LPInput := PByte(AInput); - LPOut := PByte(AOut); - while ANumChunks >= 4 do - begin - Blake3_Hash4_Sse2(LPInput, AKey, LPOut, 4, ACounter, AFlags); - System.Inc(LPInput, 4 * 1024); - System.Inc(LPOut, 4 * 32); - System.Inc(ACounter, 4); - System.Dec(ANumChunks, 4); - end; - if ANumChunks > 0 then - Blake3_HashMany_Scalar(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); -end; - -{$ENDIF HASHLIB_X86_SIMD} - -{$IFDEF HASHLIB_X86_64_ASM} - -// AVX2 hash_many: hash8 -> delegate remainder to SSE2 hash_many (x64 only; after shared Sse2) -procedure Blake3_HashMany_Avx2(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); -var - LPInput, LPOut: PByte; -begin - LPInput := PByte(AInput); - LPOut := PByte(AOut); - while ANumChunks >= 8 do - begin - Blake3_Hash8_Avx2(LPInput, AKey, LPOut, 8, ACounter, AFlags); - System.Inc(LPInput, 8 * 1024); - System.Inc(LPOut, 8 * 32); - System.Inc(ACounter, 8); - System.Dec(ANumChunks, 8); - end; - if ANumChunks > 0 then - Blake3_HashMany_Sse2(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure Blake3_Compress_Neon(AState, AMsg, ACV, ACounterFlags: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\Blake3\Blake3CompressNeon_aarch64.inc} -end; - -procedure Blake3_Hash4_Neon(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); - {$I ..\Include\Simd\Common\HlpSimdProc6Begin_aarch64.inc} - {$I ..\Include\Simd\Blake3\Blake3Hash4Neon_aarch64.inc} -end; - -// NEON hash_many: hash4 -> delegate remainder to scalar hash_many -procedure Blake3_HashMany_Neon(AInput, AKey, AOut: Pointer; - ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); -var - LPInput, LPOut: PByte; -begin - LPInput := PByte(AInput); - LPOut := PByte(AOut); - while ANumChunks >= 4 do - begin - Blake3_Hash4_Neon(LPInput, AKey, LPOut, 4, ACounter, AFlags); - System.Inc(LPInput, 4 * 1024); - System.Inc(LPOut, 4 * 32); - System.Inc(ACounter, 4); - System.Dec(ANumChunks, 4); - end; - if ANumChunks > 0 then - Blake3_HashMany_Scalar(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - Blake3_Compress := @Blake3_Compress_Scalar; - Blake3_HashMany := @Blake3_HashMany_Scalar; - Blake3_ParallelDegree := 1; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - Blake3_Compress := @Blake3_Compress_Sse2; - Blake3_HashMany := @Blake3_HashMany_Sse2; - Blake3_ParallelDegree := 4; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - Blake3_Compress := @Blake3_Compress_Avx2; - Blake3_HashMany := @Blake3_HashMany_Avx2; - Blake3_ParallelDegree := 8; - end; - TX86SimdLevel.SSE2: - begin - Blake3_Compress := @Blake3_Compress_Sse2; - Blake3_HashMany := @Blake3_HashMany_Sse2; - Blake3_ParallelDegree := 4; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - TArmSimdLevel.NEON: - begin - Blake3_Compress := @Blake3_Compress_Neon; - Blake3_HashMany := @Blake3_HashMany_Neon; - Blake3_ParallelDegree := 4; - end; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Crypto/HlpSHA1.pas b/HashLib/src/Crypto/HlpSHA1.pas index ae06b8d3..b52c80b6 100644 --- a/HashLib/src/Crypto/HlpSHA1.pas +++ b/HashLib/src/Crypto/HlpSHA1.pas @@ -8,8 +8,23 @@ interface HlpBitOperations, HlpHashLibTypes, HlpSHA0, - HlpIHash, - HlpSHA1Dispatch; + HlpIHash; + +type + TSHA1CompressProc = procedure(AState, AData: Pointer; ANumBlocks: UInt32); + +var + SHA1_Compress: TSHA1CompressProc; + +const + // K round constants, each replicated across four lanes for the SIMD kernels; + // the scalar reference reads one per group ([0], [4], [8], [12]). + K_SHA1: array [0 .. 15] of UInt32 = ( + $5A827999, $5A827999, $5A827999, $5A827999, + $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, + $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, + $CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6 + ); type TSHA1 = class sealed(TSHA0) @@ -29,6 +44,82 @@ TSHA1 = class sealed(TSHA0) implementation +uses + HlpBinaryPrimitives, + HlpSHA1Simd; + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure SHA1_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32); +var + LPState: PCardinal; + LPData: PByte; + LA, LB, LC, LD, LE, LT: UInt32; + LW: array [0 .. 79] of UInt32; + LRound: Int32; +begin + LPState := PCardinal(AState); + LPData := PByte(AData); + + while ANumBlocks > 0 do + begin + TBinaryPrimitives.CopyUInt32BigEndian(LPData, 0, @LW[0], 0, 64); + + for LRound := 16 to 79 do + begin + LT := LW[LRound - 3] xor LW[LRound - 8] xor LW[LRound - 14] + xor LW[LRound - 16]; + LW[LRound] := TBitOperations.RotateLeft32(LT, 1); + end; + + LA := LPState[0]; LB := LPState[1]; LC := LPState[2]; + LD := LPState[3]; LE := LPState[4]; + + for LRound := 0 to 19 do + begin + LT := TBitOperations.RotateLeft32(LA, 5) + (LD xor (LB and (LC xor LD))) + + LE + K_SHA1[0] + LW[LRound]; + LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); + LB := LA; LA := LT; + end; + + for LRound := 20 to 39 do + begin + LT := TBitOperations.RotateLeft32(LA, 5) + (LB xor LC xor LD) + + LE + K_SHA1[4] + LW[LRound]; + LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); + LB := LA; LA := LT; + end; + + for LRound := 40 to 59 do + begin + LT := TBitOperations.RotateLeft32(LA, 5) + + ((LB and LC) or (LD and (LB or LC))) + + LE + K_SHA1[8] + LW[LRound]; + LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); + LB := LA; LA := LT; + end; + + for LRound := 60 to 79 do + begin + LT := TBitOperations.RotateLeft32(LA, 5) + (LB xor LC xor LD) + + LE + K_SHA1[12] + LW[LRound]; + LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); + LB := LA; LA := LT; + end; + + LPState[0] := LPState[0] + LA; LPState[1] := LPState[1] + LB; + LPState[2] := LPState[2] + LC; LPState[3] := LPState[3] + LD; + LPState[4] := LPState[4] + LE; + + System.FillChar(LW, System.SizeOf(LW), 0); + System.Inc(LPData, 64); + System.Dec(ANumBlocks); + end; +end; + { TSHA1 } function TSHA1.Clone(): IHash; @@ -241,4 +332,7 @@ procedure TSHA1.Expand(AData: PCardinal); {$ENDIF USE_UNROLLED_VARIANT} end; +initialization + SHA1_Compress := TSHA1Simd.Select(@SHA1_Compress_Scalar); + end. diff --git a/HashLib/src/Crypto/HlpSHA1Dispatch.pas b/HashLib/src/Crypto/HlpSHA1Dispatch.pas deleted file mode 100644 index 0c8da7ad..00000000 --- a/HashLib/src/Crypto/HlpSHA1Dispatch.pas +++ /dev/null @@ -1,253 +0,0 @@ -unit HlpSHA1Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TSHA1CompressProc = procedure(AState, AData: Pointer; ANumBlocks: UInt32); - -var - SHA1_Compress: TSHA1CompressProc; - -const - // K round constants replicated 4x for SIMD. - // Layout: K_00_19 (16B) at 0, K_20_39 at 16, K_40_59 at 32, K_60_79 at 48. - K_SHA1: array [0 .. 15] of UInt32 = ( - $5A827999, $5A827999, $5A827999, $5A827999, - $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, - $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, - $CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6 - ); - -{$IFDEF HASHLIB_X86_SIMD} - // BSWAP32 shuffle mask for pshufb (x86 SIMD only): byte-swaps and reverses - // dword order in one shuffle (sha1rnds4 reads its four words in reverse). Not a - // SHA-1 constant; used only by the SHA-NI kernel. ARM byte-swaps with REV32 and - // needs no mask table. - BSWAP32_MASK: array [0 .. 3] of UInt32 = ( - $0C0D0E0F, $08090A0B, $04050607, $00010203 - ); - - // Doubled SHA-1 round constants plus the AVX2 byte-swap masks, shared by the - // AVX2 and SSE2 SIMD-schedule SHA-1 kernels. Each round constant fills a 128-bit - // lane (its four dwords) and is stored twice so one table feeds both the 256-bit - // AVX2 read and the 128-bit SSE2 reads (both read at a 32-byte stride, skipping - // the duplicate halves). Only the AVX2 kernel uses the appended masks: the - // byte-swap mask (BSWAP32 pattern, twice) then a whole-vector reverse mask; the - // SSE2 kernel computes its byte-swap and needs no mask. Read unaligned, so no - // special alignment is required. - K_SHA1_Doubled: array [0 .. 43] of UInt32 = ( - $5A827999, $5A827999, $5A827999, $5A827999, - $5A827999, $5A827999, $5A827999, $5A827999, - $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, - $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, - $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, - $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, - $CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6, - $CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6, - $00010203, $04050607, $08090A0B, $0C0D0E0F, - $00010203, $04050607, $08090A0B, $0C0D0E0F, - $0C0D0E0F, $08090A0B, $04050607, $00010203 - ); -{$ENDIF HASHLIB_X86_SIMD} - -implementation - -uses - HlpBinaryPrimitives, - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure SHA1_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32); -var - LPState: PCardinal; - LPData: PByte; - LA, LB, LC, LD, LE, LT: UInt32; - LW: array [0 .. 79] of UInt32; - LRound: Int32; -begin - LPState := PCardinal(AState); - LPData := PByte(AData); - - while ANumBlocks > 0 do - begin - TBinaryPrimitives.CopyUInt32BigEndian(LPData, 0, @LW[0], 0, 64); - - for LRound := 16 to 79 do - begin - LT := LW[LRound - 3] xor LW[LRound - 8] xor LW[LRound - 14] - xor LW[LRound - 16]; - LW[LRound] := TBitOperations.RotateLeft32(LT, 1); - end; - - LA := LPState[0]; LB := LPState[1]; LC := LPState[2]; - LD := LPState[3]; LE := LPState[4]; - - for LRound := 0 to 19 do - begin - LT := TBitOperations.RotateLeft32(LA, 5) + (LD xor (LB and (LC xor LD))) - + LE + $5A827999 + LW[LRound]; - LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); - LB := LA; LA := LT; - end; - - for LRound := 20 to 39 do - begin - LT := TBitOperations.RotateLeft32(LA, 5) + (LB xor LC xor LD) - + LE + $6ED9EBA1 + LW[LRound]; - LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); - LB := LA; LA := LT; - end; - - for LRound := 40 to 59 do - begin - LT := TBitOperations.RotateLeft32(LA, 5) + - ((LB and LC) or (LD and (LB or LC))) - + LE + $8F1BBCDC + LW[LRound]; - LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); - LB := LA; LA := LT; - end; - - for LRound := 60 to 79 do - begin - LT := TBitOperations.RotateLeft32(LA, 5) + (LB xor LC xor LD) - + LE + $CA62C1D6 + LW[LRound]; - LE := LD; LD := LC; LC := TBitOperations.RotateLeft32(LB, 30); - LB := LA; LA := LT; - end; - - LPState[0] := LPState[0] + LA; LPState[1] := LPState[1] + LB; - LPState[2] := LPState[2] + LC; LPState[3] := LPState[3] + LD; - LPState[4] := LPState[4] + LE; - - System.FillChar(LW, System.SizeOf(LW), 0); - System.Inc(LPData, 64); - System.Dec(ANumBlocks); - end; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: ShaNi, AVX2, SSE2 -// aarch64: SHA1 Crypto Extensions -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\SHA1\SHA1CompressSse2_i386.inc} -end; - -procedure SHA1_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA1_Compress_Sse2(AState, AData, ANumBlocks, @K_SHA1_Doubled); -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure SHA1_Compress_ShaNi(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants, AMask: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} - {$I ..\Include\Simd\SHA1\SHA1CompressShaNi_x86_64.inc} -end; - -procedure SHA1_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA1_Compress_ShaNi(AState, AData, ANumBlocks, @K_SHA1, @BSWAP32_MASK); -end; - -procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\SHA1\SHA1CompressSse2_x86_64.inc} -end; - -procedure SHA1_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA1_Compress_Sse2(AState, AData, ANumBlocks, @K_SHA1_Doubled); -end; - -procedure SHA1_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\SHA1\SHA1CompressAvx2_x86_64.inc} -end; - -procedure SHA1_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA1_Compress_Avx2(AState, AData, ANumBlocks, @K_SHA1_Doubled); -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure SHA1_Compress_CryptoExt(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\SHA1\SHA1CompressCryptoExt_aarch64.inc} -end; - -procedure SHA1_Compress_CryptoExt_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA1_Compress_CryptoExt(AState, AData, ANumBlocks, @K_SHA1); -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - SHA1_Compress := @SHA1_Compress_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - SHA1_Compress := @SHA1_Compress_Sse2_Wrap; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - if TCpuFeatures.X86.HasSHANI() then - begin - SHA1_Compress := @SHA1_Compress_ShaNi_Wrap; - Exit; - end; - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - SHA1_Compress := @SHA1_Compress_Avx2_Wrap; - end; - TX86SimdLevel.SSE2: - begin - SHA1_Compress := @SHA1_Compress_Sse2_Wrap; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - if TCpuFeatures.Arm.HasSHA1() then - begin - SHA1_Compress := @SHA1_Compress_CryptoExt_Wrap; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Crypto/HlpSHA2_256Base.pas b/HashLib/src/Crypto/HlpSHA2_256Base.pas index 3848efba..078b152e 100644 --- a/HashLib/src/Crypto/HlpSHA2_256Base.pas +++ b/HashLib/src/Crypto/HlpSHA2_256Base.pas @@ -8,8 +8,39 @@ interface HlpBinaryPrimitives, HlpHashLibTypes, HlpIHashInfo, - HlpHashCryptoNotBuildIn, - HlpSHA2_256Dispatch; + HlpHashCryptoNotBuildIn; + +type + // Block compression contract shared by the scalar reference implementation and + // every SIMD backend: hash ANumBlocks consecutive 64-byte blocks into AState. + TSHA256CompressProc = procedure(AState, AData: Pointer; ANumBlocks: UInt32); + +var + // The active SHA-256 block-compression routine. Assigned once at unit + // initialization to the best implementation the running CPU supports (see the + // SIMD facade); defaults to the scalar reference on non-accelerated targets. + SHA256_Compress: TSHA256CompressProc; + +const + // K256 round constants (64 UInt32 = 256 bytes). + K256: array [0 .. 63] of UInt32 = ( + $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, + $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, + $D807AA98, $12835B01, $243185BE, $550C7DC3, + $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, + $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, + $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, + $983E5152, $A831C66D, $B00327C8, $BF597FC7, + $C6E00BF3, $D5A79147, $06CA6351, $14292967, + $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, + $650A7354, $766A0ABB, $81C2C92E, $92722C85, + $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, + $D192E819, $D6990624, $F40E3585, $106AA070, + $19A4C116, $1E376C08, $2748774C, $34B0BCB5, + $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, + $748F82EE, $78A5636F, $84C87814, $8CC70208, + $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2 + ); type TSHA2_256Base = class abstract(TBlockHash, ICryptoNotBuildIn, ITransformBlock) @@ -31,6 +62,65 @@ TSHA2_256Base = class abstract(TBlockHash, ICryptoNotBuildIn, ITransformBlock) implementation +uses + HlpBitOperations, + HlpSHA2_256Simd; + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure SHA256_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32); +var + LPState: PCardinal; + LPData: PByte; + LA, LB, LC, LD, LE, LF, LG, LH, LT1, LT2: UInt32; + LW: array [0 .. 63] of UInt32; + LRound: Int32; +begin + LPState := PCardinal(AState); + LPData := PByte(AData); + + while ANumBlocks > 0 do + begin + TBinaryPrimitives.CopyUInt32BigEndian(LPData, 0, @LW[0], 0, 64); + + for LRound := 16 to 63 do + begin + LT1 := LW[LRound - 2]; + LT2 := LW[LRound - 15]; + LW[LRound] := (TBitOperations.RotateRight32(LT1, 17) xor TBitOperations.RotateRight32(LT1, 19) + xor (LT1 shr 10)) + LW[LRound - 7] + + (TBitOperations.RotateRight32(LT2, 7) xor TBitOperations.RotateRight32(LT2, 18) + xor (LT2 shr 3)) + LW[LRound - 16]; + end; + + LA := LPState[0]; LB := LPState[1]; LC := LPState[2]; LD := LPState[3]; + LE := LPState[4]; LF := LPState[5]; LG := LPState[6]; LH := LPState[7]; + + for LRound := 0 to 63 do + begin + LT1 := LH + (TBitOperations.RotateRight32(LE, 6) xor TBitOperations.RotateRight32(LE, 11) + xor TBitOperations.RotateRight32(LE, 25)) + ((LE and LF) xor (not LE and LG)) + + K256[LRound] + LW[LRound]; + LT2 := (TBitOperations.RotateRight32(LA, 2) xor TBitOperations.RotateRight32(LA, 13) + xor TBitOperations.RotateRight32(LA, 22)) + + ((LA and LB) xor (LA and LC) xor (LB and LC)); + LH := LG; LG := LF; LF := LE; LE := LD + LT1; + LD := LC; LC := LB; LB := LA; LA := LT1 + LT2; + end; + + LPState[0] := LPState[0] + LA; LPState[1] := LPState[1] + LB; + LPState[2] := LPState[2] + LC; LPState[3] := LPState[3] + LD; + LPState[4] := LPState[4] + LE; LPState[5] := LPState[5] + LF; + LPState[6] := LPState[6] + LG; LPState[7] := LPState[7] + LH; + + System.FillChar(LW, System.SizeOf(LW), 0); + System.Inc(LPData, 64); + System.Dec(ANumBlocks); + end; +end; + { TSHA2_256Base } constructor TSHA2_256Base.Create(AHashSize: Int32); @@ -109,4 +199,7 @@ procedure TSHA2_256Base.TransformBytes(const AData: THashLibByteArray; end; end; +initialization + SHA256_Compress := TSHA2_256Simd.Select(@SHA256_Compress_Scalar); + end. diff --git a/HashLib/src/Crypto/HlpSHA2_256Dispatch.pas b/HashLib/src/Crypto/HlpSHA2_256Dispatch.pas deleted file mode 100644 index 55c5463a..00000000 --- a/HashLib/src/Crypto/HlpSHA2_256Dispatch.pas +++ /dev/null @@ -1,272 +0,0 @@ -unit HlpSHA2_256Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TSHA256CompressProc = procedure(AState, AData: Pointer; ANumBlocks: UInt32); - -var - SHA256_Compress: TSHA256CompressProc; - -const - // K256 round constants (64 UInt32 = 256 bytes) - K256: array [0 .. 63] of UInt32 = ( - $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, - $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, - $D807AA98, $12835B01, $243185BE, $550C7DC3, - $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, - $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, - $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, - $983E5152, $A831C66D, $B00327C8, $BF597FC7, - $C6E00BF3, $D5A79147, $06CA6351, $14292967, - $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, - $650A7354, $766A0ABB, $81C2C92E, $92722C85, - $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, - $D192E819, $D6990624, $F40E3585, $106AA070, - $19A4C116, $1E376C08, $2748774C, $34B0BCB5, - $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, - $748F82EE, $78A5636F, $84C87814, $8CC70208, - $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2 - ); - -{$IFDEF HASHLIB_X86_SIMD} - // BSWAP32 shuffle mask for pshufb (x86 SIMD only): reverses bytes within each - // dword. Not a SHA-256 constant; used only by the SHA-NI kernel. ARM byte-swaps - // with REV32 and needs no mask table. - BSWAP32_MASK: array [0 .. 3] of UInt32 = ( - $00010203, $04050607, $08090A0B, $0C0D0E0F - ); - - // Doubled K256 round constants plus the three AVX2 message-schedule masks, - // shared by the AVX2 and SSE2 SIMD-schedule SHA-256 kernels. Each 128-bit K256 - // quadruple is stored twice so one table feeds both the 256-bit AVX2 lanes and - // the 128-bit SSE2 reads (both read at a 32-byte stride, skipping the duplicate - // halves). Only the AVX2 kernel uses the appended masks: the byte-swap mask - // (BSWAP32 pattern, twice) occupies [128..135] and the two schedule shuffle - // masks follow at [136..143] and [144..151]; the SSE2 kernel computes its - // byte-swap and needs no mask. Derived from K256. - K256_Doubled: array [0 .. 151] of UInt32 = ( - $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, - $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, - $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, - $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, - $D807AA98, $12835B01, $243185BE, $550C7DC3, - $D807AA98, $12835B01, $243185BE, $550C7DC3, - $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, - $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, - $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, - $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, - $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, - $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, - $983E5152, $A831C66D, $B00327C8, $BF597FC7, - $983E5152, $A831C66D, $B00327C8, $BF597FC7, - $C6E00BF3, $D5A79147, $06CA6351, $14292967, - $C6E00BF3, $D5A79147, $06CA6351, $14292967, - $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, - $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, - $650A7354, $766A0ABB, $81C2C92E, $92722C85, - $650A7354, $766A0ABB, $81C2C92E, $92722C85, - $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, - $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, - $D192E819, $D6990624, $F40E3585, $106AA070, - $D192E819, $D6990624, $F40E3585, $106AA070, - $19A4C116, $1E376C08, $2748774C, $34B0BCB5, - $19A4C116, $1E376C08, $2748774C, $34B0BCB5, - $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, - $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, - $748F82EE, $78A5636F, $84C87814, $8CC70208, - $748F82EE, $78A5636F, $84C87814, $8CC70208, - $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2, - $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2, - $00010203, $04050607, $08090A0B, $0C0D0E0F, - $00010203, $04050607, $08090A0B, $0C0D0E0F, - $03020100, $0B0A0908, $FFFFFFFF, $FFFFFFFF, - $03020100, $0B0A0908, $FFFFFFFF, $FFFFFFFF, - $FFFFFFFF, $FFFFFFFF, $03020100, $0B0A0908, - $FFFFFFFF, $FFFFFFFF, $03020100, $0B0A0908 - ); -{$ENDIF HASHLIB_X86_SIMD} - -implementation - -uses - HlpBinaryPrimitives, - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure SHA256_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32); -var - LPState: PCardinal; - LPData: PByte; - LA, LB, LC, LD, LE, LF, LG, LH, LT1, LT2: UInt32; - LW: array [0 .. 63] of UInt32; - LRound: Int32; -begin - LPState := PCardinal(AState); - LPData := PByte(AData); - - while ANumBlocks > 0 do - begin - TBinaryPrimitives.CopyUInt32BigEndian(LPData, 0, @LW[0], 0, 64); - - for LRound := 16 to 63 do - begin - LT1 := LW[LRound - 2]; - LT2 := LW[LRound - 15]; - LW[LRound] := (TBitOperations.RotateRight32(LT1, 17) xor TBitOperations.RotateRight32(LT1, 19) - xor (LT1 shr 10)) + LW[LRound - 7] + - (TBitOperations.RotateRight32(LT2, 7) xor TBitOperations.RotateRight32(LT2, 18) - xor (LT2 shr 3)) + LW[LRound - 16]; - end; - - LA := LPState[0]; LB := LPState[1]; LC := LPState[2]; LD := LPState[3]; - LE := LPState[4]; LF := LPState[5]; LG := LPState[6]; LH := LPState[7]; - - for LRound := 0 to 63 do - begin - LT1 := LH + (TBitOperations.RotateRight32(LE, 6) xor TBitOperations.RotateRight32(LE, 11) - xor TBitOperations.RotateRight32(LE, 25)) + ((LE and LF) xor (not LE and LG)) - + K256[LRound] + LW[LRound]; - LT2 := (TBitOperations.RotateRight32(LA, 2) xor TBitOperations.RotateRight32(LA, 13) - xor TBitOperations.RotateRight32(LA, 22)) + - ((LA and LB) xor (LA and LC) xor (LB and LC)); - LH := LG; LG := LF; LF := LE; LE := LD + LT1; - LD := LC; LC := LB; LB := LA; LA := LT1 + LT2; - end; - - LPState[0] := LPState[0] + LA; LPState[1] := LPState[1] + LB; - LPState[2] := LPState[2] + LC; LPState[3] := LPState[3] + LD; - LPState[4] := LPState[4] + LE; LPState[5] := LPState[5] + LF; - LPState[6] := LPState[6] + LG; LPState[7] := LPState[7] + LH; - - System.FillChar(LW, System.SizeOf(LW), 0); - System.Inc(LPData, 64); - System.Dec(ANumBlocks); - end; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: ShaNi, AVX2, SSE2 -// aarch64: SHA256 Crypto Extensions -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure SHA256_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\SHA256\SHA256CompressSse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure SHA256_Compress_ShaNi(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants, AMask: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} - {$I ..\Include\Simd\SHA256\SHA256CompressShaNi_x86_64.inc} -end; - -procedure SHA256_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA256_Compress_ShaNi(AState, AData, ANumBlocks, @K256, @BSWAP32_MASK); -end; - -procedure SHA256_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\SHA256\SHA256CompressSse2_x86_64.inc} -end; - -procedure SHA256_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\SHA256\SHA256CompressAvx2_x86_64.inc} -end; - -procedure SHA256_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA256_Compress_Avx2(AState, AData, ANumBlocks, @K256_Doubled); -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_X86_SIMD} - -procedure SHA256_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA256_Compress_Sse2(AState, AData, ANumBlocks, @K256_Doubled); -end; - -{$ENDIF HASHLIB_X86_SIMD} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure SHA256_Compress_CryptoExt(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\SHA256\SHA256CompressCryptoExt_aarch64.inc} -end; - -procedure SHA256_Compress_CryptoExt_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA256_Compress_CryptoExt(AState, AData, ANumBlocks, @K256); -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - SHA256_Compress := @SHA256_Compress_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - SHA256_Compress := @SHA256_Compress_Sse2_Wrap; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - if TCpuFeatures.X86.HasSHANI() then - begin - SHA256_Compress := @SHA256_Compress_ShaNi_Wrap; - Exit; - end; - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - SHA256_Compress := @SHA256_Compress_Avx2_Wrap; - end; - TX86SimdLevel.SSE2: - begin - SHA256_Compress := @SHA256_Compress_Sse2_Wrap; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - if TCpuFeatures.Arm.HasSHA256() then - begin - SHA256_Compress := @SHA256_Compress_CryptoExt_Wrap; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Crypto/HlpSHA2_512Base.pas b/HashLib/src/Crypto/HlpSHA2_512Base.pas index ac923d49..f26e8e1d 100644 --- a/HashLib/src/Crypto/HlpSHA2_512Base.pas +++ b/HashLib/src/Crypto/HlpSHA2_512Base.pas @@ -8,8 +8,58 @@ interface HlpBinaryPrimitives, HlpHashLibTypes, HlpIHashInfo, - HlpHashCryptoNotBuildIn, - HlpSHA2_512Dispatch; + HlpHashCryptoNotBuildIn; + +type + TSHA512CompressProc = procedure(AState, AData: Pointer; ANumBlocks: UInt32); + +var + SHA512_Compress: TSHA512CompressProc; + +const + // K512 round constants (80 UInt64 = 640 bytes). + K512: array [0 .. 79] of UInt64 = ( + UInt64($428A2F98D728AE22), UInt64($7137449123EF65CD), + UInt64($B5C0FBCFEC4D3B2F), UInt64($E9B5DBA58189DBBC), + UInt64($3956C25BF348B538), UInt64($59F111F1B605D019), + UInt64($923F82A4AF194F9B), UInt64($AB1C5ED5DA6D8118), + UInt64($D807AA98A3030242), UInt64($12835B0145706FBE), + UInt64($243185BE4EE4B28C), UInt64($550C7DC3D5FFB4E2), + UInt64($72BE5D74F27B896F), UInt64($80DEB1FE3B1696B1), + UInt64($9BDC06A725C71235), UInt64($C19BF174CF692694), + UInt64($E49B69C19EF14AD2), UInt64($EFBE4786384F25E3), + UInt64($0FC19DC68B8CD5B5), UInt64($240CA1CC77AC9C65), + UInt64($2DE92C6F592B0275), UInt64($4A7484AA6EA6E483), + UInt64($5CB0A9DCBD41FBD4), UInt64($76F988DA831153B5), + UInt64($983E5152EE66DFAB), UInt64($A831C66D2DB43210), + UInt64($B00327C898FB213F), UInt64($BF597FC7BEEF0EE4), + UInt64($C6E00BF33DA88FC2), UInt64($D5A79147930AA725), + UInt64($06CA6351E003826F), UInt64($142929670A0E6E70), + UInt64($27B70A8546D22FFC), UInt64($2E1B21385C26C926), + UInt64($4D2C6DFC5AC42AED), UInt64($53380D139D95B3DF), + UInt64($650A73548BAF63DE), UInt64($766A0ABB3C77B2A8), + UInt64($81C2C92E47EDAEE6), UInt64($92722C851482353B), + UInt64($A2BFE8A14CF10364), UInt64($A81A664BBC423001), + UInt64($C24B8B70D0F89791), UInt64($C76C51A30654BE30), + UInt64($D192E819D6EF5218), UInt64($D69906245565A910), + UInt64($F40E35855771202A), UInt64($106AA07032BBD1B8), + UInt64($19A4C116B8D2D0C8), UInt64($1E376C085141AB53), + UInt64($2748774CDF8EEB99), UInt64($34B0BCB5E19B48A8), + UInt64($391C0CB3C5C95A63), UInt64($4ED8AA4AE3418ACB), + UInt64($5B9CCA4F7763E373), UInt64($682E6FF3D6B2B8A3), + UInt64($748F82EE5DEFB2FC), UInt64($78A5636F43172F60), + UInt64($84C87814A1F0AB72), UInt64($8CC702081A6439EC), + UInt64($90BEFFFA23631E28), UInt64($A4506CEBDE82BDE9), + UInt64($BEF9A3F7B2C67915), UInt64($C67178F2E372532B), + UInt64($CA273ECEEA26619C), UInt64($D186B8C721C0C207), + UInt64($EADA7DD6CDE0EB1E), UInt64($F57D4F7FEE6ED178), + UInt64($06F067AA72176FBA), UInt64($0A637DC5A2C898A6), + UInt64($113F9804BEF90DAE), UInt64($1B710B35131C471B), + UInt64($28DB77F523047D84), UInt64($32CAAB7B40C72493), + UInt64($3C9EBE0A15C9BEBC), UInt64($431D67C49C100D4C), + UInt64($4CC5D4BECB3E42B6), UInt64($597F299CFC657E2A), + UInt64($5FCB6FAB3AD6FAEC), UInt64($6C44198C4A475817) + ); type TSHA2_512Base = class abstract(TBlockHash, ICryptoNotBuildIn, ITransformBlock) @@ -31,6 +81,65 @@ TSHA2_512Base = class abstract(TBlockHash, ICryptoNotBuildIn, ITransformBlock) implementation +uses + HlpBitOperations, + HlpSHA2_512Simd; + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure SHA512_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32); +var + LPState: PUInt64; + LPData: PByte; + LA, LB, LC, LD, LE, LF, LG, LH, LT1, LT2: UInt64; + LW: array [0 .. 79] of UInt64; + LRound: Int32; +begin + LPState := PUInt64(AState); + LPData := PByte(AData); + + while ANumBlocks > 0 do + begin + TBinaryPrimitives.CopyUInt64BigEndian(LPData, 0, @LW[0], 0, 128); + + for LRound := 16 to 79 do + begin + LT1 := LW[LRound - 2]; + LT2 := LW[LRound - 15]; + LW[LRound] := (TBitOperations.RotateRight64(LT1, 19) xor TBitOperations.RotateRight64(LT1, 61) + xor (LT1 shr 6)) + LW[LRound - 7] + + (TBitOperations.RotateRight64(LT2, 1) xor TBitOperations.RotateRight64(LT2, 8) + xor (LT2 shr 7)) + LW[LRound - 16]; + end; + + LA := LPState[0]; LB := LPState[1]; LC := LPState[2]; LD := LPState[3]; + LE := LPState[4]; LF := LPState[5]; LG := LPState[6]; LH := LPState[7]; + + for LRound := 0 to 79 do + begin + LT1 := LH + (TBitOperations.RotateRight64(LE, 14) xor TBitOperations.RotateRight64(LE, 18) + xor TBitOperations.RotateRight64(LE, 41)) + ((LE and LF) xor (not LE and LG)) + + K512[LRound] + LW[LRound]; + LT2 := (TBitOperations.RotateRight64(LA, 28) xor TBitOperations.RotateRight64(LA, 34) + xor TBitOperations.RotateRight64(LA, 39)) + + ((LA and LB) xor (LA and LC) xor (LB and LC)); + LH := LG; LG := LF; LF := LE; LE := LD + LT1; + LD := LC; LC := LB; LB := LA; LA := LT1 + LT2; + end; + + LPState[0] := LPState[0] + LA; LPState[1] := LPState[1] + LB; + LPState[2] := LPState[2] + LC; LPState[3] := LPState[3] + LD; + LPState[4] := LPState[4] + LE; LPState[5] := LPState[5] + LF; + LPState[6] := LPState[6] + LG; LPState[7] := LPState[7] + LH; + + System.FillChar(LW, System.SizeOf(LW), 0); + System.Inc(LPData, 128); + System.Dec(ANumBlocks); + end; +end; + { TSHA2_512Base } constructor TSHA2_512Base.Create(AHashSize: Int32); @@ -117,4 +226,7 @@ procedure TSHA2_512Base.TransformBytes(const AData: THashLibByteArray; end; end; +initialization + SHA512_Compress := TSHA2_512Simd.Select(@SHA512_Compress_Scalar); + end. diff --git a/HashLib/src/Crypto/HlpSHA3.pas b/HashLib/src/Crypto/HlpSHA3.pas index be739753..c23687ef 100644 --- a/HashLib/src/Crypto/HlpSHA3.pas +++ b/HashLib/src/Crypto/HlpSHA3.pas @@ -16,8 +16,33 @@ interface HlpConverters, HlpHashSize, HlpArrayUtils, - HlpHashLibTypes, - HlpSHA3Dispatch; + HlpHashLibTypes; + +type + TKeccakF1600Proc = procedure(AState: Pointer); + TKeccakF1600AbsorbProc = procedure(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); + +var + KeccakF1600_Permute: TKeccakF1600Proc; + KeccakF1600_Absorb: TKeccakF1600AbsorbProc; + +const + // Keccak-F1600 round constants (24 UInt64). Shared by the scalar reference and + // the AArch64 crypto-extension kernels. + RC: array [0 .. 23] of UInt64 = ( + UInt64($0000000000000001), UInt64($0000000000008082), + UInt64($800000000000808A), UInt64($8000000080008000), + UInt64($000000000000808B), UInt64($0000000080000001), + UInt64($8000000080008081), UInt64($8000000000008009), + UInt64($000000000000008A), UInt64($0000000000000088), + UInt64($0000000080008009), UInt64($000000008000000A), + UInt64($000000008000808B), UInt64($800000000000008B), + UInt64($8000000000008089), UInt64($8000000000008003), + UInt64($8000000000008002), UInt64($8000000000000080), + UInt64($000000000000800A), UInt64($800000008000000A), + UInt64($8000000080008081), UInt64($8000000000008080), + UInt64($0000000080000001), UInt64($8000000080008008)); resourcestring SInvalidXOFSize = @@ -379,6 +404,372 @@ TKMAC256XOF = class sealed(TKMAC256, IKMAC, IXOF, IXOFStream) implementation +uses + HlpBitOperations, + HlpSHA3Simd; + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure KeccakF1600_Scalar(AState: Pointer); +var + LPState: PUInt64; + LDa, LDe, LDi, LDo, LDu: UInt64; +{$IFDEF USE_UNROLLED_VARIANT} + Aba, Abe, Abi, Abo, Abu, Aga, Age, Agi, Ago, Agu, Aka, Ake, Aki, Ako, Aku, + Ama, Ame, Ami, Amo, Amu, Asa, Ase, Asi, Aso, Asu, BCa, BCe, BCi, BCo, BCu, + Eba, Ebe, Ebi, Ebo, Ebu, Ega, Ege, Egi, Ego, Egu, Eka, Eke, Eki, Eko, Eku, + Ema, Eme, Emi, Emo, Emu, Esa, Ese, Esi, Eso, Esu: UInt64; + LRound: Int32; +{$ELSE} + LColA, LColE, LColI, LColO, LColU: UInt64; + LTemp: array [0 .. 24] of UInt64; + LRound: Int32; +{$ENDIF USE_UNROLLED_VARIANT} +begin + LPState := PUInt64(AState); + +{$IFDEF USE_UNROLLED_VARIANT} + Aba := LPState[0]; + Abe := LPState[1]; + Abi := LPState[2]; + Abo := LPState[3]; + Abu := LPState[4]; + Aga := LPState[5]; + Age := LPState[6]; + Agi := LPState[7]; + Ago := LPState[8]; + Agu := LPState[9]; + Aka := LPState[10]; + Ake := LPState[11]; + Aki := LPState[12]; + Ako := LPState[13]; + Aku := LPState[14]; + Ama := LPState[15]; + Ame := LPState[16]; + Ami := LPState[17]; + Amo := LPState[18]; + Amu := LPState[19]; + Asa := LPState[20]; + Ase := LPState[21]; + Asi := LPState[22]; + Aso := LPState[23]; + Asu := LPState[24]; + + LRound := 0; + while LRound < 24 do + begin + BCa := Aba xor Aga xor Aka xor Ama xor Asa; + BCe := Abe xor Age xor Ake xor Ame xor Ase; + BCi := Abi xor Agi xor Aki xor Ami xor Asi; + BCo := Abo xor Ago xor Ako xor Amo xor Aso; + BCu := Abu xor Agu xor Aku xor Amu xor Asu; + + LDa := BCu xor TBitOperations.RotateLeft64(BCe, 1); + LDe := BCa xor TBitOperations.RotateLeft64(BCi, 1); + LDi := BCe xor TBitOperations.RotateLeft64(BCo, 1); + LDo := BCi xor TBitOperations.RotateLeft64(BCu, 1); + LDu := BCo xor TBitOperations.RotateLeft64(BCa, 1); + + Aba := Aba xor LDa; + BCa := Aba; + Age := Age xor LDe; + BCe := TBitOperations.RotateLeft64(Age, 44); + Aki := Aki xor LDi; + BCi := TBitOperations.RotateLeft64(Aki, 43); + Amo := Amo xor LDo; + BCo := TBitOperations.RotateLeft64(Amo, 21); + Asu := Asu xor LDu; + BCu := TBitOperations.RotateLeft64(Asu, 14); + Eba := BCa xor ((not BCe) and BCi); + Eba := Eba xor UInt64(RC[LRound]); + Ebe := BCe xor ((not BCi) and BCo); + Ebi := BCi xor ((not BCo) and BCu); + Ebo := BCo xor ((not BCu) and BCa); + Ebu := BCu xor ((not BCa) and BCe); + + Abo := Abo xor LDo; + BCa := TBitOperations.RotateLeft64(Abo, 28); + Agu := Agu xor LDu; + BCe := TBitOperations.RotateLeft64(Agu, 20); + Aka := Aka xor LDa; + BCi := TBitOperations.RotateLeft64(Aka, 3); + Ame := Ame xor LDe; + BCo := TBitOperations.RotateLeft64(Ame, 45); + Asi := Asi xor LDi; + BCu := TBitOperations.RotateLeft64(Asi, 61); + Ega := BCa xor ((not BCe) and BCi); + Ege := BCe xor ((not BCi) and BCo); + Egi := BCi xor ((not BCo) and BCu); + Ego := BCo xor ((not BCu) and BCa); + Egu := BCu xor ((not BCa) and BCe); + + Abe := Abe xor LDe; + BCa := TBitOperations.RotateLeft64(Abe, 1); + Agi := Agi xor LDi; + BCe := TBitOperations.RotateLeft64(Agi, 6); + Ako := Ako xor LDo; + BCi := TBitOperations.RotateLeft64(Ako, 25); + Amu := Amu xor LDu; + BCo := TBitOperations.RotateLeft64(Amu, 8); + Asa := Asa xor LDa; + BCu := TBitOperations.RotateLeft64(Asa, 18); + Eka := BCa xor ((not BCe) and BCi); + Eke := BCe xor ((not BCi) and BCo); + Eki := BCi xor ((not BCo) and BCu); + Eko := BCo xor ((not BCu) and BCa); + Eku := BCu xor ((not BCa) and BCe); + + Abu := Abu xor LDu; + BCa := TBitOperations.RotateLeft64(Abu, 27); + Aga := Aga xor LDa; + BCe := TBitOperations.RotateLeft64(Aga, 36); + Ake := Ake xor LDe; + BCi := TBitOperations.RotateLeft64(Ake, 10); + Ami := Ami xor LDi; + BCo := TBitOperations.RotateLeft64(Ami, 15); + Aso := Aso xor LDo; + BCu := TBitOperations.RotateLeft64(Aso, 56); + Ema := BCa xor ((not BCe) and BCi); + Eme := BCe xor ((not BCi) and BCo); + Emi := BCi xor ((not BCo) and BCu); + Emo := BCo xor ((not BCu) and BCa); + Emu := BCu xor ((not BCa) and BCe); + + Abi := Abi xor LDi; + BCa := TBitOperations.RotateLeft64(Abi, 62); + Ago := Ago xor LDo; + BCe := TBitOperations.RotateLeft64(Ago, 55); + Aku := Aku xor LDu; + BCi := TBitOperations.RotateLeft64(Aku, 39); + Ama := Ama xor LDa; + BCo := TBitOperations.RotateLeft64(Ama, 41); + Ase := Ase xor LDe; + BCu := TBitOperations.RotateLeft64(Ase, 2); + Esa := BCa xor ((not BCe) and BCi); + Ese := BCe xor ((not BCi) and BCo); + Esi := BCi xor ((not BCo) and BCu); + Eso := BCo xor ((not BCu) and BCa); + Esu := BCu xor ((not BCa) and BCe); + + BCa := Eba xor Ega xor Eka xor Ema xor Esa; + BCe := Ebe xor Ege xor Eke xor Eme xor Ese; + BCi := Ebi xor Egi xor Eki xor Emi xor Esi; + BCo := Ebo xor Ego xor Eko xor Emo xor Eso; + BCu := Ebu xor Egu xor Eku xor Emu xor Esu; + + LDa := BCu xor TBitOperations.RotateLeft64(BCe, 1); + LDe := BCa xor TBitOperations.RotateLeft64(BCi, 1); + LDi := BCe xor TBitOperations.RotateLeft64(BCo, 1); + LDo := BCi xor TBitOperations.RotateLeft64(BCu, 1); + LDu := BCo xor TBitOperations.RotateLeft64(BCa, 1); + + Eba := Eba xor LDa; + BCa := Eba; + Ege := Ege xor LDe; + BCe := TBitOperations.RotateLeft64(Ege, 44); + Eki := Eki xor LDi; + BCi := TBitOperations.RotateLeft64(Eki, 43); + Emo := Emo xor LDo; + BCo := TBitOperations.RotateLeft64(Emo, 21); + Esu := Esu xor LDu; + BCu := TBitOperations.RotateLeft64(Esu, 14); + Aba := BCa xor ((not BCe) and BCi); + Aba := Aba xor UInt64(RC[LRound + 1]); + Abe := BCe xor ((not BCi) and BCo); + Abi := BCi xor ((not BCo) and BCu); + Abo := BCo xor ((not BCu) and BCa); + Abu := BCu xor ((not BCa) and BCe); + + Ebo := Ebo xor LDo; + BCa := TBitOperations.RotateLeft64(Ebo, 28); + Egu := Egu xor LDu; + BCe := TBitOperations.RotateLeft64(Egu, 20); + Eka := Eka xor LDa; + BCi := TBitOperations.RotateLeft64(Eka, 3); + Eme := Eme xor LDe; + BCo := TBitOperations.RotateLeft64(Eme, 45); + Esi := Esi xor LDi; + BCu := TBitOperations.RotateLeft64(Esi, 61); + Aga := BCa xor ((not BCe) and BCi); + Age := BCe xor ((not BCi) and BCo); + Agi := BCi xor ((not BCo) and BCu); + Ago := BCo xor ((not BCu) and BCa); + Agu := BCu xor ((not BCa) and BCe); + + Ebe := Ebe xor LDe; + BCa := TBitOperations.RotateLeft64(Ebe, 1); + Egi := Egi xor LDi; + BCe := TBitOperations.RotateLeft64(Egi, 6); + Eko := Eko xor LDo; + BCi := TBitOperations.RotateLeft64(Eko, 25); + Emu := Emu xor LDu; + BCo := TBitOperations.RotateLeft64(Emu, 8); + Esa := Esa xor LDa; + BCu := TBitOperations.RotateLeft64(Esa, 18); + Aka := BCa xor ((not BCe) and BCi); + Ake := BCe xor ((not BCi) and BCo); + Aki := BCi xor ((not BCo) and BCu); + Ako := BCo xor ((not BCu) and BCa); + Aku := BCu xor ((not BCa) and BCe); + + Ebu := Ebu xor LDu; + BCa := TBitOperations.RotateLeft64(Ebu, 27); + Ega := Ega xor LDa; + BCe := TBitOperations.RotateLeft64(Ega, 36); + Eke := Eke xor LDe; + BCi := TBitOperations.RotateLeft64(Eke, 10); + Emi := Emi xor LDi; + BCo := TBitOperations.RotateLeft64(Emi, 15); + Eso := Eso xor LDo; + BCu := TBitOperations.RotateLeft64(Eso, 56); + Ama := BCa xor ((not BCe) and BCi); + Ame := BCe xor ((not BCi) and BCo); + Ami := BCi xor ((not BCo) and BCu); + Amo := BCo xor ((not BCu) and BCa); + Amu := BCu xor ((not BCa) and BCe); + + Ebi := Ebi xor LDi; + BCa := TBitOperations.RotateLeft64(Ebi, 62); + Ego := Ego xor LDo; + BCe := TBitOperations.RotateLeft64(Ego, 55); + Eku := Eku xor LDu; + BCi := TBitOperations.RotateLeft64(Eku, 39); + Ema := Ema xor LDa; + BCo := TBitOperations.RotateLeft64(Ema, 41); + Ese := Ese xor LDe; + BCu := TBitOperations.RotateLeft64(Ese, 2); + Asa := BCa xor ((not BCe) and BCi); + Ase := BCe xor ((not BCi) and BCo); + Asi := BCi xor ((not BCo) and BCu); + Aso := BCo xor ((not BCu) and BCa); + Asu := BCu xor ((not BCa) and BCe); + + System.Inc(LRound, 2); + end; + + LPState[0] := Aba; + LPState[1] := Abe; + LPState[2] := Abi; + LPState[3] := Abo; + LPState[4] := Abu; + LPState[5] := Aga; + LPState[6] := Age; + LPState[7] := Agi; + LPState[8] := Ago; + LPState[9] := Agu; + LPState[10] := Aka; + LPState[11] := Ake; + LPState[12] := Aki; + LPState[13] := Ako; + LPState[14] := Aku; + LPState[15] := Ama; + LPState[16] := Ame; + LPState[17] := Ami; + LPState[18] := Amo; + LPState[19] := Amu; + LPState[20] := Asa; + LPState[21] := Ase; + LPState[22] := Asi; + LPState[23] := Aso; + LPState[24] := Asu; + +{$ELSE} + for LRound := 0 to 23 do + begin + LColA := LPState[00] xor LPState[05] xor LPState[10] xor LPState[15] + xor LPState[20]; + LColE := LPState[01] xor LPState[06] xor LPState[11] xor LPState[16] + xor LPState[21]; + LColI := LPState[02] xor LPState[07] xor LPState[12] xor LPState[17] + xor LPState[22]; + LColO := LPState[03] xor LPState[08] xor LPState[13] xor LPState[18] + xor LPState[23]; + LColU := LPState[04] xor LPState[09] xor LPState[14] xor LPState[19] + xor LPState[24]; + LDa := TBitOperations.RotateLeft64(LColA, 1) xor LColO; + LDe := TBitOperations.RotateLeft64(LColE, 1) xor LColU; + LDi := TBitOperations.RotateLeft64(LColI, 1) xor LColA; + LDo := TBitOperations.RotateLeft64(LColO, 1) xor LColE; + LDu := TBitOperations.RotateLeft64(LColU, 1) xor LColI; + LTemp[00] := LPState[00] xor LDe; + LTemp[01] := TBitOperations.RotateLeft64(LPState[06] xor LDi, 44); + LTemp[02] := TBitOperations.RotateLeft64(LPState[12] xor LDo, 43); + LTemp[03] := TBitOperations.RotateLeft64(LPState[18] xor LDu, 21); + LTemp[04] := TBitOperations.RotateLeft64(LPState[24] xor LDa, 14); + LTemp[05] := TBitOperations.RotateLeft64(LPState[03] xor LDu, 28); + LTemp[06] := TBitOperations.RotateLeft64(LPState[09] xor LDa, 20); + LTemp[07] := TBitOperations.RotateLeft64(LPState[10] xor LDe, 3); + LTemp[08] := TBitOperations.RotateLeft64(LPState[16] xor LDi, 45); + LTemp[09] := TBitOperations.RotateLeft64(LPState[22] xor LDo, 61); + LTemp[10] := TBitOperations.RotateLeft64(LPState[01] xor LDi, 1); + LTemp[11] := TBitOperations.RotateLeft64(LPState[07] xor LDo, 6); + LTemp[12] := TBitOperations.RotateLeft64(LPState[13] xor LDu, 25); + LTemp[13] := TBitOperations.RotateLeft64(LPState[19] xor LDa, 8); + LTemp[14] := TBitOperations.RotateLeft64(LPState[20] xor LDe, 18); + LTemp[15] := TBitOperations.RotateLeft64(LPState[04] xor LDa, 27); + LTemp[16] := TBitOperations.RotateLeft64(LPState[05] xor LDe, 36); + LTemp[17] := TBitOperations.RotateLeft64(LPState[11] xor LDi, 10); + LTemp[18] := TBitOperations.RotateLeft64(LPState[17] xor LDo, 15); + LTemp[19] := TBitOperations.RotateLeft64(LPState[23] xor LDu, 56); + LTemp[20] := TBitOperations.RotateLeft64(LPState[02] xor LDo, 62); + LTemp[21] := TBitOperations.RotateLeft64(LPState[08] xor LDu, 55); + LTemp[22] := TBitOperations.RotateLeft64(LPState[14] xor LDa, 39); + LTemp[23] := TBitOperations.RotateLeft64(LPState[15] xor LDe, 41); + LTemp[24] := TBitOperations.RotateLeft64(LPState[21] xor LDi, 2); + LPState[00] := LTemp[00] xor ((not LTemp[01]) and LTemp[02]); + LPState[01] := LTemp[01] xor ((not LTemp[02]) and LTemp[03]); + LPState[02] := LTemp[02] xor ((not LTemp[03]) and LTemp[04]); + LPState[03] := LTemp[03] xor ((not LTemp[04]) and LTemp[00]); + LPState[04] := LTemp[04] xor ((not LTemp[00]) and LTemp[01]); + LPState[05] := LTemp[05] xor ((not LTemp[06]) and LTemp[07]); + LPState[06] := LTemp[06] xor ((not LTemp[07]) and LTemp[08]); + LPState[07] := LTemp[07] xor ((not LTemp[08]) and LTemp[09]); + LPState[08] := LTemp[08] xor ((not LTemp[09]) and LTemp[05]); + LPState[09] := LTemp[09] xor ((not LTemp[05]) and LTemp[06]); + LPState[10] := LTemp[10] xor ((not LTemp[11]) and LTemp[12]); + LPState[11] := LTemp[11] xor ((not LTemp[12]) and LTemp[13]); + LPState[12] := LTemp[12] xor ((not LTemp[13]) and LTemp[14]); + LPState[13] := LTemp[13] xor ((not LTemp[14]) and LTemp[10]); + LPState[14] := LTemp[14] xor ((not LTemp[10]) and LTemp[11]); + LPState[15] := LTemp[15] xor ((not LTemp[16]) and LTemp[17]); + LPState[16] := LTemp[16] xor ((not LTemp[17]) and LTemp[18]); + LPState[17] := LTemp[17] xor ((not LTemp[18]) and LTemp[19]); + LPState[18] := LTemp[18] xor ((not LTemp[19]) and LTemp[15]); + LPState[19] := LTemp[19] xor ((not LTemp[15]) and LTemp[16]); + LPState[20] := LTemp[20] xor ((not LTemp[21]) and LTemp[22]); + LPState[21] := LTemp[21] xor ((not LTemp[22]) and LTemp[23]); + LPState[22] := LTemp[22] xor ((not LTemp[23]) and LTemp[24]); + LPState[23] := LTemp[23] xor ((not LTemp[24]) and LTemp[20]); + LPState[24] := LTemp[24] xor ((not LTemp[20]) and LTemp[21]); + LPState[00] := LPState[00] xor RC[LRound]; + end; + + System.FillChar(LTemp, System.SizeOf(LTemp), UInt64(0)); +{$ENDIF USE_UNROLLED_VARIANT} +end; + +procedure KeccakF1600_Absorb_Scalar(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); +var + LPState: PUInt64; + LData: array [0 .. 20] of UInt64; + LBlockSizeWords, I, J: Int32; +begin + LPState := PUInt64(AState); + LBlockSizeWords := ABlockSize shr 3; + for I := 0 to ABlockCount - 1 do + begin + TBinaryPrimitives.CopyUInt64LittleEndian(AData, 0, @LData[0], 0, ABlockSize); + for J := 0 to LBlockSizeWords - 1 do + LPState[J] := LPState[J] xor LData[J]; + KeccakF1600_Scalar(AState); + System.Inc(AData, ABlockSize); + end; + System.FillChar(LData, System.SizeOf(LData), UInt64(0)); +end; + type // Self-contained SHAKE/Keccak squeeze engine. Owns a copy of the finalized // Keccak state and produces an unbounded byte stream; the size cap (if any) @@ -1496,4 +1887,8 @@ function TKeccak.GetHashMode(): TSHA3.THashMode; Result := TSHA3.THashMode.Keccak; end; +initialization + KeccakF1600_Permute := TSHA3Simd.SelectPermute(@KeccakF1600_Scalar); + KeccakF1600_Absorb := TSHA3Simd.SelectAbsorb(@KeccakF1600_Absorb_Scalar); + end. diff --git a/HashLib/src/Crypto/HlpSHA3Dispatch.pas b/HashLib/src/Crypto/HlpSHA3Dispatch.pas deleted file mode 100644 index 6381e481..00000000 --- a/HashLib/src/Crypto/HlpSHA3Dispatch.pas +++ /dev/null @@ -1,554 +0,0 @@ -unit HlpSHA3Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TKeccakF1600Proc = procedure(AState: Pointer); - TKeccakF1600AbsorbProc = procedure(AState: Pointer; AData: PByte; - ABlockCount: Int32; ABlockSize: Int32); - -var - KeccakF1600_Permute: TKeccakF1600Proc; - KeccakF1600_Absorb: TKeccakF1600AbsorbProc; - -implementation - -uses - HlpBinaryPrimitives, - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -// ============================================================================= -// Round constants -// ============================================================================= - -const - RC: array [0 .. 23] of UInt64 = ( - UInt64($0000000000000001), UInt64($0000000000008082), - UInt64($800000000000808A), UInt64($8000000080008000), - UInt64($000000000000808B), UInt64($0000000080000001), - UInt64($8000000080008081), UInt64($8000000000008009), - UInt64($000000000000008A), UInt64($0000000000000088), - UInt64($0000000080008009), UInt64($000000008000000A), - UInt64($000000008000808B), UInt64($800000000000008B), - UInt64($8000000000008089), UInt64($8000000000008003), - UInt64($8000000000008002), UInt64($8000000000000080), - UInt64($000000000000800A), UInt64($800000008000000A), - UInt64($8000000080008081), UInt64($8000000000008080), - UInt64($0000000080000001), UInt64($8000000080008008)); - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure KeccakF1600_Scalar(AState: Pointer); -var - LPState: PUInt64; - LDa, LDe, LDi, LDo, LDu: UInt64; -{$IFDEF USE_UNROLLED_VARIANT} - Aba, Abe, Abi, Abo, Abu, Aga, Age, Agi, Ago, Agu, Aka, Ake, Aki, Ako, Aku, - Ama, Ame, Ami, Amo, Amu, Asa, Ase, Asi, Aso, Asu, BCa, BCe, BCi, BCo, BCu, - Eba, Ebe, Ebi, Ebo, Ebu, Ega, Ege, Egi, Ego, Egu, Eka, Eke, Eki, Eko, Eku, - Ema, Eme, Emi, Emo, Emu, Esa, Ese, Esi, Eso, Esu: UInt64; - LRound: Int32; -{$ELSE} - LColA, LColE, LColI, LColO, LColU: UInt64; - LTemp: array [0 .. 24] of UInt64; - LRound: Int32; -{$ENDIF USE_UNROLLED_VARIANT} -begin - LPState := PUInt64(AState); - -{$IFDEF USE_UNROLLED_VARIANT} - Aba := LPState[0]; - Abe := LPState[1]; - Abi := LPState[2]; - Abo := LPState[3]; - Abu := LPState[4]; - Aga := LPState[5]; - Age := LPState[6]; - Agi := LPState[7]; - Ago := LPState[8]; - Agu := LPState[9]; - Aka := LPState[10]; - Ake := LPState[11]; - Aki := LPState[12]; - Ako := LPState[13]; - Aku := LPState[14]; - Ama := LPState[15]; - Ame := LPState[16]; - Ami := LPState[17]; - Amo := LPState[18]; - Amu := LPState[19]; - Asa := LPState[20]; - Ase := LPState[21]; - Asi := LPState[22]; - Aso := LPState[23]; - Asu := LPState[24]; - - LRound := 0; - while LRound < 24 do - begin - BCa := Aba xor Aga xor Aka xor Ama xor Asa; - BCe := Abe xor Age xor Ake xor Ame xor Ase; - BCi := Abi xor Agi xor Aki xor Ami xor Asi; - BCo := Abo xor Ago xor Ako xor Amo xor Aso; - BCu := Abu xor Agu xor Aku xor Amu xor Asu; - - LDa := BCu xor TBitOperations.RotateLeft64(BCe, 1); - LDe := BCa xor TBitOperations.RotateLeft64(BCi, 1); - LDi := BCe xor TBitOperations.RotateLeft64(BCo, 1); - LDo := BCi xor TBitOperations.RotateLeft64(BCu, 1); - LDu := BCo xor TBitOperations.RotateLeft64(BCa, 1); - - Aba := Aba xor LDa; - BCa := Aba; - Age := Age xor LDe; - BCe := TBitOperations.RotateLeft64(Age, 44); - Aki := Aki xor LDi; - BCi := TBitOperations.RotateLeft64(Aki, 43); - Amo := Amo xor LDo; - BCo := TBitOperations.RotateLeft64(Amo, 21); - Asu := Asu xor LDu; - BCu := TBitOperations.RotateLeft64(Asu, 14); - Eba := BCa xor ((not BCe) and BCi); - Eba := Eba xor UInt64(RC[LRound]); - Ebe := BCe xor ((not BCi) and BCo); - Ebi := BCi xor ((not BCo) and BCu); - Ebo := BCo xor ((not BCu) and BCa); - Ebu := BCu xor ((not BCa) and BCe); - - Abo := Abo xor LDo; - BCa := TBitOperations.RotateLeft64(Abo, 28); - Agu := Agu xor LDu; - BCe := TBitOperations.RotateLeft64(Agu, 20); - Aka := Aka xor LDa; - BCi := TBitOperations.RotateLeft64(Aka, 3); - Ame := Ame xor LDe; - BCo := TBitOperations.RotateLeft64(Ame, 45); - Asi := Asi xor LDi; - BCu := TBitOperations.RotateLeft64(Asi, 61); - Ega := BCa xor ((not BCe) and BCi); - Ege := BCe xor ((not BCi) and BCo); - Egi := BCi xor ((not BCo) and BCu); - Ego := BCo xor ((not BCu) and BCa); - Egu := BCu xor ((not BCa) and BCe); - - Abe := Abe xor LDe; - BCa := TBitOperations.RotateLeft64(Abe, 1); - Agi := Agi xor LDi; - BCe := TBitOperations.RotateLeft64(Agi, 6); - Ako := Ako xor LDo; - BCi := TBitOperations.RotateLeft64(Ako, 25); - Amu := Amu xor LDu; - BCo := TBitOperations.RotateLeft64(Amu, 8); - Asa := Asa xor LDa; - BCu := TBitOperations.RotateLeft64(Asa, 18); - Eka := BCa xor ((not BCe) and BCi); - Eke := BCe xor ((not BCi) and BCo); - Eki := BCi xor ((not BCo) and BCu); - Eko := BCo xor ((not BCu) and BCa); - Eku := BCu xor ((not BCa) and BCe); - - Abu := Abu xor LDu; - BCa := TBitOperations.RotateLeft64(Abu, 27); - Aga := Aga xor LDa; - BCe := TBitOperations.RotateLeft64(Aga, 36); - Ake := Ake xor LDe; - BCi := TBitOperations.RotateLeft64(Ake, 10); - Ami := Ami xor LDi; - BCo := TBitOperations.RotateLeft64(Ami, 15); - Aso := Aso xor LDo; - BCu := TBitOperations.RotateLeft64(Aso, 56); - Ema := BCa xor ((not BCe) and BCi); - Eme := BCe xor ((not BCi) and BCo); - Emi := BCi xor ((not BCo) and BCu); - Emo := BCo xor ((not BCu) and BCa); - Emu := BCu xor ((not BCa) and BCe); - - Abi := Abi xor LDi; - BCa := TBitOperations.RotateLeft64(Abi, 62); - Ago := Ago xor LDo; - BCe := TBitOperations.RotateLeft64(Ago, 55); - Aku := Aku xor LDu; - BCi := TBitOperations.RotateLeft64(Aku, 39); - Ama := Ama xor LDa; - BCo := TBitOperations.RotateLeft64(Ama, 41); - Ase := Ase xor LDe; - BCu := TBitOperations.RotateLeft64(Ase, 2); - Esa := BCa xor ((not BCe) and BCi); - Ese := BCe xor ((not BCi) and BCo); - Esi := BCi xor ((not BCo) and BCu); - Eso := BCo xor ((not BCu) and BCa); - Esu := BCu xor ((not BCa) and BCe); - - BCa := Eba xor Ega xor Eka xor Ema xor Esa; - BCe := Ebe xor Ege xor Eke xor Eme xor Ese; - BCi := Ebi xor Egi xor Eki xor Emi xor Esi; - BCo := Ebo xor Ego xor Eko xor Emo xor Eso; - BCu := Ebu xor Egu xor Eku xor Emu xor Esu; - - LDa := BCu xor TBitOperations.RotateLeft64(BCe, 1); - LDe := BCa xor TBitOperations.RotateLeft64(BCi, 1); - LDi := BCe xor TBitOperations.RotateLeft64(BCo, 1); - LDo := BCi xor TBitOperations.RotateLeft64(BCu, 1); - LDu := BCo xor TBitOperations.RotateLeft64(BCa, 1); - - Eba := Eba xor LDa; - BCa := Eba; - Ege := Ege xor LDe; - BCe := TBitOperations.RotateLeft64(Ege, 44); - Eki := Eki xor LDi; - BCi := TBitOperations.RotateLeft64(Eki, 43); - Emo := Emo xor LDo; - BCo := TBitOperations.RotateLeft64(Emo, 21); - Esu := Esu xor LDu; - BCu := TBitOperations.RotateLeft64(Esu, 14); - Aba := BCa xor ((not BCe) and BCi); - Aba := Aba xor UInt64(RC[LRound + 1]); - Abe := BCe xor ((not BCi) and BCo); - Abi := BCi xor ((not BCo) and BCu); - Abo := BCo xor ((not BCu) and BCa); - Abu := BCu xor ((not BCa) and BCe); - - Ebo := Ebo xor LDo; - BCa := TBitOperations.RotateLeft64(Ebo, 28); - Egu := Egu xor LDu; - BCe := TBitOperations.RotateLeft64(Egu, 20); - Eka := Eka xor LDa; - BCi := TBitOperations.RotateLeft64(Eka, 3); - Eme := Eme xor LDe; - BCo := TBitOperations.RotateLeft64(Eme, 45); - Esi := Esi xor LDi; - BCu := TBitOperations.RotateLeft64(Esi, 61); - Aga := BCa xor ((not BCe) and BCi); - Age := BCe xor ((not BCi) and BCo); - Agi := BCi xor ((not BCo) and BCu); - Ago := BCo xor ((not BCu) and BCa); - Agu := BCu xor ((not BCa) and BCe); - - Ebe := Ebe xor LDe; - BCa := TBitOperations.RotateLeft64(Ebe, 1); - Egi := Egi xor LDi; - BCe := TBitOperations.RotateLeft64(Egi, 6); - Eko := Eko xor LDo; - BCi := TBitOperations.RotateLeft64(Eko, 25); - Emu := Emu xor LDu; - BCo := TBitOperations.RotateLeft64(Emu, 8); - Esa := Esa xor LDa; - BCu := TBitOperations.RotateLeft64(Esa, 18); - Aka := BCa xor ((not BCe) and BCi); - Ake := BCe xor ((not BCi) and BCo); - Aki := BCi xor ((not BCo) and BCu); - Ako := BCo xor ((not BCu) and BCa); - Aku := BCu xor ((not BCa) and BCe); - - Ebu := Ebu xor LDu; - BCa := TBitOperations.RotateLeft64(Ebu, 27); - Ega := Ega xor LDa; - BCe := TBitOperations.RotateLeft64(Ega, 36); - Eke := Eke xor LDe; - BCi := TBitOperations.RotateLeft64(Eke, 10); - Emi := Emi xor LDi; - BCo := TBitOperations.RotateLeft64(Emi, 15); - Eso := Eso xor LDo; - BCu := TBitOperations.RotateLeft64(Eso, 56); - Ama := BCa xor ((not BCe) and BCi); - Ame := BCe xor ((not BCi) and BCo); - Ami := BCi xor ((not BCo) and BCu); - Amo := BCo xor ((not BCu) and BCa); - Amu := BCu xor ((not BCa) and BCe); - - Ebi := Ebi xor LDi; - BCa := TBitOperations.RotateLeft64(Ebi, 62); - Ego := Ego xor LDo; - BCe := TBitOperations.RotateLeft64(Ego, 55); - Eku := Eku xor LDu; - BCi := TBitOperations.RotateLeft64(Eku, 39); - Ema := Ema xor LDa; - BCo := TBitOperations.RotateLeft64(Ema, 41); - Ese := Ese xor LDe; - BCu := TBitOperations.RotateLeft64(Ese, 2); - Asa := BCa xor ((not BCe) and BCi); - Ase := BCe xor ((not BCi) and BCo); - Asi := BCi xor ((not BCo) and BCu); - Aso := BCo xor ((not BCu) and BCa); - Asu := BCu xor ((not BCa) and BCe); - - System.Inc(LRound, 2); - end; - - LPState[0] := Aba; - LPState[1] := Abe; - LPState[2] := Abi; - LPState[3] := Abo; - LPState[4] := Abu; - LPState[5] := Aga; - LPState[6] := Age; - LPState[7] := Agi; - LPState[8] := Ago; - LPState[9] := Agu; - LPState[10] := Aka; - LPState[11] := Ake; - LPState[12] := Aki; - LPState[13] := Ako; - LPState[14] := Aku; - LPState[15] := Ama; - LPState[16] := Ame; - LPState[17] := Ami; - LPState[18] := Amo; - LPState[19] := Amu; - LPState[20] := Asa; - LPState[21] := Ase; - LPState[22] := Asi; - LPState[23] := Aso; - LPState[24] := Asu; - -{$ELSE} - for LRound := 0 to 23 do - begin - LColA := LPState[00] xor LPState[05] xor LPState[10] xor LPState[15] - xor LPState[20]; - LColE := LPState[01] xor LPState[06] xor LPState[11] xor LPState[16] - xor LPState[21]; - LColI := LPState[02] xor LPState[07] xor LPState[12] xor LPState[17] - xor LPState[22]; - LColO := LPState[03] xor LPState[08] xor LPState[13] xor LPState[18] - xor LPState[23]; - LColU := LPState[04] xor LPState[09] xor LPState[14] xor LPState[19] - xor LPState[24]; - LDa := TBitOperations.RotateLeft64(LColA, 1) xor LColO; - LDe := TBitOperations.RotateLeft64(LColE, 1) xor LColU; - LDi := TBitOperations.RotateLeft64(LColI, 1) xor LColA; - LDo := TBitOperations.RotateLeft64(LColO, 1) xor LColE; - LDu := TBitOperations.RotateLeft64(LColU, 1) xor LColI; - LTemp[00] := LPState[00] xor LDe; - LTemp[01] := TBitOperations.RotateLeft64(LPState[06] xor LDi, 44); - LTemp[02] := TBitOperations.RotateLeft64(LPState[12] xor LDo, 43); - LTemp[03] := TBitOperations.RotateLeft64(LPState[18] xor LDu, 21); - LTemp[04] := TBitOperations.RotateLeft64(LPState[24] xor LDa, 14); - LTemp[05] := TBitOperations.RotateLeft64(LPState[03] xor LDu, 28); - LTemp[06] := TBitOperations.RotateLeft64(LPState[09] xor LDa, 20); - LTemp[07] := TBitOperations.RotateLeft64(LPState[10] xor LDe, 3); - LTemp[08] := TBitOperations.RotateLeft64(LPState[16] xor LDi, 45); - LTemp[09] := TBitOperations.RotateLeft64(LPState[22] xor LDo, 61); - LTemp[10] := TBitOperations.RotateLeft64(LPState[01] xor LDi, 1); - LTemp[11] := TBitOperations.RotateLeft64(LPState[07] xor LDo, 6); - LTemp[12] := TBitOperations.RotateLeft64(LPState[13] xor LDu, 25); - LTemp[13] := TBitOperations.RotateLeft64(LPState[19] xor LDa, 8); - LTemp[14] := TBitOperations.RotateLeft64(LPState[20] xor LDe, 18); - LTemp[15] := TBitOperations.RotateLeft64(LPState[04] xor LDa, 27); - LTemp[16] := TBitOperations.RotateLeft64(LPState[05] xor LDe, 36); - LTemp[17] := TBitOperations.RotateLeft64(LPState[11] xor LDi, 10); - LTemp[18] := TBitOperations.RotateLeft64(LPState[17] xor LDo, 15); - LTemp[19] := TBitOperations.RotateLeft64(LPState[23] xor LDu, 56); - LTemp[20] := TBitOperations.RotateLeft64(LPState[02] xor LDo, 62); - LTemp[21] := TBitOperations.RotateLeft64(LPState[08] xor LDu, 55); - LTemp[22] := TBitOperations.RotateLeft64(LPState[14] xor LDa, 39); - LTemp[23] := TBitOperations.RotateLeft64(LPState[15] xor LDe, 41); - LTemp[24] := TBitOperations.RotateLeft64(LPState[21] xor LDi, 2); - LPState[00] := LTemp[00] xor ((not LTemp[01]) and LTemp[02]); - LPState[01] := LTemp[01] xor ((not LTemp[02]) and LTemp[03]); - LPState[02] := LTemp[02] xor ((not LTemp[03]) and LTemp[04]); - LPState[03] := LTemp[03] xor ((not LTemp[04]) and LTemp[00]); - LPState[04] := LTemp[04] xor ((not LTemp[00]) and LTemp[01]); - LPState[05] := LTemp[05] xor ((not LTemp[06]) and LTemp[07]); - LPState[06] := LTemp[06] xor ((not LTemp[07]) and LTemp[08]); - LPState[07] := LTemp[07] xor ((not LTemp[08]) and LTemp[09]); - LPState[08] := LTemp[08] xor ((not LTemp[09]) and LTemp[05]); - LPState[09] := LTemp[09] xor ((not LTemp[05]) and LTemp[06]); - LPState[10] := LTemp[10] xor ((not LTemp[11]) and LTemp[12]); - LPState[11] := LTemp[11] xor ((not LTemp[12]) and LTemp[13]); - LPState[12] := LTemp[12] xor ((not LTemp[13]) and LTemp[14]); - LPState[13] := LTemp[13] xor ((not LTemp[14]) and LTemp[10]); - LPState[14] := LTemp[14] xor ((not LTemp[10]) and LTemp[11]); - LPState[15] := LTemp[15] xor ((not LTemp[16]) and LTemp[17]); - LPState[16] := LTemp[16] xor ((not LTemp[17]) and LTemp[18]); - LPState[17] := LTemp[17] xor ((not LTemp[18]) and LTemp[19]); - LPState[18] := LTemp[18] xor ((not LTemp[19]) and LTemp[15]); - LPState[19] := LTemp[19] xor ((not LTemp[15]) and LTemp[16]); - LPState[20] := LTemp[20] xor ((not LTemp[21]) and LTemp[22]); - LPState[21] := LTemp[21] xor ((not LTemp[22]) and LTemp[23]); - LPState[22] := LTemp[22] xor ((not LTemp[23]) and LTemp[24]); - LPState[23] := LTemp[23] xor ((not LTemp[24]) and LTemp[20]); - LPState[24] := LTemp[24] xor ((not LTemp[20]) and LTemp[21]); - LPState[00] := LPState[00] xor RC[LRound]; - end; - - System.FillChar(LTemp, System.SizeOf(LTemp), UInt64(0)); -{$ENDIF USE_UNROLLED_VARIANT} -end; - -// ============================================================================= -// Scalar absorb: XOR + permute loop (no SIMD) -// ============================================================================= - -procedure KeccakF1600_Absorb_Scalar(AState: Pointer; AData: PByte; - ABlockCount: Int32; ABlockSize: Int32); -var - LPState: PUInt64; - LData: array [0 .. 20] of UInt64; - LBlockSizeWords, I, J: Int32; -begin - LPState := PUInt64(AState); - LBlockSizeWords := ABlockSize shr 3; - for I := 0 to ABlockCount - 1 do - begin - TBinaryPrimitives.CopyUInt64LittleEndian(AData, 0, @LData[0], 0, ABlockSize); - for J := 0 to LBlockSizeWords - 1 do - LPState[J] := LPState[J] xor LData[J]; - KeccakF1600_Scalar(AState); - System.Inc(AData, ABlockSize); - end; - System.FillChar(LData, System.SizeOf(LData), UInt64(0)); -end; - -// ============================================================================= -// SIMD implementations -// -// i386: — -// x86_64: AVX2 -// aarch64: SHA3 Crypto Extensions -// ============================================================================= - -{$IFDEF HASHLIB_X86_64_ASM} - -const - K_KECCAK: packed record - RhotatesLeft: array [0..23] of UInt64; - RhotatesRight: array [0..23] of UInt64; - Iotas: array [0..95] of UInt64; - Jagged: array [0..24] of Int32; - end = ( - RhotatesLeft: ( - 3, 18, 36, 41, // ymm2: [2][0] [4][0] [1][0] [3][0] - 1, 62, 28, 27, // ymm1: [0][1] [0][2] [0][3] [0][4] - 45, 6, 56, 39, // ymm3: [3][1] [1][2] [4][3] [2][4] - 10, 61, 55, 8, // ymm4: [2][1] [4][2] [1][3] [3][4] - 2, 15, 25, 20, // ymm5: [4][1] [3][2] [2][3] [1][4] - 44, 43, 21, 14); // ymm6: [1][1] [2][2] [3][3] [4][4] - RhotatesRight: ( - 61, 46, 28, 23, - 63, 2, 36, 37, - 19, 58, 8, 25, - 54, 3, 9, 56, - 62, 49, 39, 44, - 20, 21, 43, 50); - Iotas: ( - UInt64($0000000000000001), UInt64($0000000000000001), UInt64($0000000000000001), UInt64($0000000000000001), - UInt64($0000000000008082), UInt64($0000000000008082), UInt64($0000000000008082), UInt64($0000000000008082), - UInt64($800000000000808A), UInt64($800000000000808A), UInt64($800000000000808A), UInt64($800000000000808A), - UInt64($8000000080008000), UInt64($8000000080008000), UInt64($8000000080008000), UInt64($8000000080008000), - UInt64($000000000000808B), UInt64($000000000000808B), UInt64($000000000000808B), UInt64($000000000000808B), - UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), - UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), - UInt64($8000000000008009), UInt64($8000000000008009), UInt64($8000000000008009), UInt64($8000000000008009), - UInt64($000000000000008A), UInt64($000000000000008A), UInt64($000000000000008A), UInt64($000000000000008A), - UInt64($0000000000000088), UInt64($0000000000000088), UInt64($0000000000000088), UInt64($0000000000000088), - UInt64($0000000080008009), UInt64($0000000080008009), UInt64($0000000080008009), UInt64($0000000080008009), - UInt64($000000008000000A), UInt64($000000008000000A), UInt64($000000008000000A), UInt64($000000008000000A), - UInt64($000000008000808B), UInt64($000000008000808B), UInt64($000000008000808B), UInt64($000000008000808B), - UInt64($800000000000008B), UInt64($800000000000008B), UInt64($800000000000008B), UInt64($800000000000008B), - UInt64($8000000000008089), UInt64($8000000000008089), UInt64($8000000000008089), UInt64($8000000000008089), - UInt64($8000000000008003), UInt64($8000000000008003), UInt64($8000000000008003), UInt64($8000000000008003), - UInt64($8000000000008002), UInt64($8000000000008002), UInt64($8000000000008002), UInt64($8000000000008002), - UInt64($8000000000000080), UInt64($8000000000000080), UInt64($8000000000000080), UInt64($8000000000000080), - UInt64($000000000000800A), UInt64($000000000000800A), UInt64($000000000000800A), UInt64($000000000000800A), - UInt64($800000008000000A), UInt64($800000008000000A), UInt64($800000008000000A), UInt64($800000008000000A), - UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), - UInt64($8000000000008080), UInt64($8000000000008080), UInt64($8000000000008080), UInt64($8000000000008080), - UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), - UInt64($8000000080008008), UInt64($8000000080008008), UInt64($8000000080008008), UInt64($8000000080008008)); - Jagged: ( - 0, 32, 40, 48, 56, 80, 192, 104, 144, 184, - 64, 128, 200, 176, 120, 88, 96, 168, 208, 152, - 72, 160, 136, 112, 216) - ); - -procedure KeccakF1600_Avx2(AState: Pointer; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} - {$I ..\Include\Simd\SHA3\KeccakF1600Avx2_x86_64.inc} -end; - -procedure KeccakF1600_Avx2_Wrap(AState: Pointer); -begin - KeccakF1600_Avx2(AState, @K_KECCAK); -end; - -procedure KeccakF1600_Avx2_Absorb(AState: Pointer; AData: PByte; - ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} - {$I ..\Include\Simd\SHA3\KeccakF1600Avx2Absorb_x86_64.inc} -end; - -procedure KeccakF1600_Avx2_Absorb_Wrap(AState: Pointer; AData: PByte; - ABlockCount: Int32; ABlockSize: Int32); -begin - KeccakF1600_Avx2_Absorb(AState, AData, ABlockCount, ABlockSize, @K_KECCAK); -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -// FEAT_SHA3 (ARMv8.2 crypto extension) kernels. Unlike the AVX2 path these -// reuse the plain RC round-constant table directly (the asm broadcasts each -// 64-bit iota with ld1r), so no separate packed constant block is needed. - -procedure KeccakF1600_CryptoExt(AState: Pointer; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} - {$I ..\Include\Simd\SHA3\KeccakF1600CryptoExt_aarch64.inc} -end; - -procedure KeccakF1600_CryptoExt_Wrap(AState: Pointer); -begin - KeccakF1600_CryptoExt(AState, @RC); -end; - -procedure KeccakF1600_CryptoExt_Absorb(AState: Pointer; AData: PByte; - ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc5Begin_aarch64.inc} - {$I ..\Include\Simd\SHA3\KeccakF1600CryptoExtAbsorb_aarch64.inc} -end; - -procedure KeccakF1600_CryptoExt_Absorb_Wrap(AState: Pointer; AData: PByte; - ABlockCount: Int32; ABlockSize: Int32); -begin - KeccakF1600_CryptoExt_Absorb(AState, AData, ABlockCount, ABlockSize, @RC); -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - KeccakF1600_Permute := @KeccakF1600_Scalar; - KeccakF1600_Absorb := @KeccakF1600_Absorb_Scalar; -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2]) of - TX86SimdLevel.AVX2: - begin - KeccakF1600_Permute := @KeccakF1600_Avx2_Wrap; - KeccakF1600_Absorb := @KeccakF1600_Avx2_Absorb_Wrap; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - if TCpuFeatures.Arm.HasSHA3() then - begin - KeccakF1600_Permute := @KeccakF1600_CryptoExt_Wrap; - KeccakF1600_Absorb := @KeccakF1600_CryptoExt_Absorb_Wrap; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Hash64/HlpXXHash3.pas b/HashLib/src/Hash64/HlpXXHash3.pas index 043a9197..b6ca698c 100644 --- a/HashLib/src/Hash64/HlpXXHash3.pas +++ b/HashLib/src/Hash64/HlpXXHash3.pas @@ -14,6 +14,26 @@ interface HlpIHashResult, HlpBitOperations; +type + TXXH3Accumulate512Proc = procedure(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer); + TXXH3AccumulateProc = procedure(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32); + TXXH3ScrambleAccProc = procedure(AAcc: Pointer; ASecret: Pointer); + TXXH3InitSecretProc = procedure(ACustomSecret: Pointer; + ADefaultSecret: Pointer; ASeedPtr: PUInt64); + +var + XXH3_Accumulate512: TXXH3Accumulate512Proc; + XXH3_Accumulate: TXXH3AccumulateProc; + XXH3_ScrambleAcc: TXXH3ScrambleAccProc; + XXH3_InitSecret: TXXH3InitSecretProc; + +// Generic stripe-accumulation loop, exposed so the SIMD backends can drive their +// own Accumulate512 kernel over ANbStripes stripes. +procedure XXH3_Accumulate_Loop(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32; AAcc512: TXXH3Accumulate512Proc); + resourcestring SInvalidKeyLength = 'KeyLength Must Be Equal to %d'; @@ -21,11 +41,8 @@ interface TXXH3AccArray = array [0 .. 7] of UInt64; TXXH3Core = class sealed(TObject) - public - const - XXH3_SECRET_DEFAULT_SIZE = Int32(192); XXH3_SECRET_SIZE_MIN = Int32(136); XXH_STRIPE_LEN = Int32(64); @@ -111,11 +128,9 @@ TXXH3Core = class sealed(TObject) end; TXXHash3 = class sealed(THash, IHash64, IHashWithKey, ITransformBlock) - strict private var FKey, FHash: UInt64; - const CKEY = UInt64(0); @@ -124,9 +139,7 @@ TXXHash3 = class sealed(THash, IHash64, IHashWithKey, ITransformBlock) procedure SetKey(const AValue: THashLibByteArray); inline; type - TXXH3_State = record - private var FAcc: TXXH3AccArray; @@ -180,7 +193,103 @@ TXXH3_State = record implementation uses - HlpXXHash3Dispatch; + HlpXXHash3Simd; + +const + XXH_STRIPE_LEN = 64; + XXH_ACC_NB = 8; + XXH_SECRET_CONSUME_RATE = 8; + XXH_PRIME32_1 = UInt32($9E3779B1); + +// ============================================================================= +// Scalar fallback implementations +// ============================================================================= + +procedure XXH3_Accumulate512_Scalar(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer); +var + LPAcc, LPInput, LPSecret: PByte; + LPLane: PUInt64; + I: Int32; + LDataVal, LDataKey: UInt64; +begin + LPAcc := PByte(AAcc); + LPInput := PByte(AInput); + LPSecret := PByte(ASecret); + for I := 0 to XXH_ACC_NB - 1 do + begin + LDataVal := TBinaryPrimitives.ReadUInt64LittleEndian(LPInput, I * 8); + LDataKey := LDataVal xor TBinaryPrimitives.ReadUInt64LittleEndian(LPSecret, I * 8); + LPLane := PUInt64(LPAcc + (I xor 1) * 8); + TBinaryPrimitives.StoreUInt64(LPLane, + TBinaryPrimitives.LoadUInt64(LPLane) + LDataVal); + LPLane := PUInt64(LPAcc + I * 8); + TBinaryPrimitives.StoreUInt64(LPLane, + TBinaryPrimitives.LoadUInt64(LPLane) + + UInt64(UInt32(LDataKey)) * UInt64(UInt32(LDataKey shr 32))); + end; +end; + +procedure XXH3_ScrambleAcc_Scalar(AAcc: Pointer; ASecret: Pointer); +var + LPAcc, LPSecret: PByte; + LPLane: PUInt64; + I: Int32; + LKey64, LAcc64: UInt64; +begin + LPAcc := PByte(AAcc); + LPSecret := PByte(ASecret); + for I := 0 to XXH_ACC_NB - 1 do + begin + LKey64 := TBinaryPrimitives.ReadUInt64LittleEndian(LPSecret, I * 8); + LPLane := PUInt64(LPAcc + I * 8); + LAcc64 := TBinaryPrimitives.LoadUInt64(LPLane); + LAcc64 := LAcc64 xor (LAcc64 shr 47); + LAcc64 := LAcc64 xor LKey64; + LAcc64 := LAcc64 * XXH_PRIME32_1; + TBinaryPrimitives.StoreUInt64(LPLane, LAcc64); + end; +end; + +procedure XXH3_InitSecret_Scalar(ACustomSecret: Pointer; + ADefaultSecret: Pointer; ASeedPtr: PUInt64); +var + I: Int32; + LPSrc, LPDst: PByte; + ASeed: UInt64; +begin + ASeed := ASeedPtr^; + LPSrc := PByte(ADefaultSecret); + LPDst := PByte(ACustomSecret); + + for I := 0 to (192 div 16) - 1 do + begin + TBinaryPrimitives.WriteUInt64LittleEndian(LPDst, 16 * I, + TBinaryPrimitives.ReadUInt64LittleEndian(LPSrc, 16 * I) + ASeed); + TBinaryPrimitives.WriteUInt64LittleEndian(LPDst, 16 * I + 8, + TBinaryPrimitives.ReadUInt64LittleEndian(LPSrc, 16 * I + 8) - ASeed); + end; +end; + +procedure XXH3_Accumulate_Scalar(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32); +var + N: Int32; +begin + for N := 0 to ANbStripes - 1 do + XXH3_Accumulate512_Scalar(AAcc, PByte(AInput) + N * XXH_STRIPE_LEN, + PByte(ASecret) + N * XXH_SECRET_CONSUME_RATE); +end; + +procedure XXH3_Accumulate_Loop(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32; AAcc512: TXXH3Accumulate512Proc); +var + N: Int32; +begin + for N := 0 to ANbStripes - 1 do + AAcc512(AAcc, PByte(AInput) + N * XXH_STRIPE_LEN, + PByte(ASecret) + N * XXH_SECRET_CONSUME_RATE); +end; { TXXH3Core } @@ -287,7 +396,7 @@ class procedure TXXH3Core.XXH3_scalarRound(var AAcc: TXXH3AccArray; class procedure TXXH3Core.XXH3_accumulate_512(var AAcc: TXXH3AccArray; AInput, ASecret: PByte); begin - HlpXXHash3Dispatch.XXH3_Accumulate512(@AAcc[0], AInput, ASecret); + HlpXXHash3.XXH3_Accumulate512(@AAcc[0], AInput, ASecret); end; class procedure TXXH3Core.XXH3_scalarScrambleRound(var AAcc: TXXH3AccArray; @@ -306,13 +415,13 @@ class procedure TXXH3Core.XXH3_scalarScrambleRound(var AAcc: TXXH3AccArray; class procedure TXXH3Core.XXH3_scrambleAcc(var AAcc: TXXH3AccArray; ASecret: PByte); begin - HlpXXHash3Dispatch.XXH3_ScrambleAcc(@AAcc[0], ASecret); + HlpXXHash3.XXH3_ScrambleAcc(@AAcc[0], ASecret); end; class procedure TXXH3Core.XXH3_accumulate(var AAcc: TXXH3AccArray; AInput, ASecret: PByte; ANbStripes: Int32); begin - HlpXXHash3Dispatch.XXH3_Accumulate(@AAcc[0], AInput, ASecret, ANbStripes); + HlpXXHash3.XXH3_Accumulate(@AAcc[0], AInput, ASecret, ANbStripes); end; class procedure TXXH3Core.XXH3_hashLong_internal_loop( @@ -345,7 +454,7 @@ class procedure TXXH3Core.XXH3_hashLong_internal_loop( class procedure TXXH3Core.XXH3_initCustomSecret(ACustomSecret: PByte; ASeed: UInt64); begin - HlpXXHash3Dispatch.XXH3_InitSecret(ACustomSecret, @XXH3_SECRET[0], ASeed); + HlpXXHash3.XXH3_InitSecret(ACustomSecret, @XXH3_SECRET[0], @ASeed); end; class procedure TXXH3Core.XXH3_consumeStripes(var AAcc: TXXH3AccArray; @@ -747,4 +856,10 @@ function TXXHash3.TransformFinal: IHashResult; Initialize(); end; +initialization + XXH3_Accumulate512 := TXXHash3Simd.SelectAccumulate512(@XXH3_Accumulate512_Scalar); + XXH3_Accumulate := TXXHash3Simd.SelectAccumulate(@XXH3_Accumulate_Scalar); + XXH3_ScrambleAcc := TXXHash3Simd.SelectScrambleAcc(@XXH3_ScrambleAcc_Scalar); + XXH3_InitSecret := TXXHash3Simd.SelectInitSecret(@XXH3_InitSecret_Scalar); + end. diff --git a/HashLib/src/Hash64/HlpXXHash3Dispatch.pas b/HashLib/src/Hash64/HlpXXHash3Dispatch.pas deleted file mode 100644 index 608f9c2e..00000000 --- a/HashLib/src/Hash64/HlpXXHash3Dispatch.pas +++ /dev/null @@ -1,299 +0,0 @@ -unit HlpXXHash3Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TXXH3Accumulate512Proc = procedure(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer); - TXXH3AccumulateProc = procedure(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer; ANbStripes: Int32); - TXXH3ScrambleAccProc = procedure(AAcc: Pointer; ASecret: Pointer); - TXXH3InitSecretProc = procedure(ACustomSecret: Pointer; - ADefaultSecret: Pointer; ASeed: UInt64); - -var - XXH3_Accumulate512: TXXH3Accumulate512Proc; - XXH3_Accumulate: TXXH3AccumulateProc; - XXH3_ScrambleAcc: TXXH3ScrambleAccProc; - XXH3_InitSecret: TXXH3InitSecretProc; - -implementation - -uses - HlpBinaryPrimitives, - HlpCpuFeatures, - HlpSimdLevels; - -const - XXH_STRIPE_LEN = 64; - XXH_ACC_NB = 8; - XXH_SECRET_CONSUME_RATE = 8; - XXH_PRIME32_1 = UInt32($9E3779B1); - -// ============================================================================= -// Scalar fallback implementations -// ============================================================================= - -procedure XXH3_Accumulate512_Scalar(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer); -var - LPAcc: PUInt64; - LPInput, LPSecret: PByte; - I: Int32; - LDataVal, LDataKey: UInt64; -begin - LPAcc := PUInt64(AAcc); - LPInput := PByte(AInput); - LPSecret := PByte(ASecret); - for I := 0 to XXH_ACC_NB - 1 do - begin - LDataVal := TBinaryPrimitives.ReadUInt64LittleEndian(LPInput, I * 8); - LDataKey := LDataVal xor TBinaryPrimitives.ReadUInt64LittleEndian(LPSecret, I * 8); - PUInt64(PByte(LPAcc) + (I xor 1) * 8)^ := - PUInt64(PByte(LPAcc) + (I xor 1) * 8)^ + LDataVal; - PUInt64(PByte(LPAcc) + I * 8)^ := - PUInt64(PByte(LPAcc) + I * 8)^ + - UInt64(UInt32(LDataKey)) * UInt64(UInt32(LDataKey shr 32)); - end; -end; - -procedure XXH3_ScrambleAcc_Scalar(AAcc: Pointer; ASecret: Pointer); -var - LPAcc: PUInt64; - LPSecret: PByte; - I: Int32; - LKey64, LAcc64: UInt64; -begin - LPAcc := PUInt64(AAcc); - LPSecret := PByte(ASecret); - for I := 0 to XXH_ACC_NB - 1 do - begin - LKey64 := TBinaryPrimitives.ReadUInt64LittleEndian(LPSecret, I * 8); - LAcc64 := PUInt64(PByte(LPAcc) + I * 8)^; - LAcc64 := LAcc64 xor (LAcc64 shr 47); - LAcc64 := LAcc64 xor LKey64; - LAcc64 := LAcc64 * XXH_PRIME32_1; - PUInt64(PByte(LPAcc) + I * 8)^ := LAcc64; - end; -end; - -procedure XXH3_InitSecret_Scalar(ACustomSecret: Pointer; - ADefaultSecret: Pointer; ASeed: UInt64); -var - I: Int32; - LPSrc, LPDst: PByte; -begin - LPSrc := PByte(ADefaultSecret); - LPDst := PByte(ACustomSecret); - for I := 0 to (192 div 16) - 1 do - begin - PUInt64(LPDst + 16 * I)^ := - TBinaryPrimitives.ReadUInt64LittleEndian(LPSrc, 16 * I) + ASeed; - PUInt64(LPDst + 16 * I + 8)^ := - TBinaryPrimitives.ReadUInt64LittleEndian(LPSrc, 16 * I + 8) - ASeed; - end; -end; - -procedure XXH3_Accumulate_Scalar(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer; ANbStripes: Int32); -var - N: Int32; -begin - for N := 0 to ANbStripes - 1 do - XXH3_Accumulate512_Scalar(AAcc, PByte(AInput) + N * XXH_STRIPE_LEN, - PByte(ASecret) + N * XXH_SECRET_CONSUME_RATE); -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: NEON -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure XXH3_Accumulate512_Sse2(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_i386.inc} - {$I ..\Include\Simd\XXH3\XXH3Acc512Sse2_i386.inc} -end; - -procedure XXH3_ScrambleAcc_Sse2(AAcc: Pointer; ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} - {$I ..\Include\Simd\XXH3\XXH3ScrambleSse2_i386.inc} -end; - -procedure XXH3_InitSecret_Sse2(ACustomSecret: Pointer; - ADefaultSecret: Pointer; ASeed: UInt64); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_i386.inc} - {$I ..\Include\Simd\XXH3\XXH3InitSecretSse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure XXH3_Accumulate512_Sse2(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} - {$I ..\Include\Simd\XXH3\XXH3Acc512Sse2_x86_64.inc} -end; - -procedure XXH3_ScrambleAcc_Sse2(AAcc: Pointer; ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} - {$I ..\Include\Simd\XXH3\XXH3ScrambleSse2_x86_64.inc} -end; - -procedure XXH3_InitSecret_Sse2(ACustomSecret: Pointer; - ADefaultSecret: Pointer; ASeed: UInt64); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} - {$I ..\Include\Simd\XXH3\XXH3InitSecretSse2_x86_64.inc} -end; - -procedure XXH3_Accumulate512_Avx2(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} - {$I ..\Include\Simd\XXH3\XXH3Acc512Avx2_x86_64.inc} -end; - -procedure XXH3_ScrambleAcc_Avx2(AAcc: Pointer; ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} - {$I ..\Include\Simd\XXH3\XXH3ScrambleAvx2_x86_64.inc} -end; - -procedure XXH3_InitSecret_Avx2(ACustomSecret: Pointer; - ADefaultSecret: Pointer; ASeed: UInt64); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} - {$I ..\Include\Simd\XXH3\XXH3InitSecretAvx2_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IF DEFINED(HASHLIB_X86_SIMD) OR DEFINED(HASHLIB_ARM_SIMD)} - -type - TXXH3_Accumulate512Proc = procedure(AAcc, AInput, ASecret: Pointer); - -procedure XXH3_Accumulate_Loop(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer; ANbStripes: Int32; AAcc512: TXXH3_Accumulate512Proc); -var - N: Int32; -begin - for N := 0 to ANbStripes - 1 do - AAcc512(AAcc, PByte(AInput) + N * XXH_STRIPE_LEN, - PByte(ASecret) + N * XXH_SECRET_CONSUME_RATE); -end; - -{$IFEND} - -{$IFDEF HASHLIB_X86_SIMD} - -procedure XXH3_Accumulate_Sse2(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer; ANbStripes: Int32); -begin - XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Sse2); -end; - -{$ENDIF HASHLIB_X86_SIMD} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure XXH3_Accumulate_Avx2(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer; ANbStripes: Int32); -begin - XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Avx2); -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure XXH3_Accumulate512_Neon(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_aarch64.inc} - {$I ..\Include\Simd\XXH3\XXH3Acc512Neon_aarch64.inc} -end; - -procedure XXH3_ScrambleAcc_Neon(AAcc: Pointer; ASecret: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} - {$I ..\Include\Simd\XXH3\XXH3ScrambleNeon_aarch64.inc} -end; - -procedure XXH3_InitSecret_Neon(ACustomSecret: Pointer; - ADefaultSecret: Pointer; ASeed: UInt64); - {$I ..\Include\Simd\Common\HlpSimdProc3Begin_aarch64.inc} - {$I ..\Include\Simd\XXH3\XXH3InitSecretNeon_aarch64.inc} -end; - -{$IFDEF HASHLIB_ARM_SIMD} - -procedure XXH3_Accumulate_Neon(AAcc: Pointer; AInput: Pointer; - ASecret: Pointer; ANbStripes: Int32); -begin - XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Neon); -end; - -{$ENDIF HASHLIB_ARM_SIMD} - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - XXH3_Accumulate512 := @XXH3_Accumulate512_Scalar; - XXH3_Accumulate := @XXH3_Accumulate_Scalar; - XXH3_ScrambleAcc := @XXH3_ScrambleAcc_Scalar; - XXH3_InitSecret := @XXH3_InitSecret_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - XXH3_Accumulate512 := @XXH3_Accumulate512_Sse2; - XXH3_Accumulate := @XXH3_Accumulate_Sse2; - XXH3_ScrambleAcc := @XXH3_ScrambleAcc_Sse2; - XXH3_InitSecret := @XXH3_InitSecret_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - XXH3_Accumulate512 := @XXH3_Accumulate512_Avx2; - XXH3_Accumulate := @XXH3_Accumulate_Avx2; - XXH3_ScrambleAcc := @XXH3_ScrambleAcc_Avx2; - XXH3_InitSecret := @XXH3_InitSecret_Avx2; - end; - TX86SimdLevel.SSE2: - begin - XXH3_Accumulate512 := @XXH3_Accumulate512_Sse2; - XXH3_Accumulate := @XXH3_Accumulate_Sse2; - XXH3_ScrambleAcc := @XXH3_ScrambleAcc_Sse2; - XXH3_InitSecret := @XXH3_InitSecret_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - TArmSimdLevel.NEON: - begin - XXH3_Accumulate512 := @XXH3_Accumulate512_Neon; - XXH3_Accumulate := @XXH3_Accumulate_Neon; - XXH3_ScrambleAcc := @XXH3_ScrambleAcc_Neon; - XXH3_InitSecret := @XXH3_InitSecret_Neon; - end; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_i386.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_i386.inc new file mode 100644 index 00000000..36e8f9f6 --- /dev/null +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_i386.inc @@ -0,0 +1,57 @@ +// AVX2 implementation of Adler-32 block processing (num_blocks x 32-byte blocks; +// IA-32: 256-bit ymm integer ops work in 32-bit mode and only ymm0-ymm7 exist, +// which this kernel fits). Same structure as Adler32BlocksAvx2_x86_64.inc. +// Does NOT apply mod 65521 (the caller does). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = data ptr, esi = num_blocks, +// edi = sums ptr, eax = constants ptr (moved to ecx at entry; eax then +// accumulates SumA). SumB rides xmm1 end-to-end via vmovd memory forms. +// Sums layout: [SumA: UInt32, SumB: UInt32]. +// Constants layout: [weights: 32B, ones_16: 32B] at offsets 0 and 32. +// Saves: none (uses ymm0-ymm5 only; all volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: Chromium zlib adler32_simd.c, HashLib Adler32BlocksAvx2_x86_64.inc. + + mov ecx, eax + db $C5, $E5, $EF, $DB // vpxor ymm3, ymm3, ymm3 + mov eax, dword ptr [edi] + db $C5, $F9, $6E, $4F, $04 // vmovd xmm1, dword [edi + $4] + mov edx, esi + imul edx, eax + db $C5, $F9, $6E, $D2 // vmovd xmm2, edx + db $C5, $F9, $EF, $C0 // vpxor xmm0, xmm0, xmm0 + +@loop: + db $C5, $ED, $FE, $D0 // vpaddd ymm2, ymm2, ymm0 + db $C5, $FE, $6F, $23 // vmovdqu ymm4, yword [ebx] + db $C5, $DD, $F6, $EB // vpsadbw ymm5, ymm4, ymm3 + db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 + db $C5, $FE, $6F, $29 // vmovdqu ymm5, yword [ecx] + db $C4, $E2, $5D, $04, $E5 // vpmaddubsw ymm4, ymm4, ymm5 + db $C5, $FE, $6F, $69, $20 // vmovdqu ymm5, yword [ecx + $20] + db $C5, $DD, $F5, $E5 // vpmaddwd ymm4, ymm4, ymm5 + db $C5, $F5, $FE, $CC // vpaddd ymm1, ymm1, ymm4 + add ebx, $20 + dec esi + jnz @loop + db $C5, $ED, $72, $F2, $05 // vpslld ymm2, ymm2, $5 + db $C5, $F5, $FE, $CA // vpaddd ymm1, ymm1, ymm2 + db $C4, $E3, $7D, $39, $C5, $01 // vextracti128 xmm5, ymm0, $1 + db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 + db $C5, $F9, $70, $E8, $B1 // vpshufd xmm5, xmm0, $B1 + db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 + db $C5, $F9, $70, $E8, $4E // vpshufd xmm5, xmm0, $4E + db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 + db $C5, $F9, $7E, $C2 // vmovd edx, xmm0 + add eax, edx + db $C4, $E3, $7D, $39, $CD, $01 // vextracti128 xmm5, ymm1, $1 + db $C5, $F1, $FE, $CD // vpaddd xmm1, xmm1, xmm5 + db $C5, $F9, $70, $E9, $B1 // vpshufd xmm5, xmm1, $B1 + db $C5, $F1, $FE, $CD // vpaddd xmm1, xmm1, xmm5 + db $C5, $F9, $70, $E9, $4E // vpshufd xmm5, xmm1, $4E + db $C5, $F1, $FE, $CD // vpaddd xmm1, xmm1, xmm5 + mov dword ptr [edi], eax + db $C5, $F9, $7E, $4F, $04 // vmovd dword [edi + $4], xmm1 + db $C5, $F8, $77 // vzeroupper + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_x86_64.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_x86_64.inc index aeff0cc7..ecb750b1 100644 --- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksAvx2_x86_64.inc @@ -1,12 +1,13 @@ -// AVX2 implementation of Adler-32 block processing. -// Expects MS x64 ABI: rcx = data ptr, edx = num_blocks, r8 = sums ptr, r9 = constants ptr. -// ASums layout: [SumA: UInt32, SumB: UInt32]. -// Constants layout: [weights: 32B, ones_16: 32B] at offsets 0 and 32. -// Processes num_blocks x 32-byte blocks. Does NOT apply mod 65521 (caller does it). -// Uses ymm0-ymm5 only (volatile under both ABIs; no saves needed). -// Weights and ones are reloaded from memory each iteration to avoid -// using non-volatile ymm registers. +// AVX2 implementation of Adler-32 block processing (num_blocks x 32-byte blocks). +// Does NOT apply mod 65521 (the caller does). Weights and ones are reloaded +// from memory each iteration so no non-volatile ymm register is needed. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = data ptr, edx = num_blocks, +// r8 = sums ptr, r9 = constants ptr. +// Sums layout: [SumA: UInt32, SumB: UInt32]. +// Constants layout: [weights: 32B, ones_16: 32B] at offsets 0 and 32. +// Saves: none (uses ymm0-ymm5 only; volatile under both ABIs). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: Chromium zlib adler32_simd.c. // Zero register db $C5, $E5, $EF, $DB // vpxor ymm3, ymm3, ymm3 diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksNeon_aarch64.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksNeon_aarch64.inc index 1fc83e4e..e346dfeb 100644 --- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksNeon_aarch64.inc @@ -1,10 +1,9 @@ -// Adler-32 ProcessBlocks AArch64 NEON implementation (32-byte blocks). -// Expects AAPCS64: x0 = AData, w1 = ANumBlocks, x2 = ASums, x3 = AConstants. -// ASums layout: [SumA: UInt32, SumB: UInt32]. Does NOT apply mod 65521. -// Constants layout matches Adler32Constants: bytes [32..1] at offset 0..31. -// Leaf nostackframe; caller-saved GPR/vector only. -// Reference: Chromium zlib adler32_simd.c, HashLib Adler32BlocksSsse3_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of Adler-32 block processing (num_blocks x 32-byte blocks). +// Does NOT apply mod 65521 (the caller does). +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = data ptr, w1 = num_blocks, +// x2 = sums ptr, x3 = constants ptr. +// Sums layout: [SumA: UInt32, SumB: UInt32]. +// Constants layout matches Adler32Constants: bytes [32..1] at offsets 0..31. // Register map: // w9 = block counter (from w1) // w10 = SumA in / SumA out @@ -20,6 +19,10 @@ // v24-v27 = widened weight vectors for vmlal // v28-v29 = weight byte loads from AConstants // v30 = ext scratch for weight / reduce halves +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: Chromium zlib adler32_simd.c, HashLib Adler32BlocksSsse3_x86_64.inc. + .long 0xb940004a // ldr w10, [x2] .long 0xb940044b // ldr w11, [x2, #4] .long 0x1b017d4c // mul w12, w10, w1 diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_i386.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_i386.inc index 7691003c..80537140 100644 --- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_i386.inc +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_i386.inc @@ -1,20 +1,18 @@ -// SSE2 implementation of Adler-32 block processing (IA-32). -// After HlpSimdProc4Begin_i386: ebx = AData, esi = ANumBlocks, edi = ASums, eax = AConstants. -// ASums: [SumA: UInt32, SumB: UInt32]. Constants: [weights_hi: 16B, weights_lo: 16B] (32 bytes). -// Processes num_blocks x 32-byte blocks; caller applies mod 65521. -// -// x64 uses xmm6-xmm9 for widened weights; IA-32 only has xmm0-xmm7, so widened weights -// live on stack (4 x 16 bytes). xmm6–xmm7 saved/restored defensively (volatile on i386). -// -// Same SSE2 emulation as Adler32BlocksSse2_x86_64.inc (punpcklbw/hbw + pmaddwd). -// -// Stack layout (sub esp, 96): -// [esp + 0..15]: xmm6 save -// [esp + 16..31]: xmm7 save -// [esp + 32..47]: w0 (weights_hi low) -// [esp + 48..63]: w1 (weights_hi high) -// [esp + 64..79]: w2 (weights_lo low) -// [esp + 80..95]: w3 (weights_lo high) +// SSE2 implementation of Adler-32 block processing (num_blocks x 32-byte blocks; +// IA-32). Does NOT apply mod 65521 (the caller does). Same SSE2 pmaddubsw +// emulation as Adler32BlocksSse2_x86_64.inc (punpcklbw/hbw + pmaddwd); x64 +// keeps the widened weights in xmm6-xmm9, IA-32 only has xmm0-xmm7 so they +// live on the stack (4 x 16 bytes). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = data ptr, esi = num_blocks, +// edi = sums ptr, eax = constants ptr. +// Sums layout: [SumA: UInt32, SumB: UInt32]. +// Constants layout: [weights_hi: 16B, weights_lo: 16B] (32 bytes). +// Frame (sub esp, 96): +// [esp + 0..15] = xmm6 save [esp + 48..63] = w1 (weights_hi high) +// [esp + 16..31] = xmm7 save [esp + 64..79] = w2 (weights_lo low) +// [esp + 32..47] = w0 (weights_hi low) [esp + 80..95] = w3 (weights_lo high) +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// Reference: Chromium zlib adler32_simd.c, HashLib Adler32BlocksSse2_x86_64.inc. // Preserve constants pointer (eax) before GPR reloads from ASums push eax diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc index 62605209..5b470d6a 100644 --- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc @@ -1,16 +1,16 @@ -// SSE2 implementation of Adler-32 block processing. -// Expects MS x64 ABI: rcx = data ptr, edx = num_blocks, r8 = sums ptr, r9 = constants ptr. -// ASums layout: [SumA: UInt32, SumB: UInt32]. -// Constants layout: [weights_hi: 16B, weights_lo: 16B] (only first 32 bytes used). -// Processes num_blocks x 32-byte blocks. Does NOT apply mod 65521 (caller does it). -// Uses xmm0-xmm9; xmm6-xmm9 are MS x64 non-volatile (saved/restored). -// -// Emulates SSSE3 pmaddubsw via punpcklbw/punpckhbw + pmaddwd: -// data bytes are zero-extended to i16, then multiplied with pre-widened -// weight bytes via pmaddwd (SSE2), producing the same 4 x i32 weighted -// sums per 16-byte half that pmaddubsw + pmaddwd would yield. - - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} +// SSE2 implementation of Adler-32 block processing (num_blocks x 32-byte blocks). +// Does NOT apply mod 65521 (the caller does). Emulates SSSE3 pmaddubsw via +// punpcklbw/punpckhbw + pmaddwd: data bytes are zero-extended to i16, then +// multiplied with pre-widened weight bytes via pmaddwd (SSE2), producing the +// same 4 x i32 weighted sums per 16-byte half that pmaddubsw + pmaddwd yield. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = data ptr, edx = num_blocks, +// r8 = sums ptr, r9 = constants ptr. +// Sums layout: [SumA: UInt32, SumB: UInt32]. +// Constants layout: [weights_hi: 16B, weights_lo: 16B] (first 32 bytes used). +// Saves: xmm6-xmm9 (MS x64 non-volatile) in the local frame; uses xmm0-xmm9. +// Reference: Chromium zlib adler32_simd.c. + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} // Zero constant pxor xmm3, xmm3 @@ -98,4 +98,4 @@ mov dword [r8], eax mov dword [r8 + 4], r10d - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_i386.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_i386.inc index 2fc07bd5..7392e927 100644 --- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_i386.inc +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_i386.inc @@ -1,10 +1,13 @@ -// SSSE3 Adler-32 block processing (IA-32). -// After HlpSimdProc4Begin_i386: ebx = AData, esi = ANumBlocks, edi = ASums, eax = AConstants. -// Constants: [weights_hi: 16B, weights_lo: 16B, ones_16: 16B] (48 bytes; same as x64 SSSE3). -// No xmm8 on IA-32: psadbw uses a copy in xmm5 (first half) or xmm4 (second half), then reload weights. -// Caller applies mod 65521. -// -// xmm6–xmm7 saved/restored defensively (volatile on i386). +// SSSE3 implementation of Adler-32 block processing (num_blocks x 32-byte blocks; +// IA-32). Does NOT apply mod 65521 (the caller does). No xmm8 on IA-32: +// psadbw uses a copy in xmm5 (first half) or xmm4 (second half), then the +// weights are reloaded. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = data ptr, esi = num_blocks, +// edi = sums ptr, eax = constants ptr. +// Constants layout: [weights_hi: 16B, weights_lo: 16B, ones_16: 16B] +// (48 bytes; same as the x64 SSSE3 kernel). +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// Reference: Chromium zlib adler32_simd.c, HashLib Adler32BlocksSsse3_x86_64.inc. push eax diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc index a277affe..61ebf40a 100644 --- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc +++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc @@ -1,11 +1,13 @@ -// SSSE3 implementation of Adler-32 block processing. -// Expects MS x64 ABI: rcx = data ptr, edx = num_blocks, r8 = sums ptr, r9 = constants ptr. -// ASums layout: [SumA: UInt32, SumB: UInt32]. -// Constants layout: [weights_hi: 16B, weights_lo: 16B, ones_16: 16B]. -// Processes num_blocks x 32-byte blocks. Does NOT apply mod 65521 (caller does it). -// Uses xmm0-xmm8; xmm6-xmm8 are MS x64 non-volatile (saved/restored). +// SSSE3 implementation of Adler-32 block processing (num_blocks x 32-byte blocks). +// Does NOT apply mod 65521 (the caller does). +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = data ptr, edx = num_blocks, +// r8 = sums ptr, r9 = constants ptr. +// Sums layout: [SumA: UInt32, SumB: UInt32]. +// Constants layout: [weights_hi: 16B, weights_lo: 16B, ones_16: 16B]. +// Saves: xmm6-xmm8 (MS x64 non-volatile) in the local frame; uses xmm0-xmm8. +// Reference: Chromium zlib adler32_simd.c. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} // Load constants movdqu xmm4, oword [r9] @@ -76,4 +78,4 @@ mov dword [r8], eax mov dword [r8 + 4], r10d - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_i386.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_i386.inc new file mode 100644 index 00000000..de78c4c9 --- /dev/null +++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_i386.inc @@ -0,0 +1,276 @@ +// AVX2 implementation of Argon2 FillBlock (BlaMka round function; IA-32). +// 256-bit ymm integer ops work in 32-bit mode; same algorithm and buffer +// layout as Argon2FillBlockAvx2_x86_64.inc, redesigned for the IA-32 +// register budget: +// - the byte-rotation masks are VEX memory operands (rot24 at [masks], +// rot16 at [masks+$20]; read unaligned, so the Pascal const needs no +// special alignment) - the kernel touches only ymm0-ymm4; +// - R_buf/Z_buf sit at [esp]/[esp+1024] and are addressed [esp+edx+disp] +// (edx = loop offset) - no buffer pointer registers are needed; +// - Left/Right are dead after step 1's XOR. +// Diagonalize/Undiagonalize via vpermq. vzeroupper required before return. +// ABI (after HlpSimdProc5Begin_i386.inc): ebx = Left ptr, esi = Right ptr, +// edi = Current ptr, eax = WithXor (0 or 1; live until the final step), +// ecx = byte-rotation masks ptr (ARGON2_ROT_MASKS; live throughout). +// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may +// alias; R_buf buffering handles this. +// Frame (sub esp, 2048): [esp+0..1023] R_buf, [esp+1024..2047] Z_buf. +// Saves: none (uses ymm0-ymm4 only; all volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: official Argon2 AVX2 (blamka-round-opt.h), HashLib +// Argon2FillBlockAvx2_x86_64.inc. + + sub esp, $800 + + // Step 1: R_buf = Left XOR Right at [esp] (Left/Right dead afterwards) + xor edx, edx + +@xor_loop: + db $C5, $FE, $6F, $04, $13 // vmovdqu ymm0, yword [ebx + edx*1] + db $C5, $FD, $EF, $04, $16 // vpxor ymm0, ymm0, yword [esi + edx*1] + db $C5, $FE, $7F, $04, $14 // vmovdqu yword [esp + edx*1], ymm0 + add edx, $20 + cmp edx, $400 + jb @xor_loop + + // Step 2: copy R_buf to Z_buf at [esp+1024] + xor edx, edx + +@copy_loop: + db $C5, $FE, $6F, $04, $14 // vmovdqu ymm0, yword [esp + edx*1] + db $C5, $FE, $7F, $84, $14, $00, $04, $00, $00// vmovdqu yword [esp + edx*1 + $400], ymm0 + add edx, $20 + cmp edx, $400 + jb @copy_loop + + // Step 3: column rounds on Z_buf (8 iterations, 128 bytes each) + xor edx, edx + +@col_loop: + db $C5, $FE, $6F, $84, $14, $00, $04, $00, $00// vmovdqu ymm0, yword [esp + edx*1 + $400] + db $C5, $FE, $6F, $8C, $14, $20, $04, $00, $00// vmovdqu ymm1, yword [esp + edx*1 + $420] + db $C5, $FE, $6F, $94, $14, $40, $04, $00, $00// vmovdqu ymm2, yword [esp + edx*1 + $440] + db $C5, $FE, $6F, $9C, $14, $60, $04, $00, $00// vmovdqu ymm3, yword [esp + edx*1 + $460] + // ----- Column G ----- + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 32) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 24) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx] + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 16) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20] + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 63) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + // ----- Diagonalize ----- + db $C4, $E3, $FD, $00, $C9, $39 // vpermq ymm1, ymm1, $39 + db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E + db $C4, $E3, $FD, $00, $DB, $93 // vpermq ymm3, ymm3, $93 + // ----- Diagonal G ----- + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 32) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 24) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx] + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 16) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20] + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 63) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + // ----- Undiagonalize ----- + db $C4, $E3, $FD, $00, $C9, $93 // vpermq ymm1, ymm1, $93 + db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E + db $C4, $E3, $FD, $00, $DB, $39 // vpermq ymm3, ymm3, $39 + db $C5, $FE, $7F, $84, $14, $00, $04, $00, $00// vmovdqu yword [esp + edx*1 + $400], ymm0 + db $C5, $FE, $7F, $8C, $14, $20, $04, $00, $00// vmovdqu yword [esp + edx*1 + $420], ymm1 + db $C5, $FE, $7F, $94, $14, $40, $04, $00, $00// vmovdqu yword [esp + edx*1 + $440], ymm2 + db $C5, $FE, $7F, $9C, $14, $60, $04, $00, $00// vmovdqu yword [esp + edx*1 + $460], ymm3 + add edx, $80 + cmp edx, $400 + jb @col_loop + + // Step 4: row rounds on Z_buf (8 iterations, stride-16-QWord gathers) + xor edx, edx + +@row_loop: + db $C5, $FA, $6F, $84, $14, $00, $04, $00, $00// vmovdqu xmm0, oword [esp + edx*1 + $400] + db $C4, $E3, $7D, $38, $84, $14, $80, $04, $00, $00, $01// vinserti128 ymm0, ymm0, oword [esp + edx*1 + $480], $1 + db $C5, $FA, $6F, $8C, $14, $00, $05, $00, $00// vmovdqu xmm1, oword [esp + edx*1 + $500] + db $C4, $E3, $75, $38, $8C, $14, $80, $05, $00, $00, $01// vinserti128 ymm1, ymm1, oword [esp + edx*1 + $580], $1 + db $C5, $FA, $6F, $94, $14, $00, $06, $00, $00// vmovdqu xmm2, oword [esp + edx*1 + $600] + db $C4, $E3, $6D, $38, $94, $14, $80, $06, $00, $00, $01// vinserti128 ymm2, ymm2, oword [esp + edx*1 + $680], $1 + db $C5, $FA, $6F, $9C, $14, $00, $07, $00, $00// vmovdqu xmm3, oword [esp + edx*1 + $700] + db $C4, $E3, $65, $38, $9C, $14, $80, $07, $00, $00, $01// vinserti128 ymm3, ymm3, oword [esp + edx*1 + $780], $1 + // ----- Column G ----- + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 32) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 24) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx] + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 16) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20] + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 63) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + // ----- Diagonalize ----- + db $C4, $E3, $FD, $00, $C9, $39 // vpermq ymm1, ymm1, $39 + db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E + db $C4, $E3, $FD, $00, $DB, $93 // vpermq ymm3, ymm3, $93 + // ----- Diagonal G ----- + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 32) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 24) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $09 // vpshufb ymm1, ymm1, yword [ecx] + // fBlaMka(0, 1): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + // d = rotr64(d ^ a, 16) + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $59, $20 // vpshufb ymm3, ymm3, yword [ecx + $20] + // fBlaMka(2, 3): a = a + b + 2*lo32(a)*lo32(b) + db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 + db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, $1 + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 + // b = rotr64(b ^ c, 63) + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $DD, $73, $D1, $3F // vpsrlq ymm4, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + // ----- Undiagonalize ----- + db $C4, $E3, $FD, $00, $C9, $93 // vpermq ymm1, ymm1, $93 + db $C4, $E3, $FD, $00, $D2, $4E // vpermq ymm2, ymm2, $4E + db $C4, $E3, $FD, $00, $DB, $39 // vpermq ymm3, ymm3, $39 + db $C5, $FA, $7F, $84, $14, $00, $04, $00, $00// vmovdqu oword [esp + edx*1 + $400], xmm0 + db $C4, $E3, $7D, $39, $84, $14, $80, $04, $00, $00, $01// vextracti128 oword [esp + edx*1 + $480], ymm0, $1 + db $C5, $FA, $7F, $8C, $14, $00, $05, $00, $00// vmovdqu oword [esp + edx*1 + $500], xmm1 + db $C4, $E3, $7D, $39, $8C, $14, $80, $05, $00, $00, $01// vextracti128 oword [esp + edx*1 + $580], ymm1, $1 + db $C5, $FA, $7F, $94, $14, $00, $06, $00, $00// vmovdqu oword [esp + edx*1 + $600], xmm2 + db $C4, $E3, $7D, $39, $94, $14, $80, $06, $00, $00, $01// vextracti128 oword [esp + edx*1 + $680], ymm2, $1 + db $C5, $FA, $7F, $9C, $14, $00, $07, $00, $00// vmovdqu oword [esp + edx*1 + $700], xmm3 + db $C4, $E3, $7D, $39, $9C, $14, $80, $07, $00, $00, $01// vextracti128 oword [esp + edx*1 + $780], ymm3, $1 + add edx, $10 + cmp edx, $80 + jb @row_loop + + // Step 5: Current = R_buf XOR Z_buf [XOR Current] + test eax, eax + jnz @final_with_xor + + xor edx, edx + +@final_noxor_loop: + db $C5, $FE, $6F, $04, $14 // vmovdqu ymm0, yword [esp + edx*1] + db $C5, $FD, $EF, $84, $14, $00, $04, $00, $00// vpxor ymm0, ymm0, yword [esp + edx*1 + $400] + db $C5, $FE, $7F, $04, $17 // vmovdqu yword [edi + edx*1], ymm0 + add edx, $20 + cmp edx, $400 + jb @final_noxor_loop + jmp @epilogue + + +@final_with_xor: + xor edx, edx + +@final_xor_loop: + db $C5, $FE, $6F, $04, $14 // vmovdqu ymm0, yword [esp + edx*1] + db $C5, $FD, $EF, $84, $14, $00, $04, $00, $00// vpxor ymm0, ymm0, yword [esp + edx*1 + $400] + db $C5, $FD, $EF, $04, $17 // vpxor ymm0, ymm0, yword [edi + edx*1] + db $C5, $FE, $7F, $04, $17 // vmovdqu yword [edi + edx*1], ymm0 + add edx, $20 + cmp edx, $400 + jb @final_xor_loop + + +@epilogue: + add esp, $800 + db $C5, $F8, $77 // vzeroupper + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc index 4bff0fe2..cafc467a 100644 --- a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc @@ -1,21 +1,31 @@ -// AVX2 implementation of Argon2 FillBlock. +// AVX2 implementation of Argon2 FillBlock (BlaMka round function). +// Rotations: rot32 via vpshufd, rot24/16 via vpshufb against the resident +// masks, rot63 via shift+or (not byte-aligned). Diagonalize/Undiagonalize +// via vpermq (cross-lane 64-bit permute). vzeroupper required before return. +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = Left ptr, rdx = Right ptr, +// r8 = Current ptr, r9 = WithXor (0 or 1), r10 = byte-rotation masks ptr +// (ARGON2_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the +// Pascal const needs no special alignment; consumed by the entry mask +// loads, r10 is scratch afterwards). +// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may +// alias; R_buf buffering handles this. r8/r9 survive all loops. +// Register map (G rounds): ymm0 = A(v0..v3), ymm1 = B(v4..v7), +// ymm2 = C(v8..v11), ymm3 = D(v12..v15), ymm4-ymm5 = temps, +// ymm10 = rot24 mask, ymm11 = rot16 mask. +// Frame (sub rsp, 2184): +// [rsp+0..127] = ymm6-ymm9 save area (4 x 32 bytes) +// [rsp+128..1151] = R_buf (1024 bytes) +// [rsp+1152..2175] = Z_buf (1024 bytes) +// [rsp+2176..2183] = alignment padding +// Saves: ymm6-ymm11 (MS x64 non-volatile); uses ymm0-ymm11. // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. -// Expects MS x64 ABI: rcx = Left ptr, rdx = Right ptr, r8 = Current ptr, r9 = WithXor (0 or 1). -// Each pointer addresses 128 QWords (1024 bytes). -// Uses ymm0-ymm9; ymm6-ymm9 are MS x64 non-volatile (saved/restored). -// Register map during G rounds: ymm0 = A(v0..v3), ymm1 = B(v4..v7), -// ymm2 = C(v8..v11), ymm3 = D(v12..v15), ymm4-ymm5 = temps. -// Stack layout (sub rsp, 2184): -// [rsp+0..127] ymm6-9 save area (4 * 32 = 128 bytes) -// [rsp+128..1151] R_buf (1024 bytes) -// [rsp+1152..2175] Z_buf (1024 bytes) -// [rsp+2176..2183] alignment padding -// r8 (Current) and r9 (WithXor) survive all loops (not clobbered). -// ARight and ACurrent may alias; R_buf buffering handles this. -// Diagonalize/Undiagonalize via vpermq (cross-lane 64-bit permute). -// vzeroupper required before return. - - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} +// Reference: official Argon2 AVX2 (blamka-round-opt.h). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $C4, $41, $7E, $6F, $12 // vmovdqu ymm10, yword [r10] + db $C4, $41, $7E, $6F, $5A, $20 // vmovdqu ymm11, yword [r10 + $20] sub rsp, 2056 @@ -77,9 +87,7 @@ // b = rotr64(b ^ c, 24) db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $DD, $73, $D1, $18 // vpsrlq ymm4, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10 // fBlaMka(a, b) db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 @@ -89,9 +97,7 @@ // d = rotr64(d ^ a, 16) db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $DD, $73, $D3, $10 // vpsrlq ymm4, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 + db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11 // fBlaMka(c, d) db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 @@ -129,9 +135,7 @@ db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $DD, $73, $D1, $18 // vpsrlq ymm4, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10 db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1 @@ -139,9 +143,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $DD, $73, $D3, $10 // vpsrlq ymm4, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 + db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11 db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1 @@ -207,9 +209,7 @@ db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $DD, $73, $D1, $18 // vpsrlq ymm4, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10 db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1 @@ -217,9 +217,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $DD, $73, $D3, $10 // vpsrlq ymm4, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 + db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11 db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1 @@ -252,9 +250,7 @@ db $C5, $ED, $D4, $D4 // vpaddq ymm2, ymm2, ymm4 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $DD, $73, $D1, $18 // vpsrlq ymm4, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CC // vpor ymm1, ymm1, ymm4 + db $C4, $C2, $75, $00, $CA // vpshufb ymm1, ymm1, ymm10 db $C5, $FD, $F4, $E1 // vpmuludq ymm4, ymm0, ymm1 db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1 @@ -262,9 +258,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $DD, $73, $D3, $10 // vpsrlq ymm4, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DC // vpor ymm3, ymm3, ymm4 + db $C4, $C2, $65, $00, $DB // vpshufb ymm3, ymm3, ymm11 db $C5, $ED, $F4, $E3 // vpmuludq ymm4, ymm2, ymm3 db $C5, $DD, $73, $F4, $01 // vpsllq ymm4, ymm4, 1 @@ -296,7 +290,7 @@ jb @row_loop // ========================================================================= - // Step 5: Final XOR — Current = R_buf XOR Z_buf [XOR Current] + // Step 5: Final XOR - Current = R_buf XOR Z_buf [XOR Current] // ========================================================================= test r9, r9 jnz @final_with_xor @@ -326,4 +320,4 @@ add rsp, 2056 db $C5, $F8, $77 // vzeroupper - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockNeon_aarch64.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockNeon_aarch64.inc index 7f6bc0a9..476e4089 100644 --- a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockNeon_aarch64.inc @@ -1,22 +1,27 @@ -// Argon2 FillBlock AArch64 NEON implementation (BlaMka round function). -// Expects AAPCS64: x0 = Left ptr, x1 = Right ptr, x2 = Current ptr, w3 = WithXor. -// Each pointer addresses 128 UInt64 (1024 bytes). -// Stack frame 2128 bytes; see register map. -// Steps: R=Left XOR Right; Z=copy(R); 8 column + 8 row BlaMka rounds on Z; -// Current = R XOR Z [XOR Current if WithXor]. Diagonalize via vext #8. -// Reference: jedisct1/libsodium argon2-fill-block-neon.c, HashLib Argon2FillBlockSse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of Argon2 FillBlock (BlaMka round function). +// Steps: R = Left XOR Right; Z = copy(R); 8 column + 8 row BlaMka rounds on +// Z; Current = R XOR Z [XOR Current if WithXor]. Diagonalize via vext #8. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = Left ptr, x1 = Right ptr, +// x2 = Current ptr, w3 = WithXor (0 or 1). +// Each pointer addresses 128 QWords (1024 bytes). // Register map: // v0-v1 = A0, A1 // v2-v3 = B0, B1 // v4-v5 = C0, C1 // v6-v7 = D0, D1 -// v16-v18 = fBlaMka / rotr temps +// v16-v18 = fBlaMka / rotate temps (2-op insert rotates rename B/D rows +// through v16/v17; rows return home before each slice store) // x9 = R_buf base ([sp+64]) // x10 = loop byte offset (steps 1-5) // x11 = Z_buf base ([sp+1088]) // x12 = Z slice pointer (column/row load/store) // w3 = WithXor (0 or 1) +// Frame: 2128 bytes (R_buf at [sp+64], Z_buf at [sp+1088], 16-byte SP pad). +// Saves: none (caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: jedisct1/libsodium argon2-fill-block-neon.c, HashLib +// Argon2FillBlockSse2_x86_64.inc. + sub sp, sp, #2128 add x9, sp, #64 mov x10, #0 @@ -78,138 +83,126 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680450 // ushr v16.2d, v2.2d, #24 - .long 0x4f685451 // shl v17.2d, v2.2d, #40 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x6f680470 // ushr v16.2d, v3.2d, #24 - .long 0x4f685471 // shl v17.2d, v3.2d, #40 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b - .long 0x0ea12810 // xtn v16.2s, v0.2d - .long 0x0ea12851 // xtn v17.2s, v2.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s - .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee28400 // add v0.2d, v0.2d, v2.2d + .long 0x6f685450 // sli v16.2d, v2.2d, #40 + .long 0x6f680471 // ushr v17.2d, v3.2d, #24 + .long 0x6f685471 // sli v17.2d, v3.2d, #40 + .long 0x0ea12802 // xtn v2.2s, v0.2d + .long 0x0ea12a03 // xtn v3.2s, v16.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s + .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d + .long 0x4ef08400 // add v0.2d, v0.2d, v16.2d .long 0x4ef28400 // add v0.2d, v0.2d, v18.2d - .long 0x0ea12830 // xtn v16.2s, v1.2d - .long 0x0ea12871 // xtn v17.2s, v3.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea12822 // xtn v2.2s, v1.2d + .long 0x0ea12a23 // xtn v3.2s, v17.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee38421 // add v1.2d, v1.2d, v3.2d + .long 0x4ef18421 // add v1.2d, v1.2d, v17.2d .long 0x4ef28421 // add v1.2d, v1.2d, v18.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d0 // ushr v16.2d, v6.2d, #16 - .long 0x4f7054d1 // shl v17.2d, v6.2d, #48 - .long 0x4eb11e06 // orr v6.16b, v16.16b, v17.16b - .long 0x6f7004f0 // ushr v16.2d, v7.2d, #16 - .long 0x4f7054f1 // shl v17.2d, v7.2d, #48 - .long 0x4eb11e07 // orr v7.16b, v16.16b, v17.16b - .long 0x0ea12890 // xtn v16.2s, v4.2d - .long 0x0ea128d1 // xtn v17.2s, v6.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s - .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x0ea12886 // xtn v6.2s, v4.2d + .long 0x0ea12847 // xtn v7.2s, v2.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s + .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d .long 0x4ef28484 // add v4.2d, v4.2d, v18.2d - .long 0x0ea128b0 // xtn v16.2s, v5.2d - .long 0x0ea128f1 // xtn v17.2s, v7.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea128a6 // xtn v6.2s, v5.2d + .long 0x0ea12867 // xtn v7.2s, v3.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d .long 0x4ef284a5 // add v5.2d, v5.2d, v18.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28450 // add v16.2d, v2.2d, v2.2d - .long 0x6f410451 // ushr v17.2d, v2.2d, #63 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x4ee38470 // add v16.2d, v3.2d, v3.2d - .long 0x6f410471 // ushr v17.2d, v3.2d, #63 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b - .long 0x6e034050 // ext v16.16b, v2.16b, v3.16b, #8 - .long 0x6e024071 // ext v17.16b, v3.16b, v2.16b, #8 - .long 0x4eb01e02 // orr v2.16b, v16.16b, v16.16b - .long 0x4eb11e23 // orr v3.16b, v17.16b, v17.16b + .long 0x6e241e10 // eor v16.16b, v16.16b, v4.16b + .long 0x6e251e31 // eor v17.16b, v17.16b, v5.16b + .long 0x4ef08606 // add v6.2d, v16.2d, v16.2d + .long 0x6f414606 // sri v6.2d, v16.2d, #63 + .long 0x4ef18627 // add v7.2d, v17.2d, v17.2d + .long 0x6f414627 // sri v7.2d, v17.2d, #63 + .long 0x6e0740d0 // ext v16.16b, v6.16b, v7.16b, #8 + .long 0x6e0640f1 // ext v17.16b, v7.16b, v6.16b, #8 + .long 0x4eb01e06 // orr v6.16b, v16.16b, v16.16b + .long 0x4eb11e27 // orr v7.16b, v17.16b, v17.16b .long 0x4ea41c90 // orr v16.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb01e05 // orr v5.16b, v16.16b, v16.16b - .long 0x6e0740d0 // ext v16.16b, v6.16b, v7.16b, #8 - .long 0x6e0640f1 // ext v17.16b, v7.16b, v6.16b, #8 - .long 0x4eb11e26 // orr v6.16b, v17.16b, v17.16b - .long 0x4eb01e07 // orr v7.16b, v16.16b, v16.16b + .long 0x6e034050 // ext v16.16b, v2.16b, v3.16b, #8 + .long 0x6e024071 // ext v17.16b, v3.16b, v2.16b, #8 + .long 0x4eb11e22 // orr v2.16b, v17.16b, v17.16b + .long 0x4eb01e03 // orr v3.16b, v16.16b, v16.16b .long 0x0ea12810 // xtn v16.2s, v0.2d - .long 0x0ea12851 // xtn v17.2s, v2.2d + .long 0x0ea128d1 // xtn v17.2s, v6.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee28400 // add v0.2d, v0.2d, v2.2d + .long 0x4ee68400 // add v0.2d, v0.2d, v6.2d .long 0x4ef28400 // add v0.2d, v0.2d, v18.2d .long 0x0ea12830 // xtn v16.2s, v1.2d - .long 0x0ea12871 // xtn v17.2s, v3.2d + .long 0x0ea128f1 // xtn v17.2s, v7.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee38421 // add v1.2d, v1.2d, v3.2d + .long 0x4ee78421 // add v1.2d, v1.2d, v7.2d .long 0x4ef28421 // add v1.2d, v1.2d, v18.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s .long 0x0ea12890 // xtn v16.2s, v4.2d - .long 0x0ea128d1 // xtn v17.2s, v6.2d + .long 0x0ea12851 // xtn v17.2s, v2.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d .long 0x4ef28484 // add v4.2d, v4.2d, v18.2d .long 0x0ea128b0 // xtn v16.2s, v5.2d - .long 0x0ea128f1 // xtn v17.2s, v7.2d + .long 0x0ea12871 // xtn v17.2s, v3.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d .long 0x4ef284a5 // add v5.2d, v5.2d, v18.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680450 // ushr v16.2d, v2.2d, #24 - .long 0x4f685451 // shl v17.2d, v2.2d, #40 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x6f680470 // ushr v16.2d, v3.2d, #24 - .long 0x4f685471 // shl v17.2d, v3.2d, #40 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b - .long 0x0ea12810 // xtn v16.2s, v0.2d - .long 0x0ea12851 // xtn v17.2s, v2.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s - .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee28400 // add v0.2d, v0.2d, v2.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d0 // ushr v16.2d, v6.2d, #24 + .long 0x6f6854d0 // sli v16.2d, v6.2d, #40 + .long 0x6f6804f1 // ushr v17.2d, v7.2d, #24 + .long 0x6f6854f1 // sli v17.2d, v7.2d, #40 + .long 0x0ea12806 // xtn v6.2s, v0.2d + .long 0x0ea12a07 // xtn v7.2s, v16.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s + .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d + .long 0x4ef08400 // add v0.2d, v0.2d, v16.2d .long 0x4ef28400 // add v0.2d, v0.2d, v18.2d - .long 0x0ea12830 // xtn v16.2s, v1.2d - .long 0x0ea12871 // xtn v17.2s, v3.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea12826 // xtn v6.2s, v1.2d + .long 0x0ea12a27 // xtn v7.2s, v17.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee38421 // add v1.2d, v1.2d, v3.2d + .long 0x4ef18421 // add v1.2d, v1.2d, v17.2d .long 0x4ef28421 // add v1.2d, v1.2d, v18.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d0 // ushr v16.2d, v6.2d, #16 - .long 0x4f7054d1 // shl v17.2d, v6.2d, #48 - .long 0x4eb11e06 // orr v6.16b, v16.16b, v17.16b - .long 0x6f7004f0 // ushr v16.2d, v7.2d, #16 - .long 0x4f7054f1 // shl v17.2d, v7.2d, #48 - .long 0x4eb11e07 // orr v7.16b, v16.16b, v17.16b - .long 0x0ea12890 // xtn v16.2s, v4.2d - .long 0x0ea128d1 // xtn v17.2s, v6.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 + .long 0x0ea12882 // xtn v2.2s, v4.2d + .long 0x0ea128c3 // xtn v3.2s, v6.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ef28484 // add v4.2d, v4.2d, v18.2d - .long 0x0ea128b0 // xtn v16.2s, v5.2d - .long 0x0ea128f1 // xtn v17.2s, v7.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea128a2 // xtn v2.2s, v5.2d + .long 0x0ea128e3 // xtn v3.2s, v7.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d .long 0x4ef284a5 // add v5.2d, v5.2d, v18.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28450 // add v16.2d, v2.2d, v2.2d - .long 0x6f410451 // ushr v17.2d, v2.2d, #63 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x4ee38470 // add v16.2d, v3.2d, v3.2d - .long 0x6f410471 // ushr v17.2d, v3.2d, #63 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b + .long 0x6e241e10 // eor v16.16b, v16.16b, v4.16b + .long 0x6e251e31 // eor v17.16b, v17.16b, v5.16b + .long 0x4ef08602 // add v2.2d, v16.2d, v16.2d + .long 0x6f414602 // sri v2.2d, v16.2d, #63 + .long 0x4ef18623 // add v3.2d, v17.2d, v17.2d + .long 0x6f414623 // sri v3.2d, v17.2d, #63 .long 0x6e024070 // ext v16.16b, v3.16b, v2.16b, #8 .long 0x6e034051 // ext v17.16b, v2.16b, v3.16b, #8 .long 0x4eb01e02 // orr v2.16b, v16.16b, v16.16b @@ -275,138 +268,126 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680450 // ushr v16.2d, v2.2d, #24 - .long 0x4f685451 // shl v17.2d, v2.2d, #40 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x6f680470 // ushr v16.2d, v3.2d, #24 - .long 0x4f685471 // shl v17.2d, v3.2d, #40 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b - .long 0x0ea12810 // xtn v16.2s, v0.2d - .long 0x0ea12851 // xtn v17.2s, v2.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s - .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee28400 // add v0.2d, v0.2d, v2.2d + .long 0x6f685450 // sli v16.2d, v2.2d, #40 + .long 0x6f680471 // ushr v17.2d, v3.2d, #24 + .long 0x6f685471 // sli v17.2d, v3.2d, #40 + .long 0x0ea12802 // xtn v2.2s, v0.2d + .long 0x0ea12a03 // xtn v3.2s, v16.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s + .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d + .long 0x4ef08400 // add v0.2d, v0.2d, v16.2d .long 0x4ef28400 // add v0.2d, v0.2d, v18.2d - .long 0x0ea12830 // xtn v16.2s, v1.2d - .long 0x0ea12871 // xtn v17.2s, v3.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea12822 // xtn v2.2s, v1.2d + .long 0x0ea12a23 // xtn v3.2s, v17.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee38421 // add v1.2d, v1.2d, v3.2d + .long 0x4ef18421 // add v1.2d, v1.2d, v17.2d .long 0x4ef28421 // add v1.2d, v1.2d, v18.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d0 // ushr v16.2d, v6.2d, #16 - .long 0x4f7054d1 // shl v17.2d, v6.2d, #48 - .long 0x4eb11e06 // orr v6.16b, v16.16b, v17.16b - .long 0x6f7004f0 // ushr v16.2d, v7.2d, #16 - .long 0x4f7054f1 // shl v17.2d, v7.2d, #48 - .long 0x4eb11e07 // orr v7.16b, v16.16b, v17.16b - .long 0x0ea12890 // xtn v16.2s, v4.2d - .long 0x0ea128d1 // xtn v17.2s, v6.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s - .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x0ea12886 // xtn v6.2s, v4.2d + .long 0x0ea12847 // xtn v7.2s, v2.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s + .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d .long 0x4ef28484 // add v4.2d, v4.2d, v18.2d - .long 0x0ea128b0 // xtn v16.2s, v5.2d - .long 0x0ea128f1 // xtn v17.2s, v7.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea128a6 // xtn v6.2s, v5.2d + .long 0x0ea12867 // xtn v7.2s, v3.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d .long 0x4ef284a5 // add v5.2d, v5.2d, v18.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28450 // add v16.2d, v2.2d, v2.2d - .long 0x6f410451 // ushr v17.2d, v2.2d, #63 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x4ee38470 // add v16.2d, v3.2d, v3.2d - .long 0x6f410471 // ushr v17.2d, v3.2d, #63 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b - .long 0x6e034050 // ext v16.16b, v2.16b, v3.16b, #8 - .long 0x6e024071 // ext v17.16b, v3.16b, v2.16b, #8 - .long 0x4eb01e02 // orr v2.16b, v16.16b, v16.16b - .long 0x4eb11e23 // orr v3.16b, v17.16b, v17.16b + .long 0x6e241e10 // eor v16.16b, v16.16b, v4.16b + .long 0x6e251e31 // eor v17.16b, v17.16b, v5.16b + .long 0x4ef08606 // add v6.2d, v16.2d, v16.2d + .long 0x6f414606 // sri v6.2d, v16.2d, #63 + .long 0x4ef18627 // add v7.2d, v17.2d, v17.2d + .long 0x6f414627 // sri v7.2d, v17.2d, #63 + .long 0x6e0740d0 // ext v16.16b, v6.16b, v7.16b, #8 + .long 0x6e0640f1 // ext v17.16b, v7.16b, v6.16b, #8 + .long 0x4eb01e06 // orr v6.16b, v16.16b, v16.16b + .long 0x4eb11e27 // orr v7.16b, v17.16b, v17.16b .long 0x4ea41c90 // orr v16.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb01e05 // orr v5.16b, v16.16b, v16.16b - .long 0x6e0740d0 // ext v16.16b, v6.16b, v7.16b, #8 - .long 0x6e0640f1 // ext v17.16b, v7.16b, v6.16b, #8 - .long 0x4eb11e26 // orr v6.16b, v17.16b, v17.16b - .long 0x4eb01e07 // orr v7.16b, v16.16b, v16.16b + .long 0x6e034050 // ext v16.16b, v2.16b, v3.16b, #8 + .long 0x6e024071 // ext v17.16b, v3.16b, v2.16b, #8 + .long 0x4eb11e22 // orr v2.16b, v17.16b, v17.16b + .long 0x4eb01e03 // orr v3.16b, v16.16b, v16.16b .long 0x0ea12810 // xtn v16.2s, v0.2d - .long 0x0ea12851 // xtn v17.2s, v2.2d + .long 0x0ea128d1 // xtn v17.2s, v6.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee28400 // add v0.2d, v0.2d, v2.2d + .long 0x4ee68400 // add v0.2d, v0.2d, v6.2d .long 0x4ef28400 // add v0.2d, v0.2d, v18.2d .long 0x0ea12830 // xtn v16.2s, v1.2d - .long 0x0ea12871 // xtn v17.2s, v3.2d + .long 0x0ea128f1 // xtn v17.2s, v7.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee38421 // add v1.2d, v1.2d, v3.2d + .long 0x4ee78421 // add v1.2d, v1.2d, v7.2d .long 0x4ef28421 // add v1.2d, v1.2d, v18.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s .long 0x0ea12890 // xtn v16.2s, v4.2d - .long 0x0ea128d1 // xtn v17.2s, v6.2d + .long 0x0ea12851 // xtn v17.2s, v2.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d .long 0x4ef28484 // add v4.2d, v4.2d, v18.2d .long 0x0ea128b0 // xtn v16.2s, v5.2d - .long 0x0ea128f1 // xtn v17.2s, v7.2d + .long 0x0ea12871 // xtn v17.2s, v3.2d .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d .long 0x4ef284a5 // add v5.2d, v5.2d, v18.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680450 // ushr v16.2d, v2.2d, #24 - .long 0x4f685451 // shl v17.2d, v2.2d, #40 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x6f680470 // ushr v16.2d, v3.2d, #24 - .long 0x4f685471 // shl v17.2d, v3.2d, #40 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b - .long 0x0ea12810 // xtn v16.2s, v0.2d - .long 0x0ea12851 // xtn v17.2s, v2.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s - .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee28400 // add v0.2d, v0.2d, v2.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d0 // ushr v16.2d, v6.2d, #24 + .long 0x6f6854d0 // sli v16.2d, v6.2d, #40 + .long 0x6f6804f1 // ushr v17.2d, v7.2d, #24 + .long 0x6f6854f1 // sli v17.2d, v7.2d, #40 + .long 0x0ea12806 // xtn v6.2s, v0.2d + .long 0x0ea12a07 // xtn v7.2s, v16.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s + .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d + .long 0x4ef08400 // add v0.2d, v0.2d, v16.2d .long 0x4ef28400 // add v0.2d, v0.2d, v18.2d - .long 0x0ea12830 // xtn v16.2s, v1.2d - .long 0x0ea12871 // xtn v17.2s, v3.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea12826 // xtn v6.2s, v1.2d + .long 0x0ea12a27 // xtn v7.2s, v17.2d + .long 0x2ea7c0d2 // umull v18.2d, v6.2s, v7.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d - .long 0x4ee38421 // add v1.2d, v1.2d, v3.2d + .long 0x4ef18421 // add v1.2d, v1.2d, v17.2d .long 0x4ef28421 // add v1.2d, v1.2d, v18.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d0 // ushr v16.2d, v6.2d, #16 - .long 0x4f7054d1 // shl v17.2d, v6.2d, #48 - .long 0x4eb11e06 // orr v6.16b, v16.16b, v17.16b - .long 0x6f7004f0 // ushr v16.2d, v7.2d, #16 - .long 0x4f7054f1 // shl v17.2d, v7.2d, #48 - .long 0x4eb11e07 // orr v7.16b, v16.16b, v17.16b - .long 0x0ea12890 // xtn v16.2s, v4.2d - .long 0x0ea128d1 // xtn v17.2s, v6.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 + .long 0x0ea12882 // xtn v2.2s, v4.2d + .long 0x0ea128c3 // xtn v3.2s, v6.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ef28484 // add v4.2d, v4.2d, v18.2d - .long 0x0ea128b0 // xtn v16.2s, v5.2d - .long 0x0ea128f1 // xtn v17.2s, v7.2d - .long 0x2eb1c212 // umull v18.2d, v16.2s, v17.2s + .long 0x0ea128a2 // xtn v2.2s, v5.2d + .long 0x0ea128e3 // xtn v3.2s, v7.2d + .long 0x2ea3c052 // umull v18.2d, v2.2s, v3.2s .long 0x4ef28652 // add v18.2d, v18.2d, v18.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d .long 0x4ef284a5 // add v5.2d, v5.2d, v18.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28450 // add v16.2d, v2.2d, v2.2d - .long 0x6f410451 // ushr v17.2d, v2.2d, #63 - .long 0x4eb11e02 // orr v2.16b, v16.16b, v17.16b - .long 0x4ee38470 // add v16.2d, v3.2d, v3.2d - .long 0x6f410471 // ushr v17.2d, v3.2d, #63 - .long 0x4eb11e03 // orr v3.16b, v16.16b, v17.16b + .long 0x6e241e10 // eor v16.16b, v16.16b, v4.16b + .long 0x6e251e31 // eor v17.16b, v17.16b, v5.16b + .long 0x4ef08602 // add v2.2d, v16.2d, v16.2d + .long 0x6f414602 // sri v2.2d, v16.2d, #63 + .long 0x4ef18623 // add v3.2d, v17.2d, v17.2d + .long 0x6f414623 // sri v3.2d, v17.2d, #63 .long 0x6e024070 // ext v16.16b, v3.16b, v2.16b, #8 .long 0x6e034051 // ext v17.16b, v2.16b, v3.16b, #8 .long 0x4eb01e02 // orr v2.16b, v16.16b, v16.16b diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_i386.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_i386.inc index ee688b4c..e68ca3c9 100644 --- a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_i386.inc +++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_i386.inc @@ -1,15 +1,19 @@ -// SSE2 implementation of Argon2 FillBlock. -// IA-32: after HlpSimdProc4Begin_i386 — ebx, esi, edi, eax = Left, Right, Current, WithXor -// (parallel to MS x64 ABI: rcx, rdx, r8, r9). -// Each pointer addresses 128 QWords (1024 bytes). -// Uses xmm0–xmm7; xmm6–xmm7 saved/restored defensively (volatile on i386). -// Register map during G rounds: xmm0-1 = A(0..3), xmm2-3 = B(4..7), -// xmm4-5 = C(8..11), xmm6-7 = D(12..15) / temps (same roles as x64, fewer XMM). -// IA-32 stack (sub esp, 2132): WithXor at [esp+2128]; spill slots [esp+2080],[esp+2096],[esp+2112]; -// R_buf at lea eax,[esp+32]; ECX = loop offset; EBX = Z_buf base after copy; EDI = Current. -// r8/r9 analogues (Current, WithXor) are not clobbered across loops. ARight and ACurrent may alias. -// -// Diagonalize/Undiagonalize use shufpd (see x64 header in Argon2FillBlockSse2_x86_64.inc). +// SSE2 implementation of Argon2 FillBlock (BlaMka round function; IA-32). +// Same roles as Argon2FillBlockSse2_x86_64.inc with fewer XMM registers; +// Diagonalize/Undiagonalize use shufpd (see the x64 sibling's header). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = Left ptr, esi = Right ptr, +// edi = Current ptr, eax = WithXor (0 or 1). +// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may +// alias; R_buf buffering handles this. Current/WithXor analogues are not +// clobbered across loops. +// Register map (G rounds): xmm0-1 = A(0..3), xmm2-3 = B(4..7), +// xmm4-5 = C(8..11), xmm6-7 = D(12..15) / temps. +// Frame (sub esp, 2132): WithXor at [esp+2128]; spill slots [esp+2080], +// [esp+2096], [esp+2112]; R_buf at lea eax,[esp+32]; ecx = loop offset; +// ebx = Z_buf base after the copy; edi = Current. +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// Reference: official Argon2 SSE2 (blamka-round-ref/opt), HashLib +// Argon2FillBlockSse2_x86_64.inc. sub esp, 2132 @@ -697,7 +701,7 @@ jb @row_loop // ========================================================================= - // Step 5: Final XOR — Current = R_buf XOR Z_buf [XOR Current] + // Step 5: Final XOR - Current = R_buf XOR Z_buf [XOR Current] // ========================================================================= cmp dword ptr [esp + 2128], 0 jnz @final_with_xor diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc index 4d696bee..85db6393 100644 --- a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc @@ -1,22 +1,22 @@ -// SSE2 implementation of Argon2 FillBlock. -// Expects MS x64 ABI: rcx = Left ptr, rdx = Right ptr, r8 = Current ptr, r9 = WithXor (0 or 1). -// Each pointer addresses 128 QWords (1024 bytes). -// Uses xmm0-xmm9; xmm6-xmm9 are MS x64 non-volatile (saved/restored). -// Register map during G rounds: xmm0-1 = A(0..3), xmm2-3 = B(4..7), -// xmm4-5 = C(8..11), xmm6-7 = D(12..15), xmm8-9 = temps. -// Stack layout (sub rsp, 2120): -// [rsp+0..63] xmm6-9 save area -// [rsp+64..1087] R_buf (1024 bytes) -// [rsp+1088..2111] Z_buf (1024 bytes) -// [rsp+2112..2119] alignment padding -// r8 (Current) and r9 (WithXor) survive all loops (not clobbered). -// ARight and ACurrent may alias; R_buf buffering handles this. -// +// SSE2 implementation of Argon2 FillBlock (BlaMka round function). // Diagonalize/Undiagonalize use shufpd to rotate QWord positions within -// register pairs. Right-rotates require a 3-movdqa swap since shufpd +// register pairs; right-rotates require a 3-movdqa swap since shufpd // naturally places results in the opposite register for that direction. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = Left ptr, rdx = Right ptr, +// r8 = Current ptr, r9 = WithXor (0 or 1). +// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may +// alias; R_buf buffering handles this. r8/r9 survive all loops. +// Register map (G rounds): xmm0-1 = A(0..3), xmm2-3 = B(4..7), +// xmm4-5 = C(8..11), xmm6-7 = D(12..15), xmm8-9 = temps. +// Frame (sub rsp, 2120): +// [rsp+0..63] = xmm6-xmm9 save area +// [rsp+64..1087] = R_buf (1024 bytes) +// [rsp+1088..2111] = Z_buf (1024 bytes) +// [rsp+2112..2119] = alignment padding +// Saves: xmm6-xmm9 (MS x64 non-volatile) in the frame; uses xmm0-xmm9. +// Reference: official Argon2 SSE2 (blamka-round-ref/opt). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} sub rsp, 2056 @@ -523,7 +523,7 @@ jb @row_loop // ========================================================================= - // Step 5: Final XOR — Current = R_buf XOR Z_buf [XOR Current] + // Step 5: Final XOR - Current = R_buf XOR Z_buf [XOR Current] // ========================================================================= test r9, r9 jnz @final_with_xor @@ -555,4 +555,4 @@ @epilogue: add rsp, 2056 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSsse3_x86_64.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSsse3_x86_64.inc new file mode 100644 index 00000000..0db7e32b --- /dev/null +++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSsse3_x86_64.inc @@ -0,0 +1,521 @@ +// SSSE3 implementation of Argon2 FillBlock (BlaMka round function); SSE2 +// sibling: Argon2FillBlockSse2_x86_64.inc. Unlike the sibling, the +// byte-aligned rotations (rot24/rot16) are single pshufb against masks +// resident in xmm10/xmm11; rot32 stays pshufd and rot63 stays shift+or +// (not byte-aligned). Diagonalize/Undiagonalize use shufpd (right-rotates +// via a 3-movdqa swap). +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = Left ptr, rdx = Right ptr, +// r8 = Current ptr, r9 = WithXor (0 or 1), r10 = byte-rotation masks ptr +// (ARGON2_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the +// Pascal const needs no special alignment; consumed by the entry mask +// loads, r10 is scratch afterwards). +// Each pointer addresses 128 QWords (1024 bytes). ARight and ACurrent may +// alias; R_buf buffering handles this. r8/r9 survive all loops. +// Register map (G rounds): xmm0-1 = A(0..3), xmm2-3 = B(4..7), +// xmm4-5 = C(8..11), xmm6-7 = D(12..15), xmm8-9 = temps, +// xmm10 = rot24 mask, xmm11 = rot16 mask. +// Frame (sub rsp, 2120): +// [rsp+0..63] = xmm6-xmm9 save area +// [rsp+64..1087] = R_buf (1024 bytes) +// [rsp+1088..2111] = Z_buf (1024 bytes) +// [rsp+2112..2119] = alignment padding +// Saves: xmm6-xmm11 (MS x64 non-volatile); uses xmm0-xmm11. +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: official Argon2 SSSE3 (blamka-round-opt.h). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $F3, $45, $0F, $6F, $12 // movdqu xmm10, oword [r10] + db $F3, $45, $0F, $6F, $5A, $20 // movdqu xmm11, oword [r10 + $20] + + sub rsp, 2056 + + // ========================================================================= + // Step 1: Compute R_buf = Left XOR Right, store at [rsp+64] + // ========================================================================= + lea rax, [rsp] + xor r10, r10 +@xor_loop: + movdqu xmm0, oword [rcx + r10] + movdqu xmm1, oword [rdx + r10] + pxor xmm0, xmm1 + movdqu oword [rax + r10], xmm0 + add r10, 16 + cmp r10, 1024 + jb @xor_loop + + // ========================================================================= + // Step 2: Copy R_buf to Z_buf at [rsp+1088] + // ========================================================================= + lea r11, [rsp + 1024] + xor r10, r10 +@copy_loop: + movdqu xmm0, oword [rax + r10] + movdqu oword [r11 + r10], xmm0 + add r10, 16 + cmp r10, 1024 + jb @copy_loop + + // ========================================================================= + // Step 3: Column rounds on Z_buf (8 iterations, 16 QWords = 128 bytes each) + // ========================================================================= + xor r10, r10 + +@col_loop: + movdqu xmm0, oword [r11 + r10] + movdqu xmm1, oword [r11 + r10 + $10] + movdqu xmm2, oword [r11 + r10 + $20] + movdqu xmm3, oword [r11 + r10 + $30] + movdqu xmm4, oword [r11 + r10 + $40] + movdqu xmm5, oword [r11 + r10 + $50] + movdqu xmm6, oword [r11 + r10 + $60] + movdqu xmm7, oword [r11 + r10 + $70] + + // ----- Column G: G(A0,B0,C0,D0) || G(A1,B1,C1,D1) ----- + + // fBlaMka(a, b): a = a + b + 2*lo32(a)*lo32(b) + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + // d = rotr64(d ^ a, 32) + pxor xmm6, xmm0 + pshufd xmm6, xmm6, $B1 + pxor xmm7, xmm1 + pshufd xmm7, xmm7, $B1 + + // fBlaMka(c, d) + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + // b = rotr64(b ^ c, 24) + pxor xmm2, xmm4 + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + pxor xmm3, xmm5 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // fBlaMka(a, b) + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + // d = rotr64(d ^ a, 16) + pxor xmm6, xmm0 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + // fBlaMka(c, d) + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + // b = rotr64(b ^ c, 63) + pxor xmm2, xmm4 + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + pxor xmm3, xmm5 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // ----- Diagonalize ----- + // B: rotate QWord positions left by 1 + // B0=(v4,v5)->B0'=(v5,v6), B1=(v6,v7)->B1'=(v7,v4) + movdqa xmm8, xmm2 + shufpd xmm2, xmm3, 1 + shufpd xmm3, xmm8, 1 + + // C: swap register pair c0<->c1 + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + // D: rotate QWord positions right by 1 + // D0=(v12,v13)->D0'=(v15,v12), D1=(v14,v15)->D1'=(v13,v14) + // shufpd produces results in swapped registers, so we swap afterwards + movdqa xmm8, xmm7 + shufpd xmm7, xmm6, 1 + shufpd xmm6, xmm8, 1 + movdqa xmm8, xmm6 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + // ----- Diagonal G ----- + + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + pxor xmm6, xmm0 + pshufd xmm6, xmm6, $B1 + pxor xmm7, xmm1 + pshufd xmm7, xmm7, $B1 + + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + pxor xmm2, xmm4 + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + pxor xmm3, xmm5 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + pxor xmm6, xmm0 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + pxor xmm2, xmm4 + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + pxor xmm3, xmm5 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // ----- Undiagonalize ----- + // B: rotate QWord positions right by 1 (undo left-1) + // shufpd produces results in swapped registers, so we swap afterwards + movdqa xmm8, xmm3 + shufpd xmm3, xmm2, 1 + shufpd xmm2, xmm8, 1 + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + movdqa xmm3, xmm8 + + // C: swap register pair (self-inverse) + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + // D: rotate QWord positions left by 1 (undo right-1) + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, 1 + shufpd xmm7, xmm8, 1 + + // ----- Store back to Z_buf ----- + movdqu oword [r11 + r10], xmm0 + movdqu oword [r11 + r10 + $10], xmm1 + movdqu oword [r11 + r10 + $20], xmm2 + movdqu oword [r11 + r10 + $30], xmm3 + movdqu oword [r11 + r10 + $40], xmm4 + movdqu oword [r11 + r10 + $50], xmm5 + movdqu oword [r11 + r10 + $60], xmm6 + movdqu oword [r11 + r10 + $70], xmm7 + + add r10, 128 + cmp r10, 1024 + jb @col_loop + + // ========================================================================= + // Step 4: Row rounds on Z_buf (8 iterations, stride-16 QWord access) + // ========================================================================= + xor r10, r10 + +@row_loop: + movdqu xmm0, oword [r11 + r10] + movdqu xmm1, oword [r11 + r10 + 128] + movdqu xmm2, oword [r11 + r10 + 256] + movdqu xmm3, oword [r11 + r10 + 384] + movdqu xmm4, oword [r11 + r10 + 512] + movdqu xmm5, oword [r11 + r10 + 640] + movdqu xmm6, oword [r11 + r10 + 768] + movdqu xmm7, oword [r11 + r10 + 896] + + // ----- Column G ----- + + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + pxor xmm6, xmm0 + pshufd xmm6, xmm6, $B1 + pxor xmm7, xmm1 + pshufd xmm7, xmm7, $B1 + + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + pxor xmm2, xmm4 + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + pxor xmm3, xmm5 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + pxor xmm6, xmm0 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + pxor xmm2, xmm4 + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + pxor xmm3, xmm5 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // ----- Diagonalize ----- + movdqa xmm8, xmm2 + shufpd xmm2, xmm3, 1 + shufpd xmm3, xmm8, 1 + + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm7 + shufpd xmm7, xmm6, 1 + shufpd xmm6, xmm8, 1 + movdqa xmm8, xmm6 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + // ----- Diagonal G ----- + + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + pxor xmm6, xmm0 + pshufd xmm6, xmm6, $B1 + pxor xmm7, xmm1 + pshufd xmm7, xmm7, $B1 + + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + pxor xmm2, xmm4 + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + pxor xmm3, xmm5 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + movdqa xmm8, xmm0 + pmuludq xmm8, xmm2 + psllq xmm8, 1 + paddq xmm0, xmm2 + paddq xmm0, xmm8 + movdqa xmm9, xmm1 + pmuludq xmm9, xmm3 + psllq xmm9, 1 + paddq xmm1, xmm3 + paddq xmm1, xmm9 + + pxor xmm6, xmm0 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + movdqa xmm8, xmm4 + pmuludq xmm8, xmm6 + psllq xmm8, 1 + paddq xmm4, xmm6 + paddq xmm4, xmm8 + movdqa xmm9, xmm5 + pmuludq xmm9, xmm7 + psllq xmm9, 1 + paddq xmm5, xmm7 + paddq xmm5, xmm9 + + pxor xmm2, xmm4 + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + pxor xmm3, xmm5 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // ----- Undiagonalize ----- + movdqa xmm8, xmm3 + shufpd xmm3, xmm2, 1 + shufpd xmm2, xmm8, 1 + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + movdqa xmm3, xmm8 + + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, 1 + shufpd xmm7, xmm8, 1 + + // ----- Store back to Z_buf at stride offsets ----- + movdqu oword [r11 + r10], xmm0 + movdqu oword [r11 + r10 + 128], xmm1 + movdqu oword [r11 + r10 + 256], xmm2 + movdqu oword [r11 + r10 + 384], xmm3 + movdqu oword [r11 + r10 + 512], xmm4 + movdqu oword [r11 + r10 + 640], xmm5 + movdqu oword [r11 + r10 + 768], xmm6 + movdqu oword [r11 + r10 + 896], xmm7 + + add r10, 16 + cmp r10, 128 + jb @row_loop + + // ========================================================================= + // Step 5: Final XOR - Current = R_buf XOR Z_buf [XOR Current] + // ========================================================================= + test r9, r9 + jnz @final_with_xor + + xor r10, r10 +@final_noxor_loop: + movdqu xmm0, oword [rax + r10] + movdqu xmm1, oword [r11 + r10] + pxor xmm0, xmm1 + movdqu oword [r8 + r10], xmm0 + add r10, 16 + cmp r10, 1024 + jb @final_noxor_loop + jmp @epilogue + +@final_with_xor: + xor r10, r10 +@final_xor_loop: + movdqu xmm0, oword [rax + r10] + movdqu xmm1, oword [r11 + r10] + movdqu xmm2, oword [r8 + r10] + pxor xmm0, xmm1 + pxor xmm0, xmm2 + movdqu oword [r8 + r10], xmm0 + add r10, 16 + cmp r10, 1024 + jb @final_xor_loop + +@epilogue: + add rsp, 2056 + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_i386.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_i386.inc new file mode 100644 index 00000000..366a2e9f --- /dev/null +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_i386.inc @@ -0,0 +1,976 @@ +// AVX2 implementation of BLAKE2b compress (fully unrolled 12 rounds; IA-32). +// 256-bit ymm integer ops work in 32-bit mode; the kernel fits IA-32's +// ymm0-ymm7 exactly. Same structure as Blake2BCompressAvx2_x86_64.inc, with +// the x86_64 version's memory-source vpinsrq message loads (not encodable +// in 32-bit mode) replaced 1:1 by vmovhps (same load-high-qword semantics). +// Rotations: rot32 via vpshufd, rot24/16 via vpshufb against the resident +// masks, rot63 via shift+or (not byte-aligned). Diagonalize/Undiagonalize +// via vpermq. +// ABI (after HlpSimdProc5Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = counter+flags ptr, eax = IV ptr, ecx = byte-rotation masks ptr +// (BLAKE2B_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: ymm0 = a (v0-3), ymm1 = b (v4-7), ymm2 = c (v8-11), +// ymm3 = d (v12-15), ymm4 = message temp, ymm5 = computation temp, +// ymm6 = rot24 mask, ymm7 = rot16 mask. +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: sneves/blake2-avx2 by Samuel Neves, HashLib +// Blake2BCompressAvx2_x86_64.inc. + + // Defensive save of xmm6/xmm7 (volatile on IA-32; saved like the + // SSE2 sibling to be safe under every RTL). Only the low 128 bits + // matter - the ymm uppers are dead after the trailing vzeroupper. + sub esp, $20 + db $C5, $FA, $7F, $34, $24 // vmovdqu oword [esp], xmm6 + db $C5, $FA, $7F, $7C, $24, $10 // vmovdqu oword [esp + $10], xmm7 + + + // Initialize working vector + db $C5, $FE, $6F, $03 // vmovdqu ymm0, yword [ebx] + db $C5, $FE, $6F, $4B, $20 // vmovdqu ymm1, yword [ebx + $20] + db $C5, $FE, $6F, $10 // vmovdqu ymm2, yword [eax] + db $C5, $FE, $6F, $58, $20 // vmovdqu ymm3, yword [eax + $20] + + // Load byte-rotation masks (resident for the whole kernel) + db $C5, $FE, $6F, $31 // vmovdqu ymm6, yword [ecx] + db $C5, $FE, $6F, $79, $20 // vmovdqu ymm7, yword [ecx + $20] + + // XOR d with counter and flags + db $C5, $FE, $6F, $27 // vmovdqu ymm4, yword [edi] + db $C5, $E5, $EF, $DC // vpxor ymm3, ymm3, ymm4 + + // ===== Round 0 ===== + + // G1 columns (msg: m0,m2,m4,m6) + db $C5, $FA, $7E, $26 // vmovq xmm4, qword [esi] + db $C5, $D8, $16, $66, $10 // vmovhps xmm4, xmm4, qword [esi + $10] + db $C5, $FA, $7E, $6E, $20 // vmovq xmm5, qword [esi + $20] + db $C5, $D0, $16, $6E, $30 // vmovhps xmm5, xmm5, qword [esi + $30] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m1,m3,m5,m7) + db $C5, $FA, $7E, $66, $08 // vmovq xmm4, qword [esi + $8] + db $C5, $D8, $16, $66, $18 // vmovhps xmm4, xmm4, qword [esi + $18] + db $C5, $FA, $7E, $6E, $28 // vmovq xmm5, qword [esi + $28] + db $C5, $D0, $16, $6E, $38 // vmovhps xmm5, xmm5, qword [esi + $38] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m14,m8,m10,m12) + db $C5, $FA, $7E, $66, $70 // vmovq xmm4, qword [esi + $70] + db $C5, $D8, $16, $66, $40 // vmovhps xmm4, xmm4, qword [esi + $40] + db $C5, $FA, $7E, $6E, $50 // vmovq xmm5, qword [esi + $50] + db $C5, $D0, $16, $6E, $60 // vmovhps xmm5, xmm5, qword [esi + $60] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m15,m9,m11,m13) + db $C5, $FA, $7E, $66, $78 // vmovq xmm4, qword [esi + $78] + db $C5, $D8, $16, $66, $48 // vmovhps xmm4, xmm4, qword [esi + $48] + db $C5, $FA, $7E, $6E, $58 // vmovq xmm5, qword [esi + $58] + db $C5, $D0, $16, $6E, $68 // vmovhps xmm5, xmm5, qword [esi + $68] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 1 ===== + + // G1 columns (msg: m14,m4,m9,m13) + db $C5, $FA, $7E, $66, $70 // vmovq xmm4, qword [esi + $70] + db $C5, $D8, $16, $66, $20 // vmovhps xmm4, xmm4, qword [esi + $20] + db $C5, $FA, $7E, $6E, $48 // vmovq xmm5, qword [esi + $48] + db $C5, $D0, $16, $6E, $68 // vmovhps xmm5, xmm5, qword [esi + $68] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m10,m8,m15,m6) + db $C5, $FA, $7E, $66, $50 // vmovq xmm4, qword [esi + $50] + db $C5, $D8, $16, $66, $40 // vmovhps xmm4, xmm4, qword [esi + $40] + db $C5, $FA, $7E, $6E, $78 // vmovq xmm5, qword [esi + $78] + db $C5, $D0, $16, $6E, $30 // vmovhps xmm5, xmm5, qword [esi + $30] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m5,m1,m0,m11) + db $C5, $FA, $7E, $66, $28 // vmovq xmm4, qword [esi + $28] + db $C5, $D8, $16, $66, $08 // vmovhps xmm4, xmm4, qword [esi + $8] + db $C5, $FA, $7E, $2E // vmovq xmm5, qword [esi] + db $C5, $D0, $16, $6E, $58 // vmovhps xmm5, xmm5, qword [esi + $58] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m3,m12,m2,m7) + db $C5, $FA, $7E, $66, $18 // vmovq xmm4, qword [esi + $18] + db $C5, $D8, $16, $66, $60 // vmovhps xmm4, xmm4, qword [esi + $60] + db $C5, $FA, $7E, $6E, $10 // vmovq xmm5, qword [esi + $10] + db $C5, $D0, $16, $6E, $38 // vmovhps xmm5, xmm5, qword [esi + $38] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 2 ===== + + // G1 columns (msg: m11,m12,m5,m15) + db $C5, $FA, $7E, $66, $58 // vmovq xmm4, qword [esi + $58] + db $C5, $D8, $16, $66, $60 // vmovhps xmm4, xmm4, qword [esi + $60] + db $C5, $FA, $7E, $6E, $28 // vmovq xmm5, qword [esi + $28] + db $C5, $D0, $16, $6E, $78 // vmovhps xmm5, xmm5, qword [esi + $78] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m8,m0,m2,m13) + db $C5, $FA, $7E, $66, $40 // vmovq xmm4, qword [esi + $40] + db $C5, $D8, $16, $26 // vmovhps xmm4, xmm4, qword [esi] + db $C5, $FA, $7E, $6E, $10 // vmovq xmm5, qword [esi + $10] + db $C5, $D0, $16, $6E, $68 // vmovhps xmm5, xmm5, qword [esi + $68] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m9,m10,m3,m7) + db $C5, $FA, $7E, $66, $48 // vmovq xmm4, qword [esi + $48] + db $C5, $D8, $16, $66, $50 // vmovhps xmm4, xmm4, qword [esi + $50] + db $C5, $FA, $7E, $6E, $18 // vmovq xmm5, qword [esi + $18] + db $C5, $D0, $16, $6E, $38 // vmovhps xmm5, xmm5, qword [esi + $38] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m4,m14,m6,m1) + db $C5, $FA, $7E, $66, $20 // vmovq xmm4, qword [esi + $20] + db $C5, $D8, $16, $66, $70 // vmovhps xmm4, xmm4, qword [esi + $70] + db $C5, $FA, $7E, $6E, $30 // vmovq xmm5, qword [esi + $30] + db $C5, $D0, $16, $6E, $08 // vmovhps xmm5, xmm5, qword [esi + $8] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 3 ===== + + // G1 columns (msg: m7,m3,m13,m11) + db $C5, $FA, $7E, $66, $38 // vmovq xmm4, qword [esi + $38] + db $C5, $D8, $16, $66, $18 // vmovhps xmm4, xmm4, qword [esi + $18] + db $C5, $FA, $7E, $6E, $68 // vmovq xmm5, qword [esi + $68] + db $C5, $D0, $16, $6E, $58 // vmovhps xmm5, xmm5, qword [esi + $58] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m9,m1,m12,m14) + db $C5, $FA, $7E, $66, $48 // vmovq xmm4, qword [esi + $48] + db $C5, $D8, $16, $66, $08 // vmovhps xmm4, xmm4, qword [esi + $8] + db $C5, $FA, $7E, $6E, $60 // vmovq xmm5, qword [esi + $60] + db $C5, $D0, $16, $6E, $70 // vmovhps xmm5, xmm5, qword [esi + $70] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m15,m2,m5,m4) + db $C5, $FA, $7E, $66, $78 // vmovq xmm4, qword [esi + $78] + db $C5, $D8, $16, $66, $10 // vmovhps xmm4, xmm4, qword [esi + $10] + db $C5, $FA, $7E, $6E, $28 // vmovq xmm5, qword [esi + $28] + db $C5, $D0, $16, $6E, $20 // vmovhps xmm5, xmm5, qword [esi + $20] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m8,m6,m10,m0) + db $C5, $FA, $7E, $66, $40 // vmovq xmm4, qword [esi + $40] + db $C5, $D8, $16, $66, $30 // vmovhps xmm4, xmm4, qword [esi + $30] + db $C5, $FA, $7E, $6E, $50 // vmovq xmm5, qword [esi + $50] + db $C5, $D0, $16, $2E // vmovhps xmm5, xmm5, qword [esi] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 4 ===== + + // G1 columns (msg: m9,m5,m2,m10) + db $C5, $FA, $7E, $66, $48 // vmovq xmm4, qword [esi + $48] + db $C5, $D8, $16, $66, $28 // vmovhps xmm4, xmm4, qword [esi + $28] + db $C5, $FA, $7E, $6E, $10 // vmovq xmm5, qword [esi + $10] + db $C5, $D0, $16, $6E, $50 // vmovhps xmm5, xmm5, qword [esi + $50] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m0,m7,m4,m15) + db $C5, $FA, $7E, $26 // vmovq xmm4, qword [esi] + db $C5, $D8, $16, $66, $38 // vmovhps xmm4, xmm4, qword [esi + $38] + db $C5, $FA, $7E, $6E, $20 // vmovq xmm5, qword [esi + $20] + db $C5, $D0, $16, $6E, $78 // vmovhps xmm5, xmm5, qword [esi + $78] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m3,m14,m11,m6) + db $C5, $FA, $7E, $66, $18 // vmovq xmm4, qword [esi + $18] + db $C5, $D8, $16, $66, $70 // vmovhps xmm4, xmm4, qword [esi + $70] + db $C5, $FA, $7E, $6E, $58 // vmovq xmm5, qword [esi + $58] + db $C5, $D0, $16, $6E, $30 // vmovhps xmm5, xmm5, qword [esi + $30] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m13,m1,m12,m8) + db $C5, $FA, $7E, $66, $68 // vmovq xmm4, qword [esi + $68] + db $C5, $D8, $16, $66, $08 // vmovhps xmm4, xmm4, qword [esi + $8] + db $C5, $FA, $7E, $6E, $60 // vmovq xmm5, qword [esi + $60] + db $C5, $D0, $16, $6E, $40 // vmovhps xmm5, xmm5, qword [esi + $40] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 5 ===== + + // G1 columns (msg: m2,m6,m0,m8) + db $C5, $FA, $7E, $66, $10 // vmovq xmm4, qword [esi + $10] + db $C5, $D8, $16, $66, $30 // vmovhps xmm4, xmm4, qword [esi + $30] + db $C5, $FA, $7E, $2E // vmovq xmm5, qword [esi] + db $C5, $D0, $16, $6E, $40 // vmovhps xmm5, xmm5, qword [esi + $40] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m12,m10,m11,m3) + db $C5, $FA, $7E, $66, $60 // vmovq xmm4, qword [esi + $60] + db $C5, $D8, $16, $66, $50 // vmovhps xmm4, xmm4, qword [esi + $50] + db $C5, $FA, $7E, $6E, $58 // vmovq xmm5, qword [esi + $58] + db $C5, $D0, $16, $6E, $18 // vmovhps xmm5, xmm5, qword [esi + $18] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m1,m4,m7,m15) + db $C5, $FA, $7E, $66, $08 // vmovq xmm4, qword [esi + $8] + db $C5, $D8, $16, $66, $20 // vmovhps xmm4, xmm4, qword [esi + $20] + db $C5, $FA, $7E, $6E, $38 // vmovq xmm5, qword [esi + $38] + db $C5, $D0, $16, $6E, $78 // vmovhps xmm5, xmm5, qword [esi + $78] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m9,m13,m5,m14) + db $C5, $FA, $7E, $66, $48 // vmovq xmm4, qword [esi + $48] + db $C5, $D8, $16, $66, $68 // vmovhps xmm4, xmm4, qword [esi + $68] + db $C5, $FA, $7E, $6E, $28 // vmovq xmm5, qword [esi + $28] + db $C5, $D0, $16, $6E, $70 // vmovhps xmm5, xmm5, qword [esi + $70] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 6 ===== + + // G1 columns (msg: m12,m1,m14,m4) + db $C5, $FA, $7E, $66, $60 // vmovq xmm4, qword [esi + $60] + db $C5, $D8, $16, $66, $08 // vmovhps xmm4, xmm4, qword [esi + $8] + db $C5, $FA, $7E, $6E, $70 // vmovq xmm5, qword [esi + $70] + db $C5, $D0, $16, $6E, $20 // vmovhps xmm5, xmm5, qword [esi + $20] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m5,m15,m13,m10) + db $C5, $FA, $7E, $66, $28 // vmovq xmm4, qword [esi + $28] + db $C5, $D8, $16, $66, $78 // vmovhps xmm4, xmm4, qword [esi + $78] + db $C5, $FA, $7E, $6E, $68 // vmovq xmm5, qword [esi + $68] + db $C5, $D0, $16, $6E, $50 // vmovhps xmm5, xmm5, qword [esi + $50] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m8,m0,m6,m9) + db $C5, $FA, $7E, $66, $40 // vmovq xmm4, qword [esi + $40] + db $C5, $D8, $16, $26 // vmovhps xmm4, xmm4, qword [esi] + db $C5, $FA, $7E, $6E, $30 // vmovq xmm5, qword [esi + $30] + db $C5, $D0, $16, $6E, $48 // vmovhps xmm5, xmm5, qword [esi + $48] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m11,m7,m3,m2) + db $C5, $FA, $7E, $66, $58 // vmovq xmm4, qword [esi + $58] + db $C5, $D8, $16, $66, $38 // vmovhps xmm4, xmm4, qword [esi + $38] + db $C5, $FA, $7E, $6E, $18 // vmovq xmm5, qword [esi + $18] + db $C5, $D0, $16, $6E, $10 // vmovhps xmm5, xmm5, qword [esi + $10] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 7 ===== + + // G1 columns (msg: m13,m7,m12,m3) + db $C5, $FA, $7E, $66, $68 // vmovq xmm4, qword [esi + $68] + db $C5, $D8, $16, $66, $38 // vmovhps xmm4, xmm4, qword [esi + $38] + db $C5, $FA, $7E, $6E, $60 // vmovq xmm5, qword [esi + $60] + db $C5, $D0, $16, $6E, $18 // vmovhps xmm5, xmm5, qword [esi + $18] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m11,m14,m1,m9) + db $C5, $FA, $7E, $66, $58 // vmovq xmm4, qword [esi + $58] + db $C5, $D8, $16, $66, $70 // vmovhps xmm4, xmm4, qword [esi + $70] + db $C5, $FA, $7E, $6E, $08 // vmovq xmm5, qword [esi + $8] + db $C5, $D0, $16, $6E, $48 // vmovhps xmm5, xmm5, qword [esi + $48] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m2,m5,m15,m8) + db $C5, $FA, $7E, $66, $10 // vmovq xmm4, qword [esi + $10] + db $C5, $D8, $16, $66, $28 // vmovhps xmm4, xmm4, qword [esi + $28] + db $C5, $FA, $7E, $6E, $78 // vmovq xmm5, qword [esi + $78] + db $C5, $D0, $16, $6E, $40 // vmovhps xmm5, xmm5, qword [esi + $40] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m10,m0,m4,m6) + db $C5, $FA, $7E, $66, $50 // vmovq xmm4, qword [esi + $50] + db $C5, $D8, $16, $26 // vmovhps xmm4, xmm4, qword [esi] + db $C5, $FA, $7E, $6E, $20 // vmovq xmm5, qword [esi + $20] + db $C5, $D0, $16, $6E, $30 // vmovhps xmm5, xmm5, qword [esi + $30] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 8 ===== + + // G1 columns (msg: m6,m14,m11,m0) + db $C5, $FA, $7E, $66, $30 // vmovq xmm4, qword [esi + $30] + db $C5, $D8, $16, $66, $70 // vmovhps xmm4, xmm4, qword [esi + $70] + db $C5, $FA, $7E, $6E, $58 // vmovq xmm5, qword [esi + $58] + db $C5, $D0, $16, $2E // vmovhps xmm5, xmm5, qword [esi] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m15,m9,m3,m8) + db $C5, $FA, $7E, $66, $78 // vmovq xmm4, qword [esi + $78] + db $C5, $D8, $16, $66, $48 // vmovhps xmm4, xmm4, qword [esi + $48] + db $C5, $FA, $7E, $6E, $18 // vmovq xmm5, qword [esi + $18] + db $C5, $D0, $16, $6E, $40 // vmovhps xmm5, xmm5, qword [esi + $40] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m10,m12,m13,m1) + db $C5, $FA, $7E, $66, $50 // vmovq xmm4, qword [esi + $50] + db $C5, $D8, $16, $66, $60 // vmovhps xmm4, xmm4, qword [esi + $60] + db $C5, $FA, $7E, $6E, $68 // vmovq xmm5, qword [esi + $68] + db $C5, $D0, $16, $6E, $08 // vmovhps xmm5, xmm5, qword [esi + $8] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m5,m2,m7,m4) + db $C5, $FA, $7E, $66, $28 // vmovq xmm4, qword [esi + $28] + db $C5, $D8, $16, $66, $10 // vmovhps xmm4, xmm4, qword [esi + $10] + db $C5, $FA, $7E, $6E, $38 // vmovq xmm5, qword [esi + $38] + db $C5, $D0, $16, $6E, $20 // vmovhps xmm5, xmm5, qword [esi + $20] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 9 ===== + + // G1 columns (msg: m10,m8,m7,m1) + db $C5, $FA, $7E, $66, $50 // vmovq xmm4, qword [esi + $50] + db $C5, $D8, $16, $66, $40 // vmovhps xmm4, xmm4, qword [esi + $40] + db $C5, $FA, $7E, $6E, $38 // vmovq xmm5, qword [esi + $38] + db $C5, $D0, $16, $6E, $08 // vmovhps xmm5, xmm5, qword [esi + $8] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m2,m4,m6,m5) + db $C5, $FA, $7E, $66, $10 // vmovq xmm4, qword [esi + $10] + db $C5, $D8, $16, $66, $20 // vmovhps xmm4, xmm4, qword [esi + $20] + db $C5, $FA, $7E, $6E, $30 // vmovq xmm5, qword [esi + $30] + db $C5, $D0, $16, $6E, $28 // vmovhps xmm5, xmm5, qword [esi + $28] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m13,m15,m9,m3) + db $C5, $FA, $7E, $66, $68 // vmovq xmm4, qword [esi + $68] + db $C5, $D8, $16, $66, $78 // vmovhps xmm4, xmm4, qword [esi + $78] + db $C5, $FA, $7E, $6E, $48 // vmovq xmm5, qword [esi + $48] + db $C5, $D0, $16, $6E, $18 // vmovhps xmm5, xmm5, qword [esi + $18] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m0,m11,m14,m12) + db $C5, $FA, $7E, $26 // vmovq xmm4, qword [esi] + db $C5, $D8, $16, $66, $58 // vmovhps xmm4, xmm4, qword [esi + $58] + db $C5, $FA, $7E, $6E, $70 // vmovq xmm5, qword [esi + $70] + db $C5, $D0, $16, $6E, $60 // vmovhps xmm5, xmm5, qword [esi + $60] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 10 ===== + + // G1 columns (msg: m0,m2,m4,m6) + db $C5, $FA, $7E, $26 // vmovq xmm4, qword [esi] + db $C5, $D8, $16, $66, $10 // vmovhps xmm4, xmm4, qword [esi + $10] + db $C5, $FA, $7E, $6E, $20 // vmovq xmm5, qword [esi + $20] + db $C5, $D0, $16, $6E, $30 // vmovhps xmm5, xmm5, qword [esi + $30] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m1,m3,m5,m7) + db $C5, $FA, $7E, $66, $08 // vmovq xmm4, qword [esi + $8] + db $C5, $D8, $16, $66, $18 // vmovhps xmm4, xmm4, qword [esi + $18] + db $C5, $FA, $7E, $6E, $28 // vmovq xmm5, qword [esi + $28] + db $C5, $D0, $16, $6E, $38 // vmovhps xmm5, xmm5, qword [esi + $38] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m14,m8,m10,m12) + db $C5, $FA, $7E, $66, $70 // vmovq xmm4, qword [esi + $70] + db $C5, $D8, $16, $66, $40 // vmovhps xmm4, xmm4, qword [esi + $40] + db $C5, $FA, $7E, $6E, $50 // vmovq xmm5, qword [esi + $50] + db $C5, $D0, $16, $6E, $60 // vmovhps xmm5, xmm5, qword [esi + $60] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m15,m9,m11,m13) + db $C5, $FA, $7E, $66, $78 // vmovq xmm4, qword [esi + $78] + db $C5, $D8, $16, $66, $48 // vmovhps xmm4, xmm4, qword [esi + $48] + db $C5, $FA, $7E, $6E, $58 // vmovq xmm5, qword [esi + $58] + db $C5, $D0, $16, $6E, $68 // vmovhps xmm5, xmm5, qword [esi + $68] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // ===== Round 11 ===== + + // G1 columns (msg: m14,m4,m9,m13) + db $C5, $FA, $7E, $66, $70 // vmovq xmm4, qword [esi + $70] + db $C5, $D8, $16, $66, $20 // vmovhps xmm4, xmm4, qword [esi + $20] + db $C5, $FA, $7E, $6E, $48 // vmovq xmm5, qword [esi + $48] + db $C5, $D0, $16, $6E, $68 // vmovhps xmm5, xmm5, qword [esi + $68] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 columns (msg: m10,m8,m15,m6) + db $C5, $FA, $7E, $66, $50 // vmovq xmm4, qword [esi + $50] + db $C5, $D8, $16, $66, $40 // vmovhps xmm4, xmm4, qword [esi + $40] + db $C5, $FA, $7E, $6E, $78 // vmovq xmm5, qword [esi + $78] + db $C5, $D0, $16, $6E, $30 // vmovhps xmm5, xmm5, qword [esi + $30] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Diagonalize + db $C4, $E3, $FD, $00, $C0, $93 // vpermq ymm0, ymm0, $93 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $39 // vpermq ymm2, ymm2, $39 + + // G1 diagonals (msg: m5,m1,m0,m11) + db $C5, $FA, $7E, $66, $28 // vmovq xmm4, qword [esi + $28] + db $C5, $D8, $16, $66, $08 // vmovhps xmm4, xmm4, qword [esi + $8] + db $C5, $FA, $7E, $2E // vmovq xmm5, qword [esi] + db $C5, $D0, $16, $6E, $58 // vmovhps xmm5, xmm5, qword [esi + $58] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C5, $FD, $70, $DB, $B1 // vpshufd ymm3, ymm3, $B1 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 + + // G2 diagonals (msg: m3,m12,m2,m7) + db $C5, $FA, $7E, $66, $18 // vmovq xmm4, qword [esi + $18] + db $C5, $D8, $16, $66, $60 // vmovhps xmm4, xmm4, qword [esi + $60] + db $C5, $FA, $7E, $6E, $10 // vmovq xmm5, qword [esi + $10] + db $C5, $D0, $16, $6E, $38 // vmovhps xmm5, xmm5, qword [esi + $38] + db $C4, $E3, $5D, $38, $E5, $01 // vinserti128 ymm4, ymm4, xmm5, $1 + db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 + + db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 + db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 + db $C5, $D5, $73, $D1, $3F // vpsrlq ymm5, ymm1, $3F + db $C5, $F5, $D4, $C9 // vpaddq ymm1, ymm1, ymm1 + db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + + // Undiagonalize + db $C4, $E3, $FD, $00, $C0, $39 // vpermq ymm0, ymm0, $39 + db $C4, $E3, $FD, $00, $DB, $4E // vpermq ymm3, ymm3, $4E + db $C4, $E3, $FD, $00, $D2, $93 // vpermq ymm2, ymm2, $93 + + // Finalization + db $C5, $FD, $EF, $C2 // vpxor ymm0, ymm0, ymm2 + db $C5, $F5, $EF, $CB // vpxor ymm1, ymm1, ymm3 + + // XOR with original state (re-read from memory) + db $C5, $FD, $EF, $03 // vpxor ymm0, ymm0, yword [ebx] + db $C5, $F5, $EF, $4B, $20 // vpxor ymm1, ymm1, yword [ebx + $20] + + // Store result + db $C5, $FE, $7F, $03 // vmovdqu yword [ebx], ymm0 + db $C5, $FE, $7F, $4B, $20 // vmovdqu yword [ebx + $20], ymm1 + + db $C5, $F8, $77 // vzeroupper + + + + db $C5, $FA, $6F, $34, $24 // vmovdqu xmm6, oword [esp] + db $C5, $FA, $6F, $7C, $24, $10 // vmovdqu xmm7, oword [esp + $10] + add esp, $20 + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_x86_64.inc index d507afc4..bb79a196 100644 --- a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressAvx2_x86_64.inc @@ -1,19 +1,31 @@ // AVX2 implementation of BLAKE2b compress (fully unrolled 12 rounds). -// Expects MS x64 ABI: rcx = state ptr, rdx = msg ptr, r8 = counter+flags ptr, r9 = IV ptr. -// Uses ymm0-ymm5 only (volatile under both ABIs; no saves needed). -// Register map: ymm0 = a (v0-3), ymm1 = b (v4-7), ymm2 = c (v8-11), ymm3 = d (v12-15), -// ymm4 = message temp, ymm5 = computation temp. -// Rotations: ROT32 via vpshufd, ROT16/24/63 via shift+or. -// Diagonalize/Undiagonalize via vpermq. +// Rotations: rot32 via vpshufd, rot24/16 via vpshufb against the resident +// masks, rot63 via shift+or (not byte-aligned; shifts are the known-best +// form). Diagonalize/Undiagonalize via vpermq. +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = counter+flags ptr, r9 = IV ptr, r10 = byte-rotation masks ptr +// (BLAKE2B_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: ymm0 = a (v0-3), ymm1 = b (v4-7), ymm2 = c (v8-11), +// ymm3 = d (v12-15), ymm4 = message temp, ymm5 = computation temp, +// ymm6 = rot24 mask, ymm7 = rot16 mask. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV; only xmm6/xmm7 are clobbered here - by the mask loads). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. // Reference: sneves/blake2-avx2 by Samuel Neves. + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + // Initialize working vector db $C5, $FE, $6F, $01 // vmovdqu ymm0, yword [rcx] db $C5, $FE, $6F, $49, $20 // vmovdqu ymm1, yword [rcx + $20] db $C4, $C1, $7E, $6F, $11 // vmovdqu ymm2, yword [r9] db $C4, $C1, $7E, $6F, $59, $20 // vmovdqu ymm3, yword [r9 + $20] + // Load byte-rotation masks (resident for the whole kernel) + db $C4, $C1, $7E, $6F, $32 // vmovdqu ymm6, yword [r10] + db $C4, $C1, $7E, $6F, $7A, $20 // vmovdqu ymm7, yword [r10 + $20] + // XOR d with counter and flags db $C4, $C1, $7E, $6F, $20 // vmovdqu ymm4, yword [r8] db $C5, $E5, $EF, $DC // vpxor ymm3, ymm3, ymm4 @@ -33,9 +45,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m1,m3,m5,m7) db $C4, $E1, $F9, $6E, $62, $08 // vmovq xmm4, qword [rdx + $08] @@ -46,9 +56,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -74,9 +82,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m15,m9,m11,m13) db $C4, $E1, $F9, $6E, $62, $78 // vmovq xmm4, qword [rdx + $78] @@ -87,9 +93,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -117,9 +121,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m10,m8,m15,m6) db $C4, $E1, $F9, $6E, $62, $50 // vmovq xmm4, qword [rdx + $50] @@ -130,9 +132,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -158,9 +158,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m3,m12,m2,m7) db $C4, $E1, $F9, $6E, $62, $18 // vmovq xmm4, qword [rdx + $18] @@ -171,9 +169,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -201,9 +197,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m8,m0,m2,m13) db $C4, $E1, $F9, $6E, $62, $40 // vmovq xmm4, qword [rdx + $40] @@ -214,9 +208,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -242,9 +234,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m4,m14,m6,m1) db $C4, $E1, $F9, $6E, $62, $20 // vmovq xmm4, qword [rdx + $20] @@ -255,9 +245,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -285,9 +273,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m9,m1,m12,m14) db $C4, $E1, $F9, $6E, $62, $48 // vmovq xmm4, qword [rdx + $48] @@ -298,9 +284,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -326,9 +310,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m8,m6,m10,m0) db $C4, $E1, $F9, $6E, $62, $40 // vmovq xmm4, qword [rdx + $40] @@ -339,9 +321,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -369,9 +349,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m0,m7,m4,m15) db $C4, $E1, $F9, $6E, $22 // vmovq xmm4, qword [rdx] @@ -382,9 +360,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -410,9 +386,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m13,m1,m12,m8) db $C4, $E1, $F9, $6E, $62, $68 // vmovq xmm4, qword [rdx + $68] @@ -423,9 +397,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -453,9 +425,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m12,m10,m11,m3) db $C4, $E1, $F9, $6E, $62, $60 // vmovq xmm4, qword [rdx + $60] @@ -466,9 +436,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -494,9 +462,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m9,m13,m5,m14) db $C4, $E1, $F9, $6E, $62, $48 // vmovq xmm4, qword [rdx + $48] @@ -507,9 +473,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -537,9 +501,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m5,m15,m13,m10) db $C4, $E1, $F9, $6E, $62, $28 // vmovq xmm4, qword [rdx + $28] @@ -550,9 +512,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -578,9 +538,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m11,m7,m3,m2) db $C4, $E1, $F9, $6E, $62, $58 // vmovq xmm4, qword [rdx + $58] @@ -591,9 +549,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -621,9 +577,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m11,m14,m1,m9) db $C4, $E1, $F9, $6E, $62, $58 // vmovq xmm4, qword [rdx + $58] @@ -634,9 +588,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -662,9 +614,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m10,m0,m4,m6) db $C4, $E1, $F9, $6E, $62, $50 // vmovq xmm4, qword [rdx + $50] @@ -675,9 +625,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -705,9 +653,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m15,m9,m3,m8) db $C4, $E1, $F9, $6E, $62, $78 // vmovq xmm4, qword [rdx + $78] @@ -718,9 +664,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -746,9 +690,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m5,m2,m7,m4) db $C4, $E1, $F9, $6E, $62, $28 // vmovq xmm4, qword [rdx + $28] @@ -759,9 +701,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -789,9 +729,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m2,m4,m6,m5) db $C4, $E1, $F9, $6E, $62, $10 // vmovq xmm4, qword [rdx + $10] @@ -802,9 +740,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -830,9 +766,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m0,m11,m14,m12) db $C4, $E1, $F9, $6E, $22 // vmovq xmm4, qword [rdx] @@ -843,9 +777,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -873,9 +805,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m1,m3,m5,m7) db $C4, $E1, $F9, $6E, $62, $08 // vmovq xmm4, qword [rdx + $08] @@ -886,9 +816,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -914,9 +842,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m15,m9,m11,m13) db $C4, $E1, $F9, $6E, $62, $78 // vmovq xmm4, qword [rdx + $78] @@ -927,9 +853,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -957,9 +881,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 columns (msg: m10,m8,m15,m6) db $C4, $E1, $F9, $6E, $62, $50 // vmovq xmm4, qword [rdx + $50] @@ -970,9 +892,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -998,9 +918,7 @@ db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 - db $C5, $D5, $73, $D1, $18 // vpsrlq ymm5, ymm1, 24 - db $C5, $F5, $73, $F1, $28 // vpsllq ymm1, ymm1, 40 - db $C5, $F5, $EB, $CD // vpor ymm1, ymm1, ymm5 + db $C4, $E2, $75, $00, $CE // vpshufb ymm1, ymm1, ymm6 // G2 diagonals (msg: m3,m12,m2,m7) db $C4, $E1, $F9, $6E, $62, $18 // vmovq xmm4, qword [rdx + $18] @@ -1011,9 +929,7 @@ db $C5, $FD, $D4, $C4 // vpaddq ymm0, ymm0, ymm4 db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 db $C5, $E5, $EF, $D8 // vpxor ymm3, ymm3, ymm0 - db $C5, $D5, $73, $D3, $10 // vpsrlq ymm5, ymm3, 16 - db $C5, $E5, $73, $F3, $30 // vpsllq ymm3, ymm3, 48 - db $C5, $E5, $EB, $DD // vpor ymm3, ymm3, ymm5 + db $C4, $E2, $65, $00, $DF // vpshufb ymm3, ymm3, ymm7 db $C5, $ED, $D4, $D3 // vpaddq ymm2, ymm2, ymm3 db $C5, $F5, $EF, $CA // vpxor ymm1, ymm1, ymm2 @@ -1039,3 +955,5 @@ db $C5, $FE, $7F, $49, $20 // vmovdqu yword [rcx + $20], ymm1 db $C5, $F8, $77 // vzeroupper + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc index c72e55d7..08855b8f 100644 --- a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressNeon_aarch64.inc @@ -1,9 +1,8 @@ -// BLAKE2b compress AArch64 NEON implementation. -// Expects AAPCS64: x0 = state ptr (8 x UInt64), x1 = message ptr (128 bytes), -// x2 = counter+flags ptr (32 bytes: t[0..1], f[0..1]), x3 = IV ptr (8 x UInt64). -// Leaf nostackframe; uses v8-v15 for message block (d8-d15 lanes; no save/restore). -// Reference: BLAKE2/BLAKE2 neon/blake2b-neon.c, HashLib Blake2BCompressSse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of BLAKE2b compress (fully unrolled 12 rounds). +// u64 ror-by-32 uses rev64.4s (vrev64q_u32), not rev64.16b. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (8 x UInt64), +// x1 = message ptr (128 bytes), x2 = counter+flags ptr (32 bytes: t[0..1], +// f[0..1]), x3 = IV ptr (8 x UInt64). // Register map: // v0, v1 = row1l, row1h (h[0..3] working, 2 x u64 each) // v2, v3 = row2l, row2h (h[4..7] working) @@ -12,10 +11,15 @@ // v8-v15 = message block (16 x UInt64) // v16-v19 = saved initial row1l/h, row2l/h for the final feed-forward XOR // v20 = message gather temp b0 (G function) -// v21-v23 = rotate / diagonalize temps (b1 gather uses v23) -// u64 ror-by-32 uses rev64.4s (vrev64q_u32), not rev64.16b. +// v21-v23 = rotate / diagonalize temps (2-op insert rotates rename the +// rotated row halves through v21/v22; b1 gather uses v23) +// Saves: d8-d15 via the shared HlpSimdNonVolatileSave/Restore include +// (v8-v15 hold the message block). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 neon/blake2b-neon.c, HashLib +// Blake2BCompressSse2_x86_64.inc. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} .long 0xad400400 // ldp q0, q1, [x0] .long 0xad410c02 // ldp q2, q3, [x0, #32] @@ -49,94 +53,82 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e084514 // ins v20.d[0], v8.d[1] .long 0x6e184534 // ins v20.d[1], v9.d[1] .long 0x6e084557 // ins v23.d[0], v10.d[1] .long 0x6e184577 // ins v23.d[1], v11.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e080594 // ins v20.d[0], v12.d[0] .long 0x6e1805b4 // ins v20.d[1], v13.d[0] .long 0x6e0805d7 // ins v23.d[0], v14.d[0] .long 0x6e1805f7 // ins v23.d[1], v15.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e084594 // ins v20.d[0], v12.d[1] .long 0x6e1845b4 // ins v20.d[1], v13.d[1] .long 0x6e0845d7 // ins v23.d[0], v14.d[1] .long 0x6e1845f7 // ins v23.d[1], v15.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -163,92 +155,80 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e0805b4 // ins v20.d[0], v13.d[0] .long 0x6e180594 // ins v20.d[1], v12.d[0] .long 0x6e0b41f7 // ext v23.16b, v15.16b, v11.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e084114 // ext v20.16b, v8.16b, v8.16b, #8 .long 0x6e0845b7 // ins v23.d[0], v13.d[1] .long 0x6e184557 // ins v23.d[1], v10.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e0805d4 // ins v20.d[0], v14.d[0] .long 0x6e180534 // ins v20.d[1], v9.d[0] .long 0x6e084577 // ins v23.d[0], v11.d[1] .long 0x6e184537 // ins v23.d[1], v9.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -274,93 +254,81 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e080594 // ins v20.d[0], v12.d[0] .long 0x6e180514 // ins v20.d[1], v8.d[0] .long 0x6e080537 // ins v23.d[0], v9.d[0] .long 0x6e1845d7 // ins v23.d[1], v14.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e0805b4 // ins v20.d[0], v13.d[0] .long 0x6e184534 // ins v20.d[1], v9.d[1] .long 0x6e084577 // ins v23.d[0], v11.d[1] .long 0x6e184597 // ins v23.d[1], v12.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e0805f4 // ins v20.d[0], v15.d[0] .long 0x6e180574 // ins v20.d[1], v11.d[0] .long 0x6e0a4117 // ext v23.16b, v8.16b, v10.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -387,94 +355,82 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e084594 // ins v20.d[0], v12.d[1] .long 0x6e184514 // ins v20.d[1], v8.d[1] .long 0x6e0805d7 // ins v23.d[0], v14.d[0] .long 0x6e1805f7 // ins v23.d[1], v15.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e080534 // ins v20.d[0], v9.d[0] .long 0x6e184554 // ins v20.d[1], v10.d[1] .long 0x6e080557 // ins v23.d[0], v10.d[0] .long 0x6e1845f7 // ins v23.d[1], v15.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e080574 // ins v20.d[0], v11.d[0] .long 0x6e1805b4 // ins v20.d[1], v13.d[0] .long 0x6e080517 // ins v23.d[0], v8.d[0] .long 0x6e180597 // ins v23.d[1], v12.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -501,93 +457,81 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e080514 // ins v20.d[0], v8.d[0] .long 0x6e184574 // ins v20.d[1], v11.d[1] .long 0x6e080557 // ins v23.d[0], v10.d[0] .long 0x6e1845f7 // ins v23.d[1], v15.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e0805f4 // ins v20.d[0], v15.d[0] .long 0x6e1845b4 // ins v20.d[1], v13.d[1] .long 0x6e080577 // ins v23.d[0], v11.d[0] .long 0x6e184537 // ins v23.d[1], v9.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e0e4114 // ext v20.16b, v8.16b, v14.16b, #8 .long 0x6e080597 // ins v23.d[0], v12.d[0] .long 0x6e1845d7 // ins v23.d[1], v14.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -614,94 +558,82 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e0805d4 // ins v20.d[0], v14.d[0] .long 0x6e1805b4 // ins v20.d[1], v13.d[0] .long 0x6e0845b7 // ins v23.d[0], v13.d[1] .long 0x6e184537 // ins v23.d[1], v9.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e080554 // ins v20.d[0], v10.d[0] .long 0x6e184574 // ins v20.d[1], v11.d[1] .long 0x6e0845f7 // ins v23.d[0], v15.d[1] .long 0x6e184517 // ins v23.d[1], v8.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e0845d4 // ins v20.d[0], v14.d[1] .long 0x6e184554 // ins v20.d[1], v10.d[1] .long 0x6e0805f7 // ins v23.d[0], v15.d[0] .long 0x6e184597 // ins v23.d[1], v12.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -728,92 +660,80 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e084554 // ins v20.d[0], v10.d[1] .long 0x6e1845f4 // ins v20.d[1], v15.d[1] .long 0x6e0d41d7 // ext v23.16b, v14.16b, v13.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e080514 // ins v20.d[0], v8.d[0] .long 0x6e180574 // ins v20.d[1], v11.d[0] .long 0x6e0c4197 // ext v23.16b, v12.16b, v12.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e084574 // ins v20.d[0], v11.d[1] .long 0x6e184534 // ins v20.d[1], v9.d[1] .long 0x6e080537 // ins v23.d[0], v9.d[0] .long 0x6e1845b7 // ins v23.d[1], v13.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -840,93 +760,81 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e0f41b4 // ext v20.16b, v13.16b, v15.16b, #8 .long 0x6e084517 // ins v23.d[0], v8.d[1] .long 0x6e184597 // ins v23.d[1], v12.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e084554 // ins v20.d[0], v10.d[1] .long 0x6e1845f4 // ins v20.d[1], v15.d[1] .long 0x6e080597 // ins v23.d[0], v12.d[0] .long 0x6e180537 // ins v23.d[1], v9.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e080514 // ins v20.d[0], v8.d[0] .long 0x6e180554 // ins v20.d[1], v10.d[0] .long 0x6e080577 // ins v23.d[0], v11.d[0] .long 0x6e1805b7 // ins v23.d[1], v13.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -952,90 +860,78 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e0845f4 // ins v20.d[0], v15.d[1] .long 0x6e184594 // ins v20.d[1], v12.d[1] .long 0x6e0c4137 // ext v23.16b, v9.16b, v12.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x4eae1dd4 // orr v20.16b, v14.16b, v14.16b .long 0x6e0d4117 // ext v23.16b, v8.16b, v13.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e080534 // ins v20.d[0], v9.d[0] .long 0x6e184574 // ins v20.d[1], v11.d[1] .long 0x4eaa1d57 // orr v23.16b, v10.16b, v10.16b - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -1062,93 +958,81 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e080534 // ins v20.d[0], v9.d[0] .long 0x6e180554 // ins v20.d[1], v10.d[0] .long 0x6e080577 // ins v23.d[0], v11.d[0] .long 0x6e184557 // ins v23.d[1], v10.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e0845f4 // ins v20.d[0], v15.d[1] .long 0x6e184594 // ins v20.d[1], v12.d[1] .long 0x6e084537 // ins v23.d[0], v9.d[1] .long 0x6e1845d7 // ins v23.d[1], v14.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e0f41b4 // ext v20.16b, v13.16b, v15.16b, #8 .long 0x6e0805d7 // ins v23.d[0], v14.d[0] .long 0x6e180517 // ins v23.d[1], v8.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -1175,94 +1059,82 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e084514 // ins v20.d[0], v8.d[1] .long 0x6e184534 // ins v20.d[1], v9.d[1] .long 0x6e084557 // ins v23.d[0], v10.d[1] .long 0x6e184577 // ins v23.d[1], v11.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e080594 // ins v20.d[0], v12.d[0] .long 0x6e1805b4 // ins v20.d[1], v13.d[0] .long 0x6e0805d7 // ins v23.d[0], v14.d[0] .long 0x6e1805f7 // ins v23.d[1], v15.d[0] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e084594 // ins v20.d[0], v12.d[1] .long 0x6e1845b4 // ins v20.d[1], v13.d[1] .long 0x6e0845d7 // ins v23.d[0], v14.d[1] .long 0x6e1845f7 // ins v23.d[1], v15.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -1289,92 +1161,80 @@ .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6f685455 // sli v21.2d, v2.2d, #40 + .long 0x6f680476 // ushr v22.2d, v3.2d, #24 + .long 0x6f685476 // sli v22.2d, v3.2d, #40 .long 0x6e0805b4 // ins v20.d[0], v13.d[0] .long 0x6e180594 // ins v20.d[1], v12.d[0] .long 0x6e0b41f7 // ext v23.16b, v15.16b, v11.16b, #8 - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x6e034055 // ext v21.16b, v2.16b, v3.16b, #8 - .long 0x6e024063 // ext v3.16b, v3.16b, v2.16b, #8 - .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b + .long 0x6f7004c2 // ushr v2.2d, v6.2d, #16 + .long 0x6f7054c2 // sli v2.2d, v6.2d, #48 + .long 0x6f7004e3 // ushr v3.2d, v7.2d, #16 + .long 0x6f7054e3 // sli v3.2d, v7.2d, #48 + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a6 // add v6.2d, v21.2d, v21.2d + .long 0x6f4146a6 // sri v6.2d, v21.2d, #63 + .long 0x4ef686c7 // add v7.2d, v22.2d, v22.2d + .long 0x6f4146c7 // sri v7.2d, v22.2d, #63 + .long 0x6e0740d5 // ext v21.16b, v6.16b, v7.16b, #8 + .long 0x6e0640e7 // ext v7.16b, v7.16b, v6.16b, #8 + .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b .long 0x4ea41c95 // orr v21.16b, v4.16b, v4.16b .long 0x4ea51ca4 // orr v4.16b, v5.16b, v5.16b .long 0x4eb51ea5 // orr v5.16b, v21.16b, v21.16b - .long 0x6e0640f5 // ext v21.16b, v7.16b, v6.16b, #8 - .long 0x6e0740c7 // ext v7.16b, v6.16b, v7.16b, #8 - .long 0x4eb51ea6 // orr v6.16b, v21.16b, v21.16b + .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 + .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 + .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b .long 0x6e084114 // ext v20.16b, v8.16b, v8.16b, #8 .long 0x6e0845b7 // ins v23.d[0], v13.d[1] .long 0x6e184557 // ins v23.d[1], v10.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ee68694 // add v20.2d, v20.2d, v6.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ee786f7 // add v23.2d, v23.2d, v7.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x4ea008c6 // rev64 v6.4s, v6.4s - .long 0x4ea008e7 // rev64 v7.4s, v7.4s - .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d - .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x6f680455 // ushr v21.2d, v2.2d, #24 - .long 0x4f685456 // shl v22.2d, v2.2d, #40 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x6f680475 // ushr v21.2d, v3.2d, #24 - .long 0x4f685476 // shl v22.2d, v3.2d, #40 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x4ea00842 // rev64 v2.4s, v2.4s + .long 0x4ea00863 // rev64 v3.4s, v3.4s + .long 0x4ee28484 // add v4.2d, v4.2d, v2.2d + .long 0x4ee384a5 // add v5.2d, v5.2d, v3.2d + .long 0x6e241cc6 // eor v6.16b, v6.16b, v4.16b + .long 0x6e251ce7 // eor v7.16b, v7.16b, v5.16b + .long 0x6f6804d5 // ushr v21.2d, v6.2d, #24 + .long 0x6f6854d5 // sli v21.2d, v6.2d, #40 + .long 0x6f6804f6 // ushr v22.2d, v7.2d, #24 + .long 0x6f6854f6 // sli v22.2d, v7.2d, #40 .long 0x6e0805d4 // ins v20.d[0], v14.d[0] .long 0x6e180534 // ins v20.d[1], v9.d[0] .long 0x6e084577 // ins v23.d[0], v11.d[1] .long 0x6e184537 // ins v23.d[1], v9.d[1] - .long 0x4ee28694 // add v20.2d, v20.2d, v2.2d + .long 0x4ef58694 // add v20.2d, v20.2d, v21.2d .long 0x4ef48400 // add v0.2d, v0.2d, v20.2d - .long 0x4ee386f7 // add v23.2d, v23.2d, v3.2d + .long 0x4ef686f7 // add v23.2d, v23.2d, v22.2d .long 0x4ef78421 // add v1.2d, v1.2d, v23.2d - .long 0x6e201cc6 // eor v6.16b, v6.16b, v0.16b - .long 0x6e211ce7 // eor v7.16b, v7.16b, v1.16b - .long 0x6f7004d5 // ushr v21.2d, v6.2d, #16 - .long 0x4f7054d6 // shl v22.2d, v6.2d, #48 - .long 0x4eb61ea6 // orr v6.16b, v21.16b, v22.16b - .long 0x6f7004f5 // ushr v21.2d, v7.2d, #16 - .long 0x4f7054f6 // shl v22.2d, v7.2d, #48 - .long 0x4eb61ea7 // orr v7.16b, v21.16b, v22.16b + .long 0x6e201c42 // eor v2.16b, v2.16b, v0.16b + .long 0x6e211c63 // eor v3.16b, v3.16b, v1.16b + .long 0x6f700446 // ushr v6.2d, v2.2d, #16 + .long 0x6f705446 // sli v6.2d, v2.2d, #48 + .long 0x6f700467 // ushr v7.2d, v3.2d, #16 + .long 0x6f705467 // sli v7.2d, v3.2d, #48 .long 0x4ee68484 // add v4.2d, v4.2d, v6.2d .long 0x4ee784a5 // add v5.2d, v5.2d, v7.2d - .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b - .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b - .long 0x4ee28455 // add v21.2d, v2.2d, v2.2d - .long 0x6f410456 // ushr v22.2d, v2.2d, #63 - .long 0x4eb61ea2 // orr v2.16b, v21.16b, v22.16b - .long 0x4ee38475 // add v21.2d, v3.2d, v3.2d - .long 0x6f410476 // ushr v22.2d, v3.2d, #63 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x4ef586a2 // add v2.2d, v21.2d, v21.2d + .long 0x6f4146a2 // sri v2.2d, v21.2d, #63 + .long 0x4ef686c3 // add v3.2d, v22.2d, v22.2d + .long 0x6f4146c3 // sri v3.2d, v22.2d, #63 .long 0x6e024075 // ext v21.16b, v3.16b, v2.16b, #8 .long 0x6e034043 // ext v3.16b, v2.16b, v3.16b, #8 .long 0x4eb51ea2 // orr v2.16b, v21.16b, v21.16b @@ -1395,5 +1255,5 @@ .long 0xad000400 // stp q0, q1, [x0] .long 0xad010c02 // stp q2, q3, [x0, #32] - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_i386.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_i386.inc index 18b29deb..30b7aff0 100644 --- a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_i386.inc +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_i386.inc @@ -1,10 +1,13 @@ -// SSE2 implementation of BLAKE2b compress (fully unrolled 12 rounds). -// IA-32: after HlpSimdProc4Begin_i386 — ebx = state, esi = msg, edi = counter+flags, eax = IV -// (parallel to MS x64 ABI: rcx, rdx, r8, r9). -// XMM8+ from x64 eliminated: xmm4 scratch + stack, xmm0/1 for diagonal work; uses xmm0–xmm7. -// Non-volatile xmm6–xmm7 saved in stack frame (sub esp, 128). -// Register map: xmm0-1 = row1 (v0-3), xmm2-3 = row2 (v4-7), xmm4-5 = row3 (v8-11), xmm6-7 = row4 (v12-15) -// with temps folded per IA-32 constraints. +// SSE2 implementation of BLAKE2b compress (fully unrolled 12 rounds; IA-32). +// XMM8+ from the x64 sibling eliminated: xmm4 scratch + stack, xmm0/xmm1 for +// diagonal work. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = counter+flags ptr, eax = IV ptr. +// Register map: xmm0-1 = row1 (v0-3), xmm2-3 = row2 (v4-7), +// xmm4-5 = row3 (v8-11), xmm6-7 = row4 (v12-15), temps folded per the +// IA-32 register budget. +// Frame: sub esp, 128 (spill slots + xmm6-xmm7 save). +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). // Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. sub esp, 128 diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc index 861c04fc..05a04c67 100644 --- a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc @@ -1,11 +1,12 @@ // SSE2 implementation of BLAKE2b compress (fully unrolled 12 rounds). -// Expects MS x64 ABI: rcx = state ptr, rdx = msg ptr, r8 = counter+flags ptr, r9 = IV ptr. -// Uses xmm0-xmm9; xmm6-xmm9 are MS x64 non-volatile (saved/restored). +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = counter+flags ptr, r9 = IV ptr. // Register map: xmm0-1 = row1 (v0-3), xmm2-3 = row2 (v4-7), // xmm4-5 = row3 (v8-11), xmm6-7 = row4 (v12-15), xmm8-9 = temps. +// Saves: xmm6-xmm9 (MS x64 non-volatile); uses xmm0-xmm9. // Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} // Initialize working vector movdqu xmm0, oword [rcx] @@ -1989,4 +1990,4 @@ movdqu oword [rcx + $20], xmm2 movdqu oword [rcx + $30], xmm3 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSsse3_i386.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSsse3_i386.inc new file mode 100644 index 00000000..76ea49f9 --- /dev/null +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSsse3_i386.inc @@ -0,0 +1,2074 @@ +// SSSE3 implementation of BLAKE2b compress (fully unrolled 12 rounds; IA-32); +// SSE2 sibling: Blake2BCompressSse2_i386.inc. Unlike the sibling, each pair +// of byte-aligned rotations (rot24/rot16) is one mask load into the already +// stack-spilled xmm4 scratch plus two pshufb; rot63 stays shift+or (not +// byte-aligned). XMM8+ from the x64 sibling eliminated: xmm4 scratch + +// stack, xmm0/xmm1 for diagonal work. +// ABI (after HlpSimdProc5Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = counter+flags ptr, eax = IV ptr, ecx = byte-rotation masks ptr +// (BLAKE2B_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0-1 = row1 (v0-3), xmm2-3 = row2 (v4-7), +// xmm4-5 = row3 (v8-11), xmm6-7 = row4 (v12-15), temps folded per the +// IA-32 register budget. +// Frame: sub esp, 128 (spill slots + xmm6-xmm7 save). +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. + + sub esp, 128 + movdqu oword ptr [esp], xmm6 + movdqu oword ptr [esp + $10], xmm7 + + // Initialize working vector + movdqu xmm0, oword ptr [ebx] + movdqu xmm1, oword ptr [ebx + $10] + movdqu xmm2, oword ptr [ebx + $20] + movdqu xmm3, oword ptr [ebx + $30] + movdqu xmm4, oword ptr [eax] + movdqu xmm5, oword ptr [eax + $10] + movdqu xmm6, oword ptr [eax + $20] + movdqu xmm7, oword ptr [eax + $30] + + // XOR row4 with counter and flags + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [edi] + pxor xmm6, xmm4 + movdqu xmm4, oword ptr [edi + $10] + pxor xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // ===== Round 0 ===== + + // G1 columns (msg: m0,m2 / m4,m6) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $10] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $20] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m1,m3 / m5,m7) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $28] + movhpd xmm4, qword ptr [esi + $38] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m8,m10 / m12,m14) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $40] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $70] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m9,m11 / m13,m15) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $58] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $68] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 1 ===== + + // G1 columns (msg: m14,m4 / m9,m13) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $70] + movhpd xmm4, qword ptr [esi + $20] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $68] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m10,m8 / m15,m6) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $50] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $78] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m1,m0 / m11,m5) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m12,m2 / m7,m3) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $10] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $38] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 2 ===== + + // G1 columns (msg: m11,m12 / m5,m15) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi + $60] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $28] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m8,m0 / m2,m13) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $40] + movhpd xmm4, qword ptr [esi] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $68] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m10,m3 / m7,m9) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $50] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $38] + movhpd xmm4, qword ptr [esi + $48] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m14,m6 / m1,m4) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $70] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi + $20] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 3 ===== + + // G1 columns (msg: m7,m3 / m13,m11) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $38] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $68] + movhpd xmm4, qword ptr [esi + $58] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m9,m1 / m12,m14) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $08] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $70] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m2,m5 / m4,m15) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $20] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m6,m10 / m0,m8) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $30] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 4 ===== + + // G1 columns (msg: m9,m5 / m2,m10) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m0,m7 / m4,m15) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $38] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $20] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m14,m11 / m6,m3) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $70] + movhpd xmm4, qword ptr [esi + $58] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $30] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m1,m12 / m8,m13) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi + $60] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $40] + movhpd xmm4, qword ptr [esi + $68] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 5 ===== + + // G1 columns (msg: m2,m6 / m0,m8) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m12,m10 / m11,m3) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m4,m7 / m15,m1) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $20] + movhpd xmm4, qword ptr [esi + $38] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $78] + movhpd xmm4, qword ptr [esi + $08] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m13,m5 / m14,m9) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $68] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $70] + movhpd xmm4, qword ptr [esi + $48] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 6 ===== + + // G1 columns (msg: m12,m1 / m14,m4) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $08] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $70] + movhpd xmm4, qword ptr [esi + $20] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m5,m15 / m13,m10) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $28] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $68] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m0,m6 / m9,m8) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m7,m3 / m2,m11) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $38] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $58] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 7 ===== + + // G1 columns (msg: m13,m7 / m12,m3) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $68] + movhpd xmm4, qword ptr [esi + $38] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m11,m14 / m1,m9) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi + $70] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi + $48] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m5,m15 / m8,m2) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $28] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $40] + movhpd xmm4, qword ptr [esi + $10] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m0,m4 / m6,m10) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $20] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $30] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 8 ===== + + // G1 columns (msg: m6,m14 / m11,m0) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $30] + movhpd xmm4, qword ptr [esi + $70] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m15,m9 / m3,m8) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $78] + movhpd xmm4, qword ptr [esi + $48] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $18] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m12,m13 / m1,m10) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $68] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m2,m7 / m4,m5) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $38] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $20] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 9 ===== + + // G1 columns (msg: m10,m8 / m7,m1) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $50] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $38] + movhpd xmm4, qword ptr [esi + $08] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m2,m4 / m6,m5) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $10] + movhpd xmm4, qword ptr [esi + $20] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $30] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m15,m9 / m3,m13) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $78] + movhpd xmm4, qword ptr [esi + $48] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $18] + movhpd xmm4, qword ptr [esi + $68] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m11,m14 / m12,m0) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi + $70] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 10 ===== + + // G1 columns (msg: m0,m2 / m4,m6) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi] + movhpd xmm4, qword ptr [esi + $10] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $20] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m1,m3 / m5,m7) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $28] + movhpd xmm4, qword ptr [esi + $38] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m8,m10 / m12,m14) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $40] + movhpd xmm4, qword ptr [esi + $50] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $70] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m9,m11 / m13,m15) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $58] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $68] + movhpd xmm4, qword ptr [esi + $78] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // ===== Round 11 ===== + + // G1 columns (msg: m14,m4 / m9,m13) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $70] + movhpd xmm4, qword ptr [esi + $20] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $48] + movhpd xmm4, qword ptr [esi + $68] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 columns (msg: m10,m8 / m15,m6) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $50] + movhpd xmm4, qword ptr [esi + $40] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $78] + movhpd xmm4, qword ptr [esi + $30] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Diagonalize (i386: xmm0/1 scratch; no XMM8+) + movdqu oword ptr [esp + $40], xmm0 + movdqu oword ptr [esp + $50], xmm1 + movdqa xmm0, xmm6 + movdqa xmm1, xmm2 + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + movdqa xmm6, xmm7 + shufpd xmm7, xmm0, $01 + shufpd xmm0, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm0 + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm1, $01 + movdqu xmm0, oword ptr [esp + $40] + movdqu xmm1, oword ptr [esp + $50] + + // G1 diagonals (msg: m1,m0 / m11,m5) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $08] + movhpd xmm4, qword ptr [esi] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $58] + movhpd xmm4, qword ptr [esi + $28] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx] + db $66, $0F, $38, $00, $D4 // pshufb xmm2, xmm4 + db $66, $0F, $38, $00, $DC // pshufb xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // G2 diagonals (msg: m12,m2 / m7,m3) + movdqu oword ptr [esp + $20], xmm4 + movq xmm4, qword ptr [esi + $60] + movhpd xmm4, qword ptr [esi + $10] + paddq xmm0, xmm4 + paddq xmm0, xmm2 + movq xmm4, qword ptr [esi + $38] + movhpd xmm4, qword ptr [esi + $18] + paddq xmm1, xmm4 + paddq xmm1, xmm3 + movdqu xmm4, oword ptr [esp + $20] + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ecx + $20] + db $66, $0F, $38, $00, $F4 // pshufb xmm6, xmm4 + db $66, $0F, $38, $00, $FC // pshufb xmm7, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqu oword ptr [esp + $20], xmm4 + movdqa xmm4, xmm2 + psrlq xmm4, 63 + paddq xmm2, xmm2 + por xmm2, xmm4 + movdqa xmm4, xmm3 + psrlq xmm4, 63 + paddq xmm3, xmm3 + por xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Undiagonalize (i386: memory swap + xmm0 scratch) + movdqu oword ptr [esp + $70], xmm4 + movdqa xmm4, xmm5 + movdqu xmm5, oword ptr [esp + $70] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm0, $01 + movdqu xmm0, oword ptr [esp + $40] + + movdqu oword ptr [esp + $40], xmm0 + movdqa xmm0, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm0, $01 + shufpd xmm0, xmm3, $01 + movdqa xmm3, xmm0 + movdqu xmm0, oword ptr [esp + $40] + + // Finalization + pxor xmm0, xmm4 + pxor xmm1, xmm5 + pxor xmm2, xmm6 + pxor xmm3, xmm7 + + // XOR with original state (re-read from memory via movdqu for safe unaligned access) + movdqu oword ptr [esp + $20], xmm4 + movdqu xmm4, oword ptr [ebx] + pxor xmm0, xmm4 + movdqu xmm4, oword ptr [ebx + $10] + pxor xmm1, xmm4 + movdqu xmm4, oword ptr [ebx + $20] + pxor xmm2, xmm4 + movdqu xmm4, oword ptr [ebx + $30] + pxor xmm3, xmm4 + movdqu xmm4, oword ptr [esp + $20] + + // Store result + movdqu oword ptr [ebx], xmm0 + movdqu oword ptr [ebx + $10], xmm1 + movdqu oword ptr [ebx + $20], xmm2 + movdqu oword ptr [ebx + $30], xmm3 + + movdqu xmm6, oword ptr [esp] + movdqu xmm7, oword ptr [esp + $10] + add esp, 128 + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSsse3_x86_64.inc new file mode 100644 index 00000000..9cea5130 --- /dev/null +++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSsse3_x86_64.inc @@ -0,0 +1,1716 @@ +// SSSE3 implementation of BLAKE2b compress (fully unrolled 12 rounds); SSE2 +// sibling: Blake2BCompressSse2_x86_64.inc. Unlike the sibling, the +// byte-aligned rotations (rot24/rot16) are single pshufb against masks +// resident in xmm10/xmm11; rot63 stays shift+or (not byte-aligned). +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = counter+flags ptr, r9 = IV ptr, r10 = byte-rotation masks ptr +// (BLAKE2B_ROT_MASKS: rot24 at +0, rot16 at +32; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0-1 = row1 (v0-3), xmm2-3 = row2 (v4-7), +// xmm4-5 = row3 (v8-11), xmm6-7 = row4 (v12-15), xmm8-9 = temps, +// xmm10 = rot24 mask, xmm11 = rot16 mask. +// Saves: xmm6-xmm11 (MS x64 non-volatile); uses xmm0-xmm11. +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $F3, $45, $0F, $6F, $12 // movdqu xmm10, oword [r10] + db $F3, $45, $0F, $6F, $5A, $20 // movdqu xmm11, oword [r10 + $20] + + // Initialize working vector + movdqu xmm0, oword [rcx] + movdqu xmm1, oword [rcx + $10] + movdqu xmm2, oword [rcx + $20] + movdqu xmm3, oword [rcx + $30] + movdqu xmm4, oword [r9] + movdqu xmm5, oword [r9 + $10] + movdqu xmm6, oword [r9 + $20] + movdqu xmm7, oword [r9 + $30] + + // XOR row4 with counter and flags + movdqu xmm8, oword [r8] + movdqu xmm9, oword [r8 + $10] + pxor xmm6, xmm8 + pxor xmm7, xmm9 + + // ===== Round 0 ===== + + // G1 columns (msg: m0,m2 / m4,m6) + movq xmm8, qword [rdx] + movhpd xmm8, qword [rdx + $10] + movq xmm9, qword [rdx + $20] + movhpd xmm9, qword [rdx + $30] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m1,m3 / m5,m7) + movq xmm8, qword [rdx + $08] + movhpd xmm8, qword [rdx + $18] + movq xmm9, qword [rdx + $28] + movhpd xmm9, qword [rdx + $38] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m8,m10 / m12,m14) + movq xmm8, qword [rdx + $40] + movhpd xmm8, qword [rdx + $50] + movq xmm9, qword [rdx + $60] + movhpd xmm9, qword [rdx + $70] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m9,m11 / m13,m15) + movq xmm8, qword [rdx + $48] + movhpd xmm8, qword [rdx + $58] + movq xmm9, qword [rdx + $68] + movhpd xmm9, qword [rdx + $78] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 1 ===== + + // G1 columns (msg: m14,m4 / m9,m13) + movq xmm8, qword [rdx + $70] + movhpd xmm8, qword [rdx + $20] + movq xmm9, qword [rdx + $48] + movhpd xmm9, qword [rdx + $68] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m10,m8 / m15,m6) + movq xmm8, qword [rdx + $50] + movhpd xmm8, qword [rdx + $40] + movq xmm9, qword [rdx + $78] + movhpd xmm9, qword [rdx + $30] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m1,m0 / m11,m5) + movq xmm8, qword [rdx + $08] + movhpd xmm8, qword [rdx] + movq xmm9, qword [rdx + $58] + movhpd xmm9, qword [rdx + $28] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m12,m2 / m7,m3) + movq xmm8, qword [rdx + $60] + movhpd xmm8, qword [rdx + $10] + movq xmm9, qword [rdx + $38] + movhpd xmm9, qword [rdx + $18] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 2 ===== + + // G1 columns (msg: m11,m12 / m5,m15) + movq xmm8, qword [rdx + $58] + movhpd xmm8, qword [rdx + $60] + movq xmm9, qword [rdx + $28] + movhpd xmm9, qword [rdx + $78] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m8,m0 / m2,m13) + movq xmm8, qword [rdx + $40] + movhpd xmm8, qword [rdx] + movq xmm9, qword [rdx + $10] + movhpd xmm9, qword [rdx + $68] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m10,m3 / m7,m9) + movq xmm8, qword [rdx + $50] + movhpd xmm8, qword [rdx + $18] + movq xmm9, qword [rdx + $38] + movhpd xmm9, qword [rdx + $48] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m14,m6 / m1,m4) + movq xmm8, qword [rdx + $70] + movhpd xmm8, qword [rdx + $30] + movq xmm9, qword [rdx + $08] + movhpd xmm9, qword [rdx + $20] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 3 ===== + + // G1 columns (msg: m7,m3 / m13,m11) + movq xmm8, qword [rdx + $38] + movhpd xmm8, qword [rdx + $18] + movq xmm9, qword [rdx + $68] + movhpd xmm9, qword [rdx + $58] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m9,m1 / m12,m14) + movq xmm8, qword [rdx + $48] + movhpd xmm8, qword [rdx + $08] + movq xmm9, qword [rdx + $60] + movhpd xmm9, qword [rdx + $70] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m2,m5 / m4,m15) + movq xmm8, qword [rdx + $10] + movhpd xmm8, qword [rdx + $28] + movq xmm9, qword [rdx + $20] + movhpd xmm9, qword [rdx + $78] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m6,m10 / m0,m8) + movq xmm8, qword [rdx + $30] + movhpd xmm8, qword [rdx + $50] + movq xmm9, qword [rdx] + movhpd xmm9, qword [rdx + $40] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 4 ===== + + // G1 columns (msg: m9,m5 / m2,m10) + movq xmm8, qword [rdx + $48] + movhpd xmm8, qword [rdx + $28] + movq xmm9, qword [rdx + $10] + movhpd xmm9, qword [rdx + $50] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m0,m7 / m4,m15) + movq xmm8, qword [rdx] + movhpd xmm8, qword [rdx + $38] + movq xmm9, qword [rdx + $20] + movhpd xmm9, qword [rdx + $78] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m14,m11 / m6,m3) + movq xmm8, qword [rdx + $70] + movhpd xmm8, qword [rdx + $58] + movq xmm9, qword [rdx + $30] + movhpd xmm9, qword [rdx + $18] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m1,m12 / m8,m13) + movq xmm8, qword [rdx + $08] + movhpd xmm8, qword [rdx + $60] + movq xmm9, qword [rdx + $40] + movhpd xmm9, qword [rdx + $68] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 5 ===== + + // G1 columns (msg: m2,m6 / m0,m8) + movq xmm8, qword [rdx + $10] + movhpd xmm8, qword [rdx + $30] + movq xmm9, qword [rdx] + movhpd xmm9, qword [rdx + $40] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m12,m10 / m11,m3) + movq xmm8, qword [rdx + $60] + movhpd xmm8, qword [rdx + $50] + movq xmm9, qword [rdx + $58] + movhpd xmm9, qword [rdx + $18] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m4,m7 / m15,m1) + movq xmm8, qword [rdx + $20] + movhpd xmm8, qword [rdx + $38] + movq xmm9, qword [rdx + $78] + movhpd xmm9, qword [rdx + $08] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m13,m5 / m14,m9) + movq xmm8, qword [rdx + $68] + movhpd xmm8, qword [rdx + $28] + movq xmm9, qword [rdx + $70] + movhpd xmm9, qword [rdx + $48] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 6 ===== + + // G1 columns (msg: m12,m1 / m14,m4) + movq xmm8, qword [rdx + $60] + movhpd xmm8, qword [rdx + $08] + movq xmm9, qword [rdx + $70] + movhpd xmm9, qword [rdx + $20] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m5,m15 / m13,m10) + movq xmm8, qword [rdx + $28] + movhpd xmm8, qword [rdx + $78] + movq xmm9, qword [rdx + $68] + movhpd xmm9, qword [rdx + $50] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m0,m6 / m9,m8) + movq xmm8, qword [rdx] + movhpd xmm8, qword [rdx + $30] + movq xmm9, qword [rdx + $48] + movhpd xmm9, qword [rdx + $40] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m7,m3 / m2,m11) + movq xmm8, qword [rdx + $38] + movhpd xmm8, qword [rdx + $18] + movq xmm9, qword [rdx + $10] + movhpd xmm9, qword [rdx + $58] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 7 ===== + + // G1 columns (msg: m13,m7 / m12,m3) + movq xmm8, qword [rdx + $68] + movhpd xmm8, qword [rdx + $38] + movq xmm9, qword [rdx + $60] + movhpd xmm9, qword [rdx + $18] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m11,m14 / m1,m9) + movq xmm8, qword [rdx + $58] + movhpd xmm8, qword [rdx + $70] + movq xmm9, qword [rdx + $08] + movhpd xmm9, qword [rdx + $48] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m5,m15 / m8,m2) + movq xmm8, qword [rdx + $28] + movhpd xmm8, qword [rdx + $78] + movq xmm9, qword [rdx + $40] + movhpd xmm9, qword [rdx + $10] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m0,m4 / m6,m10) + movq xmm8, qword [rdx] + movhpd xmm8, qword [rdx + $20] + movq xmm9, qword [rdx + $30] + movhpd xmm9, qword [rdx + $50] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 8 ===== + + // G1 columns (msg: m6,m14 / m11,m0) + movq xmm8, qword [rdx + $30] + movhpd xmm8, qword [rdx + $70] + movq xmm9, qword [rdx + $58] + movhpd xmm9, qword [rdx] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m15,m9 / m3,m8) + movq xmm8, qword [rdx + $78] + movhpd xmm8, qword [rdx + $48] + movq xmm9, qword [rdx + $18] + movhpd xmm9, qword [rdx + $40] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m12,m13 / m1,m10) + movq xmm8, qword [rdx + $60] + movhpd xmm8, qword [rdx + $68] + movq xmm9, qword [rdx + $08] + movhpd xmm9, qword [rdx + $50] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m2,m7 / m4,m5) + movq xmm8, qword [rdx + $10] + movhpd xmm8, qword [rdx + $38] + movq xmm9, qword [rdx + $20] + movhpd xmm9, qword [rdx + $28] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 9 ===== + + // G1 columns (msg: m10,m8 / m7,m1) + movq xmm8, qword [rdx + $50] + movhpd xmm8, qword [rdx + $40] + movq xmm9, qword [rdx + $38] + movhpd xmm9, qword [rdx + $08] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m2,m4 / m6,m5) + movq xmm8, qword [rdx + $10] + movhpd xmm8, qword [rdx + $20] + movq xmm9, qword [rdx + $30] + movhpd xmm9, qword [rdx + $28] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m15,m9 / m3,m13) + movq xmm8, qword [rdx + $78] + movhpd xmm8, qword [rdx + $48] + movq xmm9, qword [rdx + $18] + movhpd xmm9, qword [rdx + $68] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m11,m14 / m12,m0) + movq xmm8, qword [rdx + $58] + movhpd xmm8, qword [rdx + $70] + movq xmm9, qword [rdx + $60] + movhpd xmm9, qword [rdx] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 10 ===== + + // G1 columns (msg: m0,m2 / m4,m6) + movq xmm8, qword [rdx] + movhpd xmm8, qword [rdx + $10] + movq xmm9, qword [rdx + $20] + movhpd xmm9, qword [rdx + $30] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m1,m3 / m5,m7) + movq xmm8, qword [rdx + $08] + movhpd xmm8, qword [rdx + $18] + movq xmm9, qword [rdx + $28] + movhpd xmm9, qword [rdx + $38] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m8,m10 / m12,m14) + movq xmm8, qword [rdx + $40] + movhpd xmm8, qword [rdx + $50] + movq xmm9, qword [rdx + $60] + movhpd xmm9, qword [rdx + $70] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m9,m11 / m13,m15) + movq xmm8, qword [rdx + $48] + movhpd xmm8, qword [rdx + $58] + movq xmm9, qword [rdx + $68] + movhpd xmm9, qword [rdx + $78] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // ===== Round 11 ===== + + // G1 columns (msg: m14,m4 / m9,m13) + movq xmm8, qword [rdx + $70] + movhpd xmm8, qword [rdx + $20] + movq xmm9, qword [rdx + $48] + movhpd xmm9, qword [rdx + $68] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 columns (msg: m10,m8 / m15,m6) + movq xmm8, qword [rdx + $50] + movhpd xmm8, qword [rdx + $40] + movq xmm9, qword [rdx + $78] + movhpd xmm9, qword [rdx + $30] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Diagonalize + movdqa xmm8, xmm6 + movdqa xmm9, xmm2 + + movdqa xmm6, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm6 + + movdqa xmm6, xmm7 + shufpd xmm7, xmm8, $01 + shufpd xmm8, xmm6, $01 + movdqa xmm6, xmm7 + movdqa xmm7, xmm8 + + shufpd xmm2, xmm3, $01 + shufpd xmm3, xmm9, $01 + + // G1 diagonals (msg: m1,m0 / m11,m5) + movq xmm8, qword [rdx + $08] + movhpd xmm8, qword [rdx] + movq xmm9, qword [rdx + $58] + movhpd xmm9, qword [rdx + $28] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + pshufd xmm6, xmm6, $B1 + pshufd xmm7, xmm7, $B1 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + db $66, $41, $0F, $38, $00, $D2 // pshufb xmm2, xmm10 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + + // G2 diagonals (msg: m12,m2 / m7,m3) + movq xmm8, qword [rdx + $60] + movhpd xmm8, qword [rdx + $10] + movq xmm9, qword [rdx + $38] + movhpd xmm9, qword [rdx + $18] + paddq xmm0, xmm8 + paddq xmm0, xmm2 + paddq xmm1, xmm9 + paddq xmm1, xmm3 + + pxor xmm6, xmm0 + pxor xmm7, xmm1 + db $66, $41, $0F, $38, $00, $F3 // pshufb xmm6, xmm11 + db $66, $41, $0F, $38, $00, $FB // pshufb xmm7, xmm11 + + paddq xmm4, xmm6 + paddq xmm5, xmm7 + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + movdqa xmm8, xmm2 + psrlq xmm8, 63 + paddq xmm2, xmm2 + por xmm2, xmm8 + movdqa xmm9, xmm3 + psrlq xmm9, 63 + paddq xmm3, xmm3 + por xmm3, xmm9 + + // Undiagonalize + movdqa xmm8, xmm4 + movdqa xmm4, xmm5 + movdqa xmm5, xmm8 + + movdqa xmm8, xmm6 + shufpd xmm6, xmm7, $01 + shufpd xmm7, xmm8, $01 + + movdqa xmm8, xmm2 + movdqa xmm2, xmm3 + shufpd xmm2, xmm8, $01 + shufpd xmm8, xmm3, $01 + movdqa xmm3, xmm8 + + // Finalization + pxor xmm0, xmm4 + pxor xmm1, xmm5 + pxor xmm2, xmm6 + pxor xmm3, xmm7 + + // XOR with original state (re-read from memory via movdqu for safe unaligned access) + movdqu xmm8, oword [rcx] + pxor xmm0, xmm8 + movdqu xmm9, oword [rcx + $10] + pxor xmm1, xmm9 + movdqu xmm8, oword [rcx + $20] + pxor xmm2, xmm8 + movdqu xmm9, oword [rcx + $30] + pxor xmm3, xmm9 + + // Store result + movdqu oword [rcx], xmm0 + movdqu oword [rcx + $10], xmm1 + movdqu oword [rcx + $20], xmm2 + movdqu oword [rcx + $30], xmm3 + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx_i386.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx_i386.inc new file mode 100644 index 00000000..7386b05a --- /dev/null +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx_i386.inc @@ -0,0 +1,942 @@ +// AVX (VEX-128) implementation of BLAKE2s compress (fully unrolled 10 +// rounds; IA-32); the Avx name states the ISA requirement: VEX-128 +// encodings only, no 256-bit ymm. Same structure as +// Blake2SCompressAvx_x86_64.inc, with the rot8 mask read as a VEX memory +// operand per site (IA-32 has only xmm0-xmm7, so no register is free to +// keep it resident; VEX memory operands are read unaligned, so the Pascal +// const needs no special alignment). Rotations: rot16 via vpshufb against +// the resident xmm7 mask, rot8 via vpshufb against the [ecx+$10] memory +// operand, rot12/7 via shift+or (not byte-aligned). +// ABI (after HlpSimdProc5Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = counter+flags ptr, eax = IV ptr, ecx = byte-rotation masks ptr +// (BLAKE2S_ROT_MASKS: rot16 at +0, rot8 at +16). +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = message buffer, xmm5 = temp, +// xmm6 = temp (msg load), xmm7 = rot16 mask. +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves, HashLib +// Blake2SCompressAvx_x86_64.inc. + + // Defensive save of xmm6/xmm7 (volatile on IA-32; saved like the + // SSE2 sibling to be safe under every RTL) + sub esp, $20 + db $C5, $FA, $7F, $34, $24 // vmovdqu oword [esp], xmm6 + db $C5, $FA, $7F, $7C, $24, $10 // vmovdqu oword [esp + $10], xmm7 + + + // Load byte-rotation masks (resident for the whole kernel) + db $C5, $FA, $6F, $39 // vmovdqu xmm7, oword [ecx] + + // Initialize working vector + db $C5, $FA, $6F, $03 // vmovdqu xmm0, oword [ebx] + db $C5, $FA, $6F, $4B, $10 // vmovdqu xmm1, oword [ebx + $10] + db $C5, $FA, $6F, $10 // vmovdqu xmm2, oword [eax] + db $C5, $FA, $6F, $58, $10 // vmovdqu xmm3, oword [eax + $10] + + // XOR d with counter and flags + db $C5, $FA, $6F, $27 // vmovdqu xmm4, oword [edi] + db $C5, $E1, $EF, $DC // vpxor xmm3, xmm3, xmm4 + + // ===== Round 0 ===== + + // G1 columns (msg: m0,m2,m4,m6) + db $C5, $F9, $6E, $26 // vmovd xmm4, dword [esi] + db $C5, $F9, $6E, $6E, $08 // vmovd xmm5, dword [esi + $8] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $10 // vmovd xmm5, dword [esi + $10] + db $C5, $F9, $6E, $76, $18 // vmovd xmm6, dword [esi + $18] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m1,m3,m5,m7) + db $C5, $F9, $6E, $66, $04 // vmovd xmm4, dword [esi + $4] + db $C5, $F9, $6E, $6E, $0C // vmovd xmm5, dword [esi + $C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $14 // vmovd xmm5, dword [esi + $14] + db $C5, $F9, $6E, $76, $1C // vmovd xmm6, dword [esi + $1C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m14,m8,m10,m12) + db $C5, $F9, $6E, $66, $38 // vmovd xmm4, dword [esi + $38] + db $C5, $F9, $6E, $6E, $20 // vmovd xmm5, dword [esi + $20] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $28 // vmovd xmm5, dword [esi + $28] + db $C5, $F9, $6E, $76, $30 // vmovd xmm6, dword [esi + $30] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m15,m9,m11,m13) + db $C5, $F9, $6E, $66, $3C // vmovd xmm4, dword [esi + $3C] + db $C5, $F9, $6E, $6E, $24 // vmovd xmm5, dword [esi + $24] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $2C // vmovd xmm5, dword [esi + $2C] + db $C5, $F9, $6E, $76, $34 // vmovd xmm6, dword [esi + $34] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 1 ===== + + // G1 columns (msg: m14,m4,m9,m13) + db $C5, $F9, $6E, $66, $38 // vmovd xmm4, dword [esi + $38] + db $C5, $F9, $6E, $6E, $10 // vmovd xmm5, dword [esi + $10] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $24 // vmovd xmm5, dword [esi + $24] + db $C5, $F9, $6E, $76, $34 // vmovd xmm6, dword [esi + $34] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m10,m8,m15,m6) + db $C5, $F9, $6E, $66, $28 // vmovd xmm4, dword [esi + $28] + db $C5, $F9, $6E, $6E, $20 // vmovd xmm5, dword [esi + $20] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $3C // vmovd xmm5, dword [esi + $3C] + db $C5, $F9, $6E, $76, $18 // vmovd xmm6, dword [esi + $18] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m5,m1,m0,m11) + db $C5, $F9, $6E, $66, $14 // vmovd xmm4, dword [esi + $14] + db $C5, $F9, $6E, $6E, $04 // vmovd xmm5, dword [esi + $4] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $2E // vmovd xmm5, dword [esi] + db $C5, $F9, $6E, $76, $2C // vmovd xmm6, dword [esi + $2C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m3,m12,m2,m7) + db $C5, $F9, $6E, $66, $0C // vmovd xmm4, dword [esi + $C] + db $C5, $F9, $6E, $6E, $30 // vmovd xmm5, dword [esi + $30] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $08 // vmovd xmm5, dword [esi + $8] + db $C5, $F9, $6E, $76, $1C // vmovd xmm6, dword [esi + $1C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 2 ===== + + // G1 columns (msg: m11,m12,m5,m15) + db $C5, $F9, $6E, $66, $2C // vmovd xmm4, dword [esi + $2C] + db $C5, $F9, $6E, $6E, $30 // vmovd xmm5, dword [esi + $30] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $14 // vmovd xmm5, dword [esi + $14] + db $C5, $F9, $6E, $76, $3C // vmovd xmm6, dword [esi + $3C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m8,m0,m2,m13) + db $C5, $F9, $6E, $66, $20 // vmovd xmm4, dword [esi + $20] + db $C5, $F9, $6E, $2E // vmovd xmm5, dword [esi] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $08 // vmovd xmm5, dword [esi + $8] + db $C5, $F9, $6E, $76, $34 // vmovd xmm6, dword [esi + $34] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m9,m10,m3,m7) + db $C5, $F9, $6E, $66, $24 // vmovd xmm4, dword [esi + $24] + db $C5, $F9, $6E, $6E, $28 // vmovd xmm5, dword [esi + $28] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $0C // vmovd xmm5, dword [esi + $C] + db $C5, $F9, $6E, $76, $1C // vmovd xmm6, dword [esi + $1C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m4,m14,m6,m1) + db $C5, $F9, $6E, $66, $10 // vmovd xmm4, dword [esi + $10] + db $C5, $F9, $6E, $6E, $38 // vmovd xmm5, dword [esi + $38] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $18 // vmovd xmm5, dword [esi + $18] + db $C5, $F9, $6E, $76, $04 // vmovd xmm6, dword [esi + $4] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 3 ===== + + // G1 columns (msg: m7,m3,m13,m11) + db $C5, $F9, $6E, $66, $1C // vmovd xmm4, dword [esi + $1C] + db $C5, $F9, $6E, $6E, $0C // vmovd xmm5, dword [esi + $C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $34 // vmovd xmm5, dword [esi + $34] + db $C5, $F9, $6E, $76, $2C // vmovd xmm6, dword [esi + $2C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m9,m1,m12,m14) + db $C5, $F9, $6E, $66, $24 // vmovd xmm4, dword [esi + $24] + db $C5, $F9, $6E, $6E, $04 // vmovd xmm5, dword [esi + $4] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $30 // vmovd xmm5, dword [esi + $30] + db $C5, $F9, $6E, $76, $38 // vmovd xmm6, dword [esi + $38] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m15,m2,m5,m4) + db $C5, $F9, $6E, $66, $3C // vmovd xmm4, dword [esi + $3C] + db $C5, $F9, $6E, $6E, $08 // vmovd xmm5, dword [esi + $8] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $14 // vmovd xmm5, dword [esi + $14] + db $C5, $F9, $6E, $76, $10 // vmovd xmm6, dword [esi + $10] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m8,m6,m10,m0) + db $C5, $F9, $6E, $66, $20 // vmovd xmm4, dword [esi + $20] + db $C5, $F9, $6E, $6E, $18 // vmovd xmm5, dword [esi + $18] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $28 // vmovd xmm5, dword [esi + $28] + db $C5, $F9, $6E, $36 // vmovd xmm6, dword [esi] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 4 ===== + + // G1 columns (msg: m9,m5,m2,m10) + db $C5, $F9, $6E, $66, $24 // vmovd xmm4, dword [esi + $24] + db $C5, $F9, $6E, $6E, $14 // vmovd xmm5, dword [esi + $14] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $08 // vmovd xmm5, dword [esi + $8] + db $C5, $F9, $6E, $76, $28 // vmovd xmm6, dword [esi + $28] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m0,m7,m4,m15) + db $C5, $F9, $6E, $26 // vmovd xmm4, dword [esi] + db $C5, $F9, $6E, $6E, $1C // vmovd xmm5, dword [esi + $1C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $10 // vmovd xmm5, dword [esi + $10] + db $C5, $F9, $6E, $76, $3C // vmovd xmm6, dword [esi + $3C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m3,m14,m11,m6) + db $C5, $F9, $6E, $66, $0C // vmovd xmm4, dword [esi + $C] + db $C5, $F9, $6E, $6E, $38 // vmovd xmm5, dword [esi + $38] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $2C // vmovd xmm5, dword [esi + $2C] + db $C5, $F9, $6E, $76, $18 // vmovd xmm6, dword [esi + $18] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m13,m1,m12,m8) + db $C5, $F9, $6E, $66, $34 // vmovd xmm4, dword [esi + $34] + db $C5, $F9, $6E, $6E, $04 // vmovd xmm5, dword [esi + $4] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $30 // vmovd xmm5, dword [esi + $30] + db $C5, $F9, $6E, $76, $20 // vmovd xmm6, dword [esi + $20] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 5 ===== + + // G1 columns (msg: m2,m6,m0,m8) + db $C5, $F9, $6E, $66, $08 // vmovd xmm4, dword [esi + $8] + db $C5, $F9, $6E, $6E, $18 // vmovd xmm5, dword [esi + $18] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $2E // vmovd xmm5, dword [esi] + db $C5, $F9, $6E, $76, $20 // vmovd xmm6, dword [esi + $20] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m12,m10,m11,m3) + db $C5, $F9, $6E, $66, $30 // vmovd xmm4, dword [esi + $30] + db $C5, $F9, $6E, $6E, $28 // vmovd xmm5, dword [esi + $28] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $2C // vmovd xmm5, dword [esi + $2C] + db $C5, $F9, $6E, $76, $0C // vmovd xmm6, dword [esi + $C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m1,m4,m7,m15) + db $C5, $F9, $6E, $66, $04 // vmovd xmm4, dword [esi + $4] + db $C5, $F9, $6E, $6E, $10 // vmovd xmm5, dword [esi + $10] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $1C // vmovd xmm5, dword [esi + $1C] + db $C5, $F9, $6E, $76, $3C // vmovd xmm6, dword [esi + $3C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m9,m13,m5,m14) + db $C5, $F9, $6E, $66, $24 // vmovd xmm4, dword [esi + $24] + db $C5, $F9, $6E, $6E, $34 // vmovd xmm5, dword [esi + $34] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $14 // vmovd xmm5, dword [esi + $14] + db $C5, $F9, $6E, $76, $38 // vmovd xmm6, dword [esi + $38] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 6 ===== + + // G1 columns (msg: m12,m1,m14,m4) + db $C5, $F9, $6E, $66, $30 // vmovd xmm4, dword [esi + $30] + db $C5, $F9, $6E, $6E, $04 // vmovd xmm5, dword [esi + $4] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $38 // vmovd xmm5, dword [esi + $38] + db $C5, $F9, $6E, $76, $10 // vmovd xmm6, dword [esi + $10] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m5,m15,m13,m10) + db $C5, $F9, $6E, $66, $14 // vmovd xmm4, dword [esi + $14] + db $C5, $F9, $6E, $6E, $3C // vmovd xmm5, dword [esi + $3C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $34 // vmovd xmm5, dword [esi + $34] + db $C5, $F9, $6E, $76, $28 // vmovd xmm6, dword [esi + $28] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m8,m0,m6,m9) + db $C5, $F9, $6E, $66, $20 // vmovd xmm4, dword [esi + $20] + db $C5, $F9, $6E, $2E // vmovd xmm5, dword [esi] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $18 // vmovd xmm5, dword [esi + $18] + db $C5, $F9, $6E, $76, $24 // vmovd xmm6, dword [esi + $24] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m11,m7,m3,m2) + db $C5, $F9, $6E, $66, $2C // vmovd xmm4, dword [esi + $2C] + db $C5, $F9, $6E, $6E, $1C // vmovd xmm5, dword [esi + $1C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $0C // vmovd xmm5, dword [esi + $C] + db $C5, $F9, $6E, $76, $08 // vmovd xmm6, dword [esi + $8] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 7 ===== + + // G1 columns (msg: m13,m7,m12,m3) + db $C5, $F9, $6E, $66, $34 // vmovd xmm4, dword [esi + $34] + db $C5, $F9, $6E, $6E, $1C // vmovd xmm5, dword [esi + $1C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $30 // vmovd xmm5, dword [esi + $30] + db $C5, $F9, $6E, $76, $0C // vmovd xmm6, dword [esi + $C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m11,m14,m1,m9) + db $C5, $F9, $6E, $66, $2C // vmovd xmm4, dword [esi + $2C] + db $C5, $F9, $6E, $6E, $38 // vmovd xmm5, dword [esi + $38] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $04 // vmovd xmm5, dword [esi + $4] + db $C5, $F9, $6E, $76, $24 // vmovd xmm6, dword [esi + $24] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m2,m5,m15,m8) + db $C5, $F9, $6E, $66, $08 // vmovd xmm4, dword [esi + $8] + db $C5, $F9, $6E, $6E, $14 // vmovd xmm5, dword [esi + $14] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $3C // vmovd xmm5, dword [esi + $3C] + db $C5, $F9, $6E, $76, $20 // vmovd xmm6, dword [esi + $20] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m10,m0,m4,m6) + db $C5, $F9, $6E, $66, $28 // vmovd xmm4, dword [esi + $28] + db $C5, $F9, $6E, $2E // vmovd xmm5, dword [esi] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $10 // vmovd xmm5, dword [esi + $10] + db $C5, $F9, $6E, $76, $18 // vmovd xmm6, dword [esi + $18] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 8 ===== + + // G1 columns (msg: m6,m14,m11,m0) + db $C5, $F9, $6E, $66, $18 // vmovd xmm4, dword [esi + $18] + db $C5, $F9, $6E, $6E, $38 // vmovd xmm5, dword [esi + $38] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $2C // vmovd xmm5, dword [esi + $2C] + db $C5, $F9, $6E, $36 // vmovd xmm6, dword [esi] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m15,m9,m3,m8) + db $C5, $F9, $6E, $66, $3C // vmovd xmm4, dword [esi + $3C] + db $C5, $F9, $6E, $6E, $24 // vmovd xmm5, dword [esi + $24] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $0C // vmovd xmm5, dword [esi + $C] + db $C5, $F9, $6E, $76, $20 // vmovd xmm6, dword [esi + $20] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m10,m12,m13,m1) + db $C5, $F9, $6E, $66, $28 // vmovd xmm4, dword [esi + $28] + db $C5, $F9, $6E, $6E, $30 // vmovd xmm5, dword [esi + $30] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $34 // vmovd xmm5, dword [esi + $34] + db $C5, $F9, $6E, $76, $04 // vmovd xmm6, dword [esi + $4] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m5,m2,m7,m4) + db $C5, $F9, $6E, $66, $14 // vmovd xmm4, dword [esi + $14] + db $C5, $F9, $6E, $6E, $08 // vmovd xmm5, dword [esi + $8] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $1C // vmovd xmm5, dword [esi + $1C] + db $C5, $F9, $6E, $76, $10 // vmovd xmm6, dword [esi + $10] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // ===== Round 9 ===== + + // G1 columns (msg: m10,m8,m7,m1) + db $C5, $F9, $6E, $66, $28 // vmovd xmm4, dword [esi + $28] + db $C5, $F9, $6E, $6E, $20 // vmovd xmm5, dword [esi + $20] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $1C // vmovd xmm5, dword [esi + $1C] + db $C5, $F9, $6E, $76, $04 // vmovd xmm6, dword [esi + $4] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 columns (msg: m2,m4,m6,m5) + db $C5, $F9, $6E, $66, $08 // vmovd xmm4, dword [esi + $8] + db $C5, $F9, $6E, $6E, $10 // vmovd xmm5, dword [esi + $10] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $18 // vmovd xmm5, dword [esi + $18] + db $C5, $F9, $6E, $76, $14 // vmovd xmm6, dword [esi + $14] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Diagonalize + db $C5, $F9, $70, $C0, $93 // vpshufd xmm0, xmm0, $93 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $39 // vpshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m13,m15,m9,m3) + db $C5, $F9, $6E, $66, $34 // vmovd xmm4, dword [esi + $34] + db $C5, $F9, $6E, $6E, $3C // vmovd xmm5, dword [esi + $3C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $24 // vmovd xmm5, dword [esi + $24] + db $C5, $F9, $6E, $76, $0C // vmovd xmm6, dword [esi + $C] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $0C // vpsrld xmm5, xmm1, $C + db $C5, $F1, $72, $F1, $14 // vpslld xmm1, xmm1, $14 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // G2 diagonals (msg: m0,m11,m14,m12) + db $C5, $F9, $6E, $26 // vmovd xmm4, dword [esi] + db $C5, $F9, $6E, $6E, $2C // vmovd xmm5, dword [esi + $2C] + db $C5, $D9, $6C, $E5 // vpunpcklqdq xmm4, xmm4, xmm5 + db $C5, $F9, $6E, $6E, $38 // vmovd xmm5, dword [esi + $38] + db $C5, $F9, $6E, $76, $30 // vmovd xmm6, dword [esi + $30] + db $C5, $D1, $6C, $EE // vpunpcklqdq xmm5, xmm5, xmm6 + db $C5, $D8, $C6, $E5, $88 // vshufps xmm4, xmm4, xmm5, $88 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 + db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 + db $C4, $E2, $61, $00, $59, $10 // vpshufb xmm3, xmm3, oword [ecx + $10] + + db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $D1, $72, $D1, $07 // vpsrld xmm5, xmm1, $7 + db $C5, $F1, $72, $F1, $19 // vpslld xmm1, xmm1, $19 + db $C5, $F1, $EB, $CD // vpor xmm1, xmm1, xmm5 + + // Undiagonalize + db $C5, $F9, $70, $C0, $39 // vpshufd xmm0, xmm0, $39 + db $C5, $F9, $70, $DB, $4E // vpshufd xmm3, xmm3, $4E + db $C5, $F9, $70, $D2, $93 // vpshufd xmm2, xmm2, $93 + + // Finalization: state[i] ^= v[i] ^ v[i+8] + db $C5, $F9, $EF, $C2 // vpxor xmm0, xmm0, xmm2 + db $C5, $F1, $EF, $CB // vpxor xmm1, xmm1, xmm3 + db $C5, $FA, $6F, $23 // vmovdqu xmm4, oword [ebx] + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $FA, $6F, $63, $10 // vmovdqu xmm4, oword [ebx + $10] + db $C5, $F1, $EF, $CC // vpxor xmm1, xmm1, xmm4 + + // Store result + db $C5, $FA, $7F, $03 // vmovdqu oword [ebx], xmm0 + db $C5, $FA, $7F, $4B, $10 // vmovdqu oword [ebx + $10], xmm1 + + db $C5, $F8, $77 // vzeroupper + + + + db $C5, $FA, $6F, $34, $24 // vmovdqu xmm6, oword [esp] + db $C5, $FA, $6F, $7C, $24, $10 // vmovdqu xmm7, oword [esp + $10] + add esp, $20 + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx_x86_64.inc similarity index 85% rename from HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc rename to HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx_x86_64.inc index 55b78f74..f614afe2 100644 --- a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx_x86_64.inc @@ -1,13 +1,25 @@ -// AVX2 implementation of BLAKE2s compress (VEX-encoded 128-bit, fully unrolled 10 rounds). -// Expects MS x64 ABI: rcx = state ptr, rdx = msg ptr, r8 = counter+flags ptr, r9 = IV ptr. -// Uses xmm0-xmm6; xmm6 is MS x64 non-volatile (saved/restored). -// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), xmm3 = d (v12-15), -// xmm4 = message buffer, xmm5 = temp, xmm6 = temp (msg load). -// 3-operand VEX form eliminates movdqa copies needed in SSE2 rotations. +// AVX (VEX-128) implementation of BLAKE2s compress (fully unrolled 10 +// rounds); the Avx name states the ISA requirement: VEX-128 encodings only, +// no 256-bit ymm. 3-operand VEX form eliminates the movdqa copies the SSE2 +// rotations need. Rotations: rot16/8 via vpshufb against the resident +// masks, rot12/7 via shift+or (not byte-aligned; shifts are the known-best +// form). +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = counter+flags ptr, r9 = IV ptr, r10 = byte-rotation masks ptr +// (BLAKE2S_ROT_MASKS: rot16 at +0, rot8 at +16; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = message buffer, xmm5 = temp, +// xmm6 = temp (msg load), xmm7 = rot16 mask, xmm8 = rot8 mask. +// Saves: xmm6-xmm8 (MS x64 non-volatile); uses xmm0-xmm8. // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. // Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $C4, $C1, $7A, $6F, $3A // vmovdqu xmm7, oword [r10] + db $C4, $41, $7A, $6F, $42, $10 // vmovdqu xmm8, oword [r10 + $10] // Initialize working vector db $C5, $FA, $6F, $01 // vmovdqu xmm0, oword [rcx] @@ -32,9 +44,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -53,9 +63,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -79,9 +87,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -100,9 +106,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -128,9 +132,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -149,9 +151,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -175,9 +175,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -196,9 +194,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -224,9 +220,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -245,9 +239,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -271,9 +263,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -292,9 +282,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -320,9 +308,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -341,9 +327,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -367,9 +351,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -388,9 +370,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -416,9 +396,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -437,9 +415,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -463,9 +439,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -484,9 +458,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -512,9 +484,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -533,9 +503,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -559,9 +527,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -580,9 +546,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -608,9 +572,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -629,9 +591,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -655,9 +615,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -676,9 +634,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -704,9 +660,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -725,9 +679,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -751,9 +703,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -772,9 +722,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -800,9 +748,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -821,9 +767,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -847,9 +791,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -868,9 +810,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -896,9 +836,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -917,9 +855,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -943,9 +879,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $10 // vpsrld xmm5, xmm3, 16 - db $C5, $E1, $72, $F3, $10 // vpslld xmm3, xmm3, 16 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -964,9 +898,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $D1, $72, $D3, $08 // vpsrld xmm5, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + db $C4, $C2, $61, $00, $D8 // vpshufb xmm3, xmm3, xmm8 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 @@ -993,4 +925,4 @@ db $C5, $F8, $77 // vzeroupper - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressNeon_aarch64.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressNeon_aarch64.inc index 6d024a8f..3f93b5e1 100644 --- a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressNeon_aarch64.inc @@ -1,9 +1,7 @@ -// BLAKE2s compress AArch64 NEON implementation. -// Expects AAPCS64: x0 = state ptr (8 x UInt32), x1 = message ptr (64 bytes), -// x2 = counter+flags ptr (16 bytes: t[0..1], f[0..1]), x3 = IV ptr (8 x UInt32). -// Leaf nostackframe; caller-saved GPR/vector only. -// Reference: BLAKE2/BLAKE2 neon/blake2s-neon.c, HashLib Blake2SCompressSse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of BLAKE2s compress (fully unrolled 10 rounds). +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (8 x UInt32), +// x1 = message ptr (64 bytes), x2 = counter+flags ptr (16 bytes: t[0..1], +// f[0..1]), x3 = IV ptr (8 x UInt32). // Register map: // v0 = row1 (h[0..3] working) // v1 = row2 (h[4..7] working) @@ -12,7 +10,12 @@ // v4-v7 = message block (16 x UInt32) // v16, v17 = saved initial row1/row2 for the final feed-forward XOR // v20 = message gather temp (G function) -// v21-v23 = rotate temps (ushr/shl/orr) +// v21-v23 = rotate temps (2-op ushr+sli rotates rename the rotated row +// through them; rows return home before the final feed-forward) +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 neon/blake2s-neon.c, HashLib +// Blake2SCompressSse2_x86_64.inc. .long 0xad400400 // ldp q0, q1, [x0] .long 0x4ea01c10 // orr v16.16b, v0.16b, v0.16b @@ -32,85 +35,76 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e042494 // ins v20.s[0], v4.s[1] .long 0x6e0c6494 // ins v20.s[1], v4.s[3] .long 0x6e1424b4 // ins v20.s[2], v5.s[1] .long 0x6e1c64b4 // ins v20.s[3], v5.s[3] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1662d6 // ext v22.16b, v22.16b, v22.16b, #12 .long 0x6e0404d4 // ins v20.s[0], v6.s[0] .long 0x6e0c44d4 // ins v20.s[1], v6.s[2] .long 0x6e1404f4 // ins v20.s[2], v7.s[0] .long 0x6e1c44f4 // ins v20.s[3], v7.s[2] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0424d4 // ins v20.s[0], v6.s[1] .long 0x6e0c64d4 // ins v20.s[1], v6.s[3] .long 0x6e1424f4 // ins v20.s[2], v7.s[1] .long 0x6e1c64f4 // ins v20.s[3], v7.s[3] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1522b5 // ext v21.16b, v21.16b, v21.16b, #4 .long 0x6e0444f4 // ins v20.s[0], v7.s[2] .long 0x6e0c04b4 // ins v20.s[1], v5.s[0] .long 0x6e1424d4 // ins v20.s[2], v6.s[1] .long 0x6e1c24f4 // ins v20.s[3], v7.s[1] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6e600ab5 // rev32 v21.8h, v21.8h + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340436 // ushr v22.4s, v1.4s, #12 + .long 0x6f345436 // sli v22.4s, v1.4s, #20 .long 0x6e0444d4 // ins v20.s[0], v6.s[2] .long 0x6e0c04d4 // ins v20.s[1], v6.s[0] .long 0x6e1464f4 // ins v20.s[2], v7.s[3] .long 0x6e1c44b4 // ins v20.s[3], v5.s[2] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb68694 // add v20.4s, v20.4s, v22.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6f3806a3 // ushr v3.4s, v21.4s, #8 + .long 0x6f3856a3 // sli v3.4s, v21.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6e221ed6 // eor v22.16b, v22.16b, v2.16b + .long 0x6f3906c1 // ushr v1.4s, v22.4s, #7 + .long 0x6f3956c1 // sli v1.4s, v22.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 @@ -125,85 +119,76 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e0404f4 // ins v20.s[0], v7.s[0] .long 0x6e0c4494 // ins v20.s[1], v4.s[2] .long 0x6e1464b4 // ins v20.s[2], v5.s[3] .long 0x6e1c6494 // ins v20.s[3], v4.s[3] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1622d6 // ext v22.16b, v22.16b, v22.16b, #4 .long 0x6e0464d4 // ins v20.s[0], v6.s[3] .long 0x6e0c04f4 // ins v20.s[1], v7.s[0] .long 0x6e1424b4 // ins v20.s[2], v5.s[1] .long 0x6e1c64f4 // ins v20.s[3], v7.s[3] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0404d4 // ins v20.s[0], v6.s[0] .long 0x6e0c0494 // ins v20.s[1], v4.s[0] .long 0x6e144494 // ins v20.s[2], v4.s[2] .long 0x6e1c24f4 // ins v20.s[3], v7.s[1] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1562b5 // ext v21.16b, v21.16b, v21.16b, #12 .long 0x6e0444d4 // ins v20.s[0], v6.s[2] .long 0x6e0c6494 // ins v20.s[1], v4.s[3] .long 0x6e1464b4 // ins v20.s[2], v5.s[3] .long 0x6e1c24d4 // ins v20.s[3], v6.s[1] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6e600ab5 // rev32 v21.8h, v21.8h + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340436 // ushr v22.4s, v1.4s, #12 + .long 0x6f345436 // sli v22.4s, v1.4s, #20 .long 0x6e0444f4 // ins v20.s[0], v7.s[2] .long 0x6e0c44b4 // ins v20.s[1], v5.s[2] .long 0x6e142494 // ins v20.s[2], v4.s[1] .long 0x6e1c04b4 // ins v20.s[3], v5.s[0] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb68694 // add v20.4s, v20.4s, v22.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6f3806a3 // ushr v3.4s, v21.4s, #8 + .long 0x6f3856a3 // sli v3.4s, v21.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6e221ed6 // eor v22.16b, v22.16b, v2.16b + .long 0x6f3906c1 // ushr v1.4s, v22.4s, #7 + .long 0x6f3956c1 // sli v1.4s, v22.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 @@ -218,85 +203,76 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e0424d4 // ins v20.s[0], v6.s[1] .long 0x6e0c2494 // ins v20.s[1], v4.s[1] .long 0x6e1404f4 // ins v20.s[2], v7.s[0] .long 0x6e1c44f4 // ins v20.s[3], v7.s[2] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1662d6 // ext v22.16b, v22.16b, v22.16b, #12 .long 0x6e044494 // ins v20.s[0], v4.s[2] .long 0x6e0c24b4 // ins v20.s[1], v5.s[1] .long 0x6e1404b4 // ins v20.s[2], v5.s[0] .long 0x6e1c64f4 // ins v20.s[3], v7.s[3] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0444b4 // ins v20.s[0], v5.s[2] .long 0x6e0c44d4 // ins v20.s[1], v6.s[2] .long 0x6e140494 // ins v20.s[2], v4.s[0] .long 0x6e1c04d4 // ins v20.s[3], v6.s[0] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1522b5 // ext v21.16b, v21.16b, v21.16b, #4 .long 0x6e0424d4 // ins v20.s[0], v6.s[1] .long 0x6e0c24b4 // ins v20.s[1], v5.s[1] .long 0x6e144494 // ins v20.s[2], v4.s[2] .long 0x6e1c44d4 // ins v20.s[3], v6.s[2] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6e600ab5 // rev32 v21.8h, v21.8h + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340436 // ushr v22.4s, v1.4s, #12 + .long 0x6f345436 // sli v22.4s, v1.4s, #20 .long 0x6e040494 // ins v20.s[0], v4.s[0] .long 0x6e0c64b4 // ins v20.s[1], v5.s[3] .long 0x6e1404b4 // ins v20.s[2], v5.s[0] .long 0x6e1c64f4 // ins v20.s[3], v7.s[3] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb68694 // add v20.4s, v20.4s, v22.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6f3806a3 // ushr v3.4s, v21.4s, #8 + .long 0x6f3856a3 // sli v3.4s, v21.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6e221ed6 // eor v22.16b, v22.16b, v2.16b + .long 0x6f3906c1 // ushr v1.4s, v22.4s, #7 + .long 0x6f3956c1 // sli v1.4s, v22.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 @@ -311,85 +287,76 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e042494 // ins v20.s[0], v4.s[1] .long 0x6e0c04f4 // ins v20.s[1], v7.s[0] .long 0x6e1404d4 // ins v20.s[2], v6.s[0] .long 0x6e1c24f4 // ins v20.s[3], v7.s[1] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1622d6 // ext v22.16b, v22.16b, v22.16b, #4 .long 0x6e044494 // ins v20.s[0], v4.s[2] .long 0x6e0c44b4 // ins v20.s[1], v5.s[2] .long 0x6e140494 // ins v20.s[2], v4.s[0] .long 0x6e1c04d4 // ins v20.s[3], v6.s[0] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0404f4 // ins v20.s[0], v7.s[0] .long 0x6e0c44d4 // ins v20.s[1], v6.s[2] .long 0x6e1464d4 // ins v20.s[2], v6.s[3] .long 0x6e1c6494 // ins v20.s[3], v4.s[3] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1562b5 // ext v21.16b, v21.16b, v21.16b, #12 .long 0x6e0404b4 // ins v20.s[0], v5.s[0] .long 0x6e0c64b4 // ins v20.s[1], v5.s[3] .long 0x6e1464f4 // ins v20.s[2], v7.s[3] .long 0x6e1c2494 // ins v20.s[3], v4.s[1] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6e600ab5 // rev32 v21.8h, v21.8h + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340436 // ushr v22.4s, v1.4s, #12 + .long 0x6f345436 // sli v22.4s, v1.4s, #20 .long 0x6e0424f4 // ins v20.s[0], v7.s[1] .long 0x6e0c24b4 // ins v20.s[1], v5.s[1] .long 0x6e1444f4 // ins v20.s[2], v7.s[2] .long 0x6e1c24d4 // ins v20.s[3], v6.s[1] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb68694 // add v20.4s, v20.4s, v22.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6f3806a3 // ushr v3.4s, v21.4s, #8 + .long 0x6f3856a3 // sli v3.4s, v21.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6e221ed6 // eor v22.16b, v22.16b, v2.16b + .long 0x6f3906c1 // ushr v1.4s, v22.4s, #7 + .long 0x6f3956c1 // sli v1.4s, v22.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 @@ -404,85 +371,76 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e0424b4 // ins v20.s[0], v5.s[1] .long 0x6e0c64f4 // ins v20.s[1], v7.s[3] .long 0x6e1424f4 // ins v20.s[2], v7.s[1] .long 0x6e1c44d4 // ins v20.s[3], v6.s[2] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1662d6 // ext v22.16b, v22.16b, v22.16b, #12 .long 0x6e040494 // ins v20.s[0], v4.s[0] .long 0x6e0c44b4 // ins v20.s[1], v5.s[2] .long 0x6e1424d4 // ins v20.s[2], v6.s[1] .long 0x6e1c04d4 // ins v20.s[3], v6.s[0] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0464b4 // ins v20.s[0], v5.s[3] .long 0x6e0c6494 // ins v20.s[1], v4.s[3] .long 0x6e144494 // ins v20.s[2], v4.s[2] .long 0x6e1c64d4 // ins v20.s[3], v6.s[3] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1522b5 // ext v21.16b, v21.16b, v21.16b, #4 .long 0x6e0424f4 // ins v20.s[0], v7.s[1] .long 0x6e0c64b4 // ins v20.s[1], v5.s[3] .long 0x6e1404f4 // ins v20.s[2], v7.s[0] .long 0x6e1c6494 // ins v20.s[3], v4.s[3] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6e600ab5 // rev32 v21.8h, v21.8h + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340436 // ushr v22.4s, v1.4s, #12 + .long 0x6f345436 // sli v22.4s, v1.4s, #20 .long 0x6e0464d4 // ins v20.s[0], v6.s[3] .long 0x6e0c44f4 // ins v20.s[1], v7.s[2] .long 0x6e142494 // ins v20.s[2], v4.s[1] .long 0x6e1c24d4 // ins v20.s[3], v6.s[1] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb68694 // add v20.4s, v20.4s, v22.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6f3806a3 // ushr v3.4s, v21.4s, #8 + .long 0x6f3856a3 // sli v3.4s, v21.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6e221ed6 // eor v22.16b, v22.16b, v2.16b + .long 0x6f3906c1 // ushr v1.4s, v22.4s, #7 + .long 0x6f3956c1 // sli v1.4s, v22.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 @@ -497,85 +455,76 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e040494 // ins v20.s[0], v4.s[0] .long 0x6e0c04b4 // ins v20.s[1], v5.s[0] .long 0x6e1444b4 // ins v20.s[2], v5.s[2] .long 0x6e1c44d4 // ins v20.s[3], v6.s[2] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1622d6 // ext v22.16b, v22.16b, v22.16b, #4 .long 0x6e0444b4 // ins v20.s[0], v5.s[2] .long 0x6e0c44f4 // ins v20.s[1], v7.s[2] .long 0x6e1464d4 // ins v20.s[2], v6.s[3] .long 0x6e1c0494 // ins v20.s[3], v4.s[0] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0464f4 // ins v20.s[0], v7.s[3] .long 0x6e0c24d4 // ins v20.s[1], v6.s[1] .long 0x6e146494 // ins v20.s[2], v4.s[3] .long 0x6e1c04d4 // ins v20.s[3], v6.s[0] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1562b5 // ext v21.16b, v21.16b, v21.16b, #12 .long 0x6e0404f4 // ins v20.s[0], v7.s[0] .long 0x6e0c24f4 // ins v20.s[1], v7.s[1] .long 0x6e142494 // ins v20.s[2], v4.s[1] .long 0x6e1c44d4 // ins v20.s[3], v6.s[2] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6e600ab5 // rev32 v21.8h, v21.8h + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340436 // ushr v22.4s, v1.4s, #12 + .long 0x6f345436 // sli v22.4s, v1.4s, #20 .long 0x6e044494 // ins v20.s[0], v4.s[2] .long 0x6e0c64b4 // ins v20.s[1], v5.s[3] .long 0x6e1404b4 // ins v20.s[2], v5.s[0] .long 0x6e1c24b4 // ins v20.s[3], v5.s[1] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb68694 // add v20.4s, v20.4s, v22.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b + .long 0x6e201eb5 // eor v21.16b, v21.16b, v0.16b + .long 0x6f3806a3 // ushr v3.4s, v21.4s, #8 + .long 0x6f3856a3 // sli v3.4s, v21.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6e221ed6 // eor v22.16b, v22.16b, v2.16b + .long 0x6f3906c1 // ushr v1.4s, v22.4s, #7 + .long 0x6f3956c1 // sli v1.4s, v22.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 @@ -590,57 +539,52 @@ .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f345435 // sli v21.4s, v1.4s, #20 .long 0x6e044494 // ins v20.s[0], v4.s[2] .long 0x6e0c04b4 // ins v20.s[1], v5.s[0] .long 0x6e1444b4 // ins v20.s[2], v5.s[2] .long 0x6e1c24b4 // ins v20.s[3], v5.s[1] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s + .long 0x4eb58694 // add v20.4s, v20.4s, v21.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f380476 // ushr v22.4s, v3.4s, #8 + .long 0x6f385476 // sli v22.4s, v3.4s, #24 + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s + .long 0x6e221eb5 // eor v21.16b, v21.16b, v2.16b + .long 0x6f3906a1 // ushr v1.4s, v21.4s, #7 + .long 0x6f3956a1 // sli v1.4s, v21.4s, #25 .long 0x6e012021 // ext v1.16b, v1.16b, v1.16b, #4 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e036063 // ext v3.16b, v3.16b, v3.16b, #12 + .long 0x6e1662d6 // ext v22.16b, v22.16b, v22.16b, #12 .long 0x6e0464f4 // ins v20.s[0], v7.s[3] .long 0x6e0c24d4 // ins v20.s[1], v6.s[1] .long 0x6e146494 // ins v20.s[2], v4.s[3] .long 0x6e1c24f4 // ins v20.s[3], v7.s[1] .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6e600ad6 // rev32 v22.8h, v22.8h + .long 0x4eb68442 // add v2.4s, v2.4s, v22.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f340435 // ushr v21.4s, v1.4s, #12 - .long 0x4f345436 // shl v22.4s, v1.4s, #20 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x6e0464d4 // ins v20.s[0], v6.s[3] .long 0x6e0c44f4 // ins v20.s[1], v7.s[2] .long 0x6e1404f4 // ins v20.s[2], v7.s[0] .long 0x6e1c0494 // ins v20.s[3], v4.s[0] - .long 0x4ea18694 // add v20.4s, v20.4s, v1.4s - .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380475 // ushr v21.4s, v3.4s, #8 - .long 0x4f385476 // shl v22.4s, v3.4s, #24 - .long 0x4eb61ea3 // orr v3.16b, v21.16b, v22.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x6f390435 // ushr v21.4s, v1.4s, #7 - .long 0x4f395436 // shl v22.4s, v1.4s, #25 - .long 0x4eb61ea1 // orr v1.16b, v21.16b, v22.16b + .long 0x4ea38694 // add v20.4s, v20.4s, v3.4s + .long 0x4eb48400 // add v0.4s, v0.4s, v20.4s + .long 0x6e201ed6 // eor v22.16b, v22.16b, v0.16b + .long 0x6f3806d5 // ushr v21.4s, v22.4s, #8 + .long 0x6f3856d5 // sli v21.4s, v22.4s, #24 + .long 0x4eb58442 // add v2.4s, v2.4s, v21.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e016021 // ext v1.16b, v1.16b, v1.16b, #12 .long 0x6e024042 // ext v2.16b, v2.16b, v2.16b, #8 - .long 0x6e032063 // ext v3.16b, v3.16b, v3.16b, #4 + .long 0x6e1522b5 // ext v21.16b, v21.16b, v21.16b, #4 + .long 0x4eb51ea3 // orr v3.16b, v21.16b, v21.16b .long 0x6e221c00 // eor v0.16b, v0.16b, v2.16b .long 0x6e201e00 // eor v0.16b, v16.16b, v0.16b .long 0x6e231c21 // eor v1.16b, v1.16b, v3.16b diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_i386.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_i386.inc index 17c0c76d..649dd1ce 100644 --- a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_i386.inc +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_i386.inc @@ -1,12 +1,12 @@ -// SSE2 implementation of BLAKE2s compress (fully unrolled 10 rounds). -// IA-32: after HlpSimdProc4Begin_i386 — ebx = state, esi = msg, edi = counter+flags, eax = IV -// (parallel to MS x64 ABI: rcx, rdx, r8, r9). -// Uses xmm0–xmm6; xmm6 saved/restored defensively (volatile on i386). -// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), xmm3 = d (v12-15), -// xmm4 = message buffer, xmm5 = temp, xmm6 = temp (msg load). -// Rotations: ROT16/12/8/7 via psrld/pslld/por. -// Diagonalize/Undiagonalize via pshufd. -// Message loads via movd + punpcklqdq + shufps. +// SSE2 implementation of BLAKE2s compress (fully unrolled 10 rounds; IA-32). +// Rotations: rot16/12/8/7 via psrld/pslld/por. Diagonalize/Undiagonalize +// via pshufd. Message loads via movd + punpcklqdq + shufps. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = counter+flags ptr, eax = IV ptr. +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = message buffer, xmm5 = temp, +// xmm6 = temp (msg load). +// Saves: xmm6 saved/restored defensively (volatile on IA-32). // Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. sub esp, 16 diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc index 6fdfaf6c..06ce4977 100644 --- a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc @@ -1,14 +1,15 @@ // SSE2 implementation of BLAKE2s compress (fully unrolled 10 rounds). -// Expects MS x64 ABI: rcx = state ptr, rdx = msg ptr, r8 = counter+flags ptr, r9 = IV ptr. -// Uses xmm0-xmm6; xmm6 is MS x64 non-volatile (saved/restored). -// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), xmm3 = d (v12-15), -// xmm4 = message buffer, xmm5 = temp, xmm6 = temp (msg load). -// Rotations: ROT16/12/8/7 via psrld/pslld/por. -// Diagonalize/Undiagonalize via pshufd. -// Message loads via movd + punpcklqdq + shufps. +// Rotations: rot16/12/8/7 via psrld/pslld/por. Diagonalize/Undiagonalize +// via pshufd. Message loads via movd + punpcklqdq + shufps. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = counter+flags ptr, r9 = IV ptr. +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = message buffer, xmm5 = temp, +// xmm6 = temp (msg load). +// Saves: xmm6 (MS x64 non-volatile); uses xmm0-xmm6. // Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} // Initialize working vector movdqu xmm0, oword [rcx] @@ -1072,4 +1073,4 @@ movdqu oword [rcx], xmm0 movdqu oword [rcx + $10], xmm1 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSsse3_i386.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSsse3_i386.inc new file mode 100644 index 00000000..43fc2675 --- /dev/null +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSsse3_i386.inc @@ -0,0 +1,992 @@ +// SSSE3 implementation of BLAKE2s compress (fully unrolled 10 rounds; IA-32); +// SSE2 sibling: Blake2SCompressSse2_i386.inc. Rotations: rot16 via pshufb +// against the resident xmm7 mask; rot8 via a per-site mask reload into the +// xmm5 temp + pshufb (only xmm7 is free); rot12/7 via psrld/pslld/por (not +// byte-aligned). Diagonalize/Undiagonalize via pshufd. Message loads via +// movd + punpcklqdq + shufps. +// ABI (after HlpSimdProc5Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = counter+flags ptr, eax = IV ptr, ecx = byte-rotation masks ptr +// (BLAKE2S_ROT_MASKS: rot16 at +0, rot8 at +16; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = message buffer, xmm5 = temp, +// xmm6 = temp (msg load), xmm7 = rot16 mask. +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. + + sub esp, 32 + movdqu oword ptr [esp], xmm6 + movdqu oword ptr [esp + $10], xmm7 + + // Load the rot16 mask (resident; rot8 reloads the xmm5 temp per site) + movdqu xmm7, oword ptr [ecx] + + // Initialize working vector + movdqu xmm0, oword ptr [ebx] + movdqu xmm1, oword ptr [ebx + $10] + movdqu xmm2, oword ptr [eax] + movdqu xmm3, oword ptr [eax + $10] + + // XOR d with counter and flags + movdqu xmm4, oword ptr [edi] + pxor xmm3, xmm4 + + // ===== Round 0 ===== + + // G1 columns (msg: m0,m2,m4,m6) + movd xmm4, dword ptr [esi] + movd xmm5, dword ptr [esi + $08] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $10] + movd xmm6, dword ptr [esi + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m1,m3,m5,m7) + movd xmm4, dword ptr [esi + $04] + movd xmm5, dword ptr [esi + $0C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $14] + movd xmm6, dword ptr [esi + $1C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m14,m8,m10,m12) + movd xmm4, dword ptr [esi + $38] + movd xmm5, dword ptr [esi + $20] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $28] + movd xmm6, dword ptr [esi + $30] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m15,m9,m11,m13) + movd xmm4, dword ptr [esi + $3C] + movd xmm5, dword ptr [esi + $24] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $2C] + movd xmm6, dword ptr [esi + $34] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 1 ===== + + // G1 columns (msg: m14,m4,m9,m13) + movd xmm4, dword ptr [esi + $38] + movd xmm5, dword ptr [esi + $10] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $24] + movd xmm6, dword ptr [esi + $34] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m10,m8,m15,m6) + movd xmm4, dword ptr [esi + $28] + movd xmm5, dword ptr [esi + $20] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $3C] + movd xmm6, dword ptr [esi + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m5,m1,m0,m11) + movd xmm4, dword ptr [esi + $14] + movd xmm5, dword ptr [esi + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi] + movd xmm6, dword ptr [esi + $2C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m3,m12,m2,m7) + movd xmm4, dword ptr [esi + $0C] + movd xmm5, dword ptr [esi + $30] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $08] + movd xmm6, dword ptr [esi + $1C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 2 ===== + + // G1 columns (msg: m11,m12,m5,m15) + movd xmm4, dword ptr [esi + $2C] + movd xmm5, dword ptr [esi + $30] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $14] + movd xmm6, dword ptr [esi + $3C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m8,m0,m2,m13) + movd xmm4, dword ptr [esi + $20] + movd xmm5, dword ptr [esi] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $08] + movd xmm6, dword ptr [esi + $34] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m9,m10,m3,m7) + movd xmm4, dword ptr [esi + $24] + movd xmm5, dword ptr [esi + $28] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $0C] + movd xmm6, dword ptr [esi + $1C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m4,m14,m6,m1) + movd xmm4, dword ptr [esi + $10] + movd xmm5, dword ptr [esi + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $18] + movd xmm6, dword ptr [esi + $04] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 3 ===== + + // G1 columns (msg: m7,m3,m13,m11) + movd xmm4, dword ptr [esi + $1C] + movd xmm5, dword ptr [esi + $0C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $34] + movd xmm6, dword ptr [esi + $2C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m9,m1,m12,m14) + movd xmm4, dword ptr [esi + $24] + movd xmm5, dword ptr [esi + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $30] + movd xmm6, dword ptr [esi + $38] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m15,m2,m5,m4) + movd xmm4, dword ptr [esi + $3C] + movd xmm5, dword ptr [esi + $08] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $14] + movd xmm6, dword ptr [esi + $10] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m8,m6,m10,m0) + movd xmm4, dword ptr [esi + $20] + movd xmm5, dword ptr [esi + $18] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $28] + movd xmm6, dword ptr [esi] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 4 ===== + + // G1 columns (msg: m9,m5,m2,m10) + movd xmm4, dword ptr [esi + $24] + movd xmm5, dword ptr [esi + $14] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $08] + movd xmm6, dword ptr [esi + $28] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m0,m7,m4,m15) + movd xmm4, dword ptr [esi] + movd xmm5, dword ptr [esi + $1C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $10] + movd xmm6, dword ptr [esi + $3C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m3,m14,m11,m6) + movd xmm4, dword ptr [esi + $0C] + movd xmm5, dword ptr [esi + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $2C] + movd xmm6, dword ptr [esi + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m13,m1,m12,m8) + movd xmm4, dword ptr [esi + $34] + movd xmm5, dword ptr [esi + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $30] + movd xmm6, dword ptr [esi + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 5 ===== + + // G1 columns (msg: m2,m6,m0,m8) + movd xmm4, dword ptr [esi + $08] + movd xmm5, dword ptr [esi + $18] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi] + movd xmm6, dword ptr [esi + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m12,m10,m11,m3) + movd xmm4, dword ptr [esi + $30] + movd xmm5, dword ptr [esi + $28] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $2C] + movd xmm6, dword ptr [esi + $0C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m1,m4,m7,m15) + movd xmm4, dword ptr [esi + $04] + movd xmm5, dword ptr [esi + $10] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $1C] + movd xmm6, dword ptr [esi + $3C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m9,m13,m5,m14) + movd xmm4, dword ptr [esi + $24] + movd xmm5, dword ptr [esi + $34] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $14] + movd xmm6, dword ptr [esi + $38] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 6 ===== + + // G1 columns (msg: m12,m1,m14,m4) + movd xmm4, dword ptr [esi + $30] + movd xmm5, dword ptr [esi + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $38] + movd xmm6, dword ptr [esi + $10] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m5,m15,m13,m10) + movd xmm4, dword ptr [esi + $14] + movd xmm5, dword ptr [esi + $3C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $34] + movd xmm6, dword ptr [esi + $28] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m8,m0,m6,m9) + movd xmm4, dword ptr [esi + $20] + movd xmm5, dword ptr [esi] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $18] + movd xmm6, dword ptr [esi + $24] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m11,m7,m3,m2) + movd xmm4, dword ptr [esi + $2C] + movd xmm5, dword ptr [esi + $1C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $0C] + movd xmm6, dword ptr [esi + $08] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 7 ===== + + // G1 columns (msg: m13,m7,m12,m3) + movd xmm4, dword ptr [esi + $34] + movd xmm5, dword ptr [esi + $1C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $30] + movd xmm6, dword ptr [esi + $0C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m11,m14,m1,m9) + movd xmm4, dword ptr [esi + $2C] + movd xmm5, dword ptr [esi + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $04] + movd xmm6, dword ptr [esi + $24] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m2,m5,m15,m8) + movd xmm4, dword ptr [esi + $08] + movd xmm5, dword ptr [esi + $14] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $3C] + movd xmm6, dword ptr [esi + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m10,m0,m4,m6) + movd xmm4, dword ptr [esi + $28] + movd xmm5, dword ptr [esi] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $10] + movd xmm6, dword ptr [esi + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 8 ===== + + // G1 columns (msg: m6,m14,m11,m0) + movd xmm4, dword ptr [esi + $18] + movd xmm5, dword ptr [esi + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $2C] + movd xmm6, dword ptr [esi] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m15,m9,m3,m8) + movd xmm4, dword ptr [esi + $3C] + movd xmm5, dword ptr [esi + $24] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $0C] + movd xmm6, dword ptr [esi + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m10,m12,m13,m1) + movd xmm4, dword ptr [esi + $28] + movd xmm5, dword ptr [esi + $30] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $34] + movd xmm6, dword ptr [esi + $04] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m5,m2,m7,m4) + movd xmm4, dword ptr [esi + $14] + movd xmm5, dword ptr [esi + $08] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $1C] + movd xmm6, dword ptr [esi + $10] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 9 ===== + + // G1 columns (msg: m10,m8,m7,m1) + movd xmm4, dword ptr [esi + $28] + movd xmm5, dword ptr [esi + $20] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $1C] + movd xmm6, dword ptr [esi + $04] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m2,m4,m6,m5) + movd xmm4, dword ptr [esi + $08] + movd xmm5, dword ptr [esi + $10] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $18] + movd xmm6, dword ptr [esi + $14] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m13,m15,m9,m3) + movd xmm4, dword ptr [esi + $34] + movd xmm5, dword ptr [esi + $3C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $24] + movd xmm6, dword ptr [esi + $0C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m0,m11,m14,m12) + movd xmm4, dword ptr [esi] + movd xmm5, dword ptr [esi + $2C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword ptr [esi + $38] + movd xmm6, dword ptr [esi + $30] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + movdqu xmm5, oword ptr [ecx + $10] + db $66, $0F, $38, $00, $DD // pshufb xmm3, xmm5 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // Finalization: state[i] ^= v[i] ^ v[i+8] + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movdqu xmm4, oword ptr [ebx] + pxor xmm0, xmm4 + movdqu xmm4, oword ptr [ebx + $10] + pxor xmm1, xmm4 + + // Store result + movdqu oword ptr [ebx], xmm0 + movdqu oword ptr [ebx + $10], xmm1 + + movdqu xmm6, oword ptr [esp] + movdqu xmm7, oword ptr [esp + $10] + add esp, 32 + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSsse3_x86_64.inc new file mode 100644 index 00000000..ba519aa0 --- /dev/null +++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSsse3_x86_64.inc @@ -0,0 +1,965 @@ +// SSSE3 implementation of BLAKE2s compress (fully unrolled 10 rounds); SSE2 +// sibling: Blake2SCompressSse2_x86_64.inc. Rotations: rot16/8 via pshufb +// against the resident masks; rot12/7 via psrld/pslld/por (not +// byte-aligned). Diagonalize/Undiagonalize via pshufd. Message loads via +// movd + punpcklqdq + shufps. +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = counter+flags ptr, r9 = IV ptr, r10 = byte-rotation masks ptr +// (BLAKE2S_ROT_MASKS: rot16 at +0, rot8 at +16; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = message buffer, xmm5 = temp, +// xmm6 = temp (msg load), xmm7 = rot16 mask, xmm8 = rot8 mask. +// Saves: xmm6-xmm8 (MS x64 non-volatile); uses xmm0-xmm8. +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves. + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $F3, $41, $0F, $6F, $3A // movdqu xmm7, oword [r10] + db $F3, $45, $0F, $6F, $42, $10 // movdqu xmm8, oword [r10 + $10] + + // Initialize working vector + movdqu xmm0, oword [rcx] + movdqu xmm1, oword [rcx + $10] + movdqu xmm2, oword [r9] + movdqu xmm3, oword [r9 + $10] + + // XOR d with counter and flags + movdqu xmm4, oword [r8] + pxor xmm3, xmm4 + + // ===== Round 0 ===== + + // G1 columns (msg: m0,m2,m4,m6) + movd xmm4, dword [rdx] + movd xmm5, dword [rdx + $08] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $10] + movd xmm6, dword [rdx + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m1,m3,m5,m7) + movd xmm4, dword [rdx + $04] + movd xmm5, dword [rdx + $0C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $14] + movd xmm6, dword [rdx + $1C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m14,m8,m10,m12) + movd xmm4, dword [rdx + $38] + movd xmm5, dword [rdx + $20] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $28] + movd xmm6, dword [rdx + $30] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m15,m9,m11,m13) + movd xmm4, dword [rdx + $3C] + movd xmm5, dword [rdx + $24] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $2C] + movd xmm6, dword [rdx + $34] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 1 ===== + + // G1 columns (msg: m14,m4,m9,m13) + movd xmm4, dword [rdx + $38] + movd xmm5, dword [rdx + $10] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $24] + movd xmm6, dword [rdx + $34] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m10,m8,m15,m6) + movd xmm4, dword [rdx + $28] + movd xmm5, dword [rdx + $20] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $3C] + movd xmm6, dword [rdx + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m5,m1,m0,m11) + movd xmm4, dword [rdx + $14] + movd xmm5, dword [rdx + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx] + movd xmm6, dword [rdx + $2C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m3,m12,m2,m7) + movd xmm4, dword [rdx + $0C] + movd xmm5, dword [rdx + $30] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $08] + movd xmm6, dword [rdx + $1C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 2 ===== + + // G1 columns (msg: m11,m12,m5,m15) + movd xmm4, dword [rdx + $2C] + movd xmm5, dword [rdx + $30] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $14] + movd xmm6, dword [rdx + $3C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m8,m0,m2,m13) + movd xmm4, dword [rdx + $20] + movd xmm5, dword [rdx] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $08] + movd xmm6, dword [rdx + $34] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m9,m10,m3,m7) + movd xmm4, dword [rdx + $24] + movd xmm5, dword [rdx + $28] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $0C] + movd xmm6, dword [rdx + $1C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m4,m14,m6,m1) + movd xmm4, dword [rdx + $10] + movd xmm5, dword [rdx + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $18] + movd xmm6, dword [rdx + $04] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 3 ===== + + // G1 columns (msg: m7,m3,m13,m11) + movd xmm4, dword [rdx + $1C] + movd xmm5, dword [rdx + $0C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $34] + movd xmm6, dword [rdx + $2C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m9,m1,m12,m14) + movd xmm4, dword [rdx + $24] + movd xmm5, dword [rdx + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $30] + movd xmm6, dword [rdx + $38] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m15,m2,m5,m4) + movd xmm4, dword [rdx + $3C] + movd xmm5, dword [rdx + $08] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $14] + movd xmm6, dword [rdx + $10] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m8,m6,m10,m0) + movd xmm4, dword [rdx + $20] + movd xmm5, dword [rdx + $18] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $28] + movd xmm6, dword [rdx] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 4 ===== + + // G1 columns (msg: m9,m5,m2,m10) + movd xmm4, dword [rdx + $24] + movd xmm5, dword [rdx + $14] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $08] + movd xmm6, dword [rdx + $28] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m0,m7,m4,m15) + movd xmm4, dword [rdx] + movd xmm5, dword [rdx + $1C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $10] + movd xmm6, dword [rdx + $3C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m3,m14,m11,m6) + movd xmm4, dword [rdx + $0C] + movd xmm5, dword [rdx + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $2C] + movd xmm6, dword [rdx + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m13,m1,m12,m8) + movd xmm4, dword [rdx + $34] + movd xmm5, dword [rdx + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $30] + movd xmm6, dword [rdx + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 5 ===== + + // G1 columns (msg: m2,m6,m0,m8) + movd xmm4, dword [rdx + $08] + movd xmm5, dword [rdx + $18] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx] + movd xmm6, dword [rdx + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m12,m10,m11,m3) + movd xmm4, dword [rdx + $30] + movd xmm5, dword [rdx + $28] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $2C] + movd xmm6, dword [rdx + $0C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m1,m4,m7,m15) + movd xmm4, dword [rdx + $04] + movd xmm5, dword [rdx + $10] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $1C] + movd xmm6, dword [rdx + $3C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m9,m13,m5,m14) + movd xmm4, dword [rdx + $24] + movd xmm5, dword [rdx + $34] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $14] + movd xmm6, dword [rdx + $38] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 6 ===== + + // G1 columns (msg: m12,m1,m14,m4) + movd xmm4, dword [rdx + $30] + movd xmm5, dword [rdx + $04] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $38] + movd xmm6, dword [rdx + $10] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m5,m15,m13,m10) + movd xmm4, dword [rdx + $14] + movd xmm5, dword [rdx + $3C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $34] + movd xmm6, dword [rdx + $28] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m8,m0,m6,m9) + movd xmm4, dword [rdx + $20] + movd xmm5, dword [rdx] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $18] + movd xmm6, dword [rdx + $24] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m11,m7,m3,m2) + movd xmm4, dword [rdx + $2C] + movd xmm5, dword [rdx + $1C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $0C] + movd xmm6, dword [rdx + $08] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 7 ===== + + // G1 columns (msg: m13,m7,m12,m3) + movd xmm4, dword [rdx + $34] + movd xmm5, dword [rdx + $1C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $30] + movd xmm6, dword [rdx + $0C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m11,m14,m1,m9) + movd xmm4, dword [rdx + $2C] + movd xmm5, dword [rdx + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $04] + movd xmm6, dword [rdx + $24] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m2,m5,m15,m8) + movd xmm4, dword [rdx + $08] + movd xmm5, dword [rdx + $14] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $3C] + movd xmm6, dword [rdx + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m10,m0,m4,m6) + movd xmm4, dword [rdx + $28] + movd xmm5, dword [rdx] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $10] + movd xmm6, dword [rdx + $18] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 8 ===== + + // G1 columns (msg: m6,m14,m11,m0) + movd xmm4, dword [rdx + $18] + movd xmm5, dword [rdx + $38] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $2C] + movd xmm6, dword [rdx] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m15,m9,m3,m8) + movd xmm4, dword [rdx + $3C] + movd xmm5, dword [rdx + $24] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $0C] + movd xmm6, dword [rdx + $20] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m10,m12,m13,m1) + movd xmm4, dword [rdx + $28] + movd xmm5, dword [rdx + $30] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $34] + movd xmm6, dword [rdx + $04] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m5,m2,m7,m4) + movd xmm4, dword [rdx + $14] + movd xmm5, dword [rdx + $08] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $1C] + movd xmm6, dword [rdx + $10] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Round 9 ===== + + // G1 columns (msg: m10,m8,m7,m1) + movd xmm4, dword [rdx + $28] + movd xmm5, dword [rdx + $20] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $1C] + movd xmm6, dword [rdx + $04] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 columns (msg: m2,m4,m6,m5) + movd xmm4, dword [rdx + $08] + movd xmm5, dword [rdx + $10] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $18] + movd xmm6, dword [rdx + $14] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // G1 diagonals (msg: m13,m15,m9,m3) + movd xmm4, dword [rdx + $34] + movd xmm5, dword [rdx + $3C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $24] + movd xmm6, dword [rdx + $0C] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 12 + pslld xmm5, 20 + por xmm1, xmm5 + + // G2 diagonals (msg: m0,m11,m14,m12) + movd xmm4, dword [rdx] + movd xmm5, dword [rdx + $2C] + punpcklqdq xmm4, xmm5 + movd xmm5, dword [rdx + $38] + movd xmm6, dword [rdx + $30] + punpcklqdq xmm5, xmm6 + shufps xmm4, xmm5, $88 + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm5, xmm1 + psrld xmm1, 7 + pslld xmm5, 25 + por xmm1, xmm5 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // Finalization: state[i] ^= v[i] ^ v[i+8] + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movdqu xmm4, oword [rcx] + pxor xmm0, xmm4 + movdqu xmm4, oword [rcx + $10] + pxor xmm1, xmm4 + + // Store result + movdqu oword [rcx], xmm0 + movdqu oword [rcx + $10], xmm1 + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx_x86_64.inc similarity index 83% rename from HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc rename to HashLib/src/Include/Simd/Blake3/Blake3CompressAvx_x86_64.inc index c2c1023f..54443df4 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx_x86_64.inc @@ -1,22 +1,32 @@ -// AVX2 implementation of BLAKE3 compress (VEX-encoded 128-bit, 7 rounds with register-based -// message scheduling). +// AVX (VEX-128) implementation of BLAKE3 compress (7 rounds with +// register-based message scheduling); the Avx name states the ISA +// requirement: VEX-128 encodings only, no 256-bit ymm. 3-operand VEX form +// eliminates the movdqa copies the SSE2 rotations need. Rotations: rot16/8 +// via vpshufb against the resident masks, rot12/7 via shift+or (not +// byte-aligned; shifts are the known-best form). Blend emulations use +// vshufps+vpshufd (blend_0xCC) and vpunpckhdq+vshufps (blend_0xC0), +// avoiding the need for mask constants. +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = CV ptr, r9 = counterflags ptr, r10 = byte-rotation masks ptr +// (BLAKE3_ROT_MASKS: rot16 at +0, rot8 at +32; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0 = a, xmm1 = b, xmm2 = c, xmm3 = d, xmm4-xmm7 = m0-m3, +// xmm8/xmm9 = temp, xmm11/xmm14 = temp, xmm10 = rot16 mask, +// xmm12 = rot8 mask. +// Frame (sub rsp, 120): +// [rsp + 0.. 95] = xmm6/xmm7/xmm8/xmm9/xmm11/xmm14 save area (6 x 16) +// [rsp + 96..111] = IV staging area (loaded into xmm2) +// [rsp +112..119] = padding +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. // Reference: BLAKE3-team/BLAKE3 official SSE2/AVX2 implementation. -// Expects MS x64 ABI: rcx = state ptr, rdx = msg ptr, r8 = CV ptr, r9 = counterflags ptr. -// Register map: xmm0 = a, xmm1 = b, xmm2 = c, xmm3 = d, -// xmm4 = m0, xmm5 = m1, xmm6 = m2, xmm7 = m3, xmm8/xmm9 = temp, xmm11/xmm14 = temp. -// 3-operand VEX form eliminates movdqa copies needed in SSE2 rotations. -// Blend emulations use vshufps+vpshufd (blend_0xCC) and vpunpckhdq+vshufps (blend_0xC0), -// avoiding the need for mask constants. -// -// MS x64 non-volatile saves: xmm6, xmm7, xmm8, xmm9, xmm11, xmm14. -// -// Stack layout (sub rsp, 120): -// [rsp + 0.. 95]: xmm6/xmm7/xmm8/xmm9/xmm11/xmm14 save area (6 * 16) -// [rsp + 96..111]: IV staging area (loaded into xmm2) -// [rsp +112..119]: padding - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $C4, $41, $7A, $6F, $12 // vmovdqu xmm10, oword [r10] + db $C4, $41, $7A, $6F, $62, $20 // vmovdqu xmm12, oword [r10 + $20] sub rsp, 24 @@ -56,8 +66,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -67,9 +76,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -83,8 +90,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -94,9 +100,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -133,8 +137,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -144,9 +147,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -160,8 +161,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -171,9 +171,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -210,8 +208,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -221,9 +218,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -237,8 +232,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -248,9 +242,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -287,8 +279,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -298,9 +289,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -314,8 +303,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -325,9 +313,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -364,8 +350,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -375,9 +360,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -391,8 +374,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -402,9 +384,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -441,8 +421,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -452,9 +431,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -468,8 +445,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -479,9 +455,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -518,8 +492,7 @@ db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -529,9 +502,7 @@ db $C5, $F9, $FE, $C5 // vpaddd xmm0, xmm0, xmm5 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -545,8 +516,7 @@ db $C5, $F9, $FE, $C6 // vpaddd xmm0, xmm0, xmm6 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $FB, $70, $DB, $B1 // vpshuflw xmm3, xmm3, $B1 - db $C5, $FA, $70, $DB, $B1 // vpshufhw xmm3, xmm3, $B1 + db $C4, $C2, $61, $00, $DA // vpshufb xmm3, xmm3, xmm10 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $14 // vpslld xmm11, xmm1, 20 @@ -556,9 +526,7 @@ db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 db $C5, $F9, $FE, $C1 // vpaddd xmm0, xmm0, xmm1 db $C5, $E1, $EF, $D8 // vpxor xmm3, xmm3, xmm0 - db $C5, $89, $72, $D3, $08 // vpsrld xmm14, xmm3, 8 - db $C5, $E1, $72, $F3, $18 // vpslld xmm3, xmm3, 24 - db $C4, $C1, $61, $EF, $DE // vpxor xmm3, xmm3, xmm14 + db $C4, $C2, $61, $00, $DC // vpshufb xmm3, xmm3, xmm12 db $C5, $E9, $FE, $D3 // vpaddd xmm2, xmm2, xmm3 db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 db $C5, $A1, $72, $F1, $19 // vpslld xmm11, xmm1, 25 @@ -584,4 +552,4 @@ add rsp, 24 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressNeon_aarch64.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressNeon_aarch64.inc index 3a4e9b07..fcace040 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3CompressNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressNeon_aarch64.inc @@ -1,9 +1,8 @@ -// BLAKE3 compress AArch64 NEON implementation. -// Expects AAPCS64: x0 = state ptr (16 x UInt32), x1 = message ptr (64 bytes), -// x2 = CV ptr (8 x UInt32), x3 = counter+flags ptr (16 bytes: t[0..1], len, flags). -// Stack frame 128 bytes; see register map. -// Reference: BLAKE3-team/BLAKE3 c/blake3_impl.h, HashLib Blake3CompressSse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of BLAKE3 compress (7 rounds with register-based +// message scheduling). +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (16 x UInt32), +// x1 = message ptr (64 bytes), x2 = CV ptr (8 x UInt32), +// x3 = counter+flags ptr (16 bytes: t[0..1], len, flags). // Register map: // v0 = a (v[0..3]) // v1 = b (v[4..7]) @@ -12,11 +11,15 @@ // v4-v7 = message schedule registers m0..m3 // v16-v17 = permute temps t8/t9; v17 saved to stack mid-permute when shufps // needs v17 as scratch (see [sp, #0] spill below) -// v18-v19 = rotate temps (vsri path) +// v18-v19 = rotate temps (2-op ushr+sli rotates rename rows b/d through them) // v20 = shufps / non-cyclic pshufd scratch (t14); cyclic diagonalize // shuffles use in-place 'ext' instead -// Stack (128 bytes): [sp, #0] transient t9 (v17) spill during inter-round permute; -// [sp, #96..111] IV constant staging in prologue. +// Frame: 128 bytes ([sp, #0] transient t9 (v17) spill during the inter-round +// permute; [sp, #96..111] IV constant staging in the prologue). +// Saves: none (caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: BLAKE3-team/BLAKE3 c/blake3_impl.h, HashLib +// Blake3CompressSse2_x86_64.inc. sub sp, sp, #128 .long 0x529ccceb // movz w11, #0xe667, lsl #0 @@ -64,49 +67,39 @@ .long 0x6e600863 // rev32 v3.8h, v3.8h .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340432 // ushr v18.4s, v1.4s, #12 + .long 0x6f345432 // sli v18.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s + .long 0x4eb28400 // add v0.4s, v0.4s, v18.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f380473 // ushr v19.4s, v3.4s, #8 + .long 0x6f385473 // sli v19.4s, v3.4s, #24 + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s + .long 0x6e221e52 // eor v18.16b, v18.16b, v2.16b + .long 0x6f390641 // ushr v1.4s, v18.4s, #7 + .long 0x6f395641 // sli v1.4s, v18.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e134273 // ext v19.16b, v19.16b, v19.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 .long 0x4ea68400 // add v0.4s, v0.4s, v6.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6e600a73 // rev32 v19.8h, v19.8h + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x4ea38400 // add v0.4s, v0.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6f380672 // ushr v18.4s, v19.4s, #8 + .long 0x6f385672 // sli v18.4s, v19.4s, #24 + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e124252 // ext v18.16b, v18.16b, v18.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 .long 0x4ea41c90 // orr v16.16b, v4.16b, v4.16b .long 0x6e044614 // ins v20.s[0], v16.s[2] @@ -163,26 +156,21 @@ .long 0x4eb01e06 // orr v6.16b, v16.16b, v16.16b .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6e600a52 // rev32 v18.8h, v18.8h + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340433 // ushr v19.4s, v1.4s, #12 + .long 0x6f345433 // sli v19.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b + .long 0x4eb38400 // add v0.4s, v0.4s, v19.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6f380643 // ushr v3.4s, v18.4s, #8 + .long 0x6f385643 // sli v3.4s, v18.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6e221e73 // eor v19.16b, v19.16b, v2.16b + .long 0x6f390661 // ushr v1.4s, v19.4s, #7 + .long 0x6f395661 // sli v1.4s, v19.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 @@ -192,24 +180,19 @@ .long 0x6e600863 // rev32 v3.8h, v3.8h .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340432 // ushr v18.4s, v1.4s, #12 + .long 0x6f345432 // sli v18.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s + .long 0x4eb28400 // add v0.4s, v0.4s, v18.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f380473 // ushr v19.4s, v3.4s, #8 + .long 0x6f385473 // sli v19.4s, v3.4s, #24 + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s + .long 0x6e221e52 // eor v18.16b, v18.16b, v2.16b + .long 0x6f390641 // ushr v1.4s, v18.4s, #7 + .long 0x6f395641 // sli v1.4s, v18.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e134273 // ext v19.16b, v19.16b, v19.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 .long 0x4ea41c90 // orr v16.16b, v4.16b, v4.16b .long 0x6e044614 // ins v20.s[0], v16.s[2] @@ -266,51 +249,41 @@ .long 0x4eb01e06 // orr v6.16b, v16.16b, v16.16b .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6e600a73 // rev32 v19.8h, v19.8h + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x4ea38400 // add v0.4s, v0.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6f380672 // ushr v18.4s, v19.4s, #8 + .long 0x6f385672 // sli v18.4s, v19.4s, #24 + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e124252 // ext v18.16b, v18.16b, v18.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 .long 0x4ea68400 // add v0.4s, v0.4s, v6.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6e600a52 // rev32 v18.8h, v18.8h + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340433 // ushr v19.4s, v1.4s, #12 + .long 0x6f345433 // sli v19.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b + .long 0x4eb38400 // add v0.4s, v0.4s, v19.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6f380643 // ushr v3.4s, v18.4s, #8 + .long 0x6f385643 // sli v3.4s, v18.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6e221e73 // eor v19.16b, v19.16b, v2.16b + .long 0x6f390661 // ushr v1.4s, v19.4s, #7 + .long 0x6f395661 // sli v1.4s, v19.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 @@ -373,49 +346,39 @@ .long 0x6e600863 // rev32 v3.8h, v3.8h .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340432 // ushr v18.4s, v1.4s, #12 + .long 0x6f345432 // sli v18.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s + .long 0x4eb28400 // add v0.4s, v0.4s, v18.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f380473 // ushr v19.4s, v3.4s, #8 + .long 0x6f385473 // sli v19.4s, v3.4s, #24 + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s + .long 0x6e221e52 // eor v18.16b, v18.16b, v2.16b + .long 0x6f390641 // ushr v1.4s, v18.4s, #7 + .long 0x6f395641 // sli v1.4s, v18.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e134273 // ext v19.16b, v19.16b, v19.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 .long 0x4ea68400 // add v0.4s, v0.4s, v6.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6e600a73 // rev32 v19.8h, v19.8h + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x4ea38400 // add v0.4s, v0.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6f380672 // ushr v18.4s, v19.4s, #8 + .long 0x6f385672 // sli v18.4s, v19.4s, #24 + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e124252 // ext v18.16b, v18.16b, v18.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 .long 0x4ea41c90 // orr v16.16b, v4.16b, v4.16b .long 0x6e044614 // ins v20.s[0], v16.s[2] @@ -472,26 +435,21 @@ .long 0x4eb01e06 // orr v6.16b, v16.16b, v16.16b .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6e600a52 // rev32 v18.8h, v18.8h + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340433 // ushr v19.4s, v1.4s, #12 + .long 0x6f345433 // sli v19.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b + .long 0x4eb38400 // add v0.4s, v0.4s, v19.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6f380643 // ushr v3.4s, v18.4s, #8 + .long 0x6f385643 // sli v3.4s, v18.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6e221e73 // eor v19.16b, v19.16b, v2.16b + .long 0x6f390661 // ushr v1.4s, v19.4s, #7 + .long 0x6f395661 // sli v1.4s, v19.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 @@ -501,24 +459,19 @@ .long 0x6e600863 // rev32 v3.8h, v3.8h .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340432 // ushr v18.4s, v1.4s, #12 + .long 0x6f345432 // sli v18.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s + .long 0x4eb28400 // add v0.4s, v0.4s, v18.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f380473 // ushr v19.4s, v3.4s, #8 + .long 0x6f385473 // sli v19.4s, v3.4s, #24 + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s + .long 0x6e221e52 // eor v18.16b, v18.16b, v2.16b + .long 0x6f390641 // ushr v1.4s, v18.4s, #7 + .long 0x6f395641 // sli v1.4s, v18.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e134273 // ext v19.16b, v19.16b, v19.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 .long 0x4ea41c90 // orr v16.16b, v4.16b, v4.16b .long 0x6e044614 // ins v20.s[0], v16.s[2] @@ -575,51 +528,41 @@ .long 0x4eb01e06 // orr v6.16b, v16.16b, v16.16b .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6e600a73 // rev32 v19.8h, v19.8h + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x4ea38400 // add v0.4s, v0.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6f380672 // ushr v18.4s, v19.4s, #8 + .long 0x6f385672 // sli v18.4s, v19.4s, #24 + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e124252 // ext v18.16b, v18.16b, v18.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 .long 0x4ea68400 // add v0.4s, v0.4s, v6.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6e600a52 // rev32 v18.8h, v18.8h + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340433 // ushr v19.4s, v1.4s, #12 + .long 0x6f345433 // sli v19.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b + .long 0x4eb38400 // add v0.4s, v0.4s, v19.4s + .long 0x6e201e52 // eor v18.16b, v18.16b, v0.16b + .long 0x6f380643 // ushr v3.4s, v18.4s, #8 + .long 0x6f385643 // sli v3.4s, v18.4s, #24 .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6e221e73 // eor v19.16b, v19.16b, v2.16b + .long 0x6f390661 // ushr v1.4s, v19.4s, #7 + .long 0x6f395661 // sli v1.4s, v19.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 @@ -682,50 +625,41 @@ .long 0x6e600863 // rev32 v3.8h, v3.8h .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340432 // ushr v18.4s, v1.4s, #12 + .long 0x6f345432 // sli v18.4s, v1.4s, #20 .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s + .long 0x4eb28400 // add v0.4s, v0.4s, v18.4s .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f380473 // ushr v19.4s, v3.4s, #8 + .long 0x6f385473 // sli v19.4s, v3.4s, #24 + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s + .long 0x6e221e52 // eor v18.16b, v18.16b, v2.16b + .long 0x6f390641 // ushr v1.4s, v18.4s, #7 + .long 0x6f395641 // sli v1.4s, v18.4s, #25 .long 0x6e006000 // ext v0.16b, v0.16b, v0.16b, #12 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e134273 // ext v19.16b, v19.16b, v19.16b, #8 .long 0x6e022042 // ext v2.16b, v2.16b, v2.16b, #4 .long 0x4ea68400 // add v0.4s, v0.4s, v6.4s .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6e600863 // rev32 v3.8h, v3.8h - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6e600a73 // rev32 v19.8h, v19.8h + .long 0x4eb38442 // add v2.4s, v2.4s, v19.4s .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344432 // sri v18.4s, v1.4s, #12 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x6f340423 // ushr v3.4s, v1.4s, #12 + .long 0x6f345423 // sli v3.4s, v1.4s, #20 .long 0x4ea78400 // add v0.4s, v0.4s, v7.4s - .long 0x4ea18400 // add v0.4s, v0.4s, v1.4s - .long 0x6e201c63 // eor v3.16b, v3.16b, v0.16b - .long 0x6f380472 // ushr v18.4s, v3.4s, #8 - .long 0x4f385473 // shl v19.4s, v3.4s, #24 - .long 0x4eb31e43 // orr v3.16b, v18.16b, v19.16b - .long 0x4ea38442 // add v2.4s, v2.4s, v3.4s - .long 0x6e221c21 // eor v1.16b, v1.16b, v2.16b - .long 0x4ea11c32 // orr v18.16b, v1.16b, v1.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394432 // sri v18.4s, v1.4s, #7 - .long 0x4eb21e41 // orr v1.16b, v18.16b, v18.16b + .long 0x4ea38400 // add v0.4s, v0.4s, v3.4s + .long 0x6e201e73 // eor v19.16b, v19.16b, v0.16b + .long 0x6f380672 // ushr v18.4s, v19.4s, #8 + .long 0x6f385672 // sli v18.4s, v19.4s, #24 + .long 0x4eb28442 // add v2.4s, v2.4s, v18.4s + .long 0x6e221c63 // eor v3.16b, v3.16b, v2.16b + .long 0x6f390461 // ushr v1.4s, v3.4s, #7 + .long 0x6f395461 // sli v1.4s, v3.4s, #25 .long 0x6e002000 // ext v0.16b, v0.16b, v0.16b, #4 - .long 0x6e034063 // ext v3.16b, v3.16b, v3.16b, #8 + .long 0x6e124252 // ext v18.16b, v18.16b, v18.16b, #8 .long 0x6e026042 // ext v2.16b, v2.16b, v2.16b, #12 + .long 0x4eb21e43 // orr v3.16b, v18.16b, v18.16b .long 0x6e221c00 // eor v0.16b, v0.16b, v2.16b .long 0x6e231c21 // eor v1.16b, v1.16b, v3.16b .long 0xad404450 // ldp q16, q17, [x2] diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_i386.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_i386.inc index 0fee3c4b..71092ad0 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_i386.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_i386.inc @@ -1,12 +1,14 @@ -// SSE2 implementation of BLAKE3 compress (7 rounds with register-based message scheduling). -// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation (blake3_sse2_x86-64_windows_msvc.asm). -// IA-32: after HlpSimdProc4Begin_i386 — ebx = state, esi = msg, edi = CV, eax = counterflags ptr -// (parallel to MS x64 ABI: rcx, rdx, r8, r9). -// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), xmm3 = d (v12-15); -// IA-32 uses xmm0–7 only — high-XMM temps from x64 are folded into xmm0/1/2 + stack spill; -// rotations use xmm0 + [esp+$20]. +// SSE2 implementation of BLAKE3 compress (7 rounds with register-based +// message scheduling; IA-32). Message scheduling / diagonalize: same +// strategy as the x64 sibling (blend emulation, pshufd), per sneves. // Rotations: rot16 via pshuflw+pshufhw; rot12/8/7 via pslld/psrld/por. -// Message scheduling / diagonalize: same strategy as x64 (blend emulation, pshufd), per sneves. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = msg ptr, +// edi = CV ptr, eax = counterflags ptr. +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15); IA-32 uses xmm0-xmm7 only - high-XMM temps from x64 +// are folded into xmm0/1/2 + stack spill; rotations use xmm0 + [esp+$20]. +// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation +// (blake3_sse2_x86-64_windows_msvc.asm), HashLib Blake3CompressSse2_x86_64.inc. sub esp, 120 movdqu oword ptr [esp], xmm6 @@ -26,7 +28,7 @@ // Initialize state: a=CV[0..3], b=CV[4..7], c=IV[0..3], d=CounterFlags movdqu xmm0, oword ptr [edi] movdqu xmm1, oword ptr [edi + $10] - // xmm2 = IV (loaded above); eax clobbered by IV stores — reload counterflags + // xmm2 = IV (loaded above); eax clobbered by IV stores - reload counterflags mov eax, dword ptr [esp + $30] movdqu xmm3, oword ptr [eax] diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc index 396ecb5d..c3e73799 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc @@ -1,22 +1,24 @@ -// SSE2 implementation of BLAKE3 compress (7 rounds with register-based message scheduling). -// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation (blake3_sse2_x86-64_windows_msvc.asm). -// Expects MS x64 ABI: rcx = state ptr, rdx = msg ptr, r8 = CV ptr, r9 = counterflags ptr. -// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), xmm3 = d (v12-15), -// xmm4 = m0 (col G1 msg), xmm5 = m1 (col G2 msg), xmm6 = m2 (diag G1 msg), -// xmm7 = m3 (diag G2 msg), xmm8/xmm9 = temp (msg perm), xmm11/xmm14 = temp (rotations). -// Rotations: rot16 via pshuflw+pshufhw; rot12/8/7 via pslld/psrld/por. -// Message scheduling: all 16 words kept in xmm4-7, permuted between rounds using -// shufps/pshufd (blend_0xCC emulated via shufps+pshufd, blend_0xC0 via punpckhdq+shufps). -// Diagonalize/Undiagonalize via pshufd (row1 unrotated, per sneves optimization). -// -// MS x64 non-volatile saves: xmm6, xmm7, xmm8, xmm9, xmm11, xmm14. -// -// Stack layout (sub rsp, 120): -// [rsp + 0.. 95]: xmm6/xmm7/xmm8/xmm9/xmm11/xmm14 save area (6 * 16) -// [rsp + 96..111]: IV staging area (loaded into xmm2) -// [rsp +112..119]: padding - - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} +// SSE2 implementation of BLAKE3 compress (7 rounds with register-based +// message scheduling). All 16 message words stay in xmm4-xmm7, permuted +// between rounds via shufps/pshufd (blend_0xCC emulated via shufps+pshufd, +// blend_0xC0 via punpckhdq+shufps). Rotations: rot16 via pshuflw+pshufhw; +// rot12/8/7 via pslld/psrld/por. Diagonalize/Undiagonalize via pshufd +// (row1 unrotated, per sneves optimization). +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = CV ptr, r9 = counterflags ptr. +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4 = m0 (col G1 msg), xmm5 = m1 (col G2 msg), +// xmm6 = m2 (diag G1 msg), xmm7 = m3 (diag G2 msg), +// xmm8/xmm9 = temp (msg perm), xmm11/xmm14 = temp (rotations). +// Frame (sub rsp, 120): +// [rsp + 0.. 95] = xmm6/xmm7/xmm8/xmm9/xmm11/xmm14 save area (6 x 16) +// [rsp + 96..111] = IV staging area (loaded into xmm2) +// [rsp +112..119] = padding +// Saves: xmm6, xmm7, xmm8, xmm9, xmm11, xmm14 (MS x64 non-volatile). +// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation +// (blake3_sse2_x86-64_windows_msvc.asm). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} sub rsp, 24 @@ -637,4 +639,4 @@ add rsp, 24 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressSsse3_x86_64.inc new file mode 100644 index 00000000..122d926a --- /dev/null +++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressSsse3_x86_64.inc @@ -0,0 +1,590 @@ +// SSSE3 implementation of BLAKE3 compress (7 rounds with register-based +// message scheduling); SSE2 sibling: Blake3CompressSse2_x86_64.inc. +// Message scheduling: all 16 words kept in xmm4-xmm7, permuted between +// rounds via shufps/pshufd (blend_0xCC emulated via shufps+pshufd, +// blend_0xC0 via punpckhdq+shufps). Rotations: rot16/8 via pshufb against +// the resident masks; rot12/7 via pslld/psrld/por (not byte-aligned). +// Diagonalize/Undiagonalize via pshufd (row1 unrotated, per sneves +// optimization). +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = msg ptr, +// r8 = CV ptr, r9 = counterflags ptr, r10 = byte-rotation masks ptr +// (BLAKE3_ROT_MASKS: rot16 at +0, rot8 at +32; read unaligned, so the +// Pascal const needs no special alignment). +// Register map: xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15), xmm4-xmm7 = m0-m3, xmm8/xmm9 = temp (msg perm), +// xmm11/xmm14 = temp (rotations), xmm10 = rot16 mask, xmm12 = rot8 mask. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV). +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE3-team/BLAKE3 official SSE2/SSE4.1 implementations. + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $F3, $45, $0F, $6F, $12 // movdqu xmm10, oword [r10] + db $F3, $45, $0F, $6F, $62, $20 // movdqu xmm12, oword [r10 + $20] + + sub rsp, 24 + + mov eax, $6A09E667 + mov dword ptr [rsp + $00], eax + mov eax, $BB67AE85 + mov dword ptr [rsp + $04], eax + mov eax, $3C6EF372 + mov dword ptr [rsp + $08], eax + mov eax, $A54FF53A + mov dword ptr [rsp + $0C], eax + movdqa xmm2, oword [rsp + $00] + + // Initialize state: a=CV[0..3], b=CV[4..7], c=IV[0..3], d=CounterFlags + movdqu xmm0, oword [r8] + movdqu xmm1, oword [r8 + $10] + // xmm2 = IV (loaded above) + movdqu xmm3, oword [r9] + + // Load and reorder message into 4 register pairs for round structure. + // After reorder: xmm4={m0,m2,m4,m6}, xmm5={m1,m3,m5,m7}, + // xmm6={m14,m8,m10,m12} (rotated), xmm7={m15,m9,m11,m13} (rotated). + movdqu xmm4, oword [rdx] + movdqu xmm5, oword [rdx + $10] + movdqa xmm8, xmm4 + shufps xmm4, xmm5, $88 + shufps xmm8, xmm5, $DD + movdqa xmm5, xmm8 + + movdqu xmm6, oword [rdx + $20] + movdqu xmm7, oword [rdx + $30] + movdqa xmm8, xmm6 + shufps xmm6, xmm7, $88 + pshufd xmm6, xmm6, $93 + shufps xmm8, xmm7, $DD + pshufd xmm7, xmm8, $93 + + // ===== Round 0 ===== + + // Column G1 (msg xmm4) + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + // Column G2 (msg xmm5) + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + // Diagonalize + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + // Diagonal G1 (msg xmm6) + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + // Diagonal G2 (msg xmm7) + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + // Undiagonalize + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Message Permutation 0->1 ===== + movdqa xmm8, xmm4 + shufps xmm8, xmm5, $D6 + pshufd xmm9, xmm4, $0F + pshufd xmm4, xmm8, $39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, $FA + shufps xmm9, xmm8, $D8 + pshufd xmm9, xmm9, $D8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm14, xmm8 + punpckhdq xmm14, xmm6 + shufps xmm8, xmm14, $C4 + pshufd xmm8, xmm8, $78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, $1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + + // ===== Round 1 ===== + + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Message Permutation 1->2 ===== + movdqa xmm8, xmm4 + shufps xmm8, xmm5, $D6 + pshufd xmm9, xmm4, $0F + pshufd xmm4, xmm8, $39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, $FA + shufps xmm9, xmm8, $D8 + pshufd xmm9, xmm9, $D8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm14, xmm8 + punpckhdq xmm14, xmm6 + shufps xmm8, xmm14, $C4 + pshufd xmm8, xmm8, $78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, $1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + + // ===== Round 2 ===== + + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Message Permutation 2->3 ===== + movdqa xmm8, xmm4 + shufps xmm8, xmm5, $D6 + pshufd xmm9, xmm4, $0F + pshufd xmm4, xmm8, $39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, $FA + shufps xmm9, xmm8, $D8 + pshufd xmm9, xmm9, $D8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm14, xmm8 + punpckhdq xmm14, xmm6 + shufps xmm8, xmm14, $C4 + pshufd xmm8, xmm8, $78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, $1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + + // ===== Round 3 ===== + + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Message Permutation 3->4 ===== + movdqa xmm8, xmm4 + shufps xmm8, xmm5, $D6 + pshufd xmm9, xmm4, $0F + pshufd xmm4, xmm8, $39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, $FA + shufps xmm9, xmm8, $D8 + pshufd xmm9, xmm9, $D8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm14, xmm8 + punpckhdq xmm14, xmm6 + shufps xmm8, xmm14, $C4 + pshufd xmm8, xmm8, $78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, $1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + + // ===== Round 4 ===== + + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Message Permutation 4->5 ===== + movdqa xmm8, xmm4 + shufps xmm8, xmm5, $D6 + pshufd xmm9, xmm4, $0F + pshufd xmm4, xmm8, $39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, $FA + shufps xmm9, xmm8, $D8 + pshufd xmm9, xmm9, $D8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm14, xmm8 + punpckhdq xmm14, xmm6 + shufps xmm8, xmm14, $C4 + pshufd xmm8, xmm8, $78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, $1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + + // ===== Round 5 ===== + + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Message Permutation 5->6 ===== + movdqa xmm8, xmm4 + shufps xmm8, xmm5, $D6 + pshufd xmm9, xmm4, $0F + pshufd xmm4, xmm8, $39 + movdqa xmm8, xmm6 + shufps xmm8, xmm7, $FA + shufps xmm9, xmm8, $D8 + pshufd xmm9, xmm9, $D8 + movdqa xmm8, xmm7 + punpcklqdq xmm8, xmm5 + movdqa xmm14, xmm8 + punpckhdq xmm14, xmm6 + shufps xmm8, xmm14, $C4 + pshufd xmm8, xmm8, $78 + punpckhdq xmm5, xmm7 + punpckldq xmm6, xmm5 + pshufd xmm7, xmm6, $1E + movdqa xmm5, xmm9 + movdqa xmm6, xmm8 + + // ===== Round 6 ===== + + paddd xmm0, xmm4 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm5 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $93 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $39 + + paddd xmm0, xmm6 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DA // pshufb xmm3, xmm10 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 20 + psrld xmm11, 12 + por xmm1, xmm11 + + paddd xmm0, xmm7 + paddd xmm0, xmm1 + pxor xmm3, xmm0 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm2, xmm3 + pxor xmm1, xmm2 + movdqa xmm11, xmm1 + pslld xmm1, 25 + psrld xmm11, 7 + por xmm1, xmm11 + + pshufd xmm0, xmm0, $39 + pshufd xmm3, xmm3, $4E + pshufd xmm2, xmm2, $93 + + // ===== Finalization ===== + // BLAKE3: state[0..7] = a,b XOR c,d; state[8..15] = c,d XOR CV + pxor xmm0, xmm2 + pxor xmm1, xmm3 + movdqu xmm4, oword [r8] + movdqu xmm5, oword [r8 + $10] + pxor xmm2, xmm4 + pxor xmm3, xmm5 + + // Store all 16 words + movdqu oword [rcx], xmm0 + movdqu oword [rcx + $10], xmm1 + movdqu oword [rcx + $20], xmm2 + movdqu oword [rcx + $30], xmm3 + + add rsp, 24 + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc index 7907c57f..68d79f51 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Neon_aarch64.inc @@ -1,24 +1,29 @@ -// BLAKE3 hash4 AArch64 NEON implementation (4 parallel 1 KiB chunks). -// Expects AAPCS64: x0 = AInput, x1 = AKey, x2 = AOut, -// w3 = ANumChunks (unused; always 16 blocks like SSE2), x4 = ACounter, w5 = AFlags. -// Stack frame 720 bytes; see register map. -// Reference: BLAKE3-team/BLAKE3 c/blake3_impl.h, HashLib Blake3Hash4Sse2_x86_64.inc -// (output chunk order matches HashLib SSE2, not upstream blake3_neon.c store layout). -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of BLAKE3 hash4 (4 parallel 1 KiB chunks). +// Message transpose uses trn1/trn2 + zip1/zip2 (equivalent to SSE punpck / +// neon transpose_vecs_128). +// ABI (after HlpSimdProc6Begin_aarch64.inc): x0 = input ptr, x1 = key ptr, +// x2 = out ptr, w3 = num chunks (unused; always 16 blocks like SSE2), +// x4 = counter, w5 = flags. // Register map: // v0-v7 = state v0..v7 (CV rows, 4-wide) // v8-v11 = state v8..v11 (IV rows, updated each round) -// v16-v17 = G temps (v12/v13 analog; d lane spilled via stack v12-v15 slots) -// v18-v19 = rotate temps (vsri path inside G) +// v16-v17 = G temps (v12/v13 analog); every rotate is the 2-op insert +// form and the rotating values (d and b) migrate v_b -> t13 -> v_b +// within each G; the d lane spills via the stack v12-v15 slots // v20-v23 = transpose / block temps // x19 = block index (0..15), x20 = block byte offset within chunk -// Stack frame 720 bytes (712B SSE2 layout + 8B align): -// [sp, #0..255] transposed message m[0..15] (four 64-byte groups at +0,+64,+128,+192) -// [sp, #256..304] v12-v15 spills (counter lo/hi, block len, flags) -// [sp, #320..568] saved CV, counter vectors, IV, block len, AOut ptr, flags -// Message transpose uses trn1/trn2 + zip1/zip2 (equivalent to SSE punpck / neon transpose_vecs_128). +// Frame: 720 bytes (712B SSE2 layout + 8B align): +// [sp, #0..255] = transposed message m[0..15] (four 64-byte groups) +// [sp, #256..304] = v12-v15 spills (counter lo/hi, block len, flags) +// [sp, #320..568] = saved CV, counter vectors, IV, block len, AOut, flags +// Saves: d8-d15 via the shared HlpSimdNonVolatileSave/Restore include +// (v8-v15 hold IV rows / spills); x19/x20 saved in the prologue. +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: BLAKE3-team/BLAKE3 c/blake3_impl.h, HashLib +// Blake3Hash4Sse2_x86_64.inc (output chunk order matches HashLib SSE2, +// not upstream blake3_neon.c store layout). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} stp x19, x20, [sp, #-16]! sub sp, sp, #720 @@ -203,24 +208,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc007f1 // ldr q17, [sp, #16] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc007e4 // ldr q4, [sp, #16] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc00bf1 // ldr q17, [sp, #32] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -229,24 +229,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc00ff1 // ldr q17, [sp, #48] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc00fe5 // ldr q5, [sp, #48] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc013f1 // ldr q17, [sp, #64] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -255,24 +250,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc017f1 // ldr q17, [sp, #80] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc017e6 // ldr q6, [sp, #80] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc01bf1 // ldr q17, [sp, #96] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -281,24 +271,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc01ff1 // ldr q17, [sp, #112] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc01fe7 // ldr q7, [sp, #112] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc023f1 // ldr q17, [sp, #128] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -307,24 +292,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc027f1 // ldr q17, [sp, #144] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc027e5 // ldr q5, [sp, #144] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc02bf1 // ldr q17, [sp, #160] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -333,24 +313,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc02ff1 // ldr q17, [sp, #176] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc02fe6 // ldr q6, [sp, #176] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc033f1 // ldr q17, [sp, #192] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -359,24 +334,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc037f1 // ldr q17, [sp, #208] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc037e7 // ldr q7, [sp, #208] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc03bf1 // ldr q17, [sp, #224] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -385,24 +355,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc03ff1 // ldr q17, [sp, #240] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc03fe4 // ldr q4, [sp, #240] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc00bf1 // ldr q17, [sp, #32] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -411,24 +376,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc01bf1 // ldr q17, [sp, #96] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc01be4 // ldr q4, [sp, #96] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc00ff1 // ldr q17, [sp, #48] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -437,24 +397,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc02bf1 // ldr q17, [sp, #160] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc02be5 // ldr q5, [sp, #160] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc01ff1 // ldr q17, [sp, #112] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -463,24 +418,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc003f1 // ldr q17, [sp, #0] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc003e6 // ldr q6, [sp, #0] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc013f1 // ldr q17, [sp, #64] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -489,24 +439,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc037f1 // ldr q17, [sp, #208] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc037e7 // ldr q7, [sp, #208] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc007f1 // ldr q17, [sp, #16] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -515,24 +460,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc02ff1 // ldr q17, [sp, #176] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc02fe5 // ldr q5, [sp, #176] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc033f1 // ldr q17, [sp, #192] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -541,24 +481,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc017f1 // ldr q17, [sp, #80] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc017e6 // ldr q6, [sp, #80] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc027f1 // ldr q17, [sp, #144] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -567,24 +502,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc03bf1 // ldr q17, [sp, #224] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc03be7 // ldr q7, [sp, #224] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc03ff1 // ldr q17, [sp, #240] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -593,24 +523,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc023f1 // ldr q17, [sp, #128] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc023e4 // ldr q4, [sp, #128] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc00ff1 // ldr q17, [sp, #48] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -619,24 +544,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc013f1 // ldr q17, [sp, #64] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc013e4 // ldr q4, [sp, #64] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc02bf1 // ldr q17, [sp, #160] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -645,24 +565,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc033f1 // ldr q17, [sp, #192] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc033e5 // ldr q5, [sp, #192] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc037f1 // ldr q17, [sp, #208] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -671,24 +586,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc00bf1 // ldr q17, [sp, #32] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc00be6 // ldr q6, [sp, #32] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc01ff1 // ldr q17, [sp, #112] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -697,24 +607,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc03bf1 // ldr q17, [sp, #224] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc03be7 // ldr q7, [sp, #224] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc01bf1 // ldr q17, [sp, #96] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -723,24 +628,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc017f1 // ldr q17, [sp, #80] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc017e5 // ldr q5, [sp, #80] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc027f1 // ldr q17, [sp, #144] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -749,24 +649,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc003f1 // ldr q17, [sp, #0] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc003e6 // ldr q6, [sp, #0] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc02ff1 // ldr q17, [sp, #176] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -775,24 +670,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc03ff1 // ldr q17, [sp, #240] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc03fe7 // ldr q7, [sp, #240] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc023f1 // ldr q17, [sp, #128] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -801,24 +691,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc007f1 // ldr q17, [sp, #16] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc007e4 // ldr q4, [sp, #16] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc02bf1 // ldr q17, [sp, #160] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -827,24 +712,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc01ff1 // ldr q17, [sp, #112] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc01fe4 // ldr q4, [sp, #112] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc033f1 // ldr q17, [sp, #192] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -853,24 +733,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc027f1 // ldr q17, [sp, #144] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc027e5 // ldr q5, [sp, #144] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc03bf1 // ldr q17, [sp, #224] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -879,24 +754,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc00ff1 // ldr q17, [sp, #48] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc00fe6 // ldr q6, [sp, #48] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc037f1 // ldr q17, [sp, #208] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -905,24 +775,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc03ff1 // ldr q17, [sp, #240] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc03fe7 // ldr q7, [sp, #240] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc013f1 // ldr q17, [sp, #64] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -931,24 +796,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc003f1 // ldr q17, [sp, #0] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc003e5 // ldr q5, [sp, #0] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc02ff1 // ldr q17, [sp, #176] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -957,24 +817,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc00bf1 // ldr q17, [sp, #32] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc00be6 // ldr q6, [sp, #32] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc017f1 // ldr q17, [sp, #80] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -983,24 +838,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc023f1 // ldr q17, [sp, #128] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc023e7 // ldr q7, [sp, #128] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc007f1 // ldr q17, [sp, #16] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1009,24 +859,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc01bf1 // ldr q17, [sp, #96] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc01be4 // ldr q4, [sp, #96] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc033f1 // ldr q17, [sp, #192] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -1035,24 +880,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc037f1 // ldr q17, [sp, #208] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc037e4 // ldr q4, [sp, #208] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc027f1 // ldr q17, [sp, #144] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -1061,24 +901,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc02ff1 // ldr q17, [sp, #176] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc02fe5 // ldr q5, [sp, #176] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc03ff1 // ldr q17, [sp, #240] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -1087,24 +922,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc02bf1 // ldr q17, [sp, #160] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc02be6 // ldr q6, [sp, #160] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc03bf1 // ldr q17, [sp, #224] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1113,24 +943,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc023f1 // ldr q17, [sp, #128] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc023e7 // ldr q7, [sp, #128] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc01ff1 // ldr q17, [sp, #112] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -1139,24 +964,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc00bf1 // ldr q17, [sp, #32] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc00be5 // ldr q5, [sp, #32] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc017f1 // ldr q17, [sp, #80] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -1165,24 +985,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc00ff1 // ldr q17, [sp, #48] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc00fe6 // ldr q6, [sp, #48] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc003f1 // ldr q17, [sp, #0] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -1191,24 +1006,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc007f1 // ldr q17, [sp, #16] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc007e7 // ldr q7, [sp, #16] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc01bf1 // ldr q17, [sp, #96] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1217,24 +1027,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc013f1 // ldr q17, [sp, #64] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc013e4 // ldr q4, [sp, #64] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc027f1 // ldr q17, [sp, #144] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -1243,24 +1048,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc03bf1 // ldr q17, [sp, #224] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc03be4 // ldr q4, [sp, #224] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc02ff1 // ldr q17, [sp, #176] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -1269,24 +1069,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc017f1 // ldr q17, [sp, #80] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc017e5 // ldr q5, [sp, #80] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc023f1 // ldr q17, [sp, #128] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -1295,24 +1090,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc033f1 // ldr q17, [sp, #192] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc033e6 // ldr q6, [sp, #192] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc03ff1 // ldr q17, [sp, #240] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1321,24 +1111,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc007f1 // ldr q17, [sp, #16] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc007e7 // ldr q7, [sp, #16] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc037f1 // ldr q17, [sp, #208] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -1347,24 +1132,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc00ff1 // ldr q17, [sp, #48] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc00fe5 // ldr q5, [sp, #48] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc003f1 // ldr q17, [sp, #0] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -1373,24 +1153,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc02bf1 // ldr q17, [sp, #160] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc02be6 // ldr q6, [sp, #160] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc00bf1 // ldr q17, [sp, #32] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -1399,24 +1174,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc01bf1 // ldr q17, [sp, #96] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc01be7 // ldr q7, [sp, #96] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc013f1 // ldr q17, [sp, #64] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1425,24 +1195,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc01ff1 // ldr q17, [sp, #112] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc01fe4 // ldr q4, [sp, #112] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc02ff1 // ldr q17, [sp, #176] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -1451,24 +1216,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc03ff1 // ldr q17, [sp, #240] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc03fe4 // ldr q4, [sp, #240] .long 0x4ea48400 // add v0.4s, v0.4s, v4.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281c84 // eor v4.16b, v4.16b, v8.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48508 // add v8.4s, v8.4s, v4.4s + .long 0x3d8043e4 // str q4, [sp, #256] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc017f1 // ldr q17, [sp, #80] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -1477,24 +1237,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc003f1 // ldr q17, [sp, #0] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc003e5 // ldr q5, [sp, #0] .long 0x4ea58421 // add v1.4s, v1.4s, v5.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291ca5 // eor v5.16b, v5.16b, v9.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea58529 // add v9.4s, v9.4s, v5.4s + .long 0x3d8047e5 // str q5, [sp, #272] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc007f1 // ldr q17, [sp, #16] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -1503,24 +1258,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc027f1 // ldr q17, [sp, #144] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc027e6 // ldr q6, [sp, #144] .long 0x4ea68442 // add v2.4s, v2.4s, v6.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1cc6 // eor v6.16b, v6.16b, v10.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6854a // add v10.4s, v10.4s, v6.4s + .long 0x3d804be6 // str q6, [sp, #288] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc023f1 // ldr q17, [sp, #128] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1529,24 +1279,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc01bf1 // ldr q17, [sp, #96] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc01be7 // ldr q7, [sp, #96] .long 0x4ea78463 // add v3.4s, v3.4s, v7.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1ce7 // eor v7.16b, v7.16b, v11.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea7856b // add v11.4s, v11.4s, v7.4s + .long 0x3d804fe7 // str q7, [sp, #304] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04ff0 // ldr q16, [sp, #304] .long 0x3dc03bf1 // ldr q17, [sp, #224] .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s @@ -1555,24 +1300,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444b2 // sri v18.4s, v5.4s, #12 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3dc02bf1 // ldr q17, [sp, #160] - .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s + .long 0x4f3454b1 // shl v17.4s, v5.4s, #20 + .long 0x6f3444b1 // sri v17.4s, v5.4s, #12 + .long 0x3dc02be5 // ldr q5, [sp, #160] .long 0x4ea58400 // add v0.4s, v0.4s, v5.4s + .long 0x4eb18400 // add v0.4s, v0.4s, v17.4s .long 0x6e201e10 // eor v16.16b, v16.16b, v0.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0854a // add v10.4s, v10.4s, v16.4s - .long 0x6e2a1ca5 // eor v5.16b, v5.16b, v10.16b - .long 0x4ea51cb2 // orr v18.16b, v5.16b, v5.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944b2 // sri v18.4s, v5.4s, #7 - .long 0x4eb21e45 // orr v5.16b, v18.16b, v18.16b - .long 0x3d804ff0 // str q16, [sp, #304] + .long 0x6f380605 // ushr v5.4s, v16.4s, #8 + .long 0x6f385605 // sli v5.4s, v16.4s, #24 + .long 0x4ea5854a // add v10.4s, v10.4s, v5.4s + .long 0x3d804fe5 // str q5, [sp, #304] + .long 0x6e2a1e31 // eor v17.16b, v17.16b, v10.16b + .long 0x4f395625 // shl v5.4s, v17.4s, #25 + .long 0x6f394625 // sri v5.4s, v17.4s, #7 .long 0x3dc043f0 // ldr q16, [sp, #256] .long 0x3dc00bf1 // ldr q17, [sp, #32] .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s @@ -1581,24 +1321,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444d2 // sri v18.4s, v6.4s, #12 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3dc033f1 // ldr q17, [sp, #192] - .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s + .long 0x4f3454d1 // shl v17.4s, v6.4s, #20 + .long 0x6f3444d1 // sri v17.4s, v6.4s, #12 + .long 0x3dc033e6 // ldr q6, [sp, #192] .long 0x4ea68421 // add v1.4s, v1.4s, v6.4s + .long 0x4eb18421 // add v1.4s, v1.4s, v17.4s .long 0x6e211e10 // eor v16.16b, v16.16b, v1.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb0856b // add v11.4s, v11.4s, v16.4s - .long 0x6e2b1cc6 // eor v6.16b, v6.16b, v11.16b - .long 0x4ea61cd2 // orr v18.16b, v6.16b, v6.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944d2 // sri v18.4s, v6.4s, #7 - .long 0x4eb21e46 // orr v6.16b, v18.16b, v18.16b - .long 0x3d8043f0 // str q16, [sp, #256] + .long 0x6f380606 // ushr v6.4s, v16.4s, #8 + .long 0x6f385606 // sli v6.4s, v16.4s, #24 + .long 0x4ea6856b // add v11.4s, v11.4s, v6.4s + .long 0x3d8043e6 // str q6, [sp, #256] + .long 0x6e2b1e31 // eor v17.16b, v17.16b, v11.16b + .long 0x4f395626 // shl v6.4s, v17.4s, #25 + .long 0x6f394626 // sri v6.4s, v17.4s, #7 .long 0x3dc047f0 // ldr q16, [sp, #272] .long 0x3dc00ff1 // ldr q17, [sp, #48] .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s @@ -1607,24 +1342,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f3444f2 // sri v18.4s, v7.4s, #12 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3dc013f1 // ldr q17, [sp, #64] - .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s + .long 0x4f3454f1 // shl v17.4s, v7.4s, #20 + .long 0x6f3444f1 // sri v17.4s, v7.4s, #12 + .long 0x3dc013e7 // ldr q7, [sp, #64] .long 0x4ea78442 // add v2.4s, v2.4s, v7.4s + .long 0x4eb18442 // add v2.4s, v2.4s, v17.4s .long 0x6e221e10 // eor v16.16b, v16.16b, v2.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08508 // add v8.4s, v8.4s, v16.4s - .long 0x6e281ce7 // eor v7.16b, v7.16b, v8.16b - .long 0x4ea71cf2 // orr v18.16b, v7.16b, v7.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f3944f2 // sri v18.4s, v7.4s, #7 - .long 0x4eb21e47 // orr v7.16b, v18.16b, v18.16b - .long 0x3d8047f0 // str q16, [sp, #272] + .long 0x6f380607 // ushr v7.4s, v16.4s, #8 + .long 0x6f385607 // sli v7.4s, v16.4s, #24 + .long 0x4ea78508 // add v8.4s, v8.4s, v7.4s + .long 0x3d8047e7 // str q7, [sp, #272] + .long 0x6e281e31 // eor v17.16b, v17.16b, v8.16b + .long 0x4f395627 // shl v7.4s, v17.4s, #25 + .long 0x6f394627 // sri v7.4s, v17.4s, #7 .long 0x3dc04bf0 // ldr q16, [sp, #288] .long 0x3dc01ff1 // ldr q17, [sp, #112] .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s @@ -1633,24 +1363,19 @@ .long 0x6e600a10 // rev32 v16.8h, v16.8h .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f345652 // shl v18.4s, v18.4s, #20 - .long 0x6f344492 // sri v18.4s, v4.4s, #12 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3dc037f1 // ldr q17, [sp, #208] - .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s + .long 0x4f345491 // shl v17.4s, v4.4s, #20 + .long 0x6f344491 // sri v17.4s, v4.4s, #12 + .long 0x3dc037e4 // ldr q4, [sp, #208] .long 0x4ea48463 // add v3.4s, v3.4s, v4.4s + .long 0x4eb18463 // add v3.4s, v3.4s, v17.4s .long 0x6e231e10 // eor v16.16b, v16.16b, v3.16b - .long 0x6f380612 // ushr v18.4s, v16.4s, #8 - .long 0x4f385613 // shl v19.4s, v16.4s, #24 - .long 0x4eb31e50 // orr v16.16b, v18.16b, v19.16b - .long 0x4eb08529 // add v9.4s, v9.4s, v16.4s - .long 0x6e291c84 // eor v4.16b, v4.16b, v9.16b - .long 0x4ea41c92 // orr v18.16b, v4.16b, v4.16b - .long 0x4f395652 // shl v18.4s, v18.4s, #25 - .long 0x6f394492 // sri v18.4s, v4.4s, #7 - .long 0x4eb21e44 // orr v4.16b, v18.16b, v18.16b - .long 0x3d804bf0 // str q16, [sp, #288] + .long 0x6f380604 // ushr v4.4s, v16.4s, #8 + .long 0x6f385604 // sli v4.4s, v16.4s, #24 + .long 0x4ea48529 // add v9.4s, v9.4s, v4.4s + .long 0x3d804be4 // str q4, [sp, #288] + .long 0x6e291e31 // eor v17.16b, v17.16b, v9.16b + .long 0x4f395624 // shl v4.4s, v17.4s, #25 + .long 0x6f394624 // sri v4.4s, v17.4s, #7 .long 0x6e281c00 // eor v0.16b, v0.16b, v8.16b .long 0x6e291c21 // eor v1.16b, v1.16b, v9.16b .long 0x6e2a1c42 // eor v2.16b, v2.16b, v10.16b @@ -1711,5 +1436,5 @@ add sp, sp, #720 ldp x19, x20, [sp], #16 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_i386.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_i386.inc index 22904198..3461ddd7 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_i386.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_i386.inc @@ -1,23 +1,27 @@ -// SSE2 i386 implementation of BLAKE3 hash4: processes up to 4 chunks sequentially. -// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation (blake3_sse2.c / hash_one). -// i386 has only xmm0-xmm7 (8 registers), so inter-chunk parallelism is not possible; -// each chunk is processed one at a time with the inline SSE2 compress kernel. -// Register map during rounds: xmm0=a(v0-3), xmm1=b(v4-7), xmm2=c(v8-11), xmm3=d(v12-15); -// xmm4={m0,m2,m4,m6}, xmm5={m1,m3,m5,m7}, xmm6={m14,m8,m10,m12}, xmm7={m15,m9,m11,m13}. -// After HlpSimdProc6Begin_i386: ebx=P1 (input), esi=P2 (key), edi=P3 (out), eax=P4 (num chunks), -// ecx=P5_lo, edx=P5_hi (chunk counter). Prologue pushed ebx/esi/edi — epilogue pops them. -// Block index (0..15) is kept at [esp+$90] so ebp is never used as a temp (frame-friendly). -// Stack layout (148 bytes, esp-relative after sub esp,148): -// [esp+$00..0F] xmm6 save [esp+$10..1F] xmm7 save -// [esp+$20..2F] TEMP_A (xmm0 spill during rounds) -// [esp+$30..3F] TEMP_B (xmm1 spill during msg permutation — also D_VEC build slot) -// [esp+$40..4F] TEMP_C (xmm2 spill during msg permutation) -// [esp+$50..5F] IV_VEC {$6A09E667,$BB67AE85,$3C6EF372,$A54FF53A} -// [esp+$60..6F] CV_LO (running chaining value low 16 bytes) -// [esp+$70..7F] CV_HI (running chaining value high 16 bytes) -// [esp+$80] CTR_LO [esp+$84] CTR_HI [esp+$88] BASE_FLAGS [esp+$8C] SAVE_NC -// [esp+$90] BLOCK_IDX (current block within chunk, 0..15) -// P6 (flags): FPC i386 [esp+$A4] (16+148 after sub esp,148). Delphi [ebp+8]. +// SSE2 implementation of BLAKE3 hash4 (IA-32): processes up to 4 chunks +// sequentially. IA-32 has only xmm0-xmm7, so inter-chunk parallelism is not +// possible; each chunk runs one at a time through the inline SSE2 compress +// kernel. The block index (0..15) is kept at [esp+$90] so ebp is never used +// as a temp (frame-friendly). +// ABI (after HlpSimdProc6Begin_i386.inc): ebx = AInput, esi = AKey, +// edi = AOut, eax = ANumChunks, ecx = ACounter.lo32, edx = ACounter.hi32. +// P6 (AFlags) is a stack parameter: FPC i386 [esp+$A4] (16+148 after +// sub esp,148); Delphi [ebp+8]. +// Register map (rounds): xmm0 = a (v0-3), xmm1 = b (v4-7), xmm2 = c (v8-11), +// xmm3 = d (v12-15); xmm4 = {m0,m2,m4,m6}, xmm5 = {m1,m3,m5,m7}, +// xmm6 = {m14,m8,m10,m12}, xmm7 = {m15,m9,m11,m13}. +// Frame (sub esp, 148): +// [esp+$00..0F] = xmm6 save [esp+$10..1F] = xmm7 save +// [esp+$20..2F] = TEMP_A (xmm0 spill during rounds) +// [esp+$30..3F] = TEMP_B (xmm1 spill during msg permutation / D_VEC build) +// [esp+$40..4F] = TEMP_C (xmm2 spill during msg permutation) +// [esp+$50..5F] = IV_VEC {$6A09E667,$BB67AE85,$3C6EF372,$A54FF53A} +// [esp+$60..6F] = CV_LO [esp+$70..7F] = CV_HI (running chaining value) +// [esp+$80] = CTR_LO [esp+$84] = CTR_HI [esp+$88] = BASE_FLAGS +// [esp+$8C] = SAVE_NC [esp+$90] = BLOCK_IDX (current block, 0..15) +// Saves: xmm6-xmm7 saved/restored defensively (volatile on IA-32). +// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation +// (blake3_sse2.c / hash_one). sub esp, 148 @@ -63,7 +67,7 @@ @h4_block_loop: // Build D_VEC = {CTR_LO, CTR_HI, BLKLEN=64, flags} at TEMP_B slot [esp+$30] - // (TEMP_B will be overwritten by msg permutation later — that is expected) + // (TEMP_B will be overwritten by msg permutation later - that is expected) mov ecx, dword ptr [esp + $80] // CTR_LO mov edx, dword ptr [esp + $84] // CTR_HI mov eax, dword ptr [esp + $88] // BASE_FLAGS diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc index f17b54fc..a73c3093 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc @@ -1,24 +1,23 @@ -// SSE2 implementation of BLAKE3 hash4: processes 4 independent chunks simultaneously. -// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation (blake3_sse2.c / blake3_hash_many). -// Each XMM register holds the same 32-bit word from 4 different chunks (inter-chunk SIMD). -// 7 rounds per block fully unrolled; 16-block loop with branch for chunk start/end flags. -// -// After HlpSimdProc6Begin_x86_64.inc: -// rcx = AInput, rdx = AKey, r8 = AOut, -// r9 = ANumChunks, r10 = ACounter, r11 = AFlags -// +// SSE2 implementation of BLAKE3 hash4: processes 4 independent chunks +// simultaneously. Each XMM register holds the same 32-bit word from 4 +// different chunks (inter-chunk SIMD); 7 rounds per block fully unrolled; +// 16-block loop with branch for chunk start/end flags. Message transpose: +// 4x4 via punpckldq/punpckhdq + punpcklqdq/punpckhqdq (pure SSE2). +// Rotations: rot16 via pshuflw+pshufhw; rot12/8/7 via pslld/psrld/por. +// ABI (after HlpSimdProc6Begin_x86_64.inc): rcx = AInput, rdx = AKey, +// r8 = AOut, r9 = ANumChunks, r10 = ACounter, r11 = AFlags. // Register map: xmm0-xmm7 = state v0-v7, xmm8-xmm11 = state v8-v11, // xmm12-xmm13 = temp, v12-v15 spilled to stack. -// GPR: rcx = input pointer, rbx = block counter (0..15), rbp = block byte offset. -// Message transpose: 4x4 via punpckldq/punpckhdq + punpcklqdq/punpckhqdq (pure SSE2). -// Rotations: rot16 via pshuflw+pshufhw; rot12/8/7 via pslld/psrld/por. -// -// MS x64 non-volatile saves: xmm6-xmm13. +// GPR: rcx = input pointer, rbx = block counter (0..15), rbp = block +// byte offset. +// Saves: xmm6-xmm13 (MS x64 non-volatile). +// Reference: BLAKE3-team/BLAKE3 official SSE2 implementation +// (blake3_sse2.c / blake3_hash_many). push rbx push rbp - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} sub rsp, 584 @@ -1828,7 +1827,7 @@ add rsp, 584 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} pop rbp pop rbx diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Ssse3_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Ssse3_x86_64.inc new file mode 100644 index 00000000..d0596506 --- /dev/null +++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Ssse3_x86_64.inc @@ -0,0 +1,1620 @@ +// SSSE3 implementation of BLAKE3 hash4: processes 4 independent chunks +// simultaneously; SSE2 sibling: Blake3Hash4Sse2_x86_64.inc. Each XMM +// register holds the same 32-bit word from 4 different chunks (inter-chunk +// SIMD); 7 rounds per block fully unrolled; 16-block loop with branch for +// chunk start/end flags. Message transpose: 4x4 via punpckldq/punpckhdq + +// punpcklqdq/punpckhqdq (pure SSE2). Rotations: rot16/8 via pshufb against +// the resident masks; rot12/7 via pslld/psrld/por (not byte-aligned). +// ABI (after HlpSimdProc7Begin_x86_64.inc): rcx = AInput, rdx = AKey, +// r8 = AOut, r9 = ANumChunks, r10 = ACounter, r11 = AFlags, +// rax = byte-rotation masks ptr (BLAKE3_ROT_MASKS: rot16 at +0, rot8 at +// +32; read unaligned, so the Pascal const needs no special alignment; +// consumed by the entry mask loads, rax is scratch afterwards). +// Register map: xmm0-xmm7 = state v0-v7, xmm8-xmm11 = state v8-v11, +// xmm12-xmm13 = temp, xmm14 = rot16 mask, xmm15 = rot8 mask, +// v12-v15 spilled to stack. +// GPR: rcx = input pointer, rbx = block counter (0..15), rbp = block +// byte offset. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV). +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE3-team/BLAKE3 official SSE2/SSE4.1 implementations +// (blake3_hash_many). + + push rbx + push rbp + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $F3, $44, $0F, $6F, $30 // movdqu xmm14, oword [rax] + db $F3, $44, $0F, $6F, $78, $20 // movdqu xmm15, oword [rax + $20] + + sub rsp, 584 + + mov qword ptr [rsp + 560], r8 + mov dword ptr [rsp + 568], r11d + + // Broadcast key words to CV registers + movd xmm0, dword ptr [rdx + 0] + pshufd xmm0, xmm0, 0 + movd xmm1, dword ptr [rdx + 4] + pshufd xmm1, xmm1, 0 + movd xmm2, dword ptr [rdx + 8] + pshufd xmm2, xmm2, 0 + movd xmm3, dword ptr [rdx + 12] + pshufd xmm3, xmm3, 0 + movd xmm4, dword ptr [rdx + 16] + pshufd xmm4, xmm4, 0 + movd xmm5, dword ptr [rdx + 20] + pshufd xmm5, xmm5, 0 + movd xmm6, dword ptr [rdx + 24] + pshufd xmm6, xmm6, 0 + movd xmm7, dword ptr [rdx + 28] + pshufd xmm7, xmm7, 0 + + // Save initial CV to stack + movdqa oword [rsp + 320], xmm0 + movdqa oword [rsp + 336], xmm1 + movdqa oword [rsp + 352], xmm2 + movdqa oword [rsp + 368], xmm3 + movdqa oword [rsp + 384], xmm4 + movdqa oword [rsp + 400], xmm5 + movdqa oword [rsp + 416], xmm6 + movdqa oword [rsp + 432], xmm7 + + // Build counter vectors (r10 = ACounter, 64-bit) + mov rax, r10 + mov dword ptr [rsp + 448], eax + shr rax, 32 + mov dword ptr [rsp + 464], eax + lea rax, [r10 + 1] + mov dword ptr [rsp + 452], eax + shr rax, 32 + mov dword ptr [rsp + 468], eax + lea rax, [r10 + 2] + mov dword ptr [rsp + 456], eax + shr rax, 32 + mov dword ptr [rsp + 472], eax + lea rax, [r10 + 3] + mov dword ptr [rsp + 460], eax + shr rax, 32 + mov dword ptr [rsp + 476], eax + + // Pre-compute IV broadcast vectors + mov eax, $6A09E667 + movd xmm12, eax + pshufd xmm12, xmm12, 0 + movdqa oword [rsp + 480], xmm12 + mov eax, $BB67AE85 + movd xmm12, eax + pshufd xmm12, xmm12, 0 + movdqa oword [rsp + 496], xmm12 + mov eax, $3C6EF372 + movd xmm12, eax + pshufd xmm12, xmm12, 0 + movdqa oword [rsp + 512], xmm12 + mov eax, $A54FF53A + movd xmm12, eax + pshufd xmm12, xmm12, 0 + movdqa oword [rsp + 528], xmm12 + + mov eax, 64 + movd xmm12, eax + pshufd xmm12, xmm12, 0 + movdqa oword [rsp + 544], xmm12 + + xor ebx, ebx + xor ebp, ebp + +@block_loop: + + // Transpose message for current block from 4 chunks + // Words 0..3 + movdqu xmm0, oword [rcx + rbp + 0] + movdqu xmm1, oword [rcx + rbp + 1024] + movdqu xmm2, oword [rcx + rbp + 2048] + movdqu xmm3, oword [rcx + rbp + 3072] + movdqa xmm4, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + punpcklqdq xmm4, xmm5 + punpckhqdq xmm3, xmm5 + movdqa oword [rsp + 0], xmm0 + movdqa oword [rsp + 16], xmm1 + movdqa oword [rsp + 32], xmm4 + movdqa oword [rsp + 48], xmm3 + + // Words 4..7 + movdqu xmm0, oword [rcx + rbp + 16] + movdqu xmm1, oword [rcx + rbp + 1040] + movdqu xmm2, oword [rcx + rbp + 2064] + movdqu xmm3, oword [rcx + rbp + 3088] + movdqa xmm4, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + punpcklqdq xmm4, xmm5 + punpckhqdq xmm3, xmm5 + movdqa oword [rsp + 64], xmm0 + movdqa oword [rsp + 80], xmm1 + movdqa oword [rsp + 96], xmm4 + movdqa oword [rsp + 112], xmm3 + + // Words 8..11 + movdqu xmm0, oword [rcx + rbp + 32] + movdqu xmm1, oword [rcx + rbp + 1056] + movdqu xmm2, oword [rcx + rbp + 2080] + movdqu xmm3, oword [rcx + rbp + 3104] + movdqa xmm4, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + punpcklqdq xmm4, xmm5 + punpckhqdq xmm3, xmm5 + movdqa oword [rsp + 128], xmm0 + movdqa oword [rsp + 144], xmm1 + movdqa oword [rsp + 160], xmm4 + movdqa oword [rsp + 176], xmm3 + + // Words 12..15 + movdqu xmm0, oword [rcx + rbp + 48] + movdqu xmm1, oword [rcx + rbp + 1072] + movdqu xmm2, oword [rcx + rbp + 2096] + movdqu xmm3, oword [rcx + rbp + 3120] + movdqa xmm4, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + punpcklqdq xmm4, xmm5 + punpckhqdq xmm3, xmm5 + movdqa oword [rsp + 192], xmm0 + movdqa oword [rsp + 208], xmm1 + movdqa oword [rsp + 224], xmm4 + movdqa oword [rsp + 240], xmm3 + + // Initialize state: v0-v7 from CV, v8-v11 from IV + movdqa xmm0, oword [rsp + 320] + movdqa xmm1, oword [rsp + 336] + movdqa xmm2, oword [rsp + 352] + movdqa xmm3, oword [rsp + 368] + movdqa xmm4, oword [rsp + 384] + movdqa xmm5, oword [rsp + 400] + movdqa xmm6, oword [rsp + 416] + movdqa xmm7, oword [rsp + 432] + movdqa xmm8, oword [rsp + 480] + movdqa xmm9, oword [rsp + 496] + movdqa xmm10, oword [rsp + 512] + movdqa xmm11, oword [rsp + 528] + + // v12-v15: counter, block length, flags + movdqa xmm12, oword [rsp + 448] + movdqa oword [rsp + 256], xmm12 + movdqa xmm12, oword [rsp + 464] + movdqa oword [rsp + 272], xmm12 + movdqa xmm12, oword [rsp + 544] + movdqa oword [rsp + 288], xmm12 + + // Compute per-block flags and broadcast + mov eax, dword ptr [rsp + 568] + cmp ebx, 0 + jne @h4_not_first + or eax, 1 +@h4_not_first: + cmp ebx, 15 + jne @h4_not_last + or eax, 2 +@h4_not_last: + movd xmm12, eax + pshufd xmm12, xmm12, 0 + movdqa oword [rsp + 304], xmm12 + + // === Round 0 === + // G col (0,4,8,12) m[0],m[1] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 0] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 16] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[2],m[3] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 32] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 48] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[4],m[5] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 64] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 80] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[6],m[7] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 96] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 112] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[8],m[9] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 128] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 144] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[10],m[11] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 160] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 176] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[12],m[13] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 192] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 208] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[14],m[15] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 224] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 240] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // === Round 1 === + // G col (0,4,8,12) m[2],m[6] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 32] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 96] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[3],m[10] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 48] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 160] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[7],m[0] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 112] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 0] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[4],m[13] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 64] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 208] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[1],m[11] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 16] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 176] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[12],m[5] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 192] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 80] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[9],m[14] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 144] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 224] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[15],m[8] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 240] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 128] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // === Round 2 === + // G col (0,4,8,12) m[3],m[4] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 48] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 64] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[10],m[12] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 160] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 192] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[13],m[2] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 208] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 32] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[7],m[14] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 112] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 224] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[6],m[5] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 96] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 80] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[9],m[0] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 144] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 0] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[11],m[15] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 176] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 240] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[8],m[1] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 128] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 16] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // === Round 3 === + // G col (0,4,8,12) m[10],m[7] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 160] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 112] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[12],m[9] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 192] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 144] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[14],m[3] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 224] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 48] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[13],m[15] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 208] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 240] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[4],m[0] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 64] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 0] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[11],m[2] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 176] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 32] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[5],m[8] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 80] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 128] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[1],m[6] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 16] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 96] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // === Round 4 === + // G col (0,4,8,12) m[12],m[13] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 192] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 208] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[9],m[11] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 144] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 176] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[15],m[10] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 240] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 160] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[14],m[8] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 224] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 128] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[7],m[2] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 112] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 32] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[5],m[3] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 80] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 48] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[0],m[1] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 0] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 16] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[6],m[4] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 96] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 64] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // === Round 5 === + // G col (0,4,8,12) m[9],m[14] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 144] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 224] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[11],m[5] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 176] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 80] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[8],m[12] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 128] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 192] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[15],m[1] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 240] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 16] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[13],m[3] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 208] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 48] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[0],m[10] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 0] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 160] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[2],m[6] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 32] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 96] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[4],m[7] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 64] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 112] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // === Round 6 === + // G col (0,4,8,12) m[11],m[15] + movdqa xmm12, oword [rsp + 256] + paddd xmm0, oword [rsp + 176] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm0, oword [rsp + 240] + paddd xmm0, xmm4 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm4, xmm8 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + // G col (1,5,9,13) m[5],m[0] + movdqa xmm12, oword [rsp + 272] + paddd xmm1, oword [rsp + 80] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm1, oword [rsp + 0] + paddd xmm1, xmm5 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm5, xmm9 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G col (2,6,10,14) m[1],m[9] + movdqa xmm12, oword [rsp + 288] + paddd xmm2, oword [rsp + 16] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm2, oword [rsp + 144] + paddd xmm2, xmm6 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm6, xmm10 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G col (3,7,11,15) m[8],m[6] + movdqa xmm12, oword [rsp + 304] + paddd xmm3, oword [rsp + 128] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm3, oword [rsp + 96] + paddd xmm3, xmm7 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm7, xmm11 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (0,5,10,15) m[14],m[10] + movdqa xmm12, oword [rsp + 304] + paddd xmm0, oword [rsp + 224] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm10, xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 20 + psrld xmm13, 12 + por xmm5, xmm13 + paddd xmm0, oword [rsp + 160] + paddd xmm0, xmm5 + pxor xmm12, xmm0 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm10, xmm12 + movdqa oword [rsp + 304], xmm12 + pxor xmm5, xmm10 + movdqa xmm13, xmm5 + pslld xmm5, 25 + psrld xmm13, 7 + por xmm5, xmm13 + // G diag (1,6,11,12) m[2],m[12] + movdqa xmm12, oword [rsp + 256] + paddd xmm1, oword [rsp + 32] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm11, xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 20 + psrld xmm13, 12 + por xmm6, xmm13 + paddd xmm1, oword [rsp + 192] + paddd xmm1, xmm6 + pxor xmm12, xmm1 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm11, xmm12 + movdqa oword [rsp + 256], xmm12 + pxor xmm6, xmm11 + movdqa xmm13, xmm6 + pslld xmm6, 25 + psrld xmm13, 7 + por xmm6, xmm13 + // G diag (2,7,8,13) m[3],m[4] + movdqa xmm12, oword [rsp + 272] + paddd xmm2, oword [rsp + 48] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm8, xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 20 + psrld xmm13, 12 + por xmm7, xmm13 + paddd xmm2, oword [rsp + 64] + paddd xmm2, xmm7 + pxor xmm12, xmm2 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm8, xmm12 + movdqa oword [rsp + 272], xmm12 + pxor xmm7, xmm8 + movdqa xmm13, xmm7 + pslld xmm7, 25 + psrld xmm13, 7 + por xmm7, xmm13 + // G diag (3,4,9,14) m[7],m[13] + movdqa xmm12, oword [rsp + 288] + paddd xmm3, oword [rsp + 112] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E6 // pshufb xmm12, xmm14 + paddd xmm9, xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 20 + psrld xmm13, 12 + por xmm4, xmm13 + paddd xmm3, oword [rsp + 208] + paddd xmm3, xmm4 + pxor xmm12, xmm3 + db $66, $45, $0F, $38, $00, $E7 // pshufb xmm12, xmm15 + paddd xmm9, xmm12 + movdqa oword [rsp + 288], xmm12 + pxor xmm4, xmm9 + movdqa xmm13, xmm4 + pslld xmm4, 25 + psrld xmm13, 7 + por xmm4, xmm13 + + // Finalize: new_cv[i] = v[i] XOR v[i+8] + pxor xmm0, xmm8 + pxor xmm1, xmm9 + pxor xmm2, xmm10 + pxor xmm3, xmm11 + movdqa xmm12, oword [rsp + 256] + pxor xmm4, xmm12 + movdqa xmm12, oword [rsp + 272] + pxor xmm5, xmm12 + movdqa xmm12, oword [rsp + 288] + pxor xmm6, xmm12 + movdqa xmm12, oword [rsp + 304] + pxor xmm7, xmm12 + + // Store new CV + movdqa oword [rsp + 320], xmm0 + movdqa oword [rsp + 336], xmm1 + movdqa oword [rsp + 352], xmm2 + movdqa oword [rsp + 368], xmm3 + movdqa oword [rsp + 384], xmm4 + movdqa oword [rsp + 400], xmm5 + movdqa oword [rsp + 416], xmm6 + movdqa oword [rsp + 432], xmm7 + + add ebp, 64 + inc ebx + cmp ebx, 16 + jb @block_loop + + // Output: transpose 4x8 CV back to per-chunk layout + mov rax, qword ptr [rsp + 560] + + // Transpose words 0-3 + movdqa xmm0, oword [rsp + 320] + movdqa xmm1, oword [rsp + 336] + movdqa xmm2, oword [rsp + 352] + movdqa xmm3, oword [rsp + 368] + movdqa xmm4, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + punpcklqdq xmm4, xmm5 + punpckhqdq xmm3, xmm5 + movdqu oword [rax], xmm0 + movdqu oword [rax + 32], xmm1 + movdqu oword [rax + 64], xmm4 + movdqu oword [rax + 96], xmm3 + + // Transpose words 4-7 + movdqa xmm0, oword [rsp + 384] + movdqa xmm1, oword [rsp + 400] + movdqa xmm2, oword [rsp + 416] + movdqa xmm3, oword [rsp + 432] + movdqa xmm4, xmm0 + punpckldq xmm0, xmm1 + punpckhdq xmm4, xmm1 + movdqa xmm5, xmm2 + punpckldq xmm2, xmm3 + punpckhdq xmm5, xmm3 + movdqa xmm1, xmm0 + punpcklqdq xmm0, xmm2 + punpckhqdq xmm1, xmm2 + movdqa xmm3, xmm4 + punpcklqdq xmm4, xmm5 + punpckhqdq xmm3, xmm5 + movdqu oword [rax + 16], xmm0 + movdqu oword [rax + 48], xmm1 + movdqu oword [rax + 80], xmm4 + movdqu oword [rax + 112], xmm3 + + add rsp, 584 + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + + pop rbp + pop rbx diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc index 5f7060da..3acdae5e 100644 --- a/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc @@ -1,24 +1,34 @@ -// AVX2 implementation of BLAKE3 hash8: processes 8 independent chunks simultaneously. -// Reference: BLAKE3-team/BLAKE3 official AVX2 implementation (blake3_avx2.c / blake3_hash_many). -// Each YMM register holds the same 32-bit word from 8 different chunks (inter-chunk SIMD). -// All VEX instructions are db-encoded for broad assembler compatibility. -// 7 rounds per block fully unrolled; 16-block loop with branch for chunk start/end flags. -// -// After HlpSimdProc6Begin_x86_64.inc: -// rcx = AInput, rdx = AKey, r8 = AOut, -// r9 = ANumChunks, r10 = ACounter, r11 = AFlags -// +// AVX2 implementation of BLAKE3 hash8: processes 8 independent chunks +// simultaneously. Each YMM register holds the same 32-bit word from 8 +// different chunks (inter-chunk SIMD); 7 rounds per block fully unrolled; +// 16-block loop with branch for chunk start/end flags. Message transpose: +// chunks paired (0+4, 1+5, ...) via vinserti128, then per-lane 4x4. Output +// transpose: full 8x8 via vperm2i128 + vpunpckl/hdq + vpunpcklq/hqdq. +// Rotations: rot16/8 via vpshufb against the resident masks, rot12/7 via +// shift+or (not byte-aligned; shifts are the known-best form). 3-operand +// VEX form eliminates the movdqa copies the SSE2 rotations need. +// ABI (after HlpSimdProc7Begin_x86_64.inc): rcx = AInput, rdx = AKey, +// r8 = AOut, r9 = ANumChunks, r10 = ACounter, r11 = AFlags, +// rax = byte-rotation masks ptr (BLAKE3_ROT_MASKS: rot16 at +0, rot8 at +// +32; read unaligned, so the Pascal const needs no special alignment; +// consumed by the entry mask loads, rax is scratch afterwards). // Register map: ymm0-ymm7 = state v0-v7, ymm8-ymm11 = state v8-v11, -// ymm12-ymm13 = temp, v12-v15 spilled to stack. -// GPR: rcx = input pointer, rbx = block counter (0..15), rbp = block byte offset. -// Message transpose: chunks paired (0+4, 1+5, ...) via vinserti128, then per-lane 4x4. -// Output transpose: full 8x8 via vperm2i128 + vpunpckl/hdq + vpunpcklq/hqdq. -// 3-operand VEX form eliminates movdqa copies needed in SSE2 rotations. -// Stack 32-byte aligned via "and rsp, -32" for YMM movdqu stores. -// -// MS x64 non-volatile saves: xmm6-xmm13. +// ymm12-ymm13 = temp, ymm14 = rot16 mask, ymm15 = rot8 mask, +// v12-v15 spilled to stack. +// GPR: rcx = input pointer, rbx = block counter (0..15), rbp = block +// byte offset. +// Frame: stack 32-byte aligned via "and rsp, -32" for YMM movdqu stores. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: BLAKE3-team/BLAKE3 official AVX2 implementation +// (blake3_avx2.c / blake3_hash_many). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + // Load byte-rotation masks (resident for the whole kernel) + db $C5, $7E, $6F, $30 // vmovdqu ymm14, yword [rax] + db $C5, $7E, $6F, $78, $20 // vmovdqu ymm15, yword [rax + $20] push rbx push rbp @@ -246,8 +256,7 @@ db $C5, $FD, $FE, $04, $24 // vpaddd ymm0, ymm0, [rsp+0] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -256,9 +265,7 @@ db $C5, $FD, $FE, $44, $24, $20 // vpaddd ymm0, ymm0, [rsp+32] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -270,8 +277,7 @@ db $C5, $F5, $FE, $4C, $24, $40 // vpaddd ymm1, ymm1, [rsp+64] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -280,9 +286,7 @@ db $C5, $F5, $FE, $4C, $24, $60 // vpaddd ymm1, ymm1, [rsp+96] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -294,8 +298,7 @@ db $C5, $ED, $FE, $94, $24, $80, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp+128] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -304,9 +307,7 @@ db $C5, $ED, $FE, $94, $24, $A0, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp+160] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -318,8 +319,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+192] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -328,9 +328,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+224] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -342,8 +340,7 @@ db $C5, $FD, $FE, $84, $24, $00, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+256] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -352,9 +349,7 @@ db $C5, $FD, $FE, $84, $24, $20, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+288] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -366,8 +361,7 @@ db $C5, $F5, $FE, $8C, $24, $40, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+320] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -376,9 +370,7 @@ db $C5, $F5, $FE, $8C, $24, $60, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+352] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -390,8 +382,7 @@ db $C5, $ED, $FE, $94, $24, $80, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+384] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -400,9 +391,7 @@ db $C5, $ED, $FE, $94, $24, $A0, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+416] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -414,8 +403,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+448] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -424,9 +412,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+480] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -440,8 +426,7 @@ db $C5, $FD, $FE, $44, $24, $40 // vpaddd ymm0, ymm0, [rsp+64] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -450,9 +435,7 @@ db $C5, $FD, $FE, $84, $24, $C0, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+192] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -464,8 +447,7 @@ db $C5, $F5, $FE, $4C, $24, $60 // vpaddd ymm1, ymm1, [rsp+96] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -474,9 +456,7 @@ db $C5, $F5, $FE, $8C, $24, $40, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+320] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -488,8 +468,7 @@ db $C5, $ED, $FE, $94, $24, $E0, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp+224] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -498,9 +477,7 @@ db $C5, $ED, $FE, $14, $24 // vpaddd ymm2, ymm2, [rsp+0] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -512,8 +489,7 @@ db $C5, $E5, $FE, $9C, $24, $80, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+128] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -522,9 +498,7 @@ db $C5, $E5, $FE, $9C, $24, $A0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+416] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -536,8 +510,7 @@ db $C5, $FD, $FE, $44, $24, $20 // vpaddd ymm0, ymm0, [rsp+32] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -546,9 +519,7 @@ db $C5, $FD, $FE, $84, $24, $60, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+352] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -560,8 +531,7 @@ db $C5, $F5, $FE, $8C, $24, $80, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+384] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -570,9 +540,7 @@ db $C5, $F5, $FE, $8C, $24, $A0, $00, $00, $00 // vpaddd ymm1, ymm1, [rsp+160] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -584,8 +552,7 @@ db $C5, $ED, $FE, $94, $24, $20, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+288] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -594,9 +561,7 @@ db $C5, $ED, $FE, $94, $24, $C0, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+448] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -608,8 +573,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+480] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -618,9 +582,7 @@ db $C5, $E5, $FE, $9C, $24, $00, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+256] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -634,8 +596,7 @@ db $C5, $FD, $FE, $44, $24, $60 // vpaddd ymm0, ymm0, [rsp+96] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -644,9 +605,7 @@ db $C5, $FD, $FE, $84, $24, $80, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+128] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -658,8 +617,7 @@ db $C5, $F5, $FE, $8C, $24, $40, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+320] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -668,9 +626,7 @@ db $C5, $F5, $FE, $8C, $24, $80, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+384] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -682,8 +638,7 @@ db $C5, $ED, $FE, $94, $24, $A0, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+416] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -692,9 +647,7 @@ db $C5, $ED, $FE, $54, $24, $40 // vpaddd ymm2, ymm2, [rsp+64] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -706,8 +659,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+224] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -716,9 +668,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+448] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -730,8 +680,7 @@ db $C5, $FD, $FE, $84, $24, $C0, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+192] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -740,9 +689,7 @@ db $C5, $FD, $FE, $84, $24, $A0, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+160] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -754,8 +701,7 @@ db $C5, $F5, $FE, $8C, $24, $20, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+288] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -764,9 +710,7 @@ db $C5, $F5, $FE, $0C, $24 // vpaddd ymm1, ymm1, [rsp+0] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -778,8 +722,7 @@ db $C5, $ED, $FE, $94, $24, $60, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+352] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -788,9 +731,7 @@ db $C5, $ED, $FE, $94, $24, $E0, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+480] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -802,8 +743,7 @@ db $C5, $E5, $FE, $9C, $24, $00, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+256] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -812,9 +752,7 @@ db $C5, $E5, $FE, $5C, $24, $20 // vpaddd ymm3, ymm3, [rsp+32] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -828,8 +766,7 @@ db $C5, $FD, $FE, $84, $24, $40, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+320] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -838,9 +775,7 @@ db $C5, $FD, $FE, $84, $24, $E0, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+224] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -852,8 +787,7 @@ db $C5, $F5, $FE, $8C, $24, $80, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+384] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -862,9 +796,7 @@ db $C5, $F5, $FE, $8C, $24, $20, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+288] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -876,8 +808,7 @@ db $C5, $ED, $FE, $94, $24, $C0, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+448] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -886,9 +817,7 @@ db $C5, $ED, $FE, $54, $24, $60 // vpaddd ymm2, ymm2, [rsp+96] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -900,8 +829,7 @@ db $C5, $E5, $FE, $9C, $24, $A0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+416] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -910,9 +838,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+480] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -924,8 +850,7 @@ db $C5, $FD, $FE, $84, $24, $80, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+128] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -934,9 +859,7 @@ db $C5, $FD, $FE, $04, $24 // vpaddd ymm0, ymm0, [rsp+0] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -948,8 +871,7 @@ db $C5, $F5, $FE, $8C, $24, $60, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+352] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -958,9 +880,7 @@ db $C5, $F5, $FE, $4C, $24, $40 // vpaddd ymm1, ymm1, [rsp+64] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -972,8 +892,7 @@ db $C5, $ED, $FE, $94, $24, $A0, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp+160] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -982,9 +901,7 @@ db $C5, $ED, $FE, $94, $24, $00, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+256] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -996,8 +913,7 @@ db $C5, $E5, $FE, $5C, $24, $20 // vpaddd ymm3, ymm3, [rsp+32] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1006,9 +922,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+192] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -1022,8 +936,7 @@ db $C5, $FD, $FE, $84, $24, $80, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+384] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1032,9 +945,7 @@ db $C5, $FD, $FE, $84, $24, $A0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+416] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -1046,8 +957,7 @@ db $C5, $F5, $FE, $8C, $24, $20, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+288] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -1056,9 +966,7 @@ db $C5, $F5, $FE, $8C, $24, $60, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+352] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -1070,8 +978,7 @@ db $C5, $ED, $FE, $94, $24, $E0, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+480] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -1080,9 +987,7 @@ db $C5, $ED, $FE, $94, $24, $40, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+320] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -1094,8 +999,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+448] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -1104,9 +1008,7 @@ db $C5, $E5, $FE, $9C, $24, $00, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+256] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -1118,8 +1020,7 @@ db $C5, $FD, $FE, $84, $24, $E0, $00, $00, $00 // vpaddd ymm0, ymm0, [rsp+224] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -1128,9 +1029,7 @@ db $C5, $FD, $FE, $44, $24, $40 // vpaddd ymm0, ymm0, [rsp+64] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -1142,8 +1041,7 @@ db $C5, $F5, $FE, $8C, $24, $A0, $00, $00, $00 // vpaddd ymm1, ymm1, [rsp+160] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -1152,9 +1050,7 @@ db $C5, $F5, $FE, $4C, $24, $60 // vpaddd ymm1, ymm1, [rsp+96] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -1166,8 +1062,7 @@ db $C5, $ED, $FE, $14, $24 // vpaddd ymm2, ymm2, [rsp+0] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -1176,9 +1071,7 @@ db $C5, $ED, $FE, $54, $24, $20 // vpaddd ymm2, ymm2, [rsp+32] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -1190,8 +1083,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+192] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1200,9 +1092,7 @@ db $C5, $E5, $FE, $9C, $24, $80, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+128] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -1216,8 +1106,7 @@ db $C5, $FD, $FE, $84, $24, $20, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+288] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1226,9 +1115,7 @@ db $C5, $FD, $FE, $84, $24, $C0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+448] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -1240,8 +1127,7 @@ db $C5, $F5, $FE, $8C, $24, $60, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+352] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -1250,9 +1136,7 @@ db $C5, $F5, $FE, $8C, $24, $A0, $00, $00, $00 // vpaddd ymm1, ymm1, [rsp+160] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -1264,8 +1148,7 @@ db $C5, $ED, $FE, $94, $24, $00, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+256] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -1274,9 +1157,7 @@ db $C5, $ED, $FE, $94, $24, $80, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+384] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -1288,8 +1169,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+480] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -1298,9 +1178,7 @@ db $C5, $E5, $FE, $5C, $24, $20 // vpaddd ymm3, ymm3, [rsp+32] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -1312,8 +1190,7 @@ db $C5, $FD, $FE, $84, $24, $A0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+416] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -1322,9 +1199,7 @@ db $C5, $FD, $FE, $44, $24, $60 // vpaddd ymm0, ymm0, [rsp+96] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -1336,8 +1211,7 @@ db $C5, $F5, $FE, $0C, $24 // vpaddd ymm1, ymm1, [rsp+0] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -1346,9 +1220,7 @@ db $C5, $F5, $FE, $8C, $24, $40, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+320] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -1360,8 +1232,7 @@ db $C5, $ED, $FE, $54, $24, $40 // vpaddd ymm2, ymm2, [rsp+64] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -1370,9 +1241,7 @@ db $C5, $ED, $FE, $94, $24, $C0, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp+192] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -1384,8 +1253,7 @@ db $C5, $E5, $FE, $9C, $24, $80, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+128] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1394,9 +1262,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+224] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -1410,8 +1276,7 @@ db $C5, $FD, $FE, $84, $24, $60, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+352] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1420,9 +1285,7 @@ db $C5, $FD, $FE, $84, $24, $E0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+480] db $C5, $FD, $FE, $C4 // vpaddd ymm0, ymm0, ymm4 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $5D, $EF, $E0 // vpxor ymm4, ymm4, ymm8 @@ -1434,8 +1297,7 @@ db $C5, $F5, $FE, $8C, $24, $A0, $00, $00, $00 // vpaddd ymm1, ymm1, [rsp+160] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -1444,9 +1306,7 @@ db $C5, $F5, $FE, $0C, $24 // vpaddd ymm1, ymm1, [rsp+0] db $C5, $F5, $FE, $CD // vpaddd ymm1, ymm1, ymm5 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $55, $EF, $E9 // vpxor ymm5, ymm5, ymm9 @@ -1458,8 +1318,7 @@ db $C5, $ED, $FE, $54, $24, $20 // vpaddd ymm2, ymm2, [rsp+32] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -1468,9 +1327,7 @@ db $C5, $ED, $FE, $94, $24, $20, $01, $00, $00 // vpaddd ymm2, ymm2, [rsp+288] db $C5, $ED, $FE, $D6 // vpaddd ymm2, ymm2, ymm6 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $4D, $EF, $F2 // vpxor ymm6, ymm6, ymm10 @@ -1482,8 +1339,7 @@ db $C5, $E5, $FE, $9C, $24, $00, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+256] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -1492,9 +1348,7 @@ db $C5, $E5, $FE, $9C, $24, $C0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+192] db $C5, $E5, $FE, $DF // vpaddd ymm3, ymm3, ymm7 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $45, $EF, $FB // vpxor ymm7, ymm7, ymm11 @@ -1506,8 +1360,7 @@ db $C5, $FD, $FE, $84, $24, $C0, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+448] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 db $C5, $95, $72, $D5, $0C // vpsrld ymm13, ymm5, 12 @@ -1516,9 +1369,7 @@ db $C5, $FD, $FE, $84, $24, $40, $01, $00, $00 // vpaddd ymm0, ymm0, [rsp+320] db $C5, $FD, $FE, $C5 // vpaddd ymm0, ymm0, ymm5 db $C5, $1D, $EF, $E0 // vpxor ymm12, ymm12, ymm0 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $2D, $FE, $D4 // vpaddd ymm10, ymm10, ymm12 db $C5, $7E, $7F, $A4, $24, $60, $02, $00, $00 // vmovdqu [rsp+608], ymm12 db $C4, $C1, $55, $EF, $EA // vpxor ymm5, ymm5, ymm10 @@ -1530,8 +1381,7 @@ db $C5, $F5, $FE, $4C, $24, $40 // vpaddd ymm1, ymm1, [rsp+64] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 db $C5, $95, $72, $D6, $0C // vpsrld ymm13, ymm6, 12 @@ -1540,9 +1390,7 @@ db $C5, $F5, $FE, $8C, $24, $80, $01, $00, $00 // vpaddd ymm1, ymm1, [rsp+384] db $C5, $F5, $FE, $CE // vpaddd ymm1, ymm1, ymm6 db $C5, $1D, $EF, $E1 // vpxor ymm12, ymm12, ymm1 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $25, $FE, $DC // vpaddd ymm11, ymm11, ymm12 db $C5, $7E, $7F, $A4, $24, $00, $02, $00, $00 // vmovdqu [rsp+512], ymm12 db $C4, $C1, $4D, $EF, $F3 // vpxor ymm6, ymm6, ymm11 @@ -1554,8 +1402,7 @@ db $C5, $ED, $FE, $54, $24, $60 // vpaddd ymm2, ymm2, [rsp+96] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 db $C5, $95, $72, $D7, $0C // vpsrld ymm13, ymm7, 12 @@ -1564,9 +1411,7 @@ db $C5, $ED, $FE, $94, $24, $80, $00, $00, $00 // vpaddd ymm2, ymm2, [rsp+128] db $C5, $ED, $FE, $D7 // vpaddd ymm2, ymm2, ymm7 db $C5, $1D, $EF, $E2 // vpxor ymm12, ymm12, ymm2 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $3D, $FE, $C4 // vpaddd ymm8, ymm8, ymm12 db $C5, $7E, $7F, $A4, $24, $20, $02, $00, $00 // vmovdqu [rsp+544], ymm12 db $C4, $C1, $45, $EF, $F8 // vpxor ymm7, ymm7, ymm8 @@ -1578,8 +1423,7 @@ db $C5, $E5, $FE, $9C, $24, $E0, $00, $00, $00 // vpaddd ymm3, ymm3, [rsp+224] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $41, $7F, $70, $E4, $B1 // vpshuflw ymm12, ymm12, $B1 - db $C4, $41, $7E, $70, $E4, $B1 // vpshufhw ymm12, ymm12, $B1 + db $C4, $42, $1D, $00, $E6 // vpshufb ymm12, ymm12, ymm14 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 db $C5, $95, $72, $D4, $0C // vpsrld ymm13, ymm4, 12 @@ -1588,9 +1432,7 @@ db $C5, $E5, $FE, $9C, $24, $A0, $01, $00, $00 // vpaddd ymm3, ymm3, [rsp+416] db $C5, $E5, $FE, $DC // vpaddd ymm3, ymm3, ymm4 db $C5, $1D, $EF, $E3 // vpxor ymm12, ymm12, ymm3 - db $C4, $C1, $15, $72, $D4, $08 // vpsrld ymm13, ymm12, 8 - db $C4, $C1, $1D, $72, $F4, $18 // vpslld ymm12, ymm12, 24 - db $C4, $41, $1D, $EB, $E5 // vpor ymm12, ymm12, ymm13 + db $C4, $42, $1D, $00, $E7 // vpshufb ymm12, ymm12, ymm15 db $C4, $41, $35, $FE, $CC // vpaddd ymm9, ymm9, ymm12 db $C5, $7E, $7F, $A4, $24, $40, $02, $00, $00 // vmovdqu [rsp+576], ymm12 db $C4, $C1, $5D, $EF, $E1 // vpxor ymm4, ymm4, ymm9 @@ -1678,4 +1520,4 @@ pop rbp pop rbx - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardSse2_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardGpr_i386.inc similarity index 79% rename from HashLib/src/Include/Simd/CRC/CRCFoldForwardSse2_i386.inc rename to HashLib/src/Include/Simd/CRC/CRCFoldForwardGpr_i386.inc index 0f96efff..53ea7f63 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardSse2_i386.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardGpr_i386.inc @@ -1,8 +1,11 @@ -// MSB-first 64-bit CRC: 16-byte slice (IA-32; SSE2 not required — same as x64 MSB path). -// x64 counterpart: CRCFoldForwardSse2_x86_64.inc. -// After HlpSimdProc4Begin_i386: ebx = AData, esi = ALength, edi = AState, eax = Ctx. -// Ctx: FoldConstants.CrcBits at +80, TableRow at +96. CrcMask derived from width. -// Result UInt64 in edx:eax. +// Gpr implementation of the MSB-first 64-bit CRC 16-byte slice (IA-32; +// slicing-by-16); the Gpr name states the ISA requirement: none beyond +// base IA-32. x64 counterpart: CRCFoldForwardGpr_x86_64.inc. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = AData, esi = ALength, +// edi = AState, eax = Ctx (PCRCFoldRuntimeCtx). +// Ctx: FoldConstants.CrcBits at +80, TableRow at +96. CrcMask derived +// from the width. Returns the final CRC UInt64 in edx:eax. +// Reference: classic slicing-by-16 table CRC. push ebp mov ebp, eax @@ -16,13 +19,11 @@ // [esp+32] LBIdx // [esp+36] saved esi (remaining length) -@MsbSse2_outer: - cmp esi, 16 - jb @MsbSse2_done - + // Loop invariants: Width from CrcBits [ebp+80]; 64-bit mask into + // [esp+8],[esp+12]; LCrcBytes into [esp+16]; top-byte shift into [esp+20]. + // esi (length) is scratch during the mask build - park it in [esp+36]. mov dword ptr [esp + 36], esi - // Width from CrcBits [ebp+80]; build 64-bit mask into [esp+8],[esp+12] mov eax, dword ptr [ebp + 80] mov ecx, eax dec ecx @@ -50,8 +51,6 @@ mov dword ptr [esp + 8], eax mov dword ptr [esp + 12], edx - mov esi, dword ptr [esp + 36] - mov eax, dword ptr [ebp + 80] add eax, 7 shr eax, 3 @@ -61,6 +60,14 @@ sub eax, 8 mov dword ptr [esp + 20], eax + mov esi, dword ptr [esp + 36] + +@MsbSse2_outer: + cmp esi, 16 + jb @MsbSse2_done + + mov dword ptr [esp + 36], esi + mov eax, dword ptr [edi] mov edx, dword ptr [edi + 4] mov dword ptr [esp], eax diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardSse2_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardGpr_x86_64.inc similarity index 69% rename from HashLib/src/Include/Simd/CRC/CRCFoldForwardSse2_x86_64.inc rename to HashLib/src/Include/Simd/CRC/CRCFoldForwardGpr_x86_64.inc index 6bdbd6ba..c73f9856 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardGpr_x86_64.inc @@ -1,12 +1,13 @@ -// MSB-first 64-bit CRC: 16-byte slice (slicing-by-16) using SSE2 for load only. -// IA-32 counterpart: CRCFoldForwardSse2_i386.inc. -// Signature (after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Forward_Sse2(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// MS x64: rcx=AData, edx=ALength, r8=AState, r9=Ctx. -// Callee-saved: rsi/rdi must be preserved (used as LTempCopy / LBIdx scratch). -// Ctx = PCRCFoldRuntimeCtx: FoldConstants.CrcBits at +80, TableRow at +96. -// CrcMask = ((1 shl (W-1)) - 1) shl 1 or 1; W = dword [r9+80] (TCRC table path: W>=8). +// Gpr implementation of the MSB-first 64-bit CRC 16-byte slice +// (slicing-by-16); the Gpr name states the ISA requirement: none beyond +// base x86-64. IA-32 counterpart: CRCFoldForwardGpr_i386.inc. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = AData, edx = ALength, +// r8 = AState, r9 = Ctx (PCRCFoldRuntimeCtx). +// Ctx: FoldConstants.CrcBits at +80, TableRow at +96. +// CrcMask = ((1 shl (W-1)) - 1) shl 1 or 1; W = dword [r9+80] +// (TCRC table path: W >= 8). Returns the final CRC in rax. +// Saves: rsi/rdi preserved (used as LTempCopy / LBIdx scratch). +// Reference: classic slicing-by-16 table CRC. push rbx push rbp @@ -20,10 +21,7 @@ mov rbx, qword ptr [r8] // LTemp mov r15, rcx // data pointer (stable) -@MsbSse2_outer: - cmp edx, 16 - jb @MsbSse2_done - + // Loop invariants (width, mask, byte count, top-byte shift) mov r14d, dword ptr [r9 + 80] // Width from FoldConstants.CrcBits // r12 = CRCMaskFromWidth(r14d) @@ -41,13 +39,17 @@ shr eax, 3 // LCrcBytes mov r13d, eax + mov ecx, r14d + sub ecx, 8 // Width - 8, shift count for top byte + +@MsbSse2_outer: + cmp edx, 16 + jb @MsbSse2_done + xor rbp, rbp // LNewTemp mov rsi, rbx // LTempCopy xor edi, edi // LBIdx - mov ecx, r14d - sub ecx, 8 // Width - 8, shift count for top byte - @MsbSse2_m1: cmp edi, r13d jge @MsbSse2_m2 diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_i386.inc new file mode 100644 index 00000000..d3c3d6b9 --- /dev/null +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_i386.inc @@ -0,0 +1,187 @@ +// SSE2 + SSSE3 + PCLMULQDQ implementation of CRC folding + Barrett +// reduction for non-reflected (MSB-first) CRCs with width 8..64 (IA-32). +// Fold-by-4 only: the fold-by-8 upper path of the x86-64 kernel needs +// xmm8-xmm11, which IA-32 does not have. All byte sequences are verbatim +// from CRCFoldForwardPclmul_x86_64.inc (xmm0-7 forms carry no REX prefix +// and encode identically on IA-32). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = AData, esi = ALength +// (>= 16), edi = AState, eax = Ctx (PCRCFoldRuntimeCtx; FoldConstants +// at +0). +// AState: pointer to the initial CRC pre-shifted left by (64 - Width) +// as one UInt64. Returns the final CRC UInt64 in edx:eax (low qword +// of xmm0). +// Ctx: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett, [+64] BswapMask, +// [+80] CrcBits, [+88] BarrettShift. +// PCLMULQDQ/pshufb instructions are db-encoded for broad assembler +// compatibility. Encoding notes: +// pclmulqdq xmm_dst, xmm_src, imm8: 66 0F 3A 44 ModRM imm8 +// pshufb xmm_dst, xmm_src: 66 0F 38 00 ModRM +// ModRM = 11_dst_src. +// Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google), +// HashLib CRCFoldForwardPclmul_x86_64.inc. + + push ebp + mov ebp, eax // Ctx + + // BswapMask is always needed (kept in xmm6 throughout) + movdqu xmm6, oword ptr [ebp + 64] // BswapMask + + cmp esi, 64 + jae @large_input + + // --- Small input (16..63 bytes): load 1 block with byte-swap --- + movdqu xmm7, oword ptr [ebp + 16] // fold_1x128 + movdqu xmm0, oword ptr [ebx] + db $66, $0F, $38, $00, $C6 // pshufb xmm0, xmm6 + movq xmm4, qword ptr [edi] + pslldq xmm4, 8 + pxor xmm0, xmm4 + add ebx, 16 + sub esi, 16 + jmp @fold1_check + +@large_input: + // --- Large input (>= 64 bytes): load 4 blocks with byte-swap --- + movdqu xmm7, oword ptr [ebp] // fold_4x128 + + movdqu xmm0, oword ptr [ebx] + db $66, $0F, $38, $00, $C6 // pshufb xmm0, xmm6 + movdqu xmm1, oword ptr [ebx + 16] + db $66, $0F, $38, $00, $CE // pshufb xmm1, xmm6 + movdqu xmm2, oword ptr [ebx + 32] + db $66, $0F, $38, $00, $D6 // pshufb xmm2, xmm6 + movdqu xmm3, oword ptr [ebx + 48] + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + + movq xmm4, qword ptr [edi] + pslldq xmm4, 8 + pxor xmm0, xmm4 + + add ebx, 64 + sub esi, 64 + + // --- Main fold-by-4 loop --- + cmp esi, 64 + jb @fold4_done + +@fold4_loop: + // Fold xmm0 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + movdqu xmm5, oword ptr [ebx] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm0, xmm5 + + // Fold xmm1 + movdqa xmm4, xmm1 + db $66, $0F, $3A, $44, $CF, $00 // pclmulqdq xmm1, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm1, xmm4 + movdqu xmm5, oword ptr [ebx + 16] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm1, xmm5 + + // Fold xmm2 + movdqa xmm4, xmm2 + db $66, $0F, $3A, $44, $D7, $00 // pclmulqdq xmm2, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm2, xmm4 + movdqu xmm5, oword ptr [ebx + 32] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm2, xmm5 + + // Fold xmm3 + movdqa xmm4, xmm3 + db $66, $0F, $3A, $44, $DF, $00 // pclmulqdq xmm3, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm3, xmm4 + movdqu xmm5, oword ptr [ebx + 48] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm3, xmm5 + + add ebx, 64 + sub esi, 64 + cmp esi, 64 + jae @fold4_loop + +@fold4_done: + // --- Fold 4 accumulators to 1 using fold_1x128 (reload into xmm7) --- + movdqu xmm7, oword ptr [ebp + 16] // fold_1x128 + + // xmm0 = fold(xmm0) XOR xmm1 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm1 + + // xmm0 = fold(xmm0) XOR xmm2 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm2 + + // xmm0 = fold(xmm0) XOR xmm3 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm3 + + // --- Fold remaining 16-byte blocks --- +@fold1_check: + cmp esi, 16 + jb @fold1_done + +@fold1_loop: + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + movdqu xmm5, oword ptr [ebx] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm0, xmm5 + add ebx, 16 + sub esi, 16 + cmp esi, 16 + jae @fold1_loop + +@fold1_done: + // ================================================================= + // MSB Barrett reduction: 128-bit xmm0 -> CRC in edx:eax + // Two Barrett rounds, each on one 64-bit half; per Linux kernel + // crc-pclmul-template.S (Eric Biggers). xmm6 = [G : mu_lo]. + // ================================================================= + + movdqu xmm6, oword ptr [ebp + 32] // xmm6 = [G : mu_lo] + + // --- Round 1: Barrett on A_H --- + movdqa xmm1, xmm0 + db $66, $0F, $3A, $44, $CE, $01 // pclmulqdq xmm1, xmm6, $01 + pxor xmm1, xmm0 // xmm1.hi = tmp + db $66, $0F, $3A, $44, $CE, $11 // pclmulqdq xmm1, xmm6, $11 + + // Merge: crc1 in xmm1.lo bits 0..n-1; shift left by (64-n) and XOR + movq xmm2, qword ptr [ebp + 88] + psllq xmm1, xmm2 + pxor xmm0, xmm1 + + // --- Round 2: Barrett on modified A_L --- + movdqa xmm1, xmm0 + db $66, $0F, $3A, $44, $CE, $00 // pclmulqdq xmm1, xmm6, $00 + pshufd xmm0, xmm0, $4E // swap hi/lo qwords + pxor xmm0, xmm1 // xmm0.hi = tmp2 + db $66, $0F, $3A, $44, $C6, $11 // pclmulqdq xmm0, xmm6, $11 + + // Low qword of xmm0 -> edx:eax + movd eax, xmm0 + psrldq xmm0, 4 + movd edx, xmm0 + + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc index c68debc1..3a9d0511 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc @@ -1,39 +1,172 @@ -// SSE2 + SSSE3 + PCLMULQDQ CRC folding + Barrett reduction for non-reflected -// (MSB-first) CRCs with width 8..64. -// PCLMULQDQ instructions are db-encoded for broad assembler compatibility. -// -// Function signature (included after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Forward_Pclmul(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// -// Register mapping (MS x64 ABI after prologue): -// rcx = AData, edx = ALength (>= 16), r8 = AState, r9 = AConstants -// AState: [0..7] = initial CRC pre-shifted left by (64 - Width), [8..15] = 0. +// SSE2 + SSSE3 + PCLMULQDQ implementation of CRC folding + Barrett +// reduction for non-reflected (MSB-first) CRCs with width 8..64. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = AData, edx = ALength +// (>= 16), r8 = AState, r9 = AConstants. +// AState: pointer to the initial CRC pre-shifted left by (64 - Width) +// as one UInt64. Returns the final CRC in rax. // AConstants layout (TCRCFoldConstants): -// [0..15] = Fold_4x128 -// [16..31] = Fold_1x128 -// [32..47] = Barrett -// [48..63] = Fold_8x128 (unused by this path) -// [64..79] = BswapMask -// [80..87] = CrcBits -// [88..95] = BarrettShift -// Returns: final CRC in RAX. -// +// [0..15] = Fold_4x128 [48..63] = Fold_8x128 +// [16..31] = Fold_1x128 [64..79] = BswapMask +// [32..47] = Barrett [80..87] = CrcBits +// [88..95] = BarrettShift +// PCLMULQDQ/pshufb instructions are db-encoded for broad assembler +// compatibility. Encoding notes: +// pclmulqdq xmm_dst, xmm_src, imm8: 66 [REX] 0F 3A 44 ModRM imm8 +// (REX = $44 = REX.R for xmm8-15 dst); ModRM reg-reg: 11_dst_src. +// pshufb xmm_dst, xmm_src (SSSE3): 66 [REX] 0F 38 00 ModRM. // Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google). -// -// pclmulqdq xmm_dst, xmm_src, imm8 (register-register, no REX for xmm0-7): -// 66 0F 3A 44 ModRM imm8 -// ModRM for reg-reg: 11_dst_src -// xmm0=000 xmm1=001 xmm2=010 xmm3=011 xmm4=100 -// xmm5=101 xmm6=110 xmm7=111 -// pshufb xmm_dst, xmm_src (SSSE3): -// 66 0F 38 00 ModRM - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} // BswapMask is always needed (kept in xmm6 throughout) movdqu xmm6, dqword ptr [r9 + 64] // BswapMask + cmp edx, 128 + jb @check64 + + // ===== Very large input (>= 128 bytes): fold-by-8, 128 bytes/iter ===== + movdqu xmm7, dqword ptr [r9 + 48] // fold_8x128 (stride 1024) + + movdqu xmm0, dqword ptr [rcx] + db $66, $0F, $38, $00, $C6 // pshufb xmm0, xmm6 + movdqu xmm1, dqword ptr [rcx + 16] + db $66, $0F, $38, $00, $CE // pshufb xmm1, xmm6 + movdqu xmm2, dqword ptr [rcx + 32] + db $66, $0F, $38, $00, $D6 // pshufb xmm2, xmm6 + movdqu xmm3, dqword ptr [rcx + 48] + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + movdqu xmm8, dqword ptr [rcx + 64] + db $66, $44, $0F, $38, $00, $C6 // pshufb xmm8, xmm6 + movdqu xmm9, dqword ptr [rcx + 80] + db $66, $44, $0F, $38, $00, $CE // pshufb xmm9, xmm6 + movdqu xmm10, dqword ptr [rcx + 96] + db $66, $44, $0F, $38, $00, $D6 // pshufb xmm10, xmm6 + movdqu xmm11, dqword ptr [rcx + 112] + db $66, $44, $0F, $38, $00, $DE // pshufb xmm11, xmm6 + + movq xmm4, qword ptr [r8] + pslldq xmm4, 8 + pxor xmm0, xmm4 + + add rcx, 128 + sub edx, 128 + + cmp edx, 128 + jb @fold8_reduce + +@fold8_loop: + // Fold xmm0 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + movdqu xmm5, dqword ptr [rcx] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm0, xmm5 + + // Fold xmm1 + movdqa xmm4, xmm1 + db $66, $0F, $3A, $44, $CF, $00 // pclmulqdq xmm1, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm1, xmm4 + movdqu xmm5, dqword ptr [rcx + 16] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm1, xmm5 + + // Fold xmm2 + movdqa xmm4, xmm2 + db $66, $0F, $3A, $44, $D7, $00 // pclmulqdq xmm2, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm2, xmm4 + movdqu xmm5, dqword ptr [rcx + 32] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm2, xmm5 + + // Fold xmm3 + movdqa xmm4, xmm3 + db $66, $0F, $3A, $44, $DF, $00 // pclmulqdq xmm3, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm3, xmm4 + movdqu xmm5, dqword ptr [rcx + 48] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm3, xmm5 + + // Fold xmm8 + movdqa xmm4, xmm8 + db $66, $44, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm8, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm8, xmm4 + movdqu xmm5, dqword ptr [rcx + 64] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm8, xmm5 + + // Fold xmm9 + movdqa xmm4, xmm9 + db $66, $44, $0F, $3A, $44, $CF, $00 // pclmulqdq xmm9, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm9, xmm4 + movdqu xmm5, dqword ptr [rcx + 80] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm9, xmm5 + + // Fold xmm10 + movdqa xmm4, xmm10 + db $66, $44, $0F, $3A, $44, $D7, $00 // pclmulqdq xmm10, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm10, xmm4 + movdqu xmm5, dqword ptr [rcx + 96] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm10, xmm5 + + // Fold xmm11 + movdqa xmm4, xmm11 + db $66, $44, $0F, $3A, $44, $DF, $00 // pclmulqdq xmm11, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm11, xmm4 + movdqu xmm5, dqword ptr [rcx + 112] + db $66, $0F, $38, $00, $EE // pshufb xmm5, xmm6 + pxor xmm11, xmm5 + + add rcx, 128 + sub edx, 128 + cmp edx, 128 + jae @fold8_loop + +@fold8_reduce: + // --- Reduce 8 accumulators to 4 using fold_4x128 (stride 512) --- + movdqu xmm7, dqword ptr [r9] // fold_4x128 + + // xmm0 = fold(xmm0) XOR xmm8 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm8 + + // xmm1 = fold(xmm1) XOR xmm9 + movdqa xmm4, xmm1 + db $66, $0F, $3A, $44, $CF, $00 // pclmulqdq xmm1, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm1, xmm4 + pxor xmm1, xmm9 + + // xmm2 = fold(xmm2) XOR xmm10 + movdqa xmm4, xmm2 + db $66, $0F, $3A, $44, $D7, $00 // pclmulqdq xmm2, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm2, xmm4 + pxor xmm2, xmm10 + + // xmm3 = fold(xmm3) XOR xmm11 + movdqa xmm4, xmm3 + db $66, $0F, $3A, $44, $DF, $00 // pclmulqdq xmm3, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm3, xmm4 + pxor xmm3, xmm11 + + jmp @fold4_done + +@check64: cmp edx, 64 jae @large_input @@ -195,4 +328,4 @@ // movq rax, xmm0: 66 48 0F 7E C0 db $66, $48, $0F, $7E, $C0 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPmull_aarch64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPmull_aarch64.inc index 08cca4f6..07904bf5 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPmull_aarch64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPmull_aarch64.inc @@ -1,23 +1,139 @@ -// CRC folding (PMULL/PMULL2 + TBL byte-reverse) + Barrett reduction for -// non-reflected (MSB-first) CRCs with width 8..64. -// Expects AAPCS64: x0 = AData, w1 = ALength (>= 16, multiple of 16), -// x2 = AState, x3 = AConstants (PCRCFoldRuntimeCtx; FoldConstants at +0). -// AState: [0..7] = initial CRC pre-shifted left by (64 - Width), [8..15] = 0. +// PMULL implementation of CRC folding (PMULL/PMULL2 + TBL byte-reverse) + +// Barrett reduction for non-reflected (MSB-first) CRCs with width 8..64. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = AData, w1 = ALength (>= 16, +// multiple of 16), x2 = AState, x3 = AConstants (PCRCFoldRuntimeCtx; +// FoldConstants at +0). +// AState: pointer to the initial CRC pre-shifted left by (64 - Width) as +// one UInt64. Returns the final CRC in x0. // AConstants: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett, // [+64] BswapMask, [+80] CrcBits, [+88] BarrettShift. -// Returns final CRC in x0. -// Leaf nostackframe; caller-saved GPR/vector only (v0-v7, v17). -// Reference: Linux kernel crc-pclmul-template.S (Eric Biggers), -// HashLib CRCFoldForwardPclmul_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. // Register map: // v0-v3 = fold accumulators (xmm0-3) // v4 = pmull-hi temp (xmm4) // v5 = loaded 16-byte block (xmm5) // v6 = BswapMask then Barrett consts [G : mu_lo] (xmm6) -// v7 = Fold_4x128 then Fold_1x128 (xmm7) +// v7 = Fold_8x128 / Fold_4x128 then Fold_1x128 (xmm7) // v17 = dup staging for clmul hi*lo ($01) Barrett mix +// v19-v22 = fold-by-8 upper accumulators (xmm8-11) +// Saves: none (leaf nostackframe; caller-saved GPR/vector only: v0-v7, v17, +// v19-v22). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: Linux kernel crc-pclmul-template.S (Eric Biggers), HashLib +// CRCFoldForwardPclmul_x86_64.inc. + .long 0x3dc01066 // ldr q6, [x3, #64] + .long 0x7102003f // cmp w1, #128 + b.lo .Lcrc_fwd_check64 + .long 0x3dc00c67 // ldr q7, [x3, #48] + .long 0x3dc00000 // ldr q0, [x0, #0] + .long 0x4e060000 // tbl v0.16b, {v0.16b}, v6.16b + .long 0x3dc00401 // ldr q1, [x0, #16] + .long 0x4e060021 // tbl v1.16b, {v1.16b}, v6.16b + .long 0x3dc00802 // ldr q2, [x0, #32] + .long 0x4e060042 // tbl v2.16b, {v2.16b}, v6.16b + .long 0x3dc00c03 // ldr q3, [x0, #48] + .long 0x4e060063 // tbl v3.16b, {v3.16b}, v6.16b + .long 0x3dc01013 // ldr q19, [x0, #64] + .long 0x4e060273 // tbl v19.16b, {v19.16b}, v6.16b + .long 0x3dc01414 // ldr q20, [x0, #80] + .long 0x4e060294 // tbl v20.16b, {v20.16b}, v6.16b + .long 0x3dc01815 // ldr q21, [x0, #96] + .long 0x4e0602b5 // tbl v21.16b, {v21.16b}, v6.16b + .long 0x3dc01c16 // ldr q22, [x0, #112] + .long 0x4e0602d6 // tbl v22.16b, {v22.16b}, v6.16b + .long 0xf9400044 // ldr x4, [x2] + .long 0x6e241c84 // eor v4.16b, v4.16b, v4.16b + .long 0x4e181c84 // ins v4.d[1], x4 + .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b + .long 0x91020000 // add x0, x0, #128 + .long 0x51020021 // sub w1, w1, #128 + .long 0x7102003f // cmp w1, #128 + b.lo .Lcrc_fwd_fold8_reduce +.Lcrc_fwd_fold8_loop: + .long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b + .long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b + .long 0x3dc00005 // ldr q5, [x0, #0] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251c00 // eor v0.16b, v0.16b, v5.16b + .long 0x4ea11c24 // orr v4.16b, v1.16b, v1.16b + .long 0x0ee7e021 // pmull v1.1q, v1.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c21 // eor v1.16b, v1.16b, v4.16b + .long 0x3dc00405 // ldr q5, [x0, #16] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251c21 // eor v1.16b, v1.16b, v5.16b + .long 0x4ea21c44 // orr v4.16b, v2.16b, v2.16b + .long 0x0ee7e042 // pmull v2.1q, v2.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b + .long 0x3dc00805 // ldr q5, [x0, #32] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251c42 // eor v2.16b, v2.16b, v5.16b + .long 0x4ea31c64 // orr v4.16b, v3.16b, v3.16b + .long 0x0ee7e063 // pmull v3.1q, v3.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c63 // eor v3.16b, v3.16b, v4.16b + .long 0x3dc00c05 // ldr q5, [x0, #48] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b + .long 0x4eb31e64 // orr v4.16b, v19.16b, v19.16b + .long 0x0ee7e273 // pmull v19.1q, v19.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241e73 // eor v19.16b, v19.16b, v4.16b + .long 0x3dc01005 // ldr q5, [x0, #64] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251e73 // eor v19.16b, v19.16b, v5.16b + .long 0x4eb41e84 // orr v4.16b, v20.16b, v20.16b + .long 0x0ee7e294 // pmull v20.1q, v20.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241e94 // eor v20.16b, v20.16b, v4.16b + .long 0x3dc01405 // ldr q5, [x0, #80] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251e94 // eor v20.16b, v20.16b, v5.16b + .long 0x4eb51ea4 // orr v4.16b, v21.16b, v21.16b + .long 0x0ee7e2b5 // pmull v21.1q, v21.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x3dc01805 // ldr q5, [x0, #96] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251eb5 // eor v21.16b, v21.16b, v5.16b + .long 0x4eb61ec4 // orr v4.16b, v22.16b, v22.16b + .long 0x0ee7e2d6 // pmull v22.1q, v22.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241ed6 // eor v22.16b, v22.16b, v4.16b + .long 0x3dc01c05 // ldr q5, [x0, #112] + .long 0x4e0600a5 // tbl v5.16b, {v5.16b}, v6.16b + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x91020000 // add x0, x0, #128 + .long 0x51020021 // sub w1, w1, #128 + .long 0x7102003f // cmp w1, #128 + b.hs .Lcrc_fwd_fold8_loop +.Lcrc_fwd_fold8_reduce: + .long 0x3dc00067 // ldr q7, [x3] + .long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b + .long 0x0ee7e000 // pmull v0.1q, v0.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b + .long 0x6e331c00 // eor v0.16b, v0.16b, v19.16b + .long 0x4ea11c24 // orr v4.16b, v1.16b, v1.16b + .long 0x0ee7e021 // pmull v1.1q, v1.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c21 // eor v1.16b, v1.16b, v4.16b + .long 0x6e341c21 // eor v1.16b, v1.16b, v20.16b + .long 0x4ea21c44 // orr v4.16b, v2.16b, v2.16b + .long 0x0ee7e042 // pmull v2.1q, v2.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b + .long 0x6e351c42 // eor v2.16b, v2.16b, v21.16b + .long 0x4ea31c64 // orr v4.16b, v3.16b, v3.16b + .long 0x0ee7e063 // pmull v3.1q, v3.1d, v7.1d + .long 0x4ee7e084 // pmull2 v4.1q, v4.2d, v7.2d + .long 0x6e241c63 // eor v3.16b, v3.16b, v4.16b + .long 0x6e361c63 // eor v3.16b, v3.16b, v22.16b + b .Lcrc_fwd_fold4_done +.Lcrc_fwd_check64: .long 0x7101003f // cmp w1, #64 b.hs .Lcrc_fwd_large .long 0x3dc00467 // ldr q7, [x3, #16] diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_i386.inc new file mode 100644 index 00000000..dcf4ca27 --- /dev/null +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_i386.inc @@ -0,0 +1,172 @@ +// AVX2 + VPCLMULQDQ implementation of CRC folding + Barrett reduction for +// non-reflected (MSB-first) CRCs with width 8..64 (IA-32). Mirrors +// CRCFoldForwardVpclmul_x86_64.inc; ymm0-7/xmm0-7 only, so the structure +// carries over 1:1. Every register-register byte sequence is identical to +// the verified x86_64 kernel's encoding (regs 0-7 carry no REX/VEX +// extension bits); the memory forms are machine-assembled for the IA-32 +// bases. vzeroupper before returning. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = AData, esi = ALength +// (>= 16), edi = AState, eax = Ctx (PCRCFoldRuntimeCtx; FoldConstants +// at +0). +// AState: pointer to the initial CRC pre-shifted left by (64 - Width) +// as one UInt64. Returns the final CRC UInt64 in edx:eax (low qword +// of xmm0). +// Ctx: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett, +// [+48] Fold_8x128, [+64] BswapMask, [+80] CrcBits, +// [+88] BarrettShift. +// Saves: none (all vector registers are volatile on IA-32). +// AVX2/VPCLMULQDQ instructions are db-encoded for broad assembler +// compatibility. +// Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google), +// HashLib CRCFoldForwardVpclmul_x86_64.inc. + + push ebp + mov ebp, eax + cmp esi, 128 + jb @xmm_path + db $C4, $E2, $7D, $5A, $6D, $40 // vbroadcasti128 ymm5, [ebp + 64] + db $C4, $E2, $7D, $5A, $75, $30 // vbroadcasti128 ymm6, [ebp + 48] + db $C5, $FE, $6F, $03 // vmovdqu ymm0, [ebx] + db $C4, $E2, $7D, $00, $C5 // vpshufb ymm0, ymm0, ymm5 + db $C5, $FE, $6F, $4B, $20 // vmovdqu ymm1, [ebx + 32] + db $C4, $E2, $75, $00, $CD // vpshufb ymm1, ymm1, ymm5 + db $C5, $FE, $6F, $53, $40 // vmovdqu ymm2, [ebx + 64] + db $C4, $E2, $6D, $00, $D5 // vpshufb ymm2, ymm2, ymm5 + db $C5, $FE, $6F, $5B, $60 // vmovdqu ymm3, [ebx + 96] + db $C4, $E2, $65, $00, $DD // vpshufb ymm3, ymm3, ymm5 + db $C5, $FA, $7E, $27 // vmovq xmm4, qword ptr [edi] + db $C5, $D9, $73, $FC, $08 // vpslldq xmm4, xmm4, 8 + db $C5, $FD, $EF, $C4 // vpxor ymm0, ymm0, ymm4 + add ebx, 128 + sub esi, 128 + cmp esi, 128 + jb @ymm_done + +@ymm_loop: + db $C4, $E3, $7D, $44, $E6, $11 // vpclmulqdq ymm4, ymm0, ymm6, $11 + db $C4, $E3, $7D, $44, $C6, $00 // vpclmulqdq ymm0, ymm0, ymm6, $00 + db $C5, $FD, $EF, $C4 // vpxor ymm0, ymm0, ymm4 + db $C5, $FE, $6F, $3B // vmovdqu ymm7, [ebx] + db $C4, $E2, $45, $00, $FD // vpshufb ymm7, ymm7, ymm5 + db $C5, $FD, $EF, $C7 // vpxor ymm0, ymm0, ymm7 + db $C4, $E3, $75, $44, $E6, $11 // vpclmulqdq ymm4, ymm1, ymm6, $11 + db $C4, $E3, $75, $44, $CE, $00 // vpclmulqdq ymm1, ymm1, ymm6, $00 + db $C5, $F5, $EF, $CC // vpxor ymm1, ymm1, ymm4 + db $C5, $FE, $6F, $7B, $20 // vmovdqu ymm7, [ebx + 32] + db $C4, $E2, $45, $00, $FD // vpshufb ymm7, ymm7, ymm5 + db $C5, $F5, $EF, $CF // vpxor ymm1, ymm1, ymm7 + db $C4, $E3, $6D, $44, $E6, $11 // vpclmulqdq ymm4, ymm2, ymm6, $11 + db $C4, $E3, $6D, $44, $D6, $00 // vpclmulqdq ymm2, ymm2, ymm6, $00 + db $C5, $ED, $EF, $D4 // vpxor ymm2, ymm2, ymm4 + db $C5, $FE, $6F, $7B, $40 // vmovdqu ymm7, [ebx + 64] + db $C4, $E2, $45, $00, $FD // vpshufb ymm7, ymm7, ymm5 + db $C5, $ED, $EF, $D7 // vpxor ymm2, ymm2, ymm7 + db $C4, $E3, $65, $44, $E6, $11 // vpclmulqdq ymm4, ymm3, ymm6, $11 + db $C4, $E3, $65, $44, $DE, $00 // vpclmulqdq ymm3, ymm3, ymm6, $00 + db $C5, $E5, $EF, $DC // vpxor ymm3, ymm3, ymm4 + db $C5, $FE, $6F, $7B, $60 // vmovdqu ymm7, [ebx + 96] + db $C4, $E2, $45, $00, $FD // vpshufb ymm7, ymm7, ymm5 + db $C5, $E5, $EF, $DF // vpxor ymm3, ymm3, ymm7 + add ebx, 128 + sub esi, 128 + cmp esi, 128 + jae @ymm_loop + +@ymm_done: + db $C4, $E2, $7D, $5A, $75, $00 // vbroadcasti128 ymm6, [ebp] + db $C4, $E3, $7D, $44, $E6, $11 // vpclmulqdq ymm4, ymm0, ymm6, $11 + db $C4, $E3, $7D, $44, $C6, $00 // vpclmulqdq ymm0, ymm0, ymm6, $00 + db $C5, $FD, $EF, $C4 // vpxor ymm0, ymm0, ymm4 + db $C5, $FD, $EF, $C2 // vpxor ymm0, ymm0, ymm2 + db $C4, $E3, $75, $44, $E6, $11 // vpclmulqdq ymm4, ymm1, ymm6, $11 + db $C4, $E3, $75, $44, $CE, $00 // vpclmulqdq ymm1, ymm1, ymm6, $00 + db $C5, $F5, $EF, $CC // vpxor ymm1, ymm1, ymm4 + db $C5, $F5, $EF, $CB // vpxor ymm1, ymm1, ymm3 + db $C5, $F9, $6F, $D1 // vmovdqa xmm2, xmm1 + db $C4, $E3, $7D, $39, $CB, $01 // vextracti128 xmm3, ymm1, 1 + db $C4, $E3, $7D, $39, $C1, $01 // vextracti128 xmm1, ymm0, 1 + db $C5, $F8, $77 // vzeroupper + jmp @fold4to1 + +@xmm_path: + db $C5, $FA, $6F, $6D, $40 // vmovdqu xmm5, [ebp + 64] + cmp esi, 64 + jae @xmm_large + db $C5, $FA, $6F, $03 // vmovdqu xmm0, [ebx] + db $C4, $E2, $79, $00, $C5 // vpshufb xmm0, xmm0, xmm5 + db $C5, $FA, $7E, $27 // vmovq xmm4, qword ptr [edi] + db $C5, $D9, $73, $FC, $08 // vpslldq xmm4, xmm4, 8 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + add ebx, 16 + sub esi, 16 + db $C5, $FA, $6F, $7D, $10 // vmovdqu xmm7, [ebp + 16] + jmp @tail_check + +@xmm_large: + db $C5, $FA, $6F, $03 // vmovdqu xmm0, [ebx] + db $C4, $E2, $79, $00, $C5 // vpshufb xmm0, xmm0, xmm5 + db $C5, $FA, $6F, $4B, $10 // vmovdqu xmm1, [ebx + 16] + db $C4, $E2, $71, $00, $CD // vpshufb xmm1, xmm1, xmm5 + db $C5, $FA, $6F, $53, $20 // vmovdqu xmm2, [ebx + 32] + db $C4, $E2, $69, $00, $D5 // vpshufb xmm2, xmm2, xmm5 + db $C5, $FA, $6F, $5B, $30 // vmovdqu xmm3, [ebx + 48] + db $C4, $E2, $61, $00, $DD // vpshufb xmm3, xmm3, xmm5 + db $C5, $FA, $7E, $27 // vmovq xmm4, qword ptr [edi] + db $C5, $D9, $73, $FC, $08 // vpslldq xmm4, xmm4, 8 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + add ebx, 64 + sub esi, 64 + +@fold4to1: + db $C5, $FA, $6F, $7D, $10 // vmovdqu xmm7, [ebp + 16] + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $F9, $EF, $C2 // vpxor xmm0, xmm0, xmm2 + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $F9, $EF, $C3 // vpxor xmm0, xmm0, xmm3 + +@tail_check: + cmp esi, 16 + jb @tail_done + +@tail_loop: + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $FA, $6F, $33 // vmovdqu xmm6, [ebx] + db $C4, $E2, $49, $00, $F5 // vpshufb xmm6, xmm6, xmm5 + db $C5, $F9, $EF, $C6 // vpxor xmm0, xmm0, xmm6 + add ebx, 16 + sub esi, 16 + cmp esi, 16 + jae @tail_loop + +@tail_done: + db $C5, $FA, $6F, $75, $20 // vmovdqu xmm6, [ebp + 32] + db $C5, $F9, $6F, $C8 // vmovdqa xmm1, xmm0 + db $C4, $E3, $71, $44, $CE, $01 // vpclmulqdq xmm1, xmm1, xmm6, $01 + db $C5, $F1, $EF, $C8 // vpxor xmm1, xmm1, xmm0 + db $C4, $E3, $71, $44, $CE, $11 // vpclmulqdq xmm1, xmm1, xmm6, $11 + db $C5, $FA, $7E, $55, $58 // vmovq xmm2, qword ptr [ebp + 88] + db $C5, $F1, $F3, $CA // vpsllq xmm1, xmm1, xmm2 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C5, $F9, $6F, $C8 // vmovdqa xmm1, xmm0 + db $C4, $E3, $71, $44, $CE, $00 // vpclmulqdq xmm1, xmm1, xmm6, $00 + db $C5, $F9, $70, $C0, $4E // vpshufd xmm0, xmm0, $4E + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C4, $E3, $79, $44, $C6, $11 // vpclmulqdq xmm0, xmm0, xmm6, $11 + db $C5, $F8, $77 // vzeroupper + movd eax, xmm0 + psrldq xmm0, 4 + movd edx, xmm0 + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc index a00c3550..0eaa7548 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc @@ -1,32 +1,20 @@ -// AVX2 + VPCLMULQDQ CRC folding + Barrett reduction for non-reflected -// (MSB-first) CRCs with width 8..64. -// AVX2/VPCLMULQDQ/SSSE3 instructions are db-encoded for broad assembler -// compatibility. -// -// Function signature (included after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Forward_Vpclmul(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// -// Register mapping (MS x64 ABI after prologue): -// rcx = AData, edx = ALength (>= 16), r8 = AState, r9 = AConstants -// AState: [0..7] = initial CRC pre-shifted left by (64 - Width), [8..15] = 0. +// AVX2 + VPCLMULQDQ implementation of CRC folding + Barrett reduction for +// non-reflected (MSB-first) CRCs with width 8..64. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = AData, edx = ALength +// (>= 16), r8 = AState, r9 = AConstants. +// AState: pointer to the initial CRC pre-shifted left by (64 - Width) +// as one UInt64. Returns the final CRC in rax. // AConstants layout (TCRCFoldConstants): -// [0..15] = Fold_4x128 (stride 512) -// [16..31] = Fold_1x128 (stride 128) -// [32..47] = Barrett -// [48..63] = Fold_8x128 (stride 1024) -// [64..79] = BswapMask -// [80..87] = CrcBits -// [88..95] = BarrettShift -// Returns: final CRC in RAX. -// +// [0..15] = Fold_4x128 (stride 512) [48..63] = Fold_8x128 (stride 1024) +// [16..31] = Fold_1x128 (stride 128) [64..79] = BswapMask +// [32..47] = Barrett [80..87] = CrcBits +// [88..95] = BarrettShift +// AVX2/VPCLMULQDQ/SSSE3 instructions are db-encoded for broad assembler +// compatibility. Encoding notes (VEX byte layout): +// 2-byte: C5 [R.vvvv.L.pp] 3-byte: C4 [R.X.B.mmmmm] [W.vvvv.L.pp] // Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google). -// -// VEX byte layout reference: -// 2-byte: C5 [R.vvvv.L.pp] -// 3-byte: C4 [R.X.B.mmmmm] [W.vvvv.L.pp] - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} cmp edx, 128 jb @xmm_path @@ -240,4 +228,4 @@ // movq rax, xmm0: 66 48 0F 7E C0 db $66, $48, $0F, $7E, $C0 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflected32Sse2_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflected32Sse2_i386.inc deleted file mode 100644 index 186f1904..00000000 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflected32Sse2_i386.inc +++ /dev/null @@ -1,134 +0,0 @@ -// CRC32 reflected (PKZIP-style) 16-byte slice (IA-32 SSE2 + dword table XOR). -// After HlpSimdProc4Begin_i386: ebx = AData, esi = ALength, edi = AState, eax = Ctx. -// Little-endian host only. PUInt32 rows at Ctx + 96. -// d0 xor CRC saved at [esp+12]; d1 in edx; d2/d3 on stack. Result eax, edx=0. - - push ebp - mov ebp, eax - - sub esp, 32 - // [esp+12] d0 after xor CRC, [esp+16] saved esi, [esp+20] d2, [esp+24] d3 - -@L32Sse2_outer: - cmp esi, 16 - jb @L32Sse2_done - - movdqu xmm0, oword ptr [ebx] - - movd eax, xmm0 - psrldq xmm0, 4 - movd edx, xmm0 - psrldq xmm0, 4 - movd dword ptr [esp + 20], xmm0 - psrldq xmm0, 4 - movd dword ptr [esp + 24], xmm0 - - mov dword ptr [esp + 16], esi - - xor eax, dword ptr [edi] - mov dword ptr [esp + 12], eax - - xor ecx, ecx - - mov esi, dword ptr [esp + 24] - shr esi, 24 - mov eax, dword ptr [ebp + 96 + 0 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 24] - shr esi, 16 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 1 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 24] - shr esi, 8 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 2 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 24] - and esi, $FF - mov eax, dword ptr [ebp + 96 + 3 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 20] - shr esi, 24 - mov eax, dword ptr [ebp + 96 + 4 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 20] - shr esi, 16 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 5 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 20] - shr esi, 8 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 6 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 20] - and esi, $FF - mov eax, dword ptr [ebp + 96 + 7 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, edx - shr esi, 24 - mov eax, dword ptr [ebp + 96 + 8 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, edx - shr esi, 16 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 9 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, edx - shr esi, 8 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 10 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, edx - and esi, $FF - mov eax, dword ptr [ebp + 96 + 11 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 12] - shr esi, 24 - mov eax, dword ptr [ebp + 96 + 12 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 12] - shr esi, 16 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 13 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 12] - shr esi, 8 - and esi, $FF - mov eax, dword ptr [ebp + 96 + 14 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov esi, dword ptr [esp + 12] - and esi, $FF - mov eax, dword ptr [ebp + 96 + 15 * 4] - xor ecx, dword ptr [eax + esi * 4] - - mov dword ptr [edi], ecx - mov esi, dword ptr [esp + 16] - add ebx, 16 - sub esi, 16 - jmp @L32Sse2_outer - -@L32Sse2_done: - mov eax, dword ptr [edi] - xor edx, edx - add esp, 32 - pop ebp - pop edi - pop esi - pop ebx diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflected32Sse2_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflected32Sse2_x86_64.inc deleted file mode 100644 index 6a51c09a..00000000 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflected32Sse2_x86_64.inc +++ /dev/null @@ -1,134 +0,0 @@ -// CRC32 reflected (PKZIP-style) 16-byte slice: SSE2 movdqu + dword table XOR. -// Little-endian host only (x86/x64 asm reads 32-bit words directly from memory). -// Signature (after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Reflected32_Sse2(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// State: low UInt32 of qword at AState is the running CRC. -// Ctx: PCRCFoldRuntimeCtx; TableRow offset = 96. - - push rbx - push r12 - push r13 - push r14 - push r15 - - mov ebx, dword ptr [r8] - -@L32Sse2_outer: - cmp edx, 16 - jb @L32Sse2_done - - movdqu xmm0, dqword ptr [rcx] - movd r10d, xmm0 - psrldq xmm0, 4 - movd r11d, xmm0 - psrldq xmm0, 4 - movd r12d, xmm0 - psrldq xmm0, 4 - movd r13d, xmm0 - - xor r10d, ebx - - xor ebx, ebx - - mov eax, r13d - shr eax, 24 - mov r14, qword ptr [r9 + 96 + 0 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r13d - shr eax, 16 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 1 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r13d - shr eax, 8 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 2 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r13d - and eax, $FF - mov r14, qword ptr [r9 + 96 + 3 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r12d - shr eax, 24 - mov r14, qword ptr [r9 + 96 + 4 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r12d - shr eax, 16 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 5 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r12d - shr eax, 8 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 6 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r12d - and eax, $FF - mov r14, qword ptr [r9 + 96 + 7 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r11d - shr eax, 24 - mov r14, qword ptr [r9 + 96 + 8 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r11d - shr eax, 16 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 9 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r11d - shr eax, 8 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 10 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r11d - and eax, $FF - mov r14, qword ptr [r9 + 96 + 11 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r10d - shr eax, 24 - mov r14, qword ptr [r9 + 96 + 12 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r10d - shr eax, 16 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 13 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r10d - shr eax, 8 - and eax, $FF - mov r14, qword ptr [r9 + 96 + 14 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - mov eax, r10d - and eax, $FF - mov r14, qword ptr [r9 + 96 + 15 * 8] - xor ebx, dword ptr [r14 + rax * 4] - - add rcx, 16 - sub edx, 16 - jmp @L32Sse2_outer - -@L32Sse2_done: - mov dword ptr [r8], ebx - mov eax, ebx - - pop r15 - pop r14 - pop r13 - pop r12 - pop rbx diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_i386.inc new file mode 100644 index 00000000..ecbd2a43 --- /dev/null +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_i386.inc @@ -0,0 +1,183 @@ +// SSE2 + PCLMULQDQ implementation of CRC folding + Barrett reduction for +// reflected (LSB-first) CRCs with width 8..64 (IA-32). CRC-64 uses a +// special Barrett path for the 65-bit G. Fold-by-4 only: the fold-by-8 +// upper path of the x86-64 kernel needs xmm8-xmm11, which IA-32 does not +// have. All byte sequences are verbatim from +// CRCFoldReflectedPclmul_x86_64.inc (xmm0-7 forms carry no REX prefix and +// encode identically on IA-32). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = AData, esi = ALength +// (>= 16), edi = AState, eax = Ctx (PCRCFoldRuntimeCtx; FoldConstants +// at +0). +// AState: pointer to the initial CRC (reflected) as one UInt64. Returns +// the final CRC UInt64 in edx:eax (high qword of xmm0). +// Ctx: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett, [+80] CrcBits. +// PCLMULQDQ instructions are db-encoded for broad assembler compatibility. +// Encoding notes: +// pclmulqdq xmm_dst, xmm_src, imm8: 66 0F 3A 44 ModRM imm8; +// ModRM = 11_dst_src. +// Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google), +// HashLib CRCFoldReflectedPclmul_x86_64.inc. + + push ebp + mov ebp, eax // Ctx + + // fold_1x128 is always needed (fold-by-1 loop and Barrett step 1) + movdqu xmm7, oword ptr [ebp + 16] // fold_1x128 + + cmp esi, 64 + jae @large_input + + // --- Small input (16..63 bytes): load 1 block --- + movdqu xmm0, oword ptr [ebx] + movq xmm4, qword ptr [edi] + pxor xmm0, xmm4 + add ebx, 16 + sub esi, 16 + jmp @fold1_check + +@large_input: + // --- Large input (>= 64 bytes): load 4 blocks --- + movdqu xmm6, oword ptr [ebp] // fold_4x128 + + movdqu xmm0, oword ptr [ebx] + movdqu xmm1, oword ptr [ebx + 16] + movdqu xmm2, oword ptr [ebx + 32] + movdqu xmm3, oword ptr [ebx + 48] + + movq xmm4, qword ptr [edi] + pxor xmm0, xmm4 + + add ebx, 64 + sub esi, 64 + + // --- Main fold-by-4 loop --- + cmp esi, 64 + jb @fold4_done + +@fold4_loop: + // Fold xmm0 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C6, $00 // pclmulqdq xmm0, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm0, xmm4 + movdqu xmm5, oword ptr [ebx] + pxor xmm0, xmm5 + + // Fold xmm1 + movdqa xmm4, xmm1 + db $66, $0F, $3A, $44, $CE, $00 // pclmulqdq xmm1, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm1, xmm4 + movdqu xmm5, oword ptr [ebx + 16] + pxor xmm1, xmm5 + + // Fold xmm2 + movdqa xmm4, xmm2 + db $66, $0F, $3A, $44, $D6, $00 // pclmulqdq xmm2, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm2, xmm4 + movdqu xmm5, oword ptr [ebx + 32] + pxor xmm2, xmm5 + + // Fold xmm3 + movdqa xmm4, xmm3 + db $66, $0F, $3A, $44, $DE, $00 // pclmulqdq xmm3, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm3, xmm4 + movdqu xmm5, oword ptr [ebx + 48] + pxor xmm3, xmm5 + + add ebx, 64 + sub esi, 64 + cmp esi, 64 + jae @fold4_loop + +@fold4_done: + // --- Fold 4 accumulators to 1 using fold_1x128 (xmm7) --- + + // xmm0 = fold(xmm0) XOR xmm1 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm1 + + // xmm0 = fold(xmm0) XOR xmm2 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm2 + + // xmm0 = fold(xmm0) XOR xmm3 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm3 + + // --- Fold remaining 16-byte blocks --- +@fold1_check: + cmp esi, 16 + jb @fold1_done + +@fold1_loop: + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C7, $00 // pclmulqdq xmm0, xmm7, $00 + db $66, $0F, $3A, $44, $E7, $11 // pclmulqdq xmm4, xmm7, $11 + pxor xmm0, xmm4 + movdqu xmm5, oword ptr [ebx] + pxor xmm0, xmm5 + add ebx, 16 + sub esi, 16 + cmp esi, 16 + jae @fold1_loop + +@fold1_done: + // ================================================================= + // Final reduction: 128-bit xmm0 -> CRC in edx:eax + // Following Linux kernel crc-pclmul-template.S Barrett reduction. + // ================================================================= + + // Step 1: Multiply by x^n and reduce 128 bits to 64+n bits. + movdqa xmm1, xmm0 + db $66, $0F, $3A, $44, $CF, $10 // pclmulqdq xmm1, xmm7, $10 + psrldq xmm0, 8 // xmm0 = {0, A_hi} + pxor xmm0, xmm1 // xmm0 = t0 (64+n bits) + + // Step 2: Barrett reduction. + movdqu xmm6, oword ptr [ebp + 32] + + // t1 = clmul(t0.lo, barrett[0]) => q = floor(t0 / G) + movdqa xmm1, xmm0 + db $66, $0F, $3A, $44, $CE, $00 // pclmulqdq xmm1, xmm6, $00 + + // Check if CRC-64: G is 65-bit, needs split multiplication + // (CrcBits <= 64, so the low dword suffices) + cmp dword ptr [ebp + 80], 64 + je @barrett_64 + + // Standard Barrett (width < 64): single pclmulqdq covers full G + db $66, $0F, $3A, $44, $CE, $10 // pclmulqdq xmm1, xmm6, $10 + pxor xmm0, xmm1 + jmp @extract_crc + +@barrett_64: + // CRC-64 Barrett: G is 65-bit, split into 64-bit pclmulqdq + x^0 correction. + movdqa xmm2, xmm1 // save q + db $66, $0F, $3A, $44, $CE, $10 // pclmulqdq xmm1, xmm6, $10 + db $66, $0F, $6C, $D2 // punpcklqdq xmm2, xmm2 {q.lo, q.lo} + pxor xmm0, xmm1 + pxor xmm0, xmm2 + +@extract_crc: + // High qword of xmm0 -> edx:eax + psrldq xmm0, 8 + movd eax, xmm0 + psrldq xmm0, 4 + movd edx, xmm0 + + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc index 49a9521d..17e2bba5 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc @@ -1,36 +1,154 @@ -// SSE2 + PCLMULQDQ CRC folding + Barrett reduction for reflected (LSB-first) -// CRCs with width 8..64. CRC-64 uses a special Barrett path for the 65-bit G. -// PCLMULQDQ instructions are db-encoded for broad assembler compatibility. -// -// Function signature (included after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Reflected_Pclmul(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// -// Register mapping (MS x64 ABI after prologue): -// rcx = AData, edx = ALength (>= 16), r8 = AState, r9 = AConstants -// AState: [0..7] = initial CRC (reflected), [8..15] = 0. +// SSE2 + PCLMULQDQ implementation of CRC folding + Barrett reduction for +// reflected (LSB-first) CRCs with width 8..64. CRC-64 uses a special +// Barrett path for the 65-bit G. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = AData, edx = ALength +// (>= 16), r8 = AState, r9 = AConstants. +// AState: pointer to the initial CRC (reflected) as one UInt64. +// Returns the final CRC in rax. // AConstants layout (TCRCFoldConstants): -// [0..15] = Fold_4x128 -// [16..31] = Fold_1x128 -// [32..47] = Barrett -// [48..63] = Fold_8x128 (unused by this path) -// [64..79] = BswapMask (unused by this path) -// [80..87] = CrcBits -// Returns: final CRC in RAX. -// +// [0..15] = Fold_4x128 [48..63] = Fold_8x128 +// [16..31] = Fold_1x128 [64..79] = BswapMask (unused by this path) +// [32..47] = Barrett [80..87] = CrcBits +// PCLMULQDQ instructions are db-encoded for broad assembler compatibility. +// Encoding notes: +// pclmulqdq xmm_dst, xmm_src, imm8: 66 [REX] 0F 3A 44 ModRM imm8 +// (REX = $44 = REX.R for xmm8-15 dst); ModRM reg-reg: 11_dst_src. // Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google). -// -// pclmulqdq xmm_dst, xmm_src, imm8 (register-register, no REX for xmm0-7): -// 66 0F 3A 44 ModRM imm8 -// ModRM for reg-reg: 11_dst_src -// xmm0=000 xmm1=001 xmm2=010 xmm3=011 xmm4=100 -// xmm5=101 xmm6=110 xmm7=111 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} // fold_1x128 is always needed (fold-by-1 loop and Barrett step 1) movdqu xmm7, dqword ptr [r9 + 16] // fold_1x128 + cmp edx, 128 + jb @check64 + + // ===== Very large input (>= 128 bytes): fold-by-8, 128 bytes/iter ===== + movdqu xmm6, dqword ptr [r9 + 48] // fold_8x128 (stride 1024) + + movdqu xmm0, dqword ptr [rcx] + movdqu xmm1, dqword ptr [rcx + 16] + movdqu xmm2, dqword ptr [rcx + 32] + movdqu xmm3, dqword ptr [rcx + 48] + movdqu xmm8, dqword ptr [rcx + 64] + movdqu xmm9, dqword ptr [rcx + 80] + movdqu xmm10, dqword ptr [rcx + 96] + movdqu xmm11, dqword ptr [rcx + 112] + + movq xmm4, qword ptr [r8] + pxor xmm0, xmm4 + + add rcx, 128 + sub edx, 128 + + cmp edx, 128 + jb @fold8_reduce + +@fold8_loop: + // Fold xmm0 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C6, $00 // pclmulqdq xmm0, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm0, xmm4 + movdqu xmm5, dqword ptr [rcx] + pxor xmm0, xmm5 + + // Fold xmm1 + movdqa xmm4, xmm1 + db $66, $0F, $3A, $44, $CE, $00 // pclmulqdq xmm1, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm1, xmm4 + movdqu xmm5, dqword ptr [rcx + 16] + pxor xmm1, xmm5 + + // Fold xmm2 + movdqa xmm4, xmm2 + db $66, $0F, $3A, $44, $D6, $00 // pclmulqdq xmm2, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm2, xmm4 + movdqu xmm5, dqword ptr [rcx + 32] + pxor xmm2, xmm5 + + // Fold xmm3 + movdqa xmm4, xmm3 + db $66, $0F, $3A, $44, $DE, $00 // pclmulqdq xmm3, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm3, xmm4 + movdqu xmm5, dqword ptr [rcx + 48] + pxor xmm3, xmm5 + + // Fold xmm8 + movdqa xmm4, xmm8 + db $66, $44, $0F, $3A, $44, $C6, $00 // pclmulqdq xmm8, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm8, xmm4 + movdqu xmm5, dqword ptr [rcx + 64] + pxor xmm8, xmm5 + + // Fold xmm9 + movdqa xmm4, xmm9 + db $66, $44, $0F, $3A, $44, $CE, $00 // pclmulqdq xmm9, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm9, xmm4 + movdqu xmm5, dqword ptr [rcx + 80] + pxor xmm9, xmm5 + + // Fold xmm10 + movdqa xmm4, xmm10 + db $66, $44, $0F, $3A, $44, $D6, $00 // pclmulqdq xmm10, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm10, xmm4 + movdqu xmm5, dqword ptr [rcx + 96] + pxor xmm10, xmm5 + + // Fold xmm11 + movdqa xmm4, xmm11 + db $66, $44, $0F, $3A, $44, $DE, $00 // pclmulqdq xmm11, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm11, xmm4 + movdqu xmm5, dqword ptr [rcx + 112] + pxor xmm11, xmm5 + + add rcx, 128 + sub edx, 128 + cmp edx, 128 + jae @fold8_loop + +@fold8_reduce: + // --- Reduce 8 accumulators to 4 using fold_4x128 (stride 512) --- + movdqu xmm6, dqword ptr [r9] // fold_4x128 + + // xmm0 = fold(xmm0) XOR xmm8 + movdqa xmm4, xmm0 + db $66, $0F, $3A, $44, $C6, $00 // pclmulqdq xmm0, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm0, xmm4 + pxor xmm0, xmm8 + + // xmm1 = fold(xmm1) XOR xmm9 + movdqa xmm4, xmm1 + db $66, $0F, $3A, $44, $CE, $00 // pclmulqdq xmm1, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm1, xmm4 + pxor xmm1, xmm9 + + // xmm2 = fold(xmm2) XOR xmm10 + movdqa xmm4, xmm2 + db $66, $0F, $3A, $44, $D6, $00 // pclmulqdq xmm2, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm2, xmm4 + pxor xmm2, xmm10 + + // xmm3 = fold(xmm3) XOR xmm11 + movdqa xmm4, xmm3 + db $66, $0F, $3A, $44, $DE, $00 // pclmulqdq xmm3, xmm6, $00 + db $66, $0F, $3A, $44, $E6, $11 // pclmulqdq xmm4, xmm6, $11 + pxor xmm3, xmm4 + pxor xmm3, xmm11 + + jmp @fold4_done + +@check64: cmp edx, 64 jae @large_input @@ -194,4 +312,4 @@ // pextrq rax, xmm0, 1: 66 48 0F 3A 16 C0 01 db $66, $48, $0F, $3A, $16, $C0, $01 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPmull_aarch64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPmull_aarch64.inc index 06d9e96c..38ae2c14 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPmull_aarch64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPmull_aarch64.inc @@ -1,24 +1,124 @@ -// CRC folding (PMULL/PMULL2) + Barrett reduction for reflected (LSB-first) -// CRCs with width 8..64. CRC-64 uses a special Barrett path for the 65-bit G. -// Expects AAPCS64: x0 = AData, w1 = ALength (>= 16, multiple of 16), -// x2 = AState, x3 = AConstants (PCRCFoldRuntimeCtx; FoldConstants at +0). -// AState: [0..7] = initial CRC (reflected), [8..15] = 0. +// PMULL implementation of CRC folding (PMULL/PMULL2) + Barrett reduction for +// reflected (LSB-first) CRCs with width 8..64. CRC-64 uses a special Barrett +// path for the 65-bit G. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = AData, w1 = ALength (>= 16, +// multiple of 16), x2 = AState, x3 = AConstants (PCRCFoldRuntimeCtx; +// FoldConstants at +0). +// AState: pointer to the initial CRC (reflected) as one UInt64. Returns +// the final CRC in x0. // AConstants: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett, -// [+80] CrcBits. Returns final CRC in x0. -// Leaf nostackframe; caller-saved GPR/vector only (v0-v7, v16-v18). -// Reference: Linux kernel crc-pclmul-template.S (Eric Biggers), -// HashLib CRCFoldReflectedPclmul_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// [+80] CrcBits. // Register map: // v0-v3 = fold accumulators (xmm0-3) // v4 = pmull-hi temp (xmm4) // v5 = loaded 16-byte block (xmm5) -// v6 = Fold_4x128 then Barrett consts (xmm6) +// v6 = Fold_8x128 / Fold_4x128 then Barrett consts (xmm6) // v7 = Fold_1x128 (xmm7) // v16 = zeroed scratch (psrldq emulation via ext) // v17 = dup staging for clmul lo*hi ($10) Barrett mixes // v18 = saved Barrett q (CRC-64 x^0 correction) +// v19-v22 = fold-by-8 upper accumulators (xmm8-11) +// Saves: none (leaf nostackframe; caller-saved GPR/vector only: v0-v7, +// v16-v22). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: Linux kernel crc-pclmul-template.S (Eric Biggers), HashLib +// CRCFoldReflectedPclmul_x86_64.inc. + .long 0x3dc00467 // ldr q7, [x3, #16] + .long 0x7102003f // cmp w1, #128 + b.lo .Lcrc_refl_check64 + .long 0x3dc00c66 // ldr q6, [x3, #48] + .long 0x3dc00000 // ldr q0, [x0, #0] + .long 0x3dc00401 // ldr q1, [x0, #16] + .long 0x3dc00802 // ldr q2, [x0, #32] + .long 0x3dc00c03 // ldr q3, [x0, #48] + .long 0x3dc01013 // ldr q19, [x0, #64] + .long 0x3dc01414 // ldr q20, [x0, #80] + .long 0x3dc01815 // ldr q21, [x0, #96] + .long 0x3dc01c16 // ldr q22, [x0, #112] + .long 0xfd400044 // ldr d4, [x2] + .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b + .long 0x91020000 // add x0, x0, #128 + .long 0x51020021 // sub w1, w1, #128 + .long 0x7102003f // cmp w1, #128 + b.lo .Lcrc_refl_fold8_reduce +.Lcrc_refl_fold8_loop: + .long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b + .long 0x0ee6e000 // pmull v0.1q, v0.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b + .long 0x3dc00005 // ldr q5, [x0, #0] + .long 0x6e251c00 // eor v0.16b, v0.16b, v5.16b + .long 0x4ea11c24 // orr v4.16b, v1.16b, v1.16b + .long 0x0ee6e021 // pmull v1.1q, v1.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c21 // eor v1.16b, v1.16b, v4.16b + .long 0x3dc00405 // ldr q5, [x0, #16] + .long 0x6e251c21 // eor v1.16b, v1.16b, v5.16b + .long 0x4ea21c44 // orr v4.16b, v2.16b, v2.16b + .long 0x0ee6e042 // pmull v2.1q, v2.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b + .long 0x3dc00805 // ldr q5, [x0, #32] + .long 0x6e251c42 // eor v2.16b, v2.16b, v5.16b + .long 0x4ea31c64 // orr v4.16b, v3.16b, v3.16b + .long 0x0ee6e063 // pmull v3.1q, v3.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c63 // eor v3.16b, v3.16b, v4.16b + .long 0x3dc00c05 // ldr q5, [x0, #48] + .long 0x6e251c63 // eor v3.16b, v3.16b, v5.16b + .long 0x4eb31e64 // orr v4.16b, v19.16b, v19.16b + .long 0x0ee6e273 // pmull v19.1q, v19.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241e73 // eor v19.16b, v19.16b, v4.16b + .long 0x3dc01005 // ldr q5, [x0, #64] + .long 0x6e251e73 // eor v19.16b, v19.16b, v5.16b + .long 0x4eb41e84 // orr v4.16b, v20.16b, v20.16b + .long 0x0ee6e294 // pmull v20.1q, v20.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241e94 // eor v20.16b, v20.16b, v4.16b + .long 0x3dc01405 // ldr q5, [x0, #80] + .long 0x6e251e94 // eor v20.16b, v20.16b, v5.16b + .long 0x4eb51ea4 // orr v4.16b, v21.16b, v21.16b + .long 0x0ee6e2b5 // pmull v21.1q, v21.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241eb5 // eor v21.16b, v21.16b, v4.16b + .long 0x3dc01805 // ldr q5, [x0, #96] + .long 0x6e251eb5 // eor v21.16b, v21.16b, v5.16b + .long 0x4eb61ec4 // orr v4.16b, v22.16b, v22.16b + .long 0x0ee6e2d6 // pmull v22.1q, v22.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241ed6 // eor v22.16b, v22.16b, v4.16b + .long 0x3dc01c05 // ldr q5, [x0, #112] + .long 0x6e251ed6 // eor v22.16b, v22.16b, v5.16b + .long 0x91020000 // add x0, x0, #128 + .long 0x51020021 // sub w1, w1, #128 + .long 0x7102003f // cmp w1, #128 + b.hs .Lcrc_refl_fold8_loop +.Lcrc_refl_fold8_reduce: + .long 0x3dc00066 // ldr q6, [x3] + .long 0x4ea01c04 // orr v4.16b, v0.16b, v0.16b + .long 0x0ee6e000 // pmull v0.1q, v0.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b + .long 0x6e331c00 // eor v0.16b, v0.16b, v19.16b + .long 0x4ea11c24 // orr v4.16b, v1.16b, v1.16b + .long 0x0ee6e021 // pmull v1.1q, v1.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c21 // eor v1.16b, v1.16b, v4.16b + .long 0x6e341c21 // eor v1.16b, v1.16b, v20.16b + .long 0x4ea21c44 // orr v4.16b, v2.16b, v2.16b + .long 0x0ee6e042 // pmull v2.1q, v2.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c42 // eor v2.16b, v2.16b, v4.16b + .long 0x6e351c42 // eor v2.16b, v2.16b, v21.16b + .long 0x4ea31c64 // orr v4.16b, v3.16b, v3.16b + .long 0x0ee6e063 // pmull v3.1q, v3.1d, v6.1d + .long 0x4ee6e084 // pmull2 v4.1q, v4.2d, v6.2d + .long 0x6e241c63 // eor v3.16b, v3.16b, v4.16b + .long 0x6e361c63 // eor v3.16b, v3.16b, v22.16b + b .Lcrc_refl_fold4_done +.Lcrc_refl_check64: .long 0x7101003f // cmp w1, #64 b.hs .Lcrc_refl_large .long 0x3dc00000 // ldr q0, [x0] diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_i386.inc index e8612d10..7f8a2527 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_i386.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_i386.inc @@ -1,7 +1,13 @@ -// Reflected (LSB-first) 64-bit CRC: 16-byte slice (IA-32 SSE2 + GPR table XOR). -// After HlpSimdProc4Begin_i386: ebx = AData, esi = ALength, edi = AState, eax = Ctx. -// TableRow[i] at Ctx + 96; each entry is a 32-bit (untyped) Pointer. -// Result UInt64 in edx:eax (high:low). Preserves Delphi/FPC i386 frame via push/pop ebp. +// SSE2 implementation of the reflected (LSB-first) 64-bit CRC 16-byte slice +// (IA-32; SSE2 loads + GPR table XOR). GPR slicing core; SSE2 is genuinely +// required for the 128-bit loads and the CRC state shuttle - the Sse2 name +// states the ISA requirement. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = AData, esi = ALength, +// edi = AState, eax = Ctx (PCRCFoldRuntimeCtx). +// TableRow[i] at Ctx + 96; each entry is a 32-bit (untyped) Pointer. +// Returns the final CRC UInt64 in edx:eax (high:low). +// Frame: preserves the i386 frame via push/pop ebp. +// Reference: classic slicing-by-16 table CRC. push ebp mov ebp, eax diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_x86_64.inc index 3aa4034a..d6ba5de6 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedSse2_x86_64.inc @@ -1,11 +1,13 @@ -// Reflected (LSB-first) 64-bit CRC: 16-byte slice using SSE2 loads + GPR table XOR. -// Signature (after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Reflected_Sse2(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// MS x64: rcx=AData, edx=ALength, r8=AState, r9=Ctx (PCRCFoldRuntimeCtx). -// TableRow[i] begins at Ctx + SizeOf(TCRCFoldConstants) = Ctx + 96. -// -// SSE2: movdqu, movq, pxor, psrldq. No PCLMUL. +// SSE2 implementation of the reflected (LSB-first) 64-bit CRC 16-byte slice +// (SSE2 loads + GPR table XOR). GPR slicing core; SSE2 is genuinely +// required for the 128-bit loads and the CRC state shuttle - the Sse2 name +// states the ISA requirement. SSE2 ops: movdqu, movq, pxor, psrldq; +// no PCLMUL. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = AData, edx = ALength, +// r8 = AState, r9 = Ctx (PCRCFoldRuntimeCtx). +// TableRow[i] begins at Ctx + SizeOf(TCRCFoldConstants) = Ctx + 96. +// Returns the final CRC in rax. +// Reference: classic slicing-by-16 table CRC. push rbx push r12 diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_i386.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_i386.inc new file mode 100644 index 00000000..c39b09e2 --- /dev/null +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_i386.inc @@ -0,0 +1,160 @@ +// AVX2 + VPCLMULQDQ implementation of CRC folding + Barrett reduction for +// reflected (LSB-first) CRCs with width 8..64 (IA-32). CRC-64 uses a +// special Barrett path for the 65-bit G. Mirrors +// CRCFoldReflectedVpclmul_x86_64.inc; ymm0-7/xmm0-7 only, so the structure +// carries over 1:1. Every register-register byte sequence is identical to +// the verified x86_64 kernel's encoding (regs 0-7 carry no REX/VEX +// extension bits); the memory forms are machine-assembled for the IA-32 +// bases. vzeroupper before returning. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = AData, esi = ALength +// (>= 16), edi = AState, eax = Ctx (PCRCFoldRuntimeCtx; FoldConstants +// at +0). +// AState: pointer to the initial CRC (reflected) as one UInt64. Returns +// the final CRC UInt64 in edx:eax (high qword of xmm0). +// Ctx: [+0] Fold_4x128, [+16] Fold_1x128, [+32] Barrett, +// [+48] Fold_8x128, [+80] CrcBits. +// Saves: none (all vector registers are volatile on IA-32). +// AVX2/VPCLMULQDQ instructions are db-encoded for broad assembler +// compatibility. +// Reference: zlib-ng crc32_pclmulqdq_tpl.h and Linux kernel +// crc-pclmul-template.S by Eric Biggers, HashLib +// CRCFoldReflectedVpclmul_x86_64.inc. + + push ebp + mov ebp, eax + cmp esi, 128 + jb @xmm_path + db $C4, $E2, $7D, $5A, $75, $30 // vbroadcasti128 ymm6, [ebp + 48] + db $C5, $FE, $6F, $03 // vmovdqu ymm0, [ebx] + db $C5, $FE, $6F, $4B, $20 // vmovdqu ymm1, [ebx + 32] + db $C5, $FE, $6F, $53, $40 // vmovdqu ymm2, [ebx + 64] + db $C5, $FE, $6F, $5B, $60 // vmovdqu ymm3, [ebx + 96] + db $C5, $FA, $7E, $27 // vmovq xmm4, qword ptr [edi] + db $C5, $FD, $EF, $C4 // vpxor ymm0, ymm0, ymm4 + add ebx, 128 + sub esi, 128 + cmp esi, 128 + jb @ymm_done + +@ymm_loop: + db $C4, $E3, $7D, $44, $E6, $11 // vpclmulqdq ymm4, ymm0, ymm6, $11 + db $C4, $E3, $7D, $44, $C6, $00 // vpclmulqdq ymm0, ymm0, ymm6, $00 + db $C5, $FD, $EF, $C4 // vpxor ymm0, ymm0, ymm4 + db $C5, $FE, $6F, $2B // vmovdqu ymm5, [ebx] + db $C5, $FD, $EF, $C5 // vpxor ymm0, ymm0, ymm5 + db $C4, $E3, $75, $44, $E6, $11 // vpclmulqdq ymm4, ymm1, ymm6, $11 + db $C4, $E3, $75, $44, $CE, $00 // vpclmulqdq ymm1, ymm1, ymm6, $00 + db $C5, $F5, $EF, $CC // vpxor ymm1, ymm1, ymm4 + db $C5, $FE, $6F, $6B, $20 // vmovdqu ymm5, [ebx + 32] + db $C5, $F5, $EF, $CD // vpxor ymm1, ymm1, ymm5 + db $C4, $E3, $6D, $44, $E6, $11 // vpclmulqdq ymm4, ymm2, ymm6, $11 + db $C4, $E3, $6D, $44, $D6, $00 // vpclmulqdq ymm2, ymm2, ymm6, $00 + db $C5, $ED, $EF, $D4 // vpxor ymm2, ymm2, ymm4 + db $C5, $FE, $6F, $6B, $40 // vmovdqu ymm5, [ebx + 64] + db $C5, $ED, $EF, $D5 // vpxor ymm2, ymm2, ymm5 + db $C4, $E3, $65, $44, $E6, $11 // vpclmulqdq ymm4, ymm3, ymm6, $11 + db $C4, $E3, $65, $44, $DE, $00 // vpclmulqdq ymm3, ymm3, ymm6, $00 + db $C5, $E5, $EF, $DC // vpxor ymm3, ymm3, ymm4 + db $C5, $FE, $6F, $6B, $60 // vmovdqu ymm5, [ebx + 96] + db $C5, $E5, $EF, $DD // vpxor ymm3, ymm3, ymm5 + add ebx, 128 + sub esi, 128 + cmp esi, 128 + jae @ymm_loop + +@ymm_done: + db $C4, $E2, $7D, $5A, $75, $00 // vbroadcasti128 ymm6, [ebp] + db $C4, $E3, $7D, $44, $E6, $11 // vpclmulqdq ymm4, ymm0, ymm6, $11 + db $C4, $E3, $7D, $44, $C6, $00 // vpclmulqdq ymm0, ymm0, ymm6, $00 + db $C5, $FD, $EF, $C4 // vpxor ymm0, ymm0, ymm4 + db $C5, $FD, $EF, $C2 // vpxor ymm0, ymm0, ymm2 + db $C4, $E3, $75, $44, $E6, $11 // vpclmulqdq ymm4, ymm1, ymm6, $11 + db $C4, $E3, $75, $44, $CE, $00 // vpclmulqdq ymm1, ymm1, ymm6, $00 + db $C5, $F5, $EF, $CC // vpxor ymm1, ymm1, ymm4 + db $C5, $F5, $EF, $CB // vpxor ymm1, ymm1, ymm3 + db $C5, $F9, $6F, $D1 // vmovdqa xmm2, xmm1 + db $C4, $E3, $7D, $39, $CB, $01 // vextracti128 xmm3, ymm1, 1 + db $C4, $E3, $7D, $39, $C1, $01 // vextracti128 xmm1, ymm0, 1 + db $C5, $F8, $77 // vzeroupper + jmp @fold4to1 + +@xmm_path: + cmp esi, 64 + jae @xmm_large + db $C5, $FA, $6F, $03 // vmovdqu xmm0, [ebx] + db $C5, $FA, $7E, $27 // vmovq xmm4, qword ptr [edi] + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + add ebx, 16 + sub esi, 16 + db $C5, $FA, $6F, $7D, $10 // vmovdqu xmm7, [ebp + 16] + jmp @tail_check + +@xmm_large: + db $C5, $FA, $6F, $03 // vmovdqu xmm0, [ebx] + db $C5, $FA, $6F, $4B, $10 // vmovdqu xmm1, [ebx + 16] + db $C5, $FA, $6F, $53, $20 // vmovdqu xmm2, [ebx + 32] + db $C5, $FA, $6F, $5B, $30 // vmovdqu xmm3, [ebx + 48] + db $C5, $FA, $7E, $27 // vmovq xmm4, qword ptr [edi] + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + add ebx, 64 + sub esi, 64 + +@fold4to1: + db $C5, $FA, $6F, $7D, $10 // vmovdqu xmm7, [ebp + 16] + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $F9, $EF, $C2 // vpxor xmm0, xmm0, xmm2 + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $F9, $EF, $C3 // vpxor xmm0, xmm0, xmm3 + +@tail_check: + cmp esi, 16 + jb @tail_done + +@tail_loop: + db $C4, $E3, $79, $44, $E7, $11 // vpclmulqdq xmm4, xmm0, xmm7, $11 + db $C4, $E3, $79, $44, $C7, $00 // vpclmulqdq xmm0, xmm0, xmm7, $00 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $FA, $6F, $2B // vmovdqu xmm5, [ebx] + db $C5, $F9, $EF, $C5 // vpxor xmm0, xmm0, xmm5 + add ebx, 16 + sub esi, 16 + cmp esi, 16 + jae @tail_loop + +@tail_done: + db $C4, $E3, $79, $44, $CF, $10 // vpclmulqdq xmm1, xmm0, xmm7, $10 + db $C5, $F9, $73, $D8, $08 // vpsrldq xmm0, xmm0, 8 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C5, $FA, $6F, $75, $20 // vmovdqu xmm6, [ebp + 32] + db $C4, $E3, $79, $44, $CE, $00 // vpclmulqdq xmm1, xmm0, xmm6, $00 + cmp dword ptr [ebp + 80], 64 + je @barrett_64 + db $C4, $E3, $71, $44, $CE, $10 // vpclmulqdq xmm1, xmm1, xmm6, $10 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + jmp @extract_crc + +@barrett_64: + db $C5, $F9, $6F, $D1 // vmovdqa xmm2, xmm1 + db $C4, $E3, $71, $44, $CE, $10 // vpclmulqdq xmm1, xmm1, xmm6, $10 + db $C5, $E9, $6C, $D2 // vpunpcklqdq xmm2, xmm2, xmm2 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C5, $F9, $EF, $C2 // vpxor xmm0, xmm0, xmm2 + +@extract_crc: + db $C5, $F8, $77 // vzeroupper + psrldq xmm0, 8 + movd eax, xmm0 + psrldq xmm0, 4 + movd edx, xmm0 + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc index ea35440a..1c4a4824 100644 --- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc +++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc @@ -1,31 +1,21 @@ -// AVX2 + VPCLMULQDQ CRC folding + Barrett reduction for reflected (LSB-first) -// CRCs with width 8..64. CRC-64 uses a special Barrett path for the 65-bit G. -// AVX2/VPCLMULQDQ instructions are db-encoded for broad assembler compatibility. -// -// Function signature (included after HlpSimdProc4Begin_x86_64.inc): -// function CRC_Fold_Reflected_Vpclmul(AData: PByte; ALength: UInt32; -// AState: Pointer; AConstants: Pointer): UInt64; -// -// Register mapping (MS x64 ABI after prologue): -// rcx = AData, edx = ALength (>= 16), r8 = AState, r9 = AConstants -// AState: [0..7] = initial CRC (reflected), [8..15] = 0. +// AVX2 + VPCLMULQDQ implementation of CRC folding + Barrett reduction for +// reflected (LSB-first) CRCs with width 8..64. CRC-64 uses a special +// Barrett path for the 65-bit G. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = AData, edx = ALength +// (>= 16), r8 = AState, r9 = AConstants. +// AState: pointer to the initial CRC (reflected) as one UInt64. Returns +// the final CRC in rax. // AConstants layout (TCRCFoldConstants): -// [0..15] = Fold_4x128 (stride 512) -// [16..31] = Fold_1x128 (stride 128) -// [32..47] = Barrett -// [48..63] = Fold_8x128 (stride 1024) -// [64..79] = BswapMask (unused by this path) -// [80..87] = CrcBits -// Returns: final CRC in RAX. -// -// Reference: zlib-ng crc32_pclmulqdq_tpl.h (256-bit VPCLMULQDQ path) -// and Linux kernel crc-pclmul-template.S by Eric Biggers (Barrett reduction). -// -// VEX byte layout reference: -// 2-byte: C5 [R.vvvv.L.pp] -// 3-byte: C4 [R.X.B.mmmmm] [W.vvvv.L.pp] - - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} +// [0..15] = Fold_4x128 (stride 512) [48..63] = Fold_8x128 (stride 1024) +// [16..31] = Fold_1x128 (stride 128) [64..79] = BswapMask (unused here) +// [32..47] = Barrett [80..87] = CrcBits +// AVX2/VPCLMULQDQ instructions are db-encoded for broad assembler +// compatibility. Encoding notes (VEX byte layout): +// 2-byte: C5 [R.vvvv.L.pp] 3-byte: C4 [R.X.B.mmmmm] [W.vvvv.L.pp] +// Reference: zlib-ng crc32_pclmulqdq_tpl.h (256-bit VPCLMULQDQ path) and +// Linux kernel crc-pclmul-template.S by Eric Biggers (Barrett reduction). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} cmp edx, 128 jb @xmm_path @@ -216,4 +206,4 @@ db $C5, $F8, $77 // vzeroupper db $C4, $E3, $F9, $16, $C0, $01 // vpextrq rax, xmm0, 1 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Common/HlpSimdProc4Begin_aarch64.inc b/HashLib/src/Include/Simd/Common/HlpSimdProc4Begin_aarch64.inc index ec052077..4e560def 100644 --- a/HashLib/src/Include/Simd/Common/HlpSimdProc4Begin_aarch64.inc +++ b/HashLib/src/Include/Simd/Common/HlpSimdProc4Begin_aarch64.inc @@ -1,7 +1,7 @@ // Shared SIMD procedure prologue for 4-parameter assembly functions (AArch64). // After inclusion: x0 = param1, x1 = param2, w2 = param3, x3 = param4 (AAPCS64). // Matches the x86 SimdProc4/5 signature order: (AState, AData, ANumBlocks, -// AConstants[, …]); AAPCS64 already delivers the params in x0,x1,w2,x3, so no +// AConstants[, ...]); AAPCS64 already delivers the params in x0,x1,w2,x3, so no // register remapping (unlike the x86 SysV/MS variants) is needed. // Usage: // procedure MyProc(P1, P2, P3: Pointer; P4: Pointer); diff --git a/HashLib/src/Include/Simd/Common/HlpSimdProc5Begin_i386.inc b/HashLib/src/Include/Simd/Common/HlpSimdProc5Begin_i386.inc index c1d109ea..a5e06bc2 100644 --- a/HashLib/src/Include/Simd/Common/HlpSimdProc5Begin_i386.inc +++ b/HashLib/src/Include/Simd/Common/HlpSimdProc5Begin_i386.inc @@ -3,7 +3,7 @@ // (i386 register convention). // FPC: param4 and param5 are at [esp+20] and [esp+16] after push EBX/ESI/EDI (see asm). // Delphi Win32: first 3 params in EAX/EDX/ECX; param4 at [ebp+12], param5 at [ebp+8]. -// Callee-saved EBX, ESI, EDI — epilogue must pop EDI, ESI, EBX. +// Callee-saved EBX, ESI, EDI - epilogue must pop EDI, ESI, EBX. // Usage: // procedure MyProc(P1, P2: Pointer; P3, P4: Int32; P5: Pointer); // {$I HlpSimdProc5Begin_i386.inc} diff --git a/HashLib/src/Include/Simd/Common/HlpSimdProc6Begin_i386.inc b/HashLib/src/Include/Simd/Common/HlpSimdProc6Begin_i386.inc index 229122b4..769135b1 100644 --- a/HashLib/src/Include/Simd/Common/HlpSimdProc6Begin_i386.inc +++ b/HashLib/src/Include/Simd/Common/HlpSimdProc6Begin_i386.inc @@ -4,7 +4,7 @@ // FPC: param4 at [esp+28]; param5 at [esp+20]/[esp+24]; param6 at [esp+16] after push EBX/ESI/EDI (see asm). // Delphi Win32: first 3 params in EAX/EDX/ECX; param4 at [ebp+20], // param5 at [ebp+12]/[ebp+16], param6 at [ebp+8]. -// Callee-saved EBX, ESI, EDI — epilogue must pop EDI, ESI, EBX; SIMD bodies may adjust ESP for locals. +// Callee-saved EBX, ESI, EDI - epilogue must pop EDI, ESI, EBX; SIMD bodies may adjust ESP for locals. // Usage: // procedure MyProc(P1, P2, P3: Pointer; P4: Int32; P5: UInt64; P6: UInt32); // {$I HlpSimdProc6Begin_i386.inc} diff --git a/HashLib/src/Include/Simd/Common/HlpSimdProc7Begin_i386.inc b/HashLib/src/Include/Simd/Common/HlpSimdProc7Begin_i386.inc new file mode 100644 index 00000000..f4ba0ab5 --- /dev/null +++ b/HashLib/src/Include/Simd/Common/HlpSimdProc7Begin_i386.inc @@ -0,0 +1,39 @@ +// Shared SIMD procedure prologue for 7-parameter assembly functions. +// After inclusion: ebx = param1, esi = param2, edi = param3, eax = param4, +// ecx = param5 low, edx = param5 high (UInt64 split); param6/param7 are not +// loaded in this prologue (IA-32 has no free register left) - load them from +// the documented stack slots when needed. +// FPC: param4 at [esp+32]; param5 at [esp+24]/[esp+28]; param6 at [esp+20]; +// param7 at [esp+16] after push EBX/ESI/EDI (see asm). +// Delphi Win32: first 3 params in EAX/EDX/ECX; param4 at [ebp+24], +// param5 at [ebp+16]/[ebp+20], param6 at [ebp+12], param7 at [ebp+8]. +// Callee-saved EBX, ESI, EDI - epilogue must pop EDI, ESI, EBX; SIMD bodies may adjust ESP for locals. +// Usage: +// procedure MyProc(P1, P2, P3: Pointer; P4: Int32; P5: UInt64; P6: UInt32; P7: Pointer); +// {$I HlpSimdProc7Begin_i386.inc} +// // ... SIMD instructions using ebx, esi, edi, eax, ecx, edx (load param6/7 from stack if needed) ... +// end; +{$IFDEF FPC} + assembler; nostackframe; +asm + push ebx + push esi + push edi + mov ebx, eax + mov esi, edx + mov edi, ecx + mov eax, dword ptr [esp + 32] // param4 + mov ecx, dword ptr [esp + 24] // param5_lo + mov edx, dword ptr [esp + 28] // param5_hi +{$ELSE} +asm + push ebx + push esi + push edi + mov ebx, eax + mov esi, edx + mov edi, ecx + mov eax, dword ptr [ebp + 24] // param4 (first stack param = high address) + mov ecx, dword ptr [ebp + 16] // param5_lo + mov edx, dword ptr [ebp + 20] // param5_hi +{$ENDIF} diff --git a/HashLib/src/Include/Simd/Common/HlpSimdProc7Begin_x86_64.inc b/HashLib/src/Include/Simd/Common/HlpSimdProc7Begin_x86_64.inc new file mode 100644 index 00000000..9efa57c2 --- /dev/null +++ b/HashLib/src/Include/Simd/Common/HlpSimdProc7Begin_x86_64.inc @@ -0,0 +1,32 @@ +// Shared SIMD procedure prologue for 7-parameter assembly functions. +// After inclusion: rcx = param1, rdx = param2, r8 = param3, r9 = param4, +// r10 = param5, r11 = param6, rax = param7 (MS x64 ABI layout). +// When HASHLIB_SYSV_X64_ABI is defined, remaps rdi,rsi,rdx,rcx,r8,r9 -> +// rcx,rdx,r8,r9,r10,r11 and loads param7 from [rsp+8]. +// On MS x64, param5/6/7 are loaded from [rsp+40]/[rsp+48]/[rsp+56] +// (above shadow space + return addr). +// Usage: +// procedure MyProc(P1, P2, P3, P4: Pointer; P5: UInt64; P6: UInt32; P7: Pointer); +// {$I HlpSimdProc7Begin_x86_64.inc} +// // ... SIMD instructions using rcx, rdx, r8, r9, r10, r11, rax ... +// end; +{$IFDEF FPC} + assembler; nostackframe; +{$ENDIF} +asm +{$IFNDEF FPC} + .noframe +{$ENDIF} +{$IFDEF HASHLIB_SYSV_X64_ABI} + mov rax, [rsp + 8] + mov r11, r9 + mov r10, r8 + mov r9, rcx + mov r8, rdx + mov rdx, rsi + mov rcx, rdi +{$ELSE} + mov r10, [rsp + 40] + mov r11, [rsp + 48] + mov rax, [rsp + 56] +{$ENDIF} diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc index fbb5b201..155cff1d 100644 --- a/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc @@ -1,19 +1,20 @@ -// SHA-1 AVX2 two-block implementation (VEX-256), ported from OpenSSL's -// sha1-x86_64.pl (CRYPTOGAMS). Two message blocks are scheduled together in -// 256-bit lanes and the 80 rounds are interleaved with the SIMD schedule, so the -// vector and integer units run in parallel. AVX2 and BMI (rorx/andn/shlx) -// instructions are db-encoded for broad assembler compatibility (every AVX2 CPU -// also provides BMI). -// Expects MS x64 ABI (after HlpSimdProc4Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = K_SHA1_Doubled ptr (the four SHA-1 round constants each -// replicated across 256 bits, then the byte-swap mask and a reverse mask; see -// K_SHA1_Doubled). The masks are read unaligned, so the const needs no alignment. -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r14 in the local frame; r11 -// holds the caller rsp across the kernel for the unwind. +// AVX2 implementation of SHA-1 compress (VEX-256, two-block). Two message +// blocks are scheduled together in 256-bit lanes and the 80 rounds are +// interleaved with the SIMD schedule, so the vector and integer units run +// in parallel. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = K_SHA1_Doubled ptr (the four SHA-1 round +// constants each replicated across 256 bits, then the byte-swap mask and +// a reverse mask; the masks are read unaligned, so the const needs no +// special alignment). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r14 in the local frame; r11 +// holds the caller rsp across the kernel for the unwind. +// AVX2 and BMI (rorx/andn/shlx) instructions are db-encoded for broad +// assembler compatibility (every AVX2 CPU also provides BMI). +// Reference: OpenSSL CRYPTOGAMS sha1-x86_64.pl (AVX2 kernel). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} mov r11, rsp push r9 @@ -1679,4 +1680,4 @@ mov rbx, qword [r11 - $10] lea rsp, [r11] - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx_i386.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx_i386.inc new file mode 100644 index 00000000..57ffd5a3 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx_i386.inc @@ -0,0 +1,1174 @@ +// AVX (VEX-128) implementation of SHA-1 compress (SIMD schedule; IA-32); +// the Avx name states the ISA requirement: VEX-128 encodings only - there +// is no i386 AVX2 SHA-1 upstream. The 3-operand VEX forms eliminate the +// SSSE3 kernel's register-copy movdqa traffic in the interleaved message +// schedule. The four round constants and the byte-swap mask are read from +// K_SHA1_Doubled at a 32-byte stride (VEX memory operands are read +// unaligned, so the Pascal const needs no special alignment) and spilled +// to the local frame like the original; the original's PIC table setup and +// stack-argument reads are replaced by the shared-prologue registers. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = K_SHA1_Doubled ptr. +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here, giving the +// 4-deep frame the original epilogue unwinds via its saved-esp slot. +// AVX instructions are db-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha1-586.pl (AVX function). + + push ebp + mov ebp, eax + db $C5, $FC, $77 // vzeroall + db $C5, $FA, $6F, $7D, $00 // vmovdqu xmm7, oword [ebp + $0] + db $C5, $FA, $6F, $45, $20 // vmovdqu xmm0, oword [ebp + $20] + db $C5, $FA, $6F, $4D, $40 // vmovdqu xmm1, oword [ebp + $40] + db $C5, $FA, $6F, $55, $60 // vmovdqu xmm2, oword [ebp + $60] + db $C5, $FA, $6F, $B5, $80, $00, $00, $00 // vmovdqu xmm6, oword [ebp + $80] + mov edx, edi + mov edi, ebx + mov ebp, esi + mov esi, esp + sub esp, $D0 + and esp, $FFFFFFC0 + db $C5, $F9, $7F, $44, $24, $70 // vmovdqa oword [esp + $70], xmm0 + db $C5, $F9, $7F, $8C, $24, $80, $00, $00, $00 // vmovdqa oword [esp + $80], xmm1 + db $C5, $F9, $7F, $94, $24, $90, $00, $00, $00 // vmovdqa oword [esp + $90], xmm2 + shl edx, $6 + db $C5, $F9, $7F, $BC, $24, $A0, $00, $00, $00 // vmovdqa oword [esp + $A0], xmm7 + add edx, ebp + db $C5, $F9, $7F, $B4, $24, $B0, $00, $00, $00 // vmovdqa oword [esp + $B0], xmm6 + add ebp, $40 + mov dword ptr [esp + $C0], edi + mov dword ptr [esp + $C4], ebp + mov dword ptr [esp + $C8], edx + mov dword ptr [esp + $CC], esi + mov eax, dword ptr [edi] + mov ebx, dword ptr [edi + $4] + mov ecx, dword ptr [edi + $8] + mov edx, dword ptr [edi + $C] + mov edi, dword ptr [edi + $10] + mov esi, ebx + db $C5, $FA, $6F, $45, $C0 // vmovdqu xmm0, oword [ebp - $40] + db $C5, $FA, $6F, $4D, $D0 // vmovdqu xmm1, oword [ebp - $30] + db $C5, $FA, $6F, $55, $E0 // vmovdqu xmm2, oword [ebp - $20] + db $C5, $FA, $6F, $5D, $F0 // vmovdqu xmm3, oword [ebp - $10] + db $C4, $E2, $79, $00, $C6 // vpshufb xmm0, xmm0, xmm6 + db $C4, $E2, $71, $00, $CE // vpshufb xmm1, xmm1, xmm6 + db $C4, $E2, $69, $00, $D6 // vpshufb xmm2, xmm2, xmm6 + db $C5, $F9, $7F, $7C, $24, $60 // vmovdqa oword [esp + $60], xmm7 + db $C4, $E2, $61, $00, $DE // vpshufb xmm3, xmm3, xmm6 + db $C5, $F9, $FE, $E7 // vpaddd xmm4, xmm0, xmm7 + db $C5, $F1, $FE, $EF // vpaddd xmm5, xmm1, xmm7 + db $C5, $E9, $FE, $F7 // vpaddd xmm6, xmm2, xmm7 + db $C5, $F9, $7F, $24, $24 // vmovdqa oword [esp], xmm4 + mov ebp, ecx + db $C5, $F9, $7F, $6C, $24, $10 // vmovdqa oword [esp + $10], xmm5 + xor ebp, edx + db $C5, $F9, $7F, $74, $24, $20 // vmovdqa oword [esp + $20], xmm6 + and esi, ebp + jmp @loop_009 + +@loop_009: + shrd ebx, ebx, $2 + xor esi, edx + db $C4, $E3, $71, $0F, $E0, $08 // vpalignr xmm4, xmm1, xmm0, $8 + mov ebp, eax + add edi, dword ptr [esp] + db $C5, $C1, $FE, $FB // vpaddd xmm7, xmm7, xmm3 + db $C5, $F9, $7F, $44, $24, $40 // vmovdqa oword [esp + $40], xmm0 + xor ebx, ecx + shld eax, eax, $5 + db $C5, $C9, $73, $DB, $04 // vpsrldq xmm6, xmm3, $4 + add edi, esi + and ebp, ebx + db $C5, $D9, $EF, $E0 // vpxor xmm4, xmm4, xmm0 + xor ebx, ecx + add edi, eax + db $C5, $C9, $EF, $F2 // vpxor xmm6, xmm6, xmm2 + shrd eax, eax, $7 + xor ebp, ecx + db $C5, $F9, $7F, $7C, $24, $30 // vmovdqa oword [esp + $30], xmm7 + mov esi, edi + add edx, dword ptr [esp + $4] + db $C5, $D9, $EF, $E6 // vpxor xmm4, xmm4, xmm6 + xor eax, ebx + shld edi, edi, $5 + add edx, ebp + and esi, eax + db $C5, $C9, $72, $D4, $1F // vpsrld xmm6, xmm4, $1F + xor eax, ebx + add edx, edi + shrd edi, edi, $7 + xor esi, ebx + db $C5, $F9, $73, $FC, $0C // vpslldq xmm0, xmm4, $C + db $C5, $D9, $FE, $E4 // vpaddd xmm4, xmm4, xmm4 + mov ebp, edx + add ecx, dword ptr [esp + $8] + xor edi, eax + shld edx, edx, $5 + db $C5, $C1, $72, $D0, $1E // vpsrld xmm7, xmm0, $1E + db $C5, $D9, $EB, $E6 // vpor xmm4, xmm4, xmm6 + add ecx, esi + and ebp, edi + xor edi, eax + add ecx, edx + db $C5, $F9, $72, $F0, $02 // vpslld xmm0, xmm0, $2 + shrd edx, edx, $7 + xor ebp, eax + db $C5, $D9, $EF, $E7 // vpxor xmm4, xmm4, xmm7 + mov esi, ecx + add ebx, dword ptr [esp + $C] + xor edx, edi + shld ecx, ecx, $5 + db $C5, $D9, $EF, $E0 // vpxor xmm4, xmm4, xmm0 + add ebx, ebp + and esi, edx + db $C5, $F9, $6F, $44, $24, $60 // vmovdqa xmm0, oword [esp + $60] + xor edx, edi + add ebx, ecx + shrd ecx, ecx, $7 + xor esi, edi + db $C4, $E3, $69, $0F, $E9, $08 // vpalignr xmm5, xmm2, xmm1, $8 + mov ebp, ebx + add eax, dword ptr [esp + $10] + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $F9, $7F, $4C, $24, $50 // vmovdqa oword [esp + $50], xmm1 + xor ecx, edx + shld ebx, ebx, $5 + db $C5, $C1, $73, $DC, $04 // vpsrldq xmm7, xmm4, $4 + add eax, esi + and ebp, ecx + db $C5, $D1, $EF, $E9 // vpxor xmm5, xmm5, xmm1 + xor ecx, edx + add eax, ebx + db $C5, $C1, $EF, $FB // vpxor xmm7, xmm7, xmm3 + shrd ebx, ebx, $7 + xor ebp, edx + db $C5, $F9, $7F, $04, $24 // vmovdqa oword [esp], xmm0 + mov esi, eax + add edi, dword ptr [esp + $14] + db $C5, $D1, $EF, $EF // vpxor xmm5, xmm5, xmm7 + xor ebx, ecx + shld eax, eax, $5 + add edi, ebp + and esi, ebx + db $C5, $C1, $72, $D5, $1F // vpsrld xmm7, xmm5, $1F + xor ebx, ecx + add edi, eax + shrd eax, eax, $7 + xor esi, ecx + db $C5, $F1, $73, $FD, $0C // vpslldq xmm1, xmm5, $C + db $C5, $D1, $FE, $ED // vpaddd xmm5, xmm5, xmm5 + mov ebp, edi + add edx, dword ptr [esp + $18] + xor eax, ebx + shld edi, edi, $5 + db $C5, $F9, $72, $D1, $1E // vpsrld xmm0, xmm1, $1E + db $C5, $D1, $EB, $EF // vpor xmm5, xmm5, xmm7 + add edx, esi + and ebp, eax + xor eax, ebx + add edx, edi + db $C5, $F1, $72, $F1, $02 // vpslld xmm1, xmm1, $2 + shrd edi, edi, $7 + xor ebp, ebx + db $C5, $D1, $EF, $E8 // vpxor xmm5, xmm5, xmm0 + mov esi, edx + add ecx, dword ptr [esp + $1C] + xor edi, eax + shld edx, edx, $5 + db $C5, $D1, $EF, $E9 // vpxor xmm5, xmm5, xmm1 + add ecx, ebp + and esi, edi + db $C5, $F9, $6F, $4C, $24, $70 // vmovdqa xmm1, oword [esp + $70] + xor edi, eax + add ecx, edx + shrd edx, edx, $7 + xor esi, eax + db $C4, $E3, $61, $0F, $F2, $08 // vpalignr xmm6, xmm3, xmm2, $8 + mov ebp, ecx + add ebx, dword ptr [esp + $20] + db $C5, $F1, $FE, $CD // vpaddd xmm1, xmm1, xmm5 + db $C5, $F9, $7F, $54, $24, $60 // vmovdqa oword [esp + $60], xmm2 + xor edx, edi + shld ecx, ecx, $5 + db $C5, $F9, $73, $DD, $04 // vpsrldq xmm0, xmm5, $4 + add ebx, esi + and ebp, edx + db $C5, $C9, $EF, $F2 // vpxor xmm6, xmm6, xmm2 + xor edx, edi + add ebx, ecx + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + shrd ecx, ecx, $7 + xor ebp, edi + db $C5, $F9, $7F, $4C, $24, $10 // vmovdqa oword [esp + $10], xmm1 + mov esi, ebx + add eax, dword ptr [esp + $24] + db $C5, $C9, $EF, $F0 // vpxor xmm6, xmm6, xmm0 + xor ecx, edx + shld ebx, ebx, $5 + add eax, ebp + and esi, ecx + db $C5, $F9, $72, $D6, $1F // vpsrld xmm0, xmm6, $1F + xor ecx, edx + add eax, ebx + shrd ebx, ebx, $7 + xor esi, edx + db $C5, $E9, $73, $FE, $0C // vpslldq xmm2, xmm6, $C + db $C5, $C9, $FE, $F6 // vpaddd xmm6, xmm6, xmm6 + mov ebp, eax + add edi, dword ptr [esp + $28] + xor ebx, ecx + shld eax, eax, $5 + db $C5, $F1, $72, $D2, $1E // vpsrld xmm1, xmm2, $1E + db $C5, $C9, $EB, $F0 // vpor xmm6, xmm6, xmm0 + add edi, esi + and ebp, ebx + xor ebx, ecx + add edi, eax + db $C5, $E9, $72, $F2, $02 // vpslld xmm2, xmm2, $2 + db $C5, $F9, $6F, $44, $24, $40 // vmovdqa xmm0, oword [esp + $40] + shrd eax, eax, $7 + xor ebp, ecx + db $C5, $C9, $EF, $F1 // vpxor xmm6, xmm6, xmm1 + mov esi, edi + add edx, dword ptr [esp + $2C] + xor eax, ebx + shld edi, edi, $5 + db $C5, $C9, $EF, $F2 // vpxor xmm6, xmm6, xmm2 + add edx, ebp + and esi, eax + db $C5, $F9, $6F, $54, $24, $70 // vmovdqa xmm2, oword [esp + $70] + xor eax, ebx + add edx, edi + shrd edi, edi, $7 + xor esi, ebx + db $C4, $E3, $59, $0F, $FB, $08 // vpalignr xmm7, xmm4, xmm3, $8 + mov ebp, edx + add ecx, dword ptr [esp + $30] + db $C5, $E9, $FE, $D6 // vpaddd xmm2, xmm2, xmm6 + db $C5, $F9, $7F, $5C, $24, $40 // vmovdqa oword [esp + $40], xmm3 + xor edi, eax + shld edx, edx, $5 + db $C5, $F1, $73, $DE, $04 // vpsrldq xmm1, xmm6, $4 + add ecx, esi + and ebp, edi + db $C5, $C1, $EF, $FB // vpxor xmm7, xmm7, xmm3 + xor edi, eax + add ecx, edx + db $C5, $F1, $EF, $CD // vpxor xmm1, xmm1, xmm5 + shrd edx, edx, $7 + xor ebp, eax + db $C5, $F9, $7F, $54, $24, $20 // vmovdqa oword [esp + $20], xmm2 + mov esi, ecx + add ebx, dword ptr [esp + $34] + db $C5, $C1, $EF, $F9 // vpxor xmm7, xmm7, xmm1 + xor edx, edi + shld ecx, ecx, $5 + add ebx, ebp + and esi, edx + db $C5, $F1, $72, $D7, $1F // vpsrld xmm1, xmm7, $1F + xor edx, edi + add ebx, ecx + shrd ecx, ecx, $7 + xor esi, edi + db $C5, $E1, $73, $FF, $0C // vpslldq xmm3, xmm7, $C + db $C5, $C1, $FE, $FF // vpaddd xmm7, xmm7, xmm7 + mov ebp, ebx + add eax, dword ptr [esp + $38] + xor ecx, edx + shld ebx, ebx, $5 + db $C5, $E9, $72, $D3, $1E // vpsrld xmm2, xmm3, $1E + db $C5, $C1, $EB, $F9 // vpor xmm7, xmm7, xmm1 + add eax, esi + and ebp, ecx + xor ecx, edx + add eax, ebx + db $C5, $E1, $72, $F3, $02 // vpslld xmm3, xmm3, $2 + db $C5, $F9, $6F, $4C, $24, $50 // vmovdqa xmm1, oword [esp + $50] + shrd ebx, ebx, $7 + xor ebp, edx + db $C5, $C1, $EF, $FA // vpxor xmm7, xmm7, xmm2 + mov esi, eax + add edi, dword ptr [esp + $3C] + xor ebx, ecx + shld eax, eax, $5 + db $C5, $C1, $EF, $FB // vpxor xmm7, xmm7, xmm3 + add edi, ebp + and esi, ebx + db $C5, $F9, $6F, $5C, $24, $70 // vmovdqa xmm3, oword [esp + $70] + xor ebx, ecx + add edi, eax + db $C4, $E3, $41, $0F, $D6, $08 // vpalignr xmm2, xmm7, xmm6, $8 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + shrd eax, eax, $7 + xor esi, ecx + mov ebp, edi + add edx, dword ptr [esp] + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C5, $F9, $7F, $64, $24, $50 // vmovdqa oword [esp + $50], xmm4 + xor eax, ebx + shld edi, edi, $5 + db $C5, $F9, $6F, $E3 // vmovdqa xmm4, xmm3 + db $C5, $E1, $FE, $DF // vpaddd xmm3, xmm3, xmm7 + add edx, esi + and ebp, eax + db $C5, $F9, $EF, $C2 // vpxor xmm0, xmm0, xmm2 + xor eax, ebx + add edx, edi + shrd edi, edi, $7 + xor ebp, ebx + db $C5, $E9, $72, $D0, $1E // vpsrld xmm2, xmm0, $1E + db $C5, $F9, $7F, $5C, $24, $30 // vmovdqa oword [esp + $30], xmm3 + mov esi, edx + add ecx, dword ptr [esp + $4] + xor edi, eax + shld edx, edx, $5 + db $C5, $F9, $72, $F0, $02 // vpslld xmm0, xmm0, $2 + add ecx, ebp + and esi, edi + xor edi, eax + add ecx, edx + shrd edx, edx, $7 + xor esi, eax + mov ebp, ecx + add ebx, dword ptr [esp + $8] + db $C5, $F9, $EB, $C2 // vpor xmm0, xmm0, xmm2 + xor edx, edi + shld ecx, ecx, $5 + db $C5, $F9, $6F, $54, $24, $60 // vmovdqa xmm2, oword [esp + $60] + add ebx, esi + and ebp, edx + xor edx, edi + add ebx, ecx + add eax, dword ptr [esp + $C] + xor ebp, edi + mov esi, ebx + shld ebx, ebx, $5 + add eax, ebp + xor esi, edx + shrd ecx, ecx, $7 + add eax, ebx + db $C4, $E3, $79, $0F, $DF, $08 // vpalignr xmm3, xmm0, xmm7, $8 + db $C5, $F1, $EF, $CD // vpxor xmm1, xmm1, xmm5 + add edi, dword ptr [esp + $10] + xor esi, ecx + mov ebp, eax + shld eax, eax, $5 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $F9, $7F, $6C, $24, $60 // vmovdqa oword [esp + $60], xmm5 + add edi, esi + xor ebp, ecx + db $C5, $F9, $6F, $EC // vmovdqa xmm5, xmm4 + db $C5, $D9, $FE, $E0 // vpaddd xmm4, xmm4, xmm0 + shrd ebx, ebx, $7 + add edi, eax + db $C5, $F1, $EF, $CB // vpxor xmm1, xmm1, xmm3 + add edx, dword ptr [esp + $14] + xor ebp, ebx + mov esi, edi + shld edi, edi, $5 + db $C5, $E1, $72, $D1, $1E // vpsrld xmm3, xmm1, $1E + db $C5, $F9, $7F, $24, $24 // vmovdqa oword [esp], xmm4 + add edx, ebp + xor esi, ebx + shrd eax, eax, $7 + add edx, edi + db $C5, $F1, $72, $F1, $02 // vpslld xmm1, xmm1, $2 + add ecx, dword ptr [esp + $18] + xor esi, eax + mov ebp, edx + shld edx, edx, $5 + add ecx, esi + xor ebp, eax + shrd edi, edi, $7 + add ecx, edx + db $C5, $F1, $EB, $CB // vpor xmm1, xmm1, xmm3 + add ebx, dword ptr [esp + $1C] + xor ebp, edi + db $C5, $F9, $6F, $5C, $24, $40 // vmovdqa xmm3, oword [esp + $40] + mov esi, ecx + shld ecx, ecx, $5 + add ebx, ebp + xor esi, edi + shrd edx, edx, $7 + add ebx, ecx + db $C4, $E3, $71, $0F, $E0, $08 // vpalignr xmm4, xmm1, xmm0, $8 + db $C5, $E9, $EF, $D6 // vpxor xmm2, xmm2, xmm6 + add eax, dword ptr [esp + $20] + xor esi, edx + mov ebp, ebx + shld ebx, ebx, $5 + db $C5, $E9, $EF, $D3 // vpxor xmm2, xmm2, xmm3 + db $C5, $F9, $7F, $74, $24, $40 // vmovdqa oword [esp + $40], xmm6 + add eax, esi + xor ebp, edx + db $C5, $F9, $6F, $B4, $24, $80, $00, $00, $00 // vmovdqa xmm6, oword [esp + $80] + db $C5, $D1, $FE, $E9 // vpaddd xmm5, xmm5, xmm1 + shrd ecx, ecx, $7 + add eax, ebx + db $C5, $E9, $EF, $D4 // vpxor xmm2, xmm2, xmm4 + add edi, dword ptr [esp + $24] + xor ebp, ecx + mov esi, eax + shld eax, eax, $5 + db $C5, $D9, $72, $D2, $1E // vpsrld xmm4, xmm2, $1E + db $C5, $F9, $7F, $6C, $24, $10 // vmovdqa oword [esp + $10], xmm5 + add edi, ebp + xor esi, ecx + shrd ebx, ebx, $7 + add edi, eax + db $C5, $E9, $72, $F2, $02 // vpslld xmm2, xmm2, $2 + add edx, dword ptr [esp + $28] + xor esi, ebx + mov ebp, edi + shld edi, edi, $5 + add edx, esi + xor ebp, ebx + shrd eax, eax, $7 + add edx, edi + db $C5, $E9, $EB, $D4 // vpor xmm2, xmm2, xmm4 + add ecx, dword ptr [esp + $2C] + xor ebp, eax + db $C5, $F9, $6F, $64, $24, $50 // vmovdqa xmm4, oword [esp + $50] + mov esi, edx + shld edx, edx, $5 + add ecx, ebp + xor esi, eax + shrd edi, edi, $7 + add ecx, edx + db $C4, $E3, $69, $0F, $E9, $08 // vpalignr xmm5, xmm2, xmm1, $8 + db $C5, $E1, $EF, $DF // vpxor xmm3, xmm3, xmm7 + add ebx, dword ptr [esp + $30] + xor esi, edi + mov ebp, ecx + shld ecx, ecx, $5 + db $C5, $E1, $EF, $DC // vpxor xmm3, xmm3, xmm4 + db $C5, $F9, $7F, $7C, $24, $50 // vmovdqa oword [esp + $50], xmm7 + add ebx, esi + xor ebp, edi + db $C5, $F9, $6F, $FE // vmovdqa xmm7, xmm6 + db $C5, $C9, $FE, $F2 // vpaddd xmm6, xmm6, xmm2 + shrd edx, edx, $7 + add ebx, ecx + db $C5, $E1, $EF, $DD // vpxor xmm3, xmm3, xmm5 + add eax, dword ptr [esp + $34] + xor ebp, edx + mov esi, ebx + shld ebx, ebx, $5 + db $C5, $D1, $72, $D3, $1E // vpsrld xmm5, xmm3, $1E + db $C5, $F9, $7F, $74, $24, $20 // vmovdqa oword [esp + $20], xmm6 + add eax, ebp + xor esi, edx + shrd ecx, ecx, $7 + add eax, ebx + db $C5, $E1, $72, $F3, $02 // vpslld xmm3, xmm3, $2 + add edi, dword ptr [esp + $38] + xor esi, ecx + mov ebp, eax + shld eax, eax, $5 + add edi, esi + xor ebp, ecx + shrd ebx, ebx, $7 + add edi, eax + db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + add edx, dword ptr [esp + $3C] + xor ebp, ebx + db $C5, $F9, $6F, $6C, $24, $60 // vmovdqa xmm5, oword [esp + $60] + mov esi, edi + shld edi, edi, $5 + add edx, ebp + xor esi, ebx + shrd eax, eax, $7 + add edx, edi + db $C4, $E3, $61, $0F, $F2, $08 // vpalignr xmm6, xmm3, xmm2, $8 + db $C5, $D9, $EF, $E0 // vpxor xmm4, xmm4, xmm0 + add ecx, dword ptr [esp] + xor esi, eax + mov ebp, edx + shld edx, edx, $5 + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + db $C5, $F9, $7F, $44, $24, $60 // vmovdqa oword [esp + $60], xmm0 + add ecx, esi + xor ebp, eax + db $C5, $F9, $6F, $C7 // vmovdqa xmm0, xmm7 + db $C5, $C1, $FE, $FB // vpaddd xmm7, xmm7, xmm3 + shrd edi, edi, $7 + add ecx, edx + db $C5, $D9, $EF, $E6 // vpxor xmm4, xmm4, xmm6 + add ebx, dword ptr [esp + $4] + xor ebp, edi + mov esi, ecx + shld ecx, ecx, $5 + db $C5, $C9, $72, $D4, $1E // vpsrld xmm6, xmm4, $1E + db $C5, $F9, $7F, $7C, $24, $30 // vmovdqa oword [esp + $30], xmm7 + add ebx, ebp + xor esi, edi + shrd edx, edx, $7 + add ebx, ecx + db $C5, $D9, $72, $F4, $02 // vpslld xmm4, xmm4, $2 + add eax, dword ptr [esp + $8] + xor esi, edx + mov ebp, ebx + shld ebx, ebx, $5 + add eax, esi + xor ebp, edx + shrd ecx, ecx, $7 + add eax, ebx + db $C5, $D9, $EB, $E6 // vpor xmm4, xmm4, xmm6 + add edi, dword ptr [esp + $C] + xor ebp, ecx + db $C5, $F9, $6F, $74, $24, $40 // vmovdqa xmm6, oword [esp + $40] + mov esi, eax + shld eax, eax, $5 + add edi, ebp + xor esi, ecx + shrd ebx, ebx, $7 + add edi, eax + db $C4, $E3, $59, $0F, $FB, $08 // vpalignr xmm7, xmm4, xmm3, $8 + db $C5, $D1, $EF, $E9 // vpxor xmm5, xmm5, xmm1 + add edx, dword ptr [esp + $10] + xor esi, ebx + mov ebp, edi + shld edi, edi, $5 + db $C5, $D1, $EF, $EE // vpxor xmm5, xmm5, xmm6 + db $C5, $F9, $7F, $4C, $24, $40 // vmovdqa oword [esp + $40], xmm1 + add edx, esi + xor ebp, ebx + db $C5, $F9, $6F, $C8 // vmovdqa xmm1, xmm0 + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + shrd eax, eax, $7 + add edx, edi + db $C5, $D1, $EF, $EF // vpxor xmm5, xmm5, xmm7 + add ecx, dword ptr [esp + $14] + xor ebp, eax + mov esi, edx + shld edx, edx, $5 + db $C5, $C1, $72, $D5, $1E // vpsrld xmm7, xmm5, $1E + db $C5, $F9, $7F, $04, $24 // vmovdqa oword [esp], xmm0 + add ecx, ebp + xor esi, eax + shrd edi, edi, $7 + add ecx, edx + db $C5, $D1, $72, $F5, $02 // vpslld xmm5, xmm5, $2 + add ebx, dword ptr [esp + $18] + xor esi, edi + mov ebp, ecx + shld ecx, ecx, $5 + add ebx, esi + xor ebp, edi + shrd edx, edx, $7 + add ebx, ecx + db $C5, $D1, $EB, $EF // vpor xmm5, xmm5, xmm7 + add eax, dword ptr [esp + $1C] + db $C5, $F9, $6F, $7C, $24, $50 // vmovdqa xmm7, oword [esp + $50] + shrd ecx, ecx, $7 + mov esi, ebx + xor ebp, edx + shld ebx, ebx, $5 + add eax, ebp + xor esi, ecx + xor ecx, edx + add eax, ebx + db $C4, $E3, $51, $0F, $C4, $08 // vpalignr xmm0, xmm5, xmm4, $8 + db $C5, $C9, $EF, $F2 // vpxor xmm6, xmm6, xmm2 + add edi, dword ptr [esp + $20] + and esi, ecx + xor ecx, edx + shrd ebx, ebx, $7 + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + db $C5, $F9, $7F, $54, $24, $50 // vmovdqa oword [esp + $50], xmm2 + mov ebp, eax + xor esi, ecx + db $C5, $F9, $6F, $D1 // vmovdqa xmm2, xmm1 + db $C5, $F1, $FE, $CD // vpaddd xmm1, xmm1, xmm5 + shld eax, eax, $5 + add edi, esi + db $C5, $C9, $EF, $F0 // vpxor xmm6, xmm6, xmm0 + xor ebp, ebx + xor ebx, ecx + add edi, eax + add edx, dword ptr [esp + $24] + db $C5, $F9, $72, $D6, $1E // vpsrld xmm0, xmm6, $1E + db $C5, $F9, $7F, $4C, $24, $10 // vmovdqa oword [esp + $10], xmm1 + and ebp, ebx + xor ebx, ecx + shrd eax, eax, $7 + mov esi, edi + db $C5, $C9, $72, $F6, $02 // vpslld xmm6, xmm6, $2 + xor ebp, ebx + shld edi, edi, $5 + add edx, ebp + xor esi, eax + xor eax, ebx + add edx, edi + add ecx, dword ptr [esp + $28] + and esi, eax + db $C5, $C9, $EB, $F0 // vpor xmm6, xmm6, xmm0 + xor eax, ebx + shrd edi, edi, $7 + db $C5, $F9, $6F, $44, $24, $60 // vmovdqa xmm0, oword [esp + $60] + mov ebp, edx + xor esi, eax + shld edx, edx, $5 + add ecx, esi + xor ebp, edi + xor edi, eax + add ecx, edx + add ebx, dword ptr [esp + $2C] + and ebp, edi + xor edi, eax + shrd edx, edx, $7 + mov esi, ecx + xor ebp, edi + shld ecx, ecx, $5 + add ebx, ebp + xor esi, edx + xor edx, edi + add ebx, ecx + db $C4, $E3, $49, $0F, $CD, $08 // vpalignr xmm1, xmm6, xmm5, $8 + db $C5, $C1, $EF, $FB // vpxor xmm7, xmm7, xmm3 + add eax, dword ptr [esp + $30] + and esi, edx + xor edx, edi + shrd ecx, ecx, $7 + db $C5, $C1, $EF, $F8 // vpxor xmm7, xmm7, xmm0 + db $C5, $F9, $7F, $5C, $24, $60 // vmovdqa oword [esp + $60], xmm3 + mov ebp, ebx + xor esi, edx + db $C5, $F9, $6F, $9C, $24, $90, $00, $00, $00 // vmovdqa xmm3, oword [esp + $90] + db $C5, $E9, $FE, $D6 // vpaddd xmm2, xmm2, xmm6 + shld ebx, ebx, $5 + add eax, esi + db $C5, $C1, $EF, $F9 // vpxor xmm7, xmm7, xmm1 + xor ebp, ecx + xor ecx, edx + add eax, ebx + add edi, dword ptr [esp + $34] + db $C5, $F1, $72, $D7, $1E // vpsrld xmm1, xmm7, $1E + db $C5, $F9, $7F, $54, $24, $20 // vmovdqa oword [esp + $20], xmm2 + and ebp, ecx + xor ecx, edx + shrd ebx, ebx, $7 + mov esi, eax + db $C5, $C1, $72, $F7, $02 // vpslld xmm7, xmm7, $2 + xor ebp, ecx + shld eax, eax, $5 + add edi, ebp + xor esi, ebx + xor ebx, ecx + add edi, eax + add edx, dword ptr [esp + $38] + and esi, ebx + db $C5, $C1, $EB, $F9 // vpor xmm7, xmm7, xmm1 + xor ebx, ecx + shrd eax, eax, $7 + db $C5, $F9, $6F, $4C, $24, $40 // vmovdqa xmm1, oword [esp + $40] + mov ebp, edi + xor esi, ebx + shld edi, edi, $5 + add edx, esi + xor ebp, eax + xor eax, ebx + add edx, edi + add ecx, dword ptr [esp + $3C] + and ebp, eax + xor eax, ebx + shrd edi, edi, $7 + mov esi, edx + xor ebp, eax + shld edx, edx, $5 + add ecx, ebp + xor esi, edi + xor edi, eax + add ecx, edx + db $C4, $E3, $41, $0F, $D6, $08 // vpalignr xmm2, xmm7, xmm6, $8 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + add ebx, dword ptr [esp] + and esi, edi + xor edi, eax + shrd edx, edx, $7 + db $C5, $F9, $EF, $C1 // vpxor xmm0, xmm0, xmm1 + db $C5, $F9, $7F, $64, $24, $40 // vmovdqa oword [esp + $40], xmm4 + mov ebp, ecx + xor esi, edi + db $C5, $F9, $6F, $E3 // vmovdqa xmm4, xmm3 + db $C5, $E1, $FE, $DF // vpaddd xmm3, xmm3, xmm7 + shld ecx, ecx, $5 + add ebx, esi + db $C5, $F9, $EF, $C2 // vpxor xmm0, xmm0, xmm2 + xor ebp, edx + xor edx, edi + add ebx, ecx + add eax, dword ptr [esp + $4] + db $C5, $E9, $72, $D0, $1E // vpsrld xmm2, xmm0, $1E + db $C5, $F9, $7F, $5C, $24, $30 // vmovdqa oword [esp + $30], xmm3 + and ebp, edx + xor edx, edi + shrd ecx, ecx, $7 + mov esi, ebx + db $C5, $F9, $72, $F0, $02 // vpslld xmm0, xmm0, $2 + xor ebp, edx + shld ebx, ebx, $5 + add eax, ebp + xor esi, ecx + xor ecx, edx + add eax, ebx + add edi, dword ptr [esp + $8] + and esi, ecx + db $C5, $F9, $EB, $C2 // vpor xmm0, xmm0, xmm2 + xor ecx, edx + shrd ebx, ebx, $7 + db $C5, $F9, $6F, $54, $24, $50 // vmovdqa xmm2, oword [esp + $50] + mov ebp, eax + xor esi, ecx + shld eax, eax, $5 + add edi, esi + xor ebp, ebx + xor ebx, ecx + add edi, eax + add edx, dword ptr [esp + $C] + and ebp, ebx + xor ebx, ecx + shrd eax, eax, $7 + mov esi, edi + xor ebp, ebx + shld edi, edi, $5 + add edx, ebp + xor esi, eax + xor eax, ebx + add edx, edi + db $C4, $E3, $79, $0F, $DF, $08 // vpalignr xmm3, xmm0, xmm7, $8 + db $C5, $F1, $EF, $CD // vpxor xmm1, xmm1, xmm5 + add ecx, dword ptr [esp + $10] + and esi, eax + xor eax, ebx + shrd edi, edi, $7 + db $C5, $F1, $EF, $CA // vpxor xmm1, xmm1, xmm2 + db $C5, $F9, $7F, $6C, $24, $50 // vmovdqa oword [esp + $50], xmm5 + mov ebp, edx + xor esi, eax + db $C5, $F9, $6F, $EC // vmovdqa xmm5, xmm4 + db $C5, $D9, $FE, $E0 // vpaddd xmm4, xmm4, xmm0 + shld edx, edx, $5 + add ecx, esi + db $C5, $F1, $EF, $CB // vpxor xmm1, xmm1, xmm3 + xor ebp, edi + xor edi, eax + add ecx, edx + add ebx, dword ptr [esp + $14] + db $C5, $E1, $72, $D1, $1E // vpsrld xmm3, xmm1, $1E + db $C5, $F9, $7F, $24, $24 // vmovdqa oword [esp], xmm4 + and ebp, edi + xor edi, eax + shrd edx, edx, $7 + mov esi, ecx + db $C5, $F1, $72, $F1, $02 // vpslld xmm1, xmm1, $2 + xor ebp, edi + shld ecx, ecx, $5 + add ebx, ebp + xor esi, edx + xor edx, edi + add ebx, ecx + add eax, dword ptr [esp + $18] + and esi, edx + db $C5, $F1, $EB, $CB // vpor xmm1, xmm1, xmm3 + xor edx, edi + shrd ecx, ecx, $7 + db $C5, $F9, $6F, $5C, $24, $60 // vmovdqa xmm3, oword [esp + $60] + mov ebp, ebx + xor esi, edx + shld ebx, ebx, $5 + add eax, esi + xor ebp, ecx + xor ecx, edx + add eax, ebx + add edi, dword ptr [esp + $1C] + and ebp, ecx + xor ecx, edx + shrd ebx, ebx, $7 + mov esi, eax + xor ebp, ecx + shld eax, eax, $5 + add edi, ebp + xor esi, ebx + xor ebx, ecx + add edi, eax + db $C4, $E3, $71, $0F, $E0, $08 // vpalignr xmm4, xmm1, xmm0, $8 + db $C5, $E9, $EF, $D6 // vpxor xmm2, xmm2, xmm6 + add edx, dword ptr [esp + $20] + and esi, ebx + xor ebx, ecx + shrd eax, eax, $7 + db $C5, $E9, $EF, $D3 // vpxor xmm2, xmm2, xmm3 + db $C5, $F9, $7F, $74, $24, $60 // vmovdqa oword [esp + $60], xmm6 + mov ebp, edi + xor esi, ebx + db $C5, $F9, $6F, $F5 // vmovdqa xmm6, xmm5 + db $C5, $D1, $FE, $E9 // vpaddd xmm5, xmm5, xmm1 + shld edi, edi, $5 + add edx, esi + db $C5, $E9, $EF, $D4 // vpxor xmm2, xmm2, xmm4 + xor ebp, eax + xor eax, ebx + add edx, edi + add ecx, dword ptr [esp + $24] + db $C5, $D9, $72, $D2, $1E // vpsrld xmm4, xmm2, $1E + db $C5, $F9, $7F, $6C, $24, $10 // vmovdqa oword [esp + $10], xmm5 + and ebp, eax + xor eax, ebx + shrd edi, edi, $7 + mov esi, edx + db $C5, $E9, $72, $F2, $02 // vpslld xmm2, xmm2, $2 + xor ebp, eax + shld edx, edx, $5 + add ecx, ebp + xor esi, edi + xor edi, eax + add ecx, edx + add ebx, dword ptr [esp + $28] + and esi, edi + db $C5, $E9, $EB, $D4 // vpor xmm2, xmm2, xmm4 + xor edi, eax + shrd edx, edx, $7 + db $C5, $F9, $6F, $64, $24, $40 // vmovdqa xmm4, oword [esp + $40] + mov ebp, ecx + xor esi, edi + shld ecx, ecx, $5 + add ebx, esi + xor ebp, edx + xor edx, edi + add ebx, ecx + add eax, dword ptr [esp + $2C] + and ebp, edx + xor edx, edi + shrd ecx, ecx, $7 + mov esi, ebx + xor ebp, edx + shld ebx, ebx, $5 + add eax, ebp + xor esi, edx + add eax, ebx + db $C4, $E3, $69, $0F, $E9, $08 // vpalignr xmm5, xmm2, xmm1, $8 + db $C5, $E1, $EF, $DF // vpxor xmm3, xmm3, xmm7 + add edi, dword ptr [esp + $30] + xor esi, ecx + mov ebp, eax + shld eax, eax, $5 + db $C5, $E1, $EF, $DC // vpxor xmm3, xmm3, xmm4 + db $C5, $F9, $7F, $7C, $24, $40 // vmovdqa oword [esp + $40], xmm7 + add edi, esi + xor ebp, ecx + db $C5, $F9, $6F, $FE // vmovdqa xmm7, xmm6 + db $C5, $C9, $FE, $F2 // vpaddd xmm6, xmm6, xmm2 + shrd ebx, ebx, $7 + add edi, eax + db $C5, $E1, $EF, $DD // vpxor xmm3, xmm3, xmm5 + add edx, dword ptr [esp + $34] + xor ebp, ebx + mov esi, edi + shld edi, edi, $5 + db $C5, $D1, $72, $D3, $1E // vpsrld xmm5, xmm3, $1E + db $C5, $F9, $7F, $74, $24, $20 // vmovdqa oword [esp + $20], xmm6 + add edx, ebp + xor esi, ebx + shrd eax, eax, $7 + add edx, edi + db $C5, $E1, $72, $F3, $02 // vpslld xmm3, xmm3, $2 + add ecx, dword ptr [esp + $38] + xor esi, eax + mov ebp, edx + shld edx, edx, $5 + add ecx, esi + xor ebp, eax + shrd edi, edi, $7 + add ecx, edx + db $C5, $E1, $EB, $DD // vpor xmm3, xmm3, xmm5 + add ebx, dword ptr [esp + $3C] + xor ebp, edi + mov esi, ecx + shld ecx, ecx, $5 + add ebx, ebp + xor esi, edi + shrd edx, edx, $7 + add ebx, ecx + add eax, dword ptr [esp] + db $C5, $C1, $FE, $FB // vpaddd xmm7, xmm7, xmm3 + xor esi, edx + mov ebp, ebx + shld ebx, ebx, $5 + add eax, esi + db $C5, $F9, $7F, $7C, $24, $30 // vmovdqa oword [esp + $30], xmm7 + xor ebp, edx + shrd ecx, ecx, $7 + add eax, ebx + add edi, dword ptr [esp + $4] + xor ebp, ecx + mov esi, eax + shld eax, eax, $5 + add edi, ebp + xor esi, ecx + shrd ebx, ebx, $7 + add edi, eax + add edx, dword ptr [esp + $8] + xor esi, ebx + mov ebp, edi + shld edi, edi, $5 + add edx, esi + xor ebp, ebx + shrd eax, eax, $7 + add edx, edi + add ecx, dword ptr [esp + $C] + xor ebp, eax + mov esi, edx + shld edx, edx, $5 + add ecx, ebp + xor esi, eax + shrd edi, edi, $7 + add ecx, edx + mov ebp, dword ptr [esp + $C4] + cmp ebp, dword ptr [esp + $C8] + je @done_010 + db $C5, $F9, $6F, $BC, $24, $A0, $00, $00, $00 // vmovdqa xmm7, oword [esp + $A0] + db $C5, $F9, $6F, $B4, $24, $B0, $00, $00, $00 // vmovdqa xmm6, oword [esp + $B0] + db $C5, $FA, $6F, $45, $00 // vmovdqu xmm0, oword [ebp + $0] + db $C5, $FA, $6F, $4D, $10 // vmovdqu xmm1, oword [ebp + $10] + db $C5, $FA, $6F, $55, $20 // vmovdqu xmm2, oword [ebp + $20] + db $C5, $FA, $6F, $5D, $30 // vmovdqu xmm3, oword [ebp + $30] + add ebp, $40 + db $C4, $E2, $79, $00, $C6 // vpshufb xmm0, xmm0, xmm6 + mov dword ptr [esp + $C4], ebp + db $C5, $F9, $7F, $7C, $24, $60 // vmovdqa oword [esp + $60], xmm7 + add ebx, dword ptr [esp + $10] + xor esi, edi + db $C4, $E2, $71, $00, $CE // vpshufb xmm1, xmm1, xmm6 + mov ebp, ecx + shld ecx, ecx, $5 + db $C5, $F9, $FE, $E7 // vpaddd xmm4, xmm0, xmm7 + add ebx, esi + xor ebp, edi + shrd edx, edx, $7 + add ebx, ecx + db $C5, $F9, $7F, $24, $24 // vmovdqa oword [esp], xmm4 + add eax, dword ptr [esp + $14] + xor ebp, edx + mov esi, ebx + shld ebx, ebx, $5 + add eax, ebp + xor esi, edx + shrd ecx, ecx, $7 + add eax, ebx + add edi, dword ptr [esp + $18] + xor esi, ecx + mov ebp, eax + shld eax, eax, $5 + add edi, esi + xor ebp, ecx + shrd ebx, ebx, $7 + add edi, eax + add edx, dword ptr [esp + $1C] + xor ebp, ebx + mov esi, edi + shld edi, edi, $5 + add edx, ebp + xor esi, ebx + shrd eax, eax, $7 + add edx, edi + add ecx, dword ptr [esp + $20] + xor esi, eax + db $C4, $E2, $69, $00, $D6 // vpshufb xmm2, xmm2, xmm6 + mov ebp, edx + shld edx, edx, $5 + db $C5, $F1, $FE, $EF // vpaddd xmm5, xmm1, xmm7 + add ecx, esi + xor ebp, eax + shrd edi, edi, $7 + add ecx, edx + db $C5, $F9, $7F, $6C, $24, $10 // vmovdqa oword [esp + $10], xmm5 + add ebx, dword ptr [esp + $24] + xor ebp, edi + mov esi, ecx + shld ecx, ecx, $5 + add ebx, ebp + xor esi, edi + shrd edx, edx, $7 + add ebx, ecx + add eax, dword ptr [esp + $28] + xor esi, edx + mov ebp, ebx + shld ebx, ebx, $5 + add eax, esi + xor ebp, edx + shrd ecx, ecx, $7 + add eax, ebx + add edi, dword ptr [esp + $2C] + xor ebp, ecx + mov esi, eax + shld eax, eax, $5 + add edi, ebp + xor esi, ecx + shrd ebx, ebx, $7 + add edi, eax + add edx, dword ptr [esp + $30] + xor esi, ebx + db $C4, $E2, $61, $00, $DE // vpshufb xmm3, xmm3, xmm6 + mov ebp, edi + shld edi, edi, $5 + db $C5, $E9, $FE, $F7 // vpaddd xmm6, xmm2, xmm7 + add edx, esi + xor ebp, ebx + shrd eax, eax, $7 + add edx, edi + db $C5, $F9, $7F, $74, $24, $20 // vmovdqa oword [esp + $20], xmm6 + add ecx, dword ptr [esp + $34] + xor ebp, eax + mov esi, edx + shld edx, edx, $5 + add ecx, ebp + xor esi, eax + shrd edi, edi, $7 + add ecx, edx + add ebx, dword ptr [esp + $38] + xor esi, edi + mov ebp, ecx + shld ecx, ecx, $5 + add ebx, esi + xor ebp, edi + shrd edx, edx, $7 + add ebx, ecx + add eax, dword ptr [esp + $3C] + xor ebp, edx + mov esi, ebx + shld ebx, ebx, $5 + add eax, ebp + shrd ecx, ecx, $7 + add eax, ebx + mov ebp, dword ptr [esp + $C0] + add eax, dword ptr [ebp + $0] + add esi, dword ptr [ebp + $4] + add ecx, dword ptr [ebp + $8] + mov dword ptr [ebp + $0], eax + add edx, dword ptr [ebp + $C] + mov dword ptr [ebp + $4], esi + add edi, dword ptr [ebp + $10] + mov ebx, ecx + mov dword ptr [ebp + $8], ecx + xor ebx, edx + mov dword ptr [ebp + $C], edx + mov dword ptr [ebp + $10], edi + mov ebp, esi + and esi, ebx + mov ebx, ebp + jmp @loop_009 + +@done_010: + add ebx, dword ptr [esp + $10] + xor esi, edi + mov ebp, ecx + shld ecx, ecx, $5 + add ebx, esi + xor ebp, edi + shrd edx, edx, $7 + add ebx, ecx + add eax, dword ptr [esp + $14] + xor ebp, edx + mov esi, ebx + shld ebx, ebx, $5 + add eax, ebp + xor esi, edx + shrd ecx, ecx, $7 + add eax, ebx + add edi, dword ptr [esp + $18] + xor esi, ecx + mov ebp, eax + shld eax, eax, $5 + add edi, esi + xor ebp, ecx + shrd ebx, ebx, $7 + add edi, eax + add edx, dword ptr [esp + $1C] + xor ebp, ebx + mov esi, edi + shld edi, edi, $5 + add edx, ebp + xor esi, ebx + shrd eax, eax, $7 + add edx, edi + add ecx, dword ptr [esp + $20] + xor esi, eax + mov ebp, edx + shld edx, edx, $5 + add ecx, esi + xor ebp, eax + shrd edi, edi, $7 + add ecx, edx + add ebx, dword ptr [esp + $24] + xor ebp, edi + mov esi, ecx + shld ecx, ecx, $5 + add ebx, ebp + xor esi, edi + shrd edx, edx, $7 + add ebx, ecx + add eax, dword ptr [esp + $28] + xor esi, edx + mov ebp, ebx + shld ebx, ebx, $5 + add eax, esi + xor ebp, edx + shrd ecx, ecx, $7 + add eax, ebx + add edi, dword ptr [esp + $2C] + xor ebp, ecx + mov esi, eax + shld eax, eax, $5 + add edi, ebp + xor esi, ecx + shrd ebx, ebx, $7 + add edi, eax + add edx, dword ptr [esp + $30] + xor esi, ebx + mov ebp, edi + shld edi, edi, $5 + add edx, esi + xor ebp, ebx + shrd eax, eax, $7 + add edx, edi + add ecx, dword ptr [esp + $34] + xor ebp, eax + mov esi, edx + shld edx, edx, $5 + add ecx, ebp + xor esi, eax + shrd edi, edi, $7 + add ecx, edx + add ebx, dword ptr [esp + $38] + xor esi, edi + mov ebp, ecx + shld ecx, ecx, $5 + add ebx, esi + xor ebp, edi + shrd edx, edx, $7 + add ebx, ecx + add eax, dword ptr [esp + $3C] + xor ebp, edx + mov esi, ebx + shld ebx, ebx, $5 + add eax, ebp + shrd ecx, ecx, $7 + add eax, ebx + db $C5, $FC, $77 // vzeroall + mov ebp, dword ptr [esp + $C0] + add eax, dword ptr [ebp + $0] + mov esp, dword ptr [esp + $CC] + add esi, dword ptr [ebp + $4] + add ecx, dword ptr [ebp + $8] + mov dword ptr [ebp + $0], eax + add edx, dword ptr [ebp + $C] + mov dword ptr [ebp + $4], esi + add edi, dword ptr [ebp + $10] + mov dword ptr [ebp + $8], ecx + mov dword ptr [ebp + $C], edx + mov dword ptr [ebp + $10], edi + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressCryptoExt_aarch64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressCryptoExt_aarch64.inc index 7e981133..253edcfa 100644 --- a/HashLib/src/Include/Simd/SHA1/SHA1CompressCryptoExt_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressCryptoExt_aarch64.inc @@ -1,9 +1,7 @@ -// SHA-1 AArch64 FEAT_SHA1 implementation. -// Expects AAPCS64: x0 = state ptr (5 x UInt32), x1 = data ptr, -// w2 = numblocks, x3 = K_SHA1 ptr (16 UInt32: each round constant x4 lanes). -// Leaf nostackframe; caller-saved GPR/vector only. -// Reference: OpenSSL CRYPTOGAMS sha1-armv8.S. -// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// CryptoExt (FEAT_SHA1) implementation of SHA-1 compress. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (5 x UInt32), +// x1 = data ptr, w2 = numblocks, x3 = K_SHA1 ptr (16 UInt32: each round +// constant x4 lanes). // Register map: // v0 = ABCD working state // v1.s[0] = E working state (single lane; sha1c/p/m consume it as a scalar) @@ -12,6 +10,9 @@ // v16-v19 = K round constants (K_00_19, K_20_39, K_40_59, K_60_79) // v20, v21 = W+K per round // v22 = saved ABCD for the final add +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha1-armv8.S. cbz w2, .Lcryptoext_sha1_done .long 0x6e211c21 // eor v1.16b, v1.16b, v1.16b diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressGpr_aarch64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressGpr_aarch64.inc new file mode 100644 index 00000000..cfad326c --- /dev/null +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressGpr_aarch64.inc @@ -0,0 +1,1038 @@ +// Gpr implementation of SHA-1 compress (AArch64) for cores without the +// SHA-1 crypto extension; the Gpr name states the ISA requirement: none +// beyond base AArch64. Serial SHA maps better onto the 31 GPRs than onto +// NEON lanes (single dependency chain, 1-op ror, rotate-folded eor +// operands). The four round constants are materialized as movz/movk +// immediates - no K table is read. +// ABI (after HlpSimdProc3Begin_aarch64.inc): x0 = state ptr (5 x UInt32), +// x1 = data ptr, w2 = numblocks. +// Frame: 96-byte AAPCS64 frame (x29/x30 + x19-x28 callee-saved). +// AArch64 instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha1-armv8.pl (sha1_block_data_order, the +// plain path). + + .long 0x2a0203e2 // mov w2,w2 // zero-extend ANumBlocks (AAPCS64 leaves x2's top half undefined) + + .long 0xa9ba7bfd // stp x29,x30,[sp,#-96]! + .long 0x910003fd // add x29,sp,#0 + .long 0xa90153f3 // stp x19,x20,[sp,#16] + .long 0xa9025bf5 // stp x21,x22,[sp,#32] + .long 0xa90363f7 // stp x23,x24,[sp,#48] + .long 0xa9046bf9 // stp x25,x26,[sp,#64] + .long 0xa90573fb // stp x27,x28,[sp,#80] + + .long 0x29405414 // ldp w20,w21,[x0] + .long 0x29415c16 // ldp w22,w23,[x0,#8] + .long 0xb9401018 // ldr w24,[x0,#16] + + +.Lsha1_gpr_loop: + .long 0xf8440423 // ldr x3,[x1],#64 + .long 0x528f333c // movz w28,#0x7999 + .long 0xd1000442 // sub x2,x2,#1 + .long 0x72ab505c // movk w28,#0x5a82, lsl #16 + .long 0xdac00863 // rev32 x3,x3 + .long 0x0b1c0318 // add w24,w24,w28 // warm it up + .long 0x0b030318 // add w24,w24,w3 + .long 0xd360fc64 // lsr x4,x3,#32 + .long 0xf85c8025 // ldr x5,[x1,#-56] + .long 0x0a3502f9 // bic w25,w23,w21 + .long 0x0a1502da // and w26,w22,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x0b0402f7 // add w23,w23,w4 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0xdac008a5 // rev32 x5,x5 + .long 0x0a3402d9 // bic w25,w22,w20 + .long 0x0a1402ba // and w26,w21,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x0b0502d6 // add w22,w22,w5 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0xd360fca6 // lsr x6,x5,#32 + .long 0xf85d0027 // ldr x7,[x1,#-48] + .long 0x0a3802b9 // bic w25,w21,w24 + .long 0x0a18029a // and w26,w20,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x0b0602b5 // add w21,w21,w6 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0xdac008e7 // rev32 x7,x7 + .long 0x0a370299 // bic w25,w20,w23 + .long 0x0a17031a // and w26,w24,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x0b070294 // add w20,w20,w7 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0xd360fce8 // lsr x8,x7,#32 + .long 0xf85d8029 // ldr x9,[x1,#-40] + .long 0x0a360319 // bic w25,w24,w22 + .long 0x0a1602fa // and w26,w23,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x0b080318 // add w24,w24,w8 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0xdac00929 // rev32 x9,x9 + .long 0x0a3502f9 // bic w25,w23,w21 + .long 0x0a1502da // and w26,w22,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x0b0902f7 // add w23,w23,w9 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0xd360fd2a // lsr x10,x9,#32 + .long 0xf85e002b // ldr x11,[x1,#-32] + .long 0x0a3402d9 // bic w25,w22,w20 + .long 0x0a1402ba // and w26,w21,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x0b0a02d6 // add w22,w22,w10 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0xdac0096b // rev32 x11,x11 + .long 0x0a3802b9 // bic w25,w21,w24 + .long 0x0a18029a // and w26,w20,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x0b0b02b5 // add w21,w21,w11 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0xd360fd6c // lsr x12,x11,#32 + .long 0xf85e802d // ldr x13,[x1,#-24] + .long 0x0a370299 // bic w25,w20,w23 + .long 0x0a17031a // and w26,w24,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x0b0c0294 // add w20,w20,w12 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0xdac009ad // rev32 x13,x13 + .long 0x0a360319 // bic w25,w24,w22 + .long 0x0a1602fa // and w26,w23,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x0b0d0318 // add w24,w24,w13 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0xd360fdae // lsr x14,x13,#32 + .long 0xf85f002f // ldr x15,[x1,#-16] + .long 0x0a3502f9 // bic w25,w23,w21 + .long 0x0a1502da // and w26,w22,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x0b0e02f7 // add w23,w23,w14 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0xdac009ef // rev32 x15,x15 + .long 0x0a3402d9 // bic w25,w22,w20 + .long 0x0a1402ba // and w26,w21,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x0b0f02d6 // add w22,w22,w15 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0xd360fdf0 // lsr x16,x15,#32 + .long 0xf85f8031 // ldr x17,[x1,#-8] + .long 0x0a3802b9 // bic w25,w21,w24 + .long 0x0a18029a // and w26,w20,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x0b1002b5 // add w21,w21,w16 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0xdac00a31 // rev32 x17,x17 + .long 0x0a370299 // bic w25,w20,w23 + .long 0x0a17031a // and w26,w24,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x0b110294 // add w20,w20,w17 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0xd360fe33 // lsr x19,x17,#32 + .long 0x4a050063 // eor w3,w3,w5 + .long 0x0a360319 // bic w25,w24,w22 + .long 0x0a1602fa // and w26,w23,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x4a0b0063 // eor w3,w3,w11 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x4a100063 // eor w3,w3,w16 + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x0b130318 // add w24,w24,w19 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13837c63 // ror w3,w3,#31 + .long 0x4a060084 // eor w4,w4,w6 + .long 0x0a3502f9 // bic w25,w23,w21 + .long 0x0a1502da // and w26,w22,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x4a0c0084 // eor w4,w4,w12 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x4a110084 // eor w4,w4,w17 + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x0b0302f7 // add w23,w23,w3 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13847c84 // ror w4,w4,#31 + .long 0x4a0700a5 // eor w5,w5,w7 + .long 0x0a3402d9 // bic w25,w22,w20 + .long 0x0a1402ba // and w26,w21,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x4a0d00a5 // eor w5,w5,w13 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x4a1300a5 // eor w5,w5,w19 + .long 0x13940a94 // ror w20,w20,#2 + .long 0x0b0402d6 // add w22,w22,w4 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13857ca5 // ror w5,w5,#31 + .long 0x4a0800c6 // eor w6,w6,w8 + .long 0x0a3802b9 // bic w25,w21,w24 + .long 0x0a18029a // and w26,w20,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x4a0e00c6 // eor w6,w6,w14 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x4a0300c6 // eor w6,w6,w3 + .long 0x13980b18 // ror w24,w24,#2 + .long 0x0b0502b5 // add w21,w21,w5 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13867cc6 // ror w6,w6,#31 + .long 0x4a0900e7 // eor w7,w7,w9 + .long 0x0a370299 // bic w25,w20,w23 + .long 0x0a17031a // and w26,w24,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x4a0f00e7 // eor w7,w7,w15 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x4a0400e7 // eor w7,w7,w4 + .long 0x13970af7 // ror w23,w23,#2 + .long 0x0b060294 // add w20,w20,w6 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13877ce7 // ror w7,w7,#31 + .long 0x529d743c // movz w28,#0xeba1 + .long 0x72addb3c // movk w28,#0x6ed9, lsl #16 + .long 0x4a0a0108 // eor w8,w8,w10 + .long 0x0a360319 // bic w25,w24,w22 + .long 0x0a1602fa // and w26,w23,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x4a100108 // eor w8,w8,w16 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x4a050108 // eor w8,w8,w5 + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x0b070318 // add w24,w24,w7 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13887d08 // ror w8,w8,#31 + .long 0x4a0b0129 // eor w9,w9,w11 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a110129 // eor w9,w9,w17 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a060129 // eor w9,w9,w6 + .long 0x0b0802f7 // add w23,w23,w8 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13897d29 // ror w9,w9,#31 + .long 0x4a0c014a // eor w10,w10,w12 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a13014a // eor w10,w10,w19 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a07014a // eor w10,w10,w7 + .long 0x0b0902d6 // add w22,w22,w9 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x138a7d4a // ror w10,w10,#31 + .long 0x4a0d016b // eor w11,w11,w13 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a03016b // eor w11,w11,w3 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a08016b // eor w11,w11,w8 + .long 0x0b0a02b5 // add w21,w21,w10 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x138b7d6b // ror w11,w11,#31 + .long 0x4a0e018c // eor w12,w12,w14 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a04018c // eor w12,w12,w4 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a09018c // eor w12,w12,w9 + .long 0x0b0b0294 // add w20,w20,w11 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x138c7d8c // ror w12,w12,#31 + .long 0x4a0f01ad // eor w13,w13,w15 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a0501ad // eor w13,w13,w5 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0a01ad // eor w13,w13,w10 + .long 0x0b0c0318 // add w24,w24,w12 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x138d7dad // ror w13,w13,#31 + .long 0x4a1001ce // eor w14,w14,w16 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a0601ce // eor w14,w14,w6 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0b01ce // eor w14,w14,w11 + .long 0x0b0d02f7 // add w23,w23,w13 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x138e7dce // ror w14,w14,#31 + .long 0x4a1101ef // eor w15,w15,w17 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0701ef // eor w15,w15,w7 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a0c01ef // eor w15,w15,w12 + .long 0x0b0e02d6 // add w22,w22,w14 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x138f7def // ror w15,w15,#31 + .long 0x4a130210 // eor w16,w16,w19 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a080210 // eor w16,w16,w8 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a0d0210 // eor w16,w16,w13 + .long 0x0b0f02b5 // add w21,w21,w15 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13907e10 // ror w16,w16,#31 + .long 0x4a030231 // eor w17,w17,w3 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a090231 // eor w17,w17,w9 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a0e0231 // eor w17,w17,w14 + .long 0x0b100294 // add w20,w20,w16 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13917e31 // ror w17,w17,#31 + .long 0x4a040273 // eor w19,w19,w4 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a0a0273 // eor w19,w19,w10 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0f0273 // eor w19,w19,w15 + .long 0x0b110318 // add w24,w24,w17 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13937e73 // ror w19,w19,#31 + .long 0x4a050063 // eor w3,w3,w5 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a0b0063 // eor w3,w3,w11 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a100063 // eor w3,w3,w16 + .long 0x0b1302f7 // add w23,w23,w19 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13837c63 // ror w3,w3,#31 + .long 0x4a060084 // eor w4,w4,w6 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0c0084 // eor w4,w4,w12 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a110084 // eor w4,w4,w17 + .long 0x0b0302d6 // add w22,w22,w3 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13847c84 // ror w4,w4,#31 + .long 0x4a0700a5 // eor w5,w5,w7 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0d00a5 // eor w5,w5,w13 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a1300a5 // eor w5,w5,w19 + .long 0x0b0402b5 // add w21,w21,w4 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13857ca5 // ror w5,w5,#31 + .long 0x4a0800c6 // eor w6,w6,w8 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a0e00c6 // eor w6,w6,w14 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a0300c6 // eor w6,w6,w3 + .long 0x0b050294 // add w20,w20,w5 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13867cc6 // ror w6,w6,#31 + .long 0x4a0900e7 // eor w7,w7,w9 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a0f00e7 // eor w7,w7,w15 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0400e7 // eor w7,w7,w4 + .long 0x0b060318 // add w24,w24,w6 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13877ce7 // ror w7,w7,#31 + .long 0x4a0a0108 // eor w8,w8,w10 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a100108 // eor w8,w8,w16 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a050108 // eor w8,w8,w5 + .long 0x0b0702f7 // add w23,w23,w7 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13887d08 // ror w8,w8,#31 + .long 0x4a0b0129 // eor w9,w9,w11 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a110129 // eor w9,w9,w17 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a060129 // eor w9,w9,w6 + .long 0x0b0802d6 // add w22,w22,w8 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13897d29 // ror w9,w9,#31 + .long 0x4a0c014a // eor w10,w10,w12 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a13014a // eor w10,w10,w19 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a07014a // eor w10,w10,w7 + .long 0x0b0902b5 // add w21,w21,w9 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x138a7d4a // ror w10,w10,#31 + .long 0x4a0d016b // eor w11,w11,w13 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a03016b // eor w11,w11,w3 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a08016b // eor w11,w11,w8 + .long 0x0b0a0294 // add w20,w20,w10 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x138b7d6b // ror w11,w11,#31 + .long 0x52979b9c // movz w28,#0xbcdc + .long 0x72b1e37c // movk w28,#0x8f1b, lsl #16 + .long 0x4a0e018c // eor w12,w12,w14 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a04018c // eor w12,w12,w4 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a09018c // eor w12,w12,w9 + .long 0x0b0b0318 // add w24,w24,w11 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x138c7d8c // ror w12,w12,#31 + .long 0x2a1602b9 // orr w25,w21,w22 + .long 0x0a1602ba // and w26,w21,w22 + .long 0x4a0f01ad // eor w13,w13,w15 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0a170339 // and w25,w25,w23 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a0501ad // eor w13,w13,w5 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0a01ad // eor w13,w13,w10 + .long 0x0b0c02f7 // add w23,w23,w12 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x138d7dad // ror w13,w13,#31 + .long 0x2a150299 // orr w25,w20,w21 + .long 0x0a15029a // and w26,w20,w21 + .long 0x4a1001ce // eor w14,w14,w16 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0a160339 // and w25,w25,w22 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0601ce // eor w14,w14,w6 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a0b01ce // eor w14,w14,w11 + .long 0x0b0d02d6 // add w22,w22,w13 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x138e7dce // ror w14,w14,#31 + .long 0x2a140319 // orr w25,w24,w20 + .long 0x0a14031a // and w26,w24,w20 + .long 0x4a1101ef // eor w15,w15,w17 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0a150339 // and w25,w25,w21 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0701ef // eor w15,w15,w7 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a0c01ef // eor w15,w15,w12 + .long 0x0b0e02b5 // add w21,w21,w14 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x138f7def // ror w15,w15,#31 + .long 0x2a1802f9 // orr w25,w23,w24 + .long 0x0a1802fa // and w26,w23,w24 + .long 0x4a130210 // eor w16,w16,w19 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0a140339 // and w25,w25,w20 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a080210 // eor w16,w16,w8 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a0d0210 // eor w16,w16,w13 + .long 0x0b0f0294 // add w20,w20,w15 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13907e10 // ror w16,w16,#31 + .long 0x2a1702d9 // orr w25,w22,w23 + .long 0x0a1702da // and w26,w22,w23 + .long 0x4a030231 // eor w17,w17,w3 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0a180339 // and w25,w25,w24 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a090231 // eor w17,w17,w9 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0e0231 // eor w17,w17,w14 + .long 0x0b100318 // add w24,w24,w16 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13917e31 // ror w17,w17,#31 + .long 0x2a1602b9 // orr w25,w21,w22 + .long 0x0a1602ba // and w26,w21,w22 + .long 0x4a040273 // eor w19,w19,w4 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0a170339 // and w25,w25,w23 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a0a0273 // eor w19,w19,w10 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0f0273 // eor w19,w19,w15 + .long 0x0b1102f7 // add w23,w23,w17 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13937e73 // ror w19,w19,#31 + .long 0x2a150299 // orr w25,w20,w21 + .long 0x0a15029a // and w26,w20,w21 + .long 0x4a050063 // eor w3,w3,w5 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0a160339 // and w25,w25,w22 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0b0063 // eor w3,w3,w11 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a100063 // eor w3,w3,w16 + .long 0x0b1302d6 // add w22,w22,w19 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13837c63 // ror w3,w3,#31 + .long 0x2a140319 // orr w25,w24,w20 + .long 0x0a14031a // and w26,w24,w20 + .long 0x4a060084 // eor w4,w4,w6 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0a150339 // and w25,w25,w21 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0c0084 // eor w4,w4,w12 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a110084 // eor w4,w4,w17 + .long 0x0b0302b5 // add w21,w21,w3 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13847c84 // ror w4,w4,#31 + .long 0x2a1802f9 // orr w25,w23,w24 + .long 0x0a1802fa // and w26,w23,w24 + .long 0x4a0700a5 // eor w5,w5,w7 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0a140339 // and w25,w25,w20 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a0d00a5 // eor w5,w5,w13 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a1300a5 // eor w5,w5,w19 + .long 0x0b040294 // add w20,w20,w4 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13857ca5 // ror w5,w5,#31 + .long 0x2a1702d9 // orr w25,w22,w23 + .long 0x0a1702da // and w26,w22,w23 + .long 0x4a0800c6 // eor w6,w6,w8 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0a180339 // and w25,w25,w24 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a0e00c6 // eor w6,w6,w14 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0300c6 // eor w6,w6,w3 + .long 0x0b050318 // add w24,w24,w5 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13867cc6 // ror w6,w6,#31 + .long 0x2a1602b9 // orr w25,w21,w22 + .long 0x0a1602ba // and w26,w21,w22 + .long 0x4a0900e7 // eor w7,w7,w9 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0a170339 // and w25,w25,w23 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a0f00e7 // eor w7,w7,w15 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0400e7 // eor w7,w7,w4 + .long 0x0b0602f7 // add w23,w23,w6 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13877ce7 // ror w7,w7,#31 + .long 0x2a150299 // orr w25,w20,w21 + .long 0x0a15029a // and w26,w20,w21 + .long 0x4a0a0108 // eor w8,w8,w10 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0a160339 // and w25,w25,w22 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a100108 // eor w8,w8,w16 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a050108 // eor w8,w8,w5 + .long 0x0b0702d6 // add w22,w22,w7 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13887d08 // ror w8,w8,#31 + .long 0x2a140319 // orr w25,w24,w20 + .long 0x0a14031a // and w26,w24,w20 + .long 0x4a0b0129 // eor w9,w9,w11 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0a150339 // and w25,w25,w21 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a110129 // eor w9,w9,w17 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a060129 // eor w9,w9,w6 + .long 0x0b0802b5 // add w21,w21,w8 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13897d29 // ror w9,w9,#31 + .long 0x2a1802f9 // orr w25,w23,w24 + .long 0x0a1802fa // and w26,w23,w24 + .long 0x4a0c014a // eor w10,w10,w12 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0a140339 // and w25,w25,w20 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a13014a // eor w10,w10,w19 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a07014a // eor w10,w10,w7 + .long 0x0b090294 // add w20,w20,w9 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x138a7d4a // ror w10,w10,#31 + .long 0x2a1702d9 // orr w25,w22,w23 + .long 0x0a1702da // and w26,w22,w23 + .long 0x4a0d016b // eor w11,w11,w13 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0a180339 // and w25,w25,w24 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a03016b // eor w11,w11,w3 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a08016b // eor w11,w11,w8 + .long 0x0b0a0318 // add w24,w24,w10 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x138b7d6b // ror w11,w11,#31 + .long 0x2a1602b9 // orr w25,w21,w22 + .long 0x0a1602ba // and w26,w21,w22 + .long 0x4a0e018c // eor w12,w12,w14 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0a170339 // and w25,w25,w23 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a04018c // eor w12,w12,w4 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a09018c // eor w12,w12,w9 + .long 0x0b0b02f7 // add w23,w23,w11 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x138c7d8c // ror w12,w12,#31 + .long 0x2a150299 // orr w25,w20,w21 + .long 0x0a15029a // and w26,w20,w21 + .long 0x4a0f01ad // eor w13,w13,w15 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0a160339 // and w25,w25,w22 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0501ad // eor w13,w13,w5 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a0a01ad // eor w13,w13,w10 + .long 0x0b0c02d6 // add w22,w22,w12 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x138d7dad // ror w13,w13,#31 + .long 0x2a140319 // orr w25,w24,w20 + .long 0x0a14031a // and w26,w24,w20 + .long 0x4a1001ce // eor w14,w14,w16 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0a150339 // and w25,w25,w21 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0601ce // eor w14,w14,w6 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a0b01ce // eor w14,w14,w11 + .long 0x0b0d02b5 // add w21,w21,w13 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x138e7dce // ror w14,w14,#31 + .long 0x2a1802f9 // orr w25,w23,w24 + .long 0x0a1802fa // and w26,w23,w24 + .long 0x4a1101ef // eor w15,w15,w17 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0a140339 // and w25,w25,w20 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a0701ef // eor w15,w15,w7 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a0c01ef // eor w15,w15,w12 + .long 0x0b0e0294 // add w20,w20,w14 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x138f7def // ror w15,w15,#31 + .long 0x52983adc // movz w28,#0xc1d6 + .long 0x72b94c5c // movk w28,#0xca62, lsl #16 + .long 0x2a1702d9 // orr w25,w22,w23 + .long 0x0a1702da // and w26,w22,w23 + .long 0x4a130210 // eor w16,w16,w19 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0a180339 // and w25,w25,w24 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a080210 // eor w16,w16,w8 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x2a1a0339 // orr w25,w25,w26 + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0d0210 // eor w16,w16,w13 + .long 0x0b0f0318 // add w24,w24,w15 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13907e10 // ror w16,w16,#31 + .long 0x4a030231 // eor w17,w17,w3 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a090231 // eor w17,w17,w9 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0e0231 // eor w17,w17,w14 + .long 0x0b1002f7 // add w23,w23,w16 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13917e31 // ror w17,w17,#31 + .long 0x4a040273 // eor w19,w19,w4 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0a0273 // eor w19,w19,w10 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a0f0273 // eor w19,w19,w15 + .long 0x0b1102d6 // add w22,w22,w17 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13937e73 // ror w19,w19,#31 + .long 0x4a050063 // eor w3,w3,w5 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0b0063 // eor w3,w3,w11 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a100063 // eor w3,w3,w16 + .long 0x0b1302b5 // add w21,w21,w19 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13837c63 // ror w3,w3,#31 + .long 0x4a060084 // eor w4,w4,w6 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a0c0084 // eor w4,w4,w12 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a110084 // eor w4,w4,w17 + .long 0x0b030294 // add w20,w20,w3 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13847c84 // ror w4,w4,#31 + .long 0x4a0700a5 // eor w5,w5,w7 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a0d00a5 // eor w5,w5,w13 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a1300a5 // eor w5,w5,w19 + .long 0x0b040318 // add w24,w24,w4 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x13857ca5 // ror w5,w5,#31 + .long 0x4a0800c6 // eor w6,w6,w8 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a0e00c6 // eor w6,w6,w14 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0300c6 // eor w6,w6,w3 + .long 0x0b0502f7 // add w23,w23,w5 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13867cc6 // ror w6,w6,#31 + .long 0x4a0900e7 // eor w7,w7,w9 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a0f00e7 // eor w7,w7,w15 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a0400e7 // eor w7,w7,w4 + .long 0x0b0602d6 // add w22,w22,w6 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13877ce7 // ror w7,w7,#31 + .long 0x4a0a0108 // eor w8,w8,w10 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a100108 // eor w8,w8,w16 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a050108 // eor w8,w8,w5 + .long 0x0b0702b5 // add w21,w21,w7 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13887d08 // ror w8,w8,#31 + .long 0x4a0b0129 // eor w9,w9,w11 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a110129 // eor w9,w9,w17 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a060129 // eor w9,w9,w6 + .long 0x0b080294 // add w20,w20,w8 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x13897d29 // ror w9,w9,#31 + .long 0x4a0c014a // eor w10,w10,w12 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a13014a // eor w10,w10,w19 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a07014a // eor w10,w10,w7 + .long 0x0b090318 // add w24,w24,w9 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x138a7d4a // ror w10,w10,#31 + .long 0x4a0d016b // eor w11,w11,w13 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a03016b // eor w11,w11,w3 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a08016b // eor w11,w11,w8 + .long 0x0b0a02f7 // add w23,w23,w10 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x138b7d6b // ror w11,w11,#31 + .long 0x4a0e018c // eor w12,w12,w14 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a04018c // eor w12,w12,w4 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a09018c // eor w12,w12,w9 + .long 0x0b0b02d6 // add w22,w22,w11 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x138c7d8c // ror w12,w12,#31 + .long 0x4a0f01ad // eor w13,w13,w15 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0501ad // eor w13,w13,w5 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a0a01ad // eor w13,w13,w10 + .long 0x0b0c02b5 // add w21,w21,w12 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x138d7dad // ror w13,w13,#31 + .long 0x4a1001ce // eor w14,w14,w16 + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a0601ce // eor w14,w14,w6 + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x4a0b01ce // eor w14,w14,w11 + .long 0x0b0d0294 // add w20,w20,w13 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x138e7dce // ror w14,w14,#31 + .long 0x4a1101ef // eor w15,w15,w17 + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x0b1c0318 // add w24,w24,w28 // future e+=K + .long 0x4a0701ef // eor w15,w15,w7 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0x4a0c01ef // eor w15,w15,w12 + .long 0x0b0e0318 // add w24,w24,w14 // future e+=X[i] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x138f7def // ror w15,w15,#31 + .long 0x4a130210 // eor w16,w16,w19 + .long 0x4a1502f9 // eor w25,w23,w21 + .long 0x13946e9b // ror w27,w20,#27 + .long 0x0b1c02f7 // add w23,w23,w28 // future e+=K + .long 0x4a080210 // eor w16,w16,w8 + .long 0x4a160339 // eor w25,w25,w22 + .long 0x0b1b0318 // add w24,w24,w27 // e+=rot(a,5) + .long 0x13950ab5 // ror w21,w21,#2 + .long 0x4a0d0210 // eor w16,w16,w13 + .long 0x0b0f02f7 // add w23,w23,w15 // future e+=X[i] + .long 0x0b190318 // add w24,w24,w25 // e+=F(b,c,d) + .long 0x13907e10 // ror w16,w16,#31 + .long 0x4a030231 // eor w17,w17,w3 + .long 0x4a1402d9 // eor w25,w22,w20 + .long 0x13986f1b // ror w27,w24,#27 + .long 0x0b1c02d6 // add w22,w22,w28 // future e+=K + .long 0x4a090231 // eor w17,w17,w9 + .long 0x4a150339 // eor w25,w25,w21 + .long 0x0b1b02f7 // add w23,w23,w27 // e+=rot(a,5) + .long 0x13940a94 // ror w20,w20,#2 + .long 0x4a0e0231 // eor w17,w17,w14 + .long 0x0b1002d6 // add w22,w22,w16 // future e+=X[i] + .long 0x0b1902f7 // add w23,w23,w25 // e+=F(b,c,d) + .long 0x13917e31 // ror w17,w17,#31 + .long 0x4a040273 // eor w19,w19,w4 + .long 0x4a1802b9 // eor w25,w21,w24 + .long 0x13976efb // ror w27,w23,#27 + .long 0x0b1c02b5 // add w21,w21,w28 // future e+=K + .long 0x4a0a0273 // eor w19,w19,w10 + .long 0x4a140339 // eor w25,w25,w20 + .long 0x0b1b02d6 // add w22,w22,w27 // e+=rot(a,5) + .long 0x13980b18 // ror w24,w24,#2 + .long 0x4a0f0273 // eor w19,w19,w15 + .long 0x0b1102b5 // add w21,w21,w17 // future e+=X[i] + .long 0x0b1902d6 // add w22,w22,w25 // e+=F(b,c,d) + .long 0x13937e73 // ror w19,w19,#31 + .long 0x29401404 // ldp w4,w5,[x0] + .long 0x4a170299 // eor w25,w20,w23 + .long 0x13966edb // ror w27,w22,#27 + .long 0x0b1c0294 // add w20,w20,w28 // future e+=K + .long 0x4a180339 // eor w25,w25,w24 + .long 0x0b1b02b5 // add w21,w21,w27 // e+=rot(a,5) + .long 0x13970af7 // ror w23,w23,#2 + .long 0x0b130294 // add w20,w20,w19 // future e+=X[i] + .long 0x0b1902b5 // add w21,w21,w25 // e+=F(b,c,d) + .long 0x29411c06 // ldp w6,w7,[x0,#8] + .long 0x4a160319 // eor w25,w24,w22 + .long 0x13956ebb // ror w27,w21,#27 + .long 0x4a170339 // eor w25,w25,w23 + .long 0x0b1b0294 // add w20,w20,w27 // e+=rot(a,5) + .long 0x13960ad6 // ror w22,w22,#2 + .long 0xb9401008 // ldr w8,[x0,#16] + .long 0x0b190294 // add w20,w20,w25 // e+=F(b,c,d) + .long 0x0b0502b5 // add w21,w21,w5 + .long 0x0b0602d6 // add w22,w22,w6 + .long 0x0b040294 // add w20,w20,w4 + .long 0x0b0702f7 // add w23,w23,w7 + .long 0x0b080318 // add w24,w24,w8 + .long 0x29005414 // stp w20,w21,[x0] + .long 0x29015c16 // stp w22,w23,[x0,#8] + .long 0xb9001018 // str w24,[x0,#16] + cbnz x2,.Lsha1_gpr_loop + + .long 0xa94153f3 // ldp x19,x20,[sp,#16] + .long 0xa9425bf5 // ldp x21,x22,[sp,#32] + .long 0xa94363f7 // ldp x23,x24,[sp,#48] + .long 0xa9446bf9 // ldp x25,x26,[sp,#64] + .long 0xa94573fb // ldp x27,x28,[sp,#80] + .long 0xf84607fd // ldr x29,[sp],#96 + ret diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_i386.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_i386.inc new file mode 100644 index 00000000..769191ee --- /dev/null +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_i386.inc @@ -0,0 +1,172 @@ +// SHA-NI implementation of SHA-1 compress (IA-32; Intel SHA Extensions +// work in 32-bit mode). The twenty sha1rnds4 steps carry their round +// constants as immediates; the only table read is the 16-byte flip mask, +// taken from K_SHA1_Doubled at +160 (byte-identical to the original's +// .LK_XX_XX+80; read unaligned - Pascal consts have no alignment +// guarantee). The original's PIC table setup and stack-argument reads are +// replaced by the shared-prologue registers. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = K_SHA1_Doubled ptr. +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here, giving the +// 4-deep frame the original epilogue unwinds via its saved-esp register. +// SHA-NI and shuffle instructions are db-encoded for broad assembler +// compatibility. +// Reference: OpenSSL CRYPTOGAMS sha1-586.pl (shaext function). + + push ebp + mov ebp, eax + mov ecx, edi + mov edi, ebx + mov ebx, esp + sub esp, $20 + movdqu xmm0, oword ptr [edi] + movd xmm1, dword ptr [edi + $10] + and esp, $FFFFFFE0 + movdqu xmm3, oword ptr [ebp + $A0] + movdqu xmm4, oword ptr [esi] + pshufd xmm0, xmm0, $1B + movdqu xmm5, oword ptr [esi + $10] + pshufd xmm1, xmm1, $1B + movdqu xmm6, oword ptr [esi + $20] + db $66, $0F, $38, $00, $E3 // pshufb xmm4, xmm3 + movdqu xmm7, oword ptr [esi + $30] + db $66, $0F, $38, $00, $EB // pshufb xmm5, xmm3 + db $66, $0F, $38, $00, $F3 // pshufb xmm6, xmm3 + db $66, $0F, $38, $00, $FB // pshufb xmm7, xmm3 + jmp @loop_shaext_004 + +@loop_shaext_004: + dec ecx + lea eax, [esi + $40] + movdqa oword ptr [esp], xmm1 + paddd xmm1, xmm4 + cmovne esi, eax + movdqa oword ptr [esp + $10], xmm0 + db $0F, $38, $C9, $E5 // sha1msg1 xmm4, xmm5 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $00 // sha1rnds4 xmm0, xmm1, $0 + db $0F, $38, $C8, $D5 // sha1nexte xmm2, xmm5 + pxor xmm4, xmm6 + db $0F, $38, $C9, $EE // sha1msg1 xmm5, xmm6 + db $0F, $38, $CA, $E7 // sha1msg2 xmm4, xmm7 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $00 // sha1rnds4 xmm0, xmm2, $0 + db $0F, $38, $C8, $CE // sha1nexte xmm1, xmm6 + pxor xmm5, xmm7 + db $0F, $38, $CA, $EC // sha1msg2 xmm5, xmm4 + db $0F, $38, $C9, $F7 // sha1msg1 xmm6, xmm7 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $00 // sha1rnds4 xmm0, xmm1, $0 + db $0F, $38, $C8, $D7 // sha1nexte xmm2, xmm7 + pxor xmm6, xmm4 + db $0F, $38, $C9, $FC // sha1msg1 xmm7, xmm4 + db $0F, $38, $CA, $F5 // sha1msg2 xmm6, xmm5 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $00 // sha1rnds4 xmm0, xmm2, $0 + db $0F, $38, $C8, $CC // sha1nexte xmm1, xmm4 + pxor xmm7, xmm5 + db $0F, $38, $CA, $FE // sha1msg2 xmm7, xmm6 + db $0F, $38, $C9, $E5 // sha1msg1 xmm4, xmm5 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $00 // sha1rnds4 xmm0, xmm1, $0 + db $0F, $38, $C8, $D5 // sha1nexte xmm2, xmm5 + pxor xmm4, xmm6 + db $0F, $38, $C9, $EE // sha1msg1 xmm5, xmm6 + db $0F, $38, $CA, $E7 // sha1msg2 xmm4, xmm7 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $01 // sha1rnds4 xmm0, xmm2, $1 + db $0F, $38, $C8, $CE // sha1nexte xmm1, xmm6 + pxor xmm5, xmm7 + db $0F, $38, $CA, $EC // sha1msg2 xmm5, xmm4 + db $0F, $38, $C9, $F7 // sha1msg1 xmm6, xmm7 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $01 // sha1rnds4 xmm0, xmm1, $1 + db $0F, $38, $C8, $D7 // sha1nexte xmm2, xmm7 + pxor xmm6, xmm4 + db $0F, $38, $C9, $FC // sha1msg1 xmm7, xmm4 + db $0F, $38, $CA, $F5 // sha1msg2 xmm6, xmm5 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $01 // sha1rnds4 xmm0, xmm2, $1 + db $0F, $38, $C8, $CC // sha1nexte xmm1, xmm4 + pxor xmm7, xmm5 + db $0F, $38, $CA, $FE // sha1msg2 xmm7, xmm6 + db $0F, $38, $C9, $E5 // sha1msg1 xmm4, xmm5 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $01 // sha1rnds4 xmm0, xmm1, $1 + db $0F, $38, $C8, $D5 // sha1nexte xmm2, xmm5 + pxor xmm4, xmm6 + db $0F, $38, $C9, $EE // sha1msg1 xmm5, xmm6 + db $0F, $38, $CA, $E7 // sha1msg2 xmm4, xmm7 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $01 // sha1rnds4 xmm0, xmm2, $1 + db $0F, $38, $C8, $CE // sha1nexte xmm1, xmm6 + pxor xmm5, xmm7 + db $0F, $38, $CA, $EC // sha1msg2 xmm5, xmm4 + db $0F, $38, $C9, $F7 // sha1msg1 xmm6, xmm7 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $02 // sha1rnds4 xmm0, xmm1, $2 + db $0F, $38, $C8, $D7 // sha1nexte xmm2, xmm7 + pxor xmm6, xmm4 + db $0F, $38, $C9, $FC // sha1msg1 xmm7, xmm4 + db $0F, $38, $CA, $F5 // sha1msg2 xmm6, xmm5 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $02 // sha1rnds4 xmm0, xmm2, $2 + db $0F, $38, $C8, $CC // sha1nexte xmm1, xmm4 + pxor xmm7, xmm5 + db $0F, $38, $CA, $FE // sha1msg2 xmm7, xmm6 + db $0F, $38, $C9, $E5 // sha1msg1 xmm4, xmm5 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $02 // sha1rnds4 xmm0, xmm1, $2 + db $0F, $38, $C8, $D5 // sha1nexte xmm2, xmm5 + pxor xmm4, xmm6 + db $0F, $38, $C9, $EE // sha1msg1 xmm5, xmm6 + db $0F, $38, $CA, $E7 // sha1msg2 xmm4, xmm7 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $02 // sha1rnds4 xmm0, xmm2, $2 + db $0F, $38, $C8, $CE // sha1nexte xmm1, xmm6 + pxor xmm5, xmm7 + db $0F, $38, $CA, $EC // sha1msg2 xmm5, xmm4 + db $0F, $38, $C9, $F7 // sha1msg1 xmm6, xmm7 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $02 // sha1rnds4 xmm0, xmm1, $2 + db $0F, $38, $C8, $D7 // sha1nexte xmm2, xmm7 + pxor xmm6, xmm4 + db $0F, $38, $C9, $FC // sha1msg1 xmm7, xmm4 + db $0F, $38, $CA, $F5 // sha1msg2 xmm6, xmm5 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $03 // sha1rnds4 xmm0, xmm2, $3 + db $0F, $38, $C8, $CC // sha1nexte xmm1, xmm4 + pxor xmm7, xmm5 + db $0F, $38, $CA, $FE // sha1msg2 xmm7, xmm6 + movdqu xmm4, oword ptr [esi] + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $03 // sha1rnds4 xmm0, xmm1, $3 + db $0F, $38, $C8, $D5 // sha1nexte xmm2, xmm5 + movdqu xmm5, oword ptr [esi + $10] + db $66, $0F, $38, $00, $E3 // pshufb xmm4, xmm3 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $03 // sha1rnds4 xmm0, xmm2, $3 + db $0F, $38, $C8, $CE // sha1nexte xmm1, xmm6 + movdqu xmm6, oword ptr [esi + $20] + db $66, $0F, $38, $00, $EB // pshufb xmm5, xmm3 + movdqa xmm2, xmm0 + db $0F, $3A, $CC, $C1, $03 // sha1rnds4 xmm0, xmm1, $3 + db $0F, $38, $C8, $D7 // sha1nexte xmm2, xmm7 + movdqu xmm7, oword ptr [esi + $30] + db $66, $0F, $38, $00, $F3 // pshufb xmm6, xmm3 + movdqa xmm1, xmm0 + db $0F, $3A, $CC, $C2, $03 // sha1rnds4 xmm0, xmm2, $3 + movdqa xmm2, oword ptr [esp] + db $66, $0F, $38, $00, $FB // pshufb xmm7, xmm3 + db $0F, $38, $C8, $CA // sha1nexte xmm1, xmm2 + paddd xmm0, oword ptr [esp + $10] + jnz @loop_shaext_004 + pshufd xmm0, xmm0, $1B + pshufd xmm1, xmm1, $1B + movdqu oword ptr [edi], xmm0 + movd dword ptr [edi + $10], xmm1 + mov esp, ebx + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc index 3f8fa1bd..8dcd1baa 100644 --- a/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc @@ -1,40 +1,30 @@ -// SHA-1 SHA-NI implementation (Intel SHA Extensions), the same design as -// OpenSSL's x86_64 sha1_block_data_order_shaext (CRYPTOGAMS). The 20 sha1rnds4 -// groups run with the message schedule (sha1msg1/sha1msg2/sha1nexte) interleaved. -// The input is byte-swapped with a single pshufb against a full-reverse mask -// (byte-swap and dword-reverse folded into one mask, loaded once into xmm3), so -// no per-word pshufd is needed. The next block's message loads are software- -// pipelined into rounds 64-79 and the data pointer is advanced branchlessly -// (cmovnz). -// -// Expects MS x64 ABI (after HlpSimdProc5Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = round-constant ptr (unused - sha1rnds4 has the round -// constants built in), r10 = byte-swap mask ptr (16 bytes; see BSWAP32_MASK, -// matching OpenSSL K_XX_XX+0xa0). -// -// SHA-NI instructions, db-encoded for assembler compatibility: -// sha1rnds4 dst, src, imm = $0F $3A $CC -// sha1nexte dst, src = $0F $38 $C8 -// sha1msg1 dst, src = $0F $38 $C9 -// sha1msg2 dst, src = $0F $38 $CA -// pshufb dst, src = $66 $0F $38 $00 -// -// Register map (identical to OpenSSL): -// xmm0 = ABCD state -// xmm1 = E -// xmm2 = E' (ping-pong E register) -// xmm3 = byte-swap mask (loaded once) -// xmm4 = MSG0 -// xmm5 = MSG1 -// xmm6 = MSG2 -// xmm7 = MSG3 -// xmm8 = ABCD_SAVE -// xmm9 = E_SAVE -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); the kernel itself uses only volatile GPRs (rcx/rdx/r8-r11). - - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} +// SHA-NI implementation of SHA-1 compress (Intel SHA Extensions). The 20 +// sha1rnds4 groups run with the message schedule (sha1msg1/sha1msg2/ +// sha1nexte) interleaved. The input is byte-swapped with a single pshufb +// against a full-reverse mask (byte-swap and dword-reverse folded into one +// mask, loaded once into xmm3), so no per-word pshufd is needed. The next +// block's message loads are software-pipelined into rounds 64-79 and the +// data pointer is advanced branchlessly (cmovnz). +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = round-constant ptr (unused - sha1rnds4 has the +// round constants built in), r10 = byte-swap mask ptr (16 bytes; see +// BSWAP32_MASK, matching OpenSSL K_XX_XX+0xa0). +// Register map (identical to OpenSSL): xmm0 = ABCD state, xmm1 = E, +// xmm2 = E' (ping-pong), xmm3 = byte-swap mask (loaded once), +// xmm4-xmm7 = MSG0-MSG3, xmm8 = ABCD_SAVE, xmm9 = E_SAVE. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); the kernel itself uses only volatile GPRs +// (rcx/rdx/r8-r11). +// SHA-NI instructions are db-encoded for broad assembler compatibility. +// Encoding notes: +// sha1rnds4 dst, src, imm = $0F $3A $CC +// sha1nexte dst, src = $0F $38 $C8 +// sha1msg1 dst, src = $0F $38 $C9 +// sha1msg2 dst, src = $0F $38 $CA +// pshufb dst, src = $66 $0F $38 $00 +// Reference: OpenSSL CRYPTOGAMS x86_64 sha1_block_data_order_shaext. + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} test r8d, r8d jz @sha1ni_done @@ -203,4 +193,4 @@ @sha1ni_done: - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_i386.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_i386.inc index 3cfb9fd4..420c4be3 100644 --- a/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_i386.inc +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_i386.inc @@ -1,14 +1,15 @@ -// SHA-1 SSE2 SIMD-schedule implementation (IA-32), derived from OpenSSL's -// sha1-586 (CRYPTOGAMS) SSSE3 kernel. The message schedule runs in SSE2 -// (pxor/pshufd/psrldq/pslldq) interleaved with the GPR compression rounds. -// OpenSSL's only SSSE3-only op is the pshufb input byte-swap, emulated in SSE2 as -// psrlw/psllw/por + pshuflw/pshufhw (the freed byte-swap-mask register xmm6 is the -// scratch); the W-schedule uses no palignr. -// Expects (after HlpSimdProc4Begin_i386): ebx = state, esi = data, edi = numblocks, -// eax = doubled-K_SHA1 ptr (read at a 32-byte stride so the duplicated halves -// are skipped; see K_SHA1_Doubled). HlpSimdProc4Begin pushes ebx/esi/edi; ebp is -// pushed here as the K pointer, then reused as the data pointer and for the -// state writeback. +// SSE2 implementation of SHA-1 compress (SIMD schedule; IA-32). The message +// schedule runs in SSE2 (pxor/pshufd/psrldq/pslldq) interleaved with the +// GPR compression rounds. The original's only SSSE3-only op is the pshufb +// input byte-swap, emulated in SSE2 as psrlw/psllw/por + pshuflw/pshufhw +// (the freed byte-swap-mask register xmm6 is the scratch); the W-schedule +// uses no palignr. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = doubled-K_SHA1 ptr (read at a 32-byte stride so +// the duplicated halves are skipped; see K_SHA1_Doubled). +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K +// pointer, then reused as the data pointer and for the state writeback. +// Reference: OpenSSL CRYPTOGAMS sha1-586.pl (SSSE3 kernel). push ebp mov ebp, eax diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc index 7562a955..e44db045 100644 --- a/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc @@ -1,19 +1,20 @@ -// SHA-1 SSE2 SIMD-schedule implementation, derived from OpenSSL's SHA-1 SSSE3 -// kernel in sha1-x86_64.pl (CRYPTOGAMS). The message schedule runs in SSE2 -// (pxor/pshufd/psrldq/pslldq) interleaved with the GPR compression rounds, so the -// vector and integer units run in parallel. OpenSSL's only SSSE3-only op here is -// the pshufb input byte-swap, which is emulated in SSE2 (psrlw/psllw/por + -// pshuflw/pshufhw; xmm12 is a free scratch); the W-schedule uses no palignr. The -// appended masks go unused - the byte-swap is computed, not shuffled. -// Expects MS x64 ABI (after HlpSimdProc4Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = doubled-K_SHA1 ptr (read 128-bit at a 32-byte stride, -// r14 = K + 64, so the duplicated halves are skipped; see K_SHA1_Doubled). -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r14 in the local frame; r11 -// holds the caller rsp across the kernel for the unwind. +// SSE2 implementation of SHA-1 compress (SIMD schedule). The message +// schedule runs in SSE2 (pxor/pshufd/psrldq/pslldq) interleaved with the +// GPR compression rounds, so the vector and integer units run in parallel. +// The original's only SSSE3-only op here is the pshufb input byte-swap, +// emulated in SSE2 (psrlw/psllw/por + pshuflw/pshufhw; xmm12 is a free +// scratch); the W-schedule uses no palignr. The appended masks go unused - +// the byte-swap is computed, not shuffled. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K_SHA1 ptr (read 128-bit at a 32-byte +// stride, r14 = K + 64, so the duplicated halves are skipped; see +// K_SHA1_Doubled). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r14 in the local frame; r11 +// holds the caller rsp across the kernel for the unwind. +// Reference: OpenSSL CRYPTOGAMS sha1-x86_64.pl (SSSE3 kernel). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} mov r11, rsp push r9 @@ -1228,4 +1229,4 @@ mov rbx, qword [r11 - $10] lea rsp, [r11] - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_i386.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_i386.inc new file mode 100644 index 00000000..ff65a72a --- /dev/null +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_i386.inc @@ -0,0 +1,1221 @@ +// SSSE3 implementation of SHA-1 compress (SIMD schedule; IA-32); SSE2 +// sibling: SHA1CompressSse2_i386.inc. The message schedule runs in SIMD +// (pxor/pshufd/psrldq/pslldq) interleaved with the GPR compression rounds. +// Unlike the SSE2 sibling the input byte-swaps are real pshufb against the +// mask kept resident in xmm6; the W-schedule uses no palignr. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = doubled-K_SHA1 ptr (read at a 32-byte stride so +// the duplicated halves are skipped; see K_SHA1_Doubled). +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K +// pointer, then reused as the data pointer and for the state writeback. +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha1-586.pl (SSSE3 kernel). + + push ebp + mov ebp, eax + + movdqu xmm7, oword [ebp + $0] + movdqu xmm0, oword [ebp + $20] + movdqu xmm1, oword [ebp + $40] + movdqu xmm2, oword [ebp + $60] + movdqu xmm6, oword [ebp + $80] + mov edx, edi + mov edi, ebx + mov ebp, esi + mov esi, esp + sub esp, $D0 + and esp, $FFFFFFC0 + movdqa oword [esp + $70], xmm0 + movdqa oword [esp + $80], xmm1 + movdqa oword [esp + $90], xmm2 + shl edx, $6 + movdqa oword [esp + $A0], xmm7 + add edx, ebp + movdqa oword [esp + $B0], xmm6 + add ebp, $40 + mov dword [esp + $C0], edi + mov dword [esp + $C4], ebp + mov dword [esp + $C8], edx + mov dword [esp + $CC], esi + mov eax, dword [edi] + mov ebx, dword [edi + $4] + mov ecx, dword [edi + $8] + mov edx, dword [edi + $C] + mov edi, dword [edi + $10] + mov esi, ebx + movdqu xmm0, oword [ebp - $40] + movdqu xmm1, oword [ebp - $30] + movdqu xmm2, oword [ebp - $20] + movdqu xmm3, oword [ebp - $10] + db $66, $0F, $38, $00, $C6 // pshufb xmm0, xmm6 + db $66, $0F, $38, $00, $CE // pshufb xmm1, xmm6 + db $66, $0F, $38, $00, $D6 // pshufb xmm2, xmm6 + movdqa oword [esp + $60], xmm7 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + paddd xmm0, xmm7 + paddd xmm1, xmm7 + paddd xmm2, xmm7 + movdqa oword [esp], xmm0 + psubd xmm0, xmm7 + movdqa oword [esp + $10], xmm1 + psubd xmm1, xmm7 + movdqa oword [esp + $20], xmm2 + mov ebp, ecx + psubd xmm2, xmm7 + xor ebp, edx + pshufd xmm4, xmm0, $EE + and esi, ebp + jmp @block_loop + +@block_loop: + ror ebx, $2 + xor esi, edx + mov ebp, eax + punpcklqdq xmm4, xmm1 + movdqa xmm6, xmm3 + add edi, dword [esp] + xor ebx, ecx + paddd xmm7, xmm3 + movdqa oword [esp + $40], xmm0 + rol eax, $5 + add edi, esi + psrldq xmm6, $4 + and ebp, ebx + xor ebx, ecx + pxor xmm4, xmm0 + add edi, eax + ror eax, $7 + pxor xmm6, xmm2 + xor ebp, ecx + mov esi, edi + add edx, dword [esp + $4] + pxor xmm4, xmm6 + xor eax, ebx + rol edi, $5 + movdqa oword [esp + $30], xmm7 + add edx, ebp + and esi, eax + movdqa xmm0, xmm4 + xor eax, ebx + add edx, edi + ror edi, $7 + movdqa xmm6, xmm4 + xor esi, ebx + pslldq xmm0, $C + paddd xmm4, xmm4 + mov ebp, edx + add ecx, dword [esp + $8] + psrld xmm6, $1F + xor edi, eax + rol edx, $5 + movdqa xmm7, xmm0 + add ecx, esi + and ebp, edi + xor edi, eax + psrld xmm0, $1E + add ecx, edx + ror edx, $7 + por xmm4, xmm6 + xor ebp, eax + mov esi, ecx + add ebx, dword [esp + $C] + pslld xmm7, $2 + xor edx, edi + rol ecx, $5 + pxor xmm4, xmm0 + movdqa xmm0, oword [esp + $60] + add ebx, ebp + and esi, edx + pxor xmm4, xmm7 + pshufd xmm5, xmm1, $EE + xor edx, edi + add ebx, ecx + ror ecx, $7 + xor esi, edi + mov ebp, ebx + punpcklqdq xmm5, xmm2 + movdqa xmm7, xmm4 + add eax, dword [esp + $10] + xor ecx, edx + paddd xmm0, xmm4 + movdqa oword [esp + $50], xmm1 + rol ebx, $5 + add eax, esi + psrldq xmm7, $4 + and ebp, ecx + xor ecx, edx + pxor xmm5, xmm1 + add eax, ebx + ror ebx, $7 + pxor xmm7, xmm3 + xor ebp, edx + mov esi, eax + add edi, dword [esp + $14] + pxor xmm5, xmm7 + xor ebx, ecx + rol eax, $5 + movdqa oword [esp], xmm0 + add edi, ebp + and esi, ebx + movdqa xmm1, xmm5 + xor ebx, ecx + add edi, eax + ror eax, $7 + movdqa xmm7, xmm5 + xor esi, ecx + pslldq xmm1, $C + paddd xmm5, xmm5 + mov ebp, edi + add edx, dword [esp + $18] + psrld xmm7, $1F + xor eax, ebx + rol edi, $5 + movdqa xmm0, xmm1 + add edx, esi + and ebp, eax + xor eax, ebx + psrld xmm1, $1E + add edx, edi + ror edi, $7 + por xmm5, xmm7 + xor ebp, ebx + mov esi, edx + add ecx, dword [esp + $1C] + pslld xmm0, $2 + xor edi, eax + rol edx, $5 + pxor xmm5, xmm1 + movdqa xmm1, oword [esp + $70] + add ecx, ebp + and esi, edi + pxor xmm5, xmm0 + pshufd xmm6, xmm2, $EE + xor edi, eax + add ecx, edx + ror edx, $7 + xor esi, eax + mov ebp, ecx + punpcklqdq xmm6, xmm3 + movdqa xmm0, xmm5 + add ebx, dword [esp + $20] + xor edx, edi + paddd xmm1, xmm5 + movdqa oword [esp + $60], xmm2 + rol ecx, $5 + add ebx, esi + psrldq xmm0, $4 + and ebp, edx + xor edx, edi + pxor xmm6, xmm2 + add ebx, ecx + ror ecx, $7 + pxor xmm0, xmm4 + xor ebp, edi + mov esi, ebx + add eax, dword [esp + $24] + pxor xmm6, xmm0 + xor ecx, edx + rol ebx, $5 + movdqa oword [esp + $10], xmm1 + add eax, ebp + and esi, ecx + movdqa xmm2, xmm6 + xor ecx, edx + add eax, ebx + ror ebx, $7 + movdqa xmm0, xmm6 + xor esi, edx + pslldq xmm2, $C + paddd xmm6, xmm6 + mov ebp, eax + add edi, dword [esp + $28] + psrld xmm0, $1F + xor ebx, ecx + rol eax, $5 + movdqa xmm1, xmm2 + add edi, esi + and ebp, ebx + xor ebx, ecx + psrld xmm2, $1E + add edi, eax + ror eax, $7 + por xmm6, xmm0 + xor ebp, ecx + movdqa xmm0, oword [esp + $40] + mov esi, edi + add edx, dword [esp + $2C] + pslld xmm1, $2 + xor eax, ebx + rol edi, $5 + pxor xmm6, xmm2 + movdqa xmm2, oword [esp + $70] + add edx, ebp + and esi, eax + pxor xmm6, xmm1 + pshufd xmm7, xmm3, $EE + xor eax, ebx + add edx, edi + ror edi, $7 + xor esi, ebx + mov ebp, edx + punpcklqdq xmm7, xmm4 + movdqa xmm1, xmm6 + add ecx, dword [esp + $30] + xor edi, eax + paddd xmm2, xmm6 + movdqa oword [esp + $40], xmm3 + rol edx, $5 + add ecx, esi + psrldq xmm1, $4 + and ebp, edi + xor edi, eax + pxor xmm7, xmm3 + add ecx, edx + ror edx, $7 + pxor xmm1, xmm5 + xor ebp, eax + mov esi, ecx + add ebx, dword [esp + $34] + pxor xmm7, xmm1 + xor edx, edi + rol ecx, $5 + movdqa oword [esp + $20], xmm2 + add ebx, ebp + and esi, edx + movdqa xmm3, xmm7 + xor edx, edi + add ebx, ecx + ror ecx, $7 + movdqa xmm1, xmm7 + xor esi, edi + pslldq xmm3, $C + paddd xmm7, xmm7 + mov ebp, ebx + add eax, dword [esp + $38] + psrld xmm1, $1F + xor ecx, edx + rol ebx, $5 + movdqa xmm2, xmm3 + add eax, esi + and ebp, ecx + xor ecx, edx + psrld xmm3, $1E + add eax, ebx + ror ebx, $7 + por xmm7, xmm1 + xor ebp, edx + movdqa xmm1, oword [esp + $50] + mov esi, eax + add edi, dword [esp + $3C] + pslld xmm2, $2 + xor ebx, ecx + rol eax, $5 + pxor xmm7, xmm3 + movdqa xmm3, oword [esp + $70] + add edi, ebp + and esi, ebx + pxor xmm7, xmm2 + pshufd xmm2, xmm6, $EE + xor ebx, ecx + add edi, eax + ror eax, $7 + pxor xmm0, xmm4 + punpcklqdq xmm2, xmm7 + xor esi, ecx + mov ebp, edi + add edx, dword [esp] + pxor xmm0, xmm1 + movdqa oword [esp + $50], xmm4 + xor eax, ebx + rol edi, $5 + movdqa xmm4, xmm3 + add edx, esi + paddd xmm3, xmm7 + and ebp, eax + pxor xmm0, xmm2 + xor eax, ebx + add edx, edi + ror edi, $7 + xor ebp, ebx + movdqa xmm2, xmm0 + movdqa oword [esp + $30], xmm3 + mov esi, edx + add ecx, dword [esp + $4] + xor edi, eax + rol edx, $5 + pslld xmm0, $2 + add ecx, ebp + and esi, edi + psrld xmm2, $1E + xor edi, eax + add ecx, edx + ror edx, $7 + xor esi, eax + mov ebp, ecx + add ebx, dword [esp + $8] + xor edx, edi + rol ecx, $5 + por xmm0, xmm2 + add ebx, esi + and ebp, edx + movdqa xmm2, oword [esp + $60] + xor edx, edi + add ebx, ecx + add eax, dword [esp + $C] + xor ebp, edi + mov esi, ebx + pshufd xmm3, xmm7, $EE + rol ebx, $5 + add eax, ebp + xor esi, edx + ror ecx, $7 + add eax, ebx + add edi, dword [esp + $10] + pxor xmm1, xmm5 + punpcklqdq xmm3, xmm0 + xor esi, ecx + mov ebp, eax + rol eax, $5 + pxor xmm1, xmm2 + movdqa oword [esp + $60], xmm5 + add edi, esi + xor ebp, ecx + movdqa xmm5, xmm4 + ror ebx, $7 + paddd xmm4, xmm0 + add edi, eax + pxor xmm1, xmm3 + add edx, dword [esp + $14] + xor ebp, ebx + mov esi, edi + rol edi, $5 + movdqa xmm3, xmm1 + movdqa oword [esp], xmm4 + add edx, ebp + xor esi, ebx + ror eax, $7 + add edx, edi + pslld xmm1, $2 + add ecx, dword [esp + $18] + xor esi, eax + psrld xmm3, $1E + mov ebp, edx + rol edx, $5 + add ecx, esi + xor ebp, eax + ror edi, $7 + add ecx, edx + por xmm1, xmm3 + add ebx, dword [esp + $1C] + xor ebp, edi + movdqa xmm3, oword [esp + $40] + mov esi, ecx + rol ecx, $5 + add ebx, ebp + xor esi, edi + ror edx, $7 + pshufd xmm4, xmm0, $EE + add ebx, ecx + add eax, dword [esp + $20] + pxor xmm2, xmm6 + punpcklqdq xmm4, xmm1 + xor esi, edx + mov ebp, ebx + rol ebx, $5 + pxor xmm2, xmm3 + movdqa oword [esp + $40], xmm6 + add eax, esi + xor ebp, edx + movdqa xmm6, oword [esp + $80] + ror ecx, $7 + paddd xmm5, xmm1 + add eax, ebx + pxor xmm2, xmm4 + add edi, dword [esp + $24] + xor ebp, ecx + mov esi, eax + rol eax, $5 + movdqa xmm4, xmm2 + movdqa oword [esp + $10], xmm5 + add edi, ebp + xor esi, ecx + ror ebx, $7 + add edi, eax + pslld xmm2, $2 + add edx, dword [esp + $28] + xor esi, ebx + psrld xmm4, $1E + mov ebp, edi + rol edi, $5 + add edx, esi + xor ebp, ebx + ror eax, $7 + add edx, edi + por xmm2, xmm4 + add ecx, dword [esp + $2C] + xor ebp, eax + movdqa xmm4, oword [esp + $50] + mov esi, edx + rol edx, $5 + add ecx, ebp + xor esi, eax + ror edi, $7 + pshufd xmm5, xmm1, $EE + add ecx, edx + add ebx, dword [esp + $30] + pxor xmm3, xmm7 + punpcklqdq xmm5, xmm2 + xor esi, edi + mov ebp, ecx + rol ecx, $5 + pxor xmm3, xmm4 + movdqa oword [esp + $50], xmm7 + add ebx, esi + xor ebp, edi + movdqa xmm7, xmm6 + ror edx, $7 + paddd xmm6, xmm2 + add ebx, ecx + pxor xmm3, xmm5 + add eax, dword [esp + $34] + xor ebp, edx + mov esi, ebx + rol ebx, $5 + movdqa xmm5, xmm3 + movdqa oword [esp + $20], xmm6 + add eax, ebp + xor esi, edx + ror ecx, $7 + add eax, ebx + pslld xmm3, $2 + add edi, dword [esp + $38] + xor esi, ecx + psrld xmm5, $1E + mov ebp, eax + rol eax, $5 + add edi, esi + xor ebp, ecx + ror ebx, $7 + add edi, eax + por xmm3, xmm5 + add edx, dword [esp + $3C] + xor ebp, ebx + movdqa xmm5, oword [esp + $60] + mov esi, edi + rol edi, $5 + add edx, ebp + xor esi, ebx + ror eax, $7 + pshufd xmm6, xmm2, $EE + add edx, edi + add ecx, dword [esp] + pxor xmm4, xmm0 + punpcklqdq xmm6, xmm3 + xor esi, eax + mov ebp, edx + rol edx, $5 + pxor xmm4, xmm5 + movdqa oword [esp + $60], xmm0 + add ecx, esi + xor ebp, eax + movdqa xmm0, xmm7 + ror edi, $7 + paddd xmm7, xmm3 + add ecx, edx + pxor xmm4, xmm6 + add ebx, dword [esp + $4] + xor ebp, edi + mov esi, ecx + rol ecx, $5 + movdqa xmm6, xmm4 + movdqa oword [esp + $30], xmm7 + add ebx, ebp + xor esi, edi + ror edx, $7 + add ebx, ecx + pslld xmm4, $2 + add eax, dword [esp + $8] + xor esi, edx + psrld xmm6, $1E + mov ebp, ebx + rol ebx, $5 + add eax, esi + xor ebp, edx + ror ecx, $7 + add eax, ebx + por xmm4, xmm6 + add edi, dword [esp + $C] + xor ebp, ecx + movdqa xmm6, oword [esp + $40] + mov esi, eax + rol eax, $5 + add edi, ebp + xor esi, ecx + ror ebx, $7 + pshufd xmm7, xmm3, $EE + add edi, eax + add edx, dword [esp + $10] + pxor xmm5, xmm1 + punpcklqdq xmm7, xmm4 + xor esi, ebx + mov ebp, edi + rol edi, $5 + pxor xmm5, xmm6 + movdqa oword [esp + $40], xmm1 + add edx, esi + xor ebp, ebx + movdqa xmm1, xmm0 + ror eax, $7 + paddd xmm0, xmm4 + add edx, edi + pxor xmm5, xmm7 + add ecx, dword [esp + $14] + xor ebp, eax + mov esi, edx + rol edx, $5 + movdqa xmm7, xmm5 + movdqa oword [esp], xmm0 + add ecx, ebp + xor esi, eax + ror edi, $7 + add ecx, edx + pslld xmm5, $2 + add ebx, dword [esp + $18] + xor esi, edi + psrld xmm7, $1E + mov ebp, ecx + rol ecx, $5 + add ebx, esi + xor ebp, edi + ror edx, $7 + add ebx, ecx + por xmm5, xmm7 + add eax, dword [esp + $1C] + movdqa xmm7, oword [esp + $50] + ror ecx, $7 + mov esi, ebx + xor ebp, edx + rol ebx, $5 + pshufd xmm0, xmm4, $EE + add eax, ebp + xor esi, ecx + xor ecx, edx + add eax, ebx + add edi, dword [esp + $20] + pxor xmm6, xmm2 + punpcklqdq xmm0, xmm5 + and esi, ecx + xor ecx, edx + ror ebx, $7 + pxor xmm6, xmm7 + movdqa oword [esp + $50], xmm2 + mov ebp, eax + xor esi, ecx + rol eax, $5 + movdqa xmm2, xmm1 + add edi, esi + paddd xmm1, xmm5 + xor ebp, ebx + pxor xmm6, xmm0 + xor ebx, ecx + add edi, eax + add edx, dword [esp + $24] + and ebp, ebx + movdqa xmm0, xmm6 + movdqa oword [esp + $10], xmm1 + xor ebx, ecx + ror eax, $7 + mov esi, edi + xor ebp, ebx + rol edi, $5 + pslld xmm6, $2 + add edx, ebp + xor esi, eax + psrld xmm0, $1E + xor eax, ebx + add edx, edi + add ecx, dword [esp + $28] + and esi, eax + xor eax, ebx + ror edi, $7 + por xmm6, xmm0 + mov ebp, edx + xor esi, eax + movdqa xmm0, oword [esp + $60] + rol edx, $5 + add ecx, esi + xor ebp, edi + xor edi, eax + add ecx, edx + pshufd xmm1, xmm5, $EE + add ebx, dword [esp + $2C] + and ebp, edi + xor edi, eax + ror edx, $7 + mov esi, ecx + xor ebp, edi + rol ecx, $5 + add ebx, ebp + xor esi, edx + xor edx, edi + add ebx, ecx + add eax, dword [esp + $30] + pxor xmm7, xmm3 + punpcklqdq xmm1, xmm6 + and esi, edx + xor edx, edi + ror ecx, $7 + pxor xmm7, xmm0 + movdqa oword [esp + $60], xmm3 + mov ebp, ebx + xor esi, edx + rol ebx, $5 + movdqa xmm3, oword [esp + $90] + add eax, esi + paddd xmm2, xmm6 + xor ebp, ecx + pxor xmm7, xmm1 + xor ecx, edx + add eax, ebx + add edi, dword [esp + $34] + and ebp, ecx + movdqa xmm1, xmm7 + movdqa oword [esp + $20], xmm2 + xor ecx, edx + ror ebx, $7 + mov esi, eax + xor ebp, ecx + rol eax, $5 + pslld xmm7, $2 + add edi, ebp + xor esi, ebx + psrld xmm1, $1E + xor ebx, ecx + add edi, eax + add edx, dword [esp + $38] + and esi, ebx + xor ebx, ecx + ror eax, $7 + por xmm7, xmm1 + mov ebp, edi + xor esi, ebx + movdqa xmm1, oword [esp + $40] + rol edi, $5 + add edx, esi + xor ebp, eax + xor eax, ebx + add edx, edi + pshufd xmm2, xmm6, $EE + add ecx, dword [esp + $3C] + and ebp, eax + xor eax, ebx + ror edi, $7 + mov esi, edx + xor ebp, eax + rol edx, $5 + add ecx, ebp + xor esi, edi + xor edi, eax + add ecx, edx + add ebx, dword [esp] + pxor xmm0, xmm4 + punpcklqdq xmm2, xmm7 + and esi, edi + xor edi, eax + ror edx, $7 + pxor xmm0, xmm1 + movdqa oword [esp + $40], xmm4 + mov ebp, ecx + xor esi, edi + rol ecx, $5 + movdqa xmm4, xmm3 + add ebx, esi + paddd xmm3, xmm7 + xor ebp, edx + pxor xmm0, xmm2 + xor edx, edi + add ebx, ecx + add eax, dword [esp + $4] + and ebp, edx + movdqa xmm2, xmm0 + movdqa oword [esp + $30], xmm3 + xor edx, edi + ror ecx, $7 + mov esi, ebx + xor ebp, edx + rol ebx, $5 + pslld xmm0, $2 + add eax, ebp + xor esi, ecx + psrld xmm2, $1E + xor ecx, edx + add eax, ebx + add edi, dword [esp + $8] + and esi, ecx + xor ecx, edx + ror ebx, $7 + por xmm0, xmm2 + mov ebp, eax + xor esi, ecx + movdqa xmm2, oword [esp + $50] + rol eax, $5 + add edi, esi + xor ebp, ebx + xor ebx, ecx + add edi, eax + pshufd xmm3, xmm7, $EE + add edx, dword [esp + $C] + and ebp, ebx + xor ebx, ecx + ror eax, $7 + mov esi, edi + xor ebp, ebx + rol edi, $5 + add edx, ebp + xor esi, eax + xor eax, ebx + add edx, edi + add ecx, dword [esp + $10] + pxor xmm1, xmm5 + punpcklqdq xmm3, xmm0 + and esi, eax + xor eax, ebx + ror edi, $7 + pxor xmm1, xmm2 + movdqa oword [esp + $50], xmm5 + mov ebp, edx + xor esi, eax + rol edx, $5 + movdqa xmm5, xmm4 + add ecx, esi + paddd xmm4, xmm0 + xor ebp, edi + pxor xmm1, xmm3 + xor edi, eax + add ecx, edx + add ebx, dword [esp + $14] + and ebp, edi + movdqa xmm3, xmm1 + movdqa oword [esp], xmm4 + xor edi, eax + ror edx, $7 + mov esi, ecx + xor ebp, edi + rol ecx, $5 + pslld xmm1, $2 + add ebx, ebp + xor esi, edx + psrld xmm3, $1E + xor edx, edi + add ebx, ecx + add eax, dword [esp + $18] + and esi, edx + xor edx, edi + ror ecx, $7 + por xmm1, xmm3 + mov ebp, ebx + xor esi, edx + movdqa xmm3, oword [esp + $60] + rol ebx, $5 + add eax, esi + xor ebp, ecx + xor ecx, edx + add eax, ebx + pshufd xmm4, xmm0, $EE + add edi, dword [esp + $1C] + and ebp, ecx + xor ecx, edx + ror ebx, $7 + mov esi, eax + xor ebp, ecx + rol eax, $5 + add edi, ebp + xor esi, ebx + xor ebx, ecx + add edi, eax + add edx, dword [esp + $20] + pxor xmm2, xmm6 + punpcklqdq xmm4, xmm1 + and esi, ebx + xor ebx, ecx + ror eax, $7 + pxor xmm2, xmm3 + movdqa oword [esp + $60], xmm6 + mov ebp, edi + xor esi, ebx + rol edi, $5 + movdqa xmm6, xmm5 + add edx, esi + paddd xmm5, xmm1 + xor ebp, eax + pxor xmm2, xmm4 + xor eax, ebx + add edx, edi + add ecx, dword [esp + $24] + and ebp, eax + movdqa xmm4, xmm2 + movdqa oword [esp + $10], xmm5 + xor eax, ebx + ror edi, $7 + mov esi, edx + xor ebp, eax + rol edx, $5 + pslld xmm2, $2 + add ecx, ebp + xor esi, edi + psrld xmm4, $1E + xor edi, eax + add ecx, edx + add ebx, dword [esp + $28] + and esi, edi + xor edi, eax + ror edx, $7 + por xmm2, xmm4 + mov ebp, ecx + xor esi, edi + movdqa xmm4, oword [esp + $40] + rol ecx, $5 + add ebx, esi + xor ebp, edx + xor edx, edi + add ebx, ecx + pshufd xmm5, xmm1, $EE + add eax, dword [esp + $2C] + and ebp, edx + xor edx, edi + ror ecx, $7 + mov esi, ebx + xor ebp, edx + rol ebx, $5 + add eax, ebp + xor esi, edx + add eax, ebx + add edi, dword [esp + $30] + pxor xmm3, xmm7 + punpcklqdq xmm5, xmm2 + xor esi, ecx + mov ebp, eax + rol eax, $5 + pxor xmm3, xmm4 + movdqa oword [esp + $40], xmm7 + add edi, esi + xor ebp, ecx + movdqa xmm7, xmm6 + ror ebx, $7 + paddd xmm6, xmm2 + add edi, eax + pxor xmm3, xmm5 + add edx, dword [esp + $34] + xor ebp, ebx + mov esi, edi + rol edi, $5 + movdqa xmm5, xmm3 + movdqa oword [esp + $20], xmm6 + add edx, ebp + xor esi, ebx + ror eax, $7 + add edx, edi + pslld xmm3, $2 + add ecx, dword [esp + $38] + xor esi, eax + psrld xmm5, $1E + mov ebp, edx + rol edx, $5 + add ecx, esi + xor ebp, eax + ror edi, $7 + add ecx, edx + por xmm3, xmm5 + add ebx, dword [esp + $3C] + xor ebp, edi + mov esi, ecx + rol ecx, $5 + add ebx, ebp + xor esi, edi + ror edx, $7 + add ebx, ecx + add eax, dword [esp] + xor esi, edx + mov ebp, ebx + rol ebx, $5 + add eax, esi + xor ebp, edx + ror ecx, $7 + paddd xmm7, xmm3 + add eax, ebx + add edi, dword [esp + $4] + xor ebp, ecx + mov esi, eax + movdqa oword [esp + $30], xmm7 + rol eax, $5 + add edi, ebp + xor esi, ecx + ror ebx, $7 + add edi, eax + add edx, dword [esp + $8] + xor esi, ebx + mov ebp, edi + rol edi, $5 + add edx, esi + xor ebp, ebx + ror eax, $7 + add edx, edi + add ecx, dword [esp + $C] + xor ebp, eax + mov esi, edx + rol edx, $5 + add ecx, ebp + xor esi, eax + ror edi, $7 + add ecx, edx + mov ebp, dword [esp + $C4] + cmp ebp, dword [esp + $C8] + je @done + movdqa xmm7, oword [esp + $A0] + movdqa xmm6, oword [esp + $B0] + movdqu xmm0, oword [ebp + $0] + movdqu xmm1, oword [ebp + $10] + movdqu xmm2, oword [ebp + $20] + movdqu xmm3, oword [ebp + $30] + add ebp, $40 + db $66, $0F, $38, $00, $C6 // pshufb xmm0, xmm6 + mov dword [esp + $C4], ebp + movdqa oword [esp + $60], xmm7 + add ebx, dword [esp + $10] + xor esi, edi + mov ebp, ecx + rol ecx, $5 + add ebx, esi + xor ebp, edi + ror edx, $7 + db $66, $0F, $38, $00, $CE // pshufb xmm1, xmm6 + add ebx, ecx + add eax, dword [esp + $14] + xor ebp, edx + mov esi, ebx + paddd xmm0, xmm7 + rol ebx, $5 + add eax, ebp + xor esi, edx + ror ecx, $7 + movdqa oword [esp], xmm0 + add eax, ebx + add edi, dword [esp + $18] + xor esi, ecx + mov ebp, eax + psubd xmm0, xmm7 + rol eax, $5 + add edi, esi + xor ebp, ecx + ror ebx, $7 + add edi, eax + add edx, dword [esp + $1C] + xor ebp, ebx + mov esi, edi + rol edi, $5 + add edx, ebp + xor esi, ebx + ror eax, $7 + add edx, edi + add ecx, dword [esp + $20] + xor esi, eax + mov ebp, edx + rol edx, $5 + add ecx, esi + xor ebp, eax + ror edi, $7 + db $66, $0F, $38, $00, $D6 // pshufb xmm2, xmm6 + add ecx, edx + add ebx, dword [esp + $24] + xor ebp, edi + mov esi, ecx + paddd xmm1, xmm7 + rol ecx, $5 + add ebx, ebp + xor esi, edi + ror edx, $7 + movdqa oword [esp + $10], xmm1 + add ebx, ecx + add eax, dword [esp + $28] + xor esi, edx + mov ebp, ebx + psubd xmm1, xmm7 + rol ebx, $5 + add eax, esi + xor ebp, edx + ror ecx, $7 + add eax, ebx + add edi, dword [esp + $2C] + xor ebp, ecx + mov esi, eax + rol eax, $5 + add edi, ebp + xor esi, ecx + ror ebx, $7 + add edi, eax + add edx, dword [esp + $30] + xor esi, ebx + mov ebp, edi + rol edi, $5 + add edx, esi + xor ebp, ebx + ror eax, $7 + db $66, $0F, $38, $00, $DE // pshufb xmm3, xmm6 + add edx, edi + add ecx, dword [esp + $34] + xor ebp, eax + mov esi, edx + paddd xmm2, xmm7 + rol edx, $5 + add ecx, ebp + xor esi, eax + ror edi, $7 + movdqa oword [esp + $20], xmm2 + add ecx, edx + add ebx, dword [esp + $38] + xor esi, edi + mov ebp, ecx + psubd xmm2, xmm7 + rol ecx, $5 + add ebx, esi + xor ebp, edi + ror edx, $7 + add ebx, ecx + add eax, dword [esp + $3C] + xor ebp, edx + mov esi, ebx + rol ebx, $5 + add eax, ebp + ror ecx, $7 + add eax, ebx + mov ebp, dword [esp + $C0] + add eax, dword [ebp + $0] + add esi, dword [ebp + $4] + add ecx, dword [ebp + $8] + mov dword [ebp + $0], eax + add edx, dword [ebp + $C] + mov dword [ebp + $4], esi + add edi, dword [ebp + $10] + mov dword [ebp + $8], ecx + mov ebx, ecx + mov dword [ebp + $C], edx + xor ebx, edx + mov dword [ebp + $10], edi + mov ebp, esi + pshufd xmm4, xmm0, $EE + and esi, ebx + mov ebx, ebp + jmp @block_loop + +@done: + add ebx, dword [esp + $10] + xor esi, edi + mov ebp, ecx + rol ecx, $5 + add ebx, esi + xor ebp, edi + ror edx, $7 + add ebx, ecx + add eax, dword [esp + $14] + xor ebp, edx + mov esi, ebx + rol ebx, $5 + add eax, ebp + xor esi, edx + ror ecx, $7 + add eax, ebx + add edi, dword [esp + $18] + xor esi, ecx + mov ebp, eax + rol eax, $5 + add edi, esi + xor ebp, ecx + ror ebx, $7 + add edi, eax + add edx, dword [esp + $1C] + xor ebp, ebx + mov esi, edi + rol edi, $5 + add edx, ebp + xor esi, ebx + ror eax, $7 + add edx, edi + add ecx, dword [esp + $20] + xor esi, eax + mov ebp, edx + rol edx, $5 + add ecx, esi + xor ebp, eax + ror edi, $7 + add ecx, edx + add ebx, dword [esp + $24] + xor ebp, edi + mov esi, ecx + rol ecx, $5 + add ebx, ebp + xor esi, edi + ror edx, $7 + add ebx, ecx + add eax, dword [esp + $28] + xor esi, edx + mov ebp, ebx + rol ebx, $5 + add eax, esi + xor ebp, edx + ror ecx, $7 + add eax, ebx + add edi, dword [esp + $2C] + xor ebp, ecx + mov esi, eax + rol eax, $5 + add edi, ebp + xor esi, ecx + ror ebx, $7 + add edi, eax + add edx, dword [esp + $30] + xor esi, ebx + mov ebp, edi + rol edi, $5 + add edx, esi + xor ebp, ebx + ror eax, $7 + add edx, edi + add ecx, dword [esp + $34] + xor ebp, eax + mov esi, edx + rol edx, $5 + add ecx, ebp + xor esi, eax + ror edi, $7 + add ecx, edx + add ebx, dword [esp + $38] + xor esi, edi + mov ebp, ecx + rol ecx, $5 + add ebx, esi + xor ebp, edi + ror edx, $7 + add ebx, ecx + add eax, dword [esp + $3C] + xor ebp, edx + mov esi, ebx + rol ebx, $5 + add eax, ebp + ror ecx, $7 + add eax, ebx + mov ebp, dword [esp + $C0] + add eax, dword [ebp + $0] + mov esp, dword [esp + $CC] + add esi, dword [ebp + $4] + add ecx, dword [ebp + $8] + mov dword [ebp + $0], eax + add edx, dword [ebp + $C] + mov dword [ebp + $4], esi + add edi, dword [ebp + $10] + mov dword [ebp + $8], ecx + mov dword [ebp + $C], edx + mov dword [ebp + $10], edi + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc new file mode 100644 index 00000000..6cb779a9 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc @@ -0,0 +1,1193 @@ +// SSSE3 implementation of SHA-1 compress (SIMD schedule); SSE2 sibling: +// SHA1CompressSse2_x86_64.inc. The message schedule runs in SIMD +// (pxor/pshufd/psrldq/pslldq) interleaved with the GPR compression rounds, +// so the vector and integer units run in parallel. Unlike the SSE2 sibling +// the input byte-swaps are real pshufb against the mask kept resident in +// xmm12 (K_SHA1_Doubled + 128); the W-schedule uses no palignr. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K_SHA1 ptr (read 128-bit at a 32-byte +// stride, r14 = K + 64, so the duplicated halves are skipped; see +// K_SHA1_Doubled). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r14 in the local frame; r11 +// holds the caller rsp across the kernel for the unwind. +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha1-x86_64.pl (SSSE3 kernel). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + mov r11, rsp + push r9 + push rbx + push rbp + push r12 + push r13 + push r14 + push rdi + push rsi + mov rdi, rcx + mov rsi, rdx + mov rdx, r8 + lea rsp, [rsp - $A0] + and rsp, $FFFFFFFFFFFFFFC0 + mov r8, rdi + mov r9, rsi + mov r10, rdx + shl r10, $6 + add r10, r9 + mov r14, qword [r11 - $8] + movdqu xmm12, oword [r14 + $80] // byte-swap mask (K_SHA1_Doubled + 128) + add r14, $40 + mov eax, dword [r8] + mov ebx, dword [r8 + $4] + mov ecx, dword [r8 + $8] + mov edx, dword [r8 + $C] + mov esi, ebx + mov ebp, dword [r8 + $10] + mov edi, ecx + xor edi, edx + and esi, edi + movdqu xmm6, oword [r14 + $40] + movdqu xmm9, oword [r14 - $40] + movdqu xmm0, oword [r9] + movdqu xmm1, oword [r9 + $10] + movdqu xmm2, oword [r9 + $20] + movdqu xmm3, oword [r9 + $30] + db $66, $41, $0F, $38, $00, $C4 // pshufb xmm0, xmm12 + db $66, $41, $0F, $38, $00, $CC // pshufb xmm1, xmm12 + db $66, $41, $0F, $38, $00, $D4 // pshufb xmm2, xmm12 + add r9, $40 + paddd xmm0, xmm9 + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + paddd xmm1, xmm9 + paddd xmm2, xmm9 + movdqa oword [rsp], xmm0 + psubd xmm0, xmm9 + movdqa oword [rsp + $10], xmm1 + psubd xmm1, xmm9 + movdqa oword [rsp + $20], xmm2 + psubd xmm2, xmm9 + jmp @block_loop + +@block_loop: + ror ebx, $2 + pshufd xmm4, xmm0, $EE + xor esi, edx + movdqa xmm8, xmm3 + paddd xmm9, xmm3 + mov edi, eax + add ebp, dword [rsp] + punpcklqdq xmm4, xmm1 + xor ebx, ecx + rol eax, $5 + add ebp, esi + psrldq xmm8, $4 + and edi, ebx + xor ebx, ecx + pxor xmm4, xmm0 + add ebp, eax + ror eax, $7 + pxor xmm8, xmm2 + xor edi, ecx + mov esi, ebp + add edx, dword [rsp + $4] + pxor xmm4, xmm8 + xor eax, ebx + rol ebp, $5 + movdqa oword [rsp + $30], xmm9 + add edx, edi + and esi, eax + movdqa xmm10, xmm4 + xor eax, ebx + add edx, ebp + ror ebp, $7 + movdqa xmm8, xmm4 + xor esi, ebx + pslldq xmm10, $C + paddd xmm4, xmm4 + mov edi, edx + add ecx, dword [rsp + $8] + psrld xmm8, $1F + xor ebp, eax + rol edx, $5 + add ecx, esi + movdqa xmm9, xmm10 + and edi, ebp + xor ebp, eax + psrld xmm10, $1E + add ecx, edx + ror edx, $7 + por xmm4, xmm8 + xor edi, eax + mov esi, ecx + add ebx, dword [rsp + $C] + pslld xmm9, $2 + pxor xmm4, xmm10 + xor edx, ebp + movdqu xmm10, oword [r14 - $40] + rol ecx, $5 + add ebx, edi + and esi, edx + pxor xmm4, xmm9 + xor edx, ebp + add ebx, ecx + ror ecx, $7 + pshufd xmm5, xmm1, $EE + xor esi, ebp + movdqa xmm9, xmm4 + paddd xmm10, xmm4 + mov edi, ebx + add eax, dword [rsp + $10] + punpcklqdq xmm5, xmm2 + xor ecx, edx + rol ebx, $5 + add eax, esi + psrldq xmm9, $4 + and edi, ecx + xor ecx, edx + pxor xmm5, xmm1 + add eax, ebx + ror ebx, $7 + pxor xmm9, xmm3 + xor edi, edx + mov esi, eax + add ebp, dword [rsp + $14] + pxor xmm5, xmm9 + xor ebx, ecx + rol eax, $5 + movdqa oword [rsp], xmm10 + add ebp, edi + and esi, ebx + movdqa xmm8, xmm5 + xor ebx, ecx + add ebp, eax + ror eax, $7 + movdqa xmm9, xmm5 + xor esi, ecx + pslldq xmm8, $C + paddd xmm5, xmm5 + mov edi, ebp + add edx, dword [rsp + $18] + psrld xmm9, $1F + xor eax, ebx + rol ebp, $5 + add edx, esi + movdqa xmm10, xmm8 + and edi, eax + xor eax, ebx + psrld xmm8, $1E + add edx, ebp + ror ebp, $7 + por xmm5, xmm9 + xor edi, ebx + mov esi, edx + add ecx, dword [rsp + $1C] + pslld xmm10, $2 + pxor xmm5, xmm8 + xor ebp, eax + movdqu xmm8, oword [r14 - $20] + rol edx, $5 + add ecx, edi + and esi, ebp + pxor xmm5, xmm10 + xor ebp, eax + add ecx, edx + ror edx, $7 + pshufd xmm6, xmm2, $EE + xor esi, eax + movdqa xmm10, xmm5 + paddd xmm8, xmm5 + mov edi, ecx + add ebx, dword [rsp + $20] + punpcklqdq xmm6, xmm3 + xor edx, ebp + rol ecx, $5 + add ebx, esi + psrldq xmm10, $4 + and edi, edx + xor edx, ebp + pxor xmm6, xmm2 + add ebx, ecx + ror ecx, $7 + pxor xmm10, xmm4 + xor edi, ebp + mov esi, ebx + add eax, dword [rsp + $24] + pxor xmm6, xmm10 + xor ecx, edx + rol ebx, $5 + movdqa oword [rsp + $10], xmm8 + add eax, edi + and esi, ecx + movdqa xmm9, xmm6 + xor ecx, edx + add eax, ebx + ror ebx, $7 + movdqa xmm10, xmm6 + xor esi, edx + pslldq xmm9, $C + paddd xmm6, xmm6 + mov edi, eax + add ebp, dword [rsp + $28] + psrld xmm10, $1F + xor ebx, ecx + rol eax, $5 + add ebp, esi + movdqa xmm8, xmm9 + and edi, ebx + xor ebx, ecx + psrld xmm9, $1E + add ebp, eax + ror eax, $7 + por xmm6, xmm10 + xor edi, ecx + mov esi, ebp + add edx, dword [rsp + $2C] + pslld xmm8, $2 + pxor xmm6, xmm9 + xor eax, ebx + movdqu xmm9, oword [r14 - $20] + rol ebp, $5 + add edx, edi + and esi, eax + pxor xmm6, xmm8 + xor eax, ebx + add edx, ebp + ror ebp, $7 + pshufd xmm7, xmm3, $EE + xor esi, ebx + movdqa xmm8, xmm6 + paddd xmm9, xmm6 + mov edi, edx + add ecx, dword [rsp + $30] + punpcklqdq xmm7, xmm4 + xor ebp, eax + rol edx, $5 + add ecx, esi + psrldq xmm8, $4 + and edi, ebp + xor ebp, eax + pxor xmm7, xmm3 + add ecx, edx + ror edx, $7 + pxor xmm8, xmm5 + xor edi, eax + mov esi, ecx + add ebx, dword [rsp + $34] + pxor xmm7, xmm8 + xor edx, ebp + rol ecx, $5 + movdqa oword [rsp + $20], xmm9 + add ebx, edi + and esi, edx + movdqa xmm10, xmm7 + xor edx, ebp + add ebx, ecx + ror ecx, $7 + movdqa xmm8, xmm7 + xor esi, ebp + pslldq xmm10, $C + paddd xmm7, xmm7 + mov edi, ebx + add eax, dword [rsp + $38] + psrld xmm8, $1F + xor ecx, edx + rol ebx, $5 + add eax, esi + movdqa xmm9, xmm10 + and edi, ecx + xor ecx, edx + psrld xmm10, $1E + add eax, ebx + ror ebx, $7 + por xmm7, xmm8 + xor edi, edx + mov esi, eax + add ebp, dword [rsp + $3C] + pslld xmm9, $2 + pxor xmm7, xmm10 + xor ebx, ecx + movdqu xmm10, oword [r14 - $20] + rol eax, $5 + add ebp, edi + and esi, ebx + pxor xmm7, xmm9 + pshufd xmm9, xmm6, $EE + xor ebx, ecx + add ebp, eax + ror eax, $7 + pxor xmm0, xmm4 + xor esi, ecx + mov edi, ebp + add edx, dword [rsp] + punpcklqdq xmm9, xmm7 + xor eax, ebx + rol ebp, $5 + pxor xmm0, xmm1 + add edx, esi + and edi, eax + movdqa xmm8, xmm10 + xor eax, ebx + paddd xmm10, xmm7 + add edx, ebp + pxor xmm0, xmm9 + ror ebp, $7 + xor edi, ebx + mov esi, edx + add ecx, dword [rsp + $4] + movdqa xmm9, xmm0 + xor ebp, eax + rol edx, $5 + movdqa oword [rsp + $30], xmm10 + add ecx, edi + and esi, ebp + xor ebp, eax + pslld xmm0, $2 + add ecx, edx + ror edx, $7 + psrld xmm9, $1E + xor esi, eax + mov edi, ecx + add ebx, dword [rsp + $8] + por xmm0, xmm9 + xor edx, ebp + rol ecx, $5 + pshufd xmm10, xmm7, $EE + add ebx, esi + and edi, edx + xor edx, ebp + add ebx, ecx + add eax, dword [rsp + $C] + xor edi, ebp + mov esi, ebx + rol ebx, $5 + add eax, edi + xor esi, edx + ror ecx, $7 + add eax, ebx + pxor xmm1, xmm5 + add ebp, dword [rsp + $10] + xor esi, ecx + punpcklqdq xmm10, xmm0 + mov edi, eax + rol eax, $5 + pxor xmm1, xmm2 + add ebp, esi + xor edi, ecx + movdqa xmm9, xmm8 + ror ebx, $7 + paddd xmm8, xmm0 + add ebp, eax + pxor xmm1, xmm10 + add edx, dword [rsp + $14] + xor edi, ebx + mov esi, ebp + rol ebp, $5 + movdqa xmm10, xmm1 + add edx, edi + xor esi, ebx + movdqa oword [rsp], xmm8 + ror eax, $7 + add edx, ebp + add ecx, dword [rsp + $18] + pslld xmm1, $2 + xor esi, eax + mov edi, edx + psrld xmm10, $1E + rol edx, $5 + add ecx, esi + xor edi, eax + ror ebp, $7 + por xmm1, xmm10 + add ecx, edx + add ebx, dword [rsp + $1C] + pshufd xmm8, xmm0, $EE + xor edi, ebp + mov esi, ecx + rol ecx, $5 + add ebx, edi + xor esi, ebp + ror edx, $7 + add ebx, ecx + pxor xmm2, xmm6 + add eax, dword [rsp + $20] + xor esi, edx + punpcklqdq xmm8, xmm1 + mov edi, ebx + rol ebx, $5 + pxor xmm2, xmm3 + add eax, esi + xor edi, edx + movdqu xmm10, oword [r14] + ror ecx, $7 + paddd xmm9, xmm1 + add eax, ebx + pxor xmm2, xmm8 + add ebp, dword [rsp + $24] + xor edi, ecx + mov esi, eax + rol eax, $5 + movdqa xmm8, xmm2 + add ebp, edi + xor esi, ecx + movdqa oword [rsp + $10], xmm9 + ror ebx, $7 + add ebp, eax + add edx, dword [rsp + $28] + pslld xmm2, $2 + xor esi, ebx + mov edi, ebp + psrld xmm8, $1E + rol ebp, $5 + add edx, esi + xor edi, ebx + ror eax, $7 + por xmm2, xmm8 + add edx, ebp + add ecx, dword [rsp + $2C] + pshufd xmm9, xmm1, $EE + xor edi, eax + mov esi, edx + rol edx, $5 + add ecx, edi + xor esi, eax + ror ebp, $7 + add ecx, edx + pxor xmm3, xmm7 + add ebx, dword [rsp + $30] + xor esi, ebp + punpcklqdq xmm9, xmm2 + mov edi, ecx + rol ecx, $5 + pxor xmm3, xmm4 + add ebx, esi + xor edi, ebp + movdqa xmm8, xmm10 + ror edx, $7 + paddd xmm10, xmm2 + add ebx, ecx + pxor xmm3, xmm9 + add eax, dword [rsp + $34] + xor edi, edx + mov esi, ebx + rol ebx, $5 + movdqa xmm9, xmm3 + add eax, edi + xor esi, edx + movdqa oword [rsp + $20], xmm10 + ror ecx, $7 + add eax, ebx + add ebp, dword [rsp + $38] + pslld xmm3, $2 + xor esi, ecx + mov edi, eax + psrld xmm9, $1E + rol eax, $5 + add ebp, esi + xor edi, ecx + ror ebx, $7 + por xmm3, xmm9 + add ebp, eax + add edx, dword [rsp + $3C] + pshufd xmm10, xmm2, $EE + xor edi, ebx + mov esi, ebp + rol ebp, $5 + add edx, edi + xor esi, ebx + ror eax, $7 + add edx, ebp + pxor xmm4, xmm0 + add ecx, dword [rsp] + xor esi, eax + punpcklqdq xmm10, xmm3 + mov edi, edx + rol edx, $5 + pxor xmm4, xmm5 + add ecx, esi + xor edi, eax + movdqa xmm9, xmm8 + ror ebp, $7 + paddd xmm8, xmm3 + add ecx, edx + pxor xmm4, xmm10 + add ebx, dword [rsp + $4] + xor edi, ebp + mov esi, ecx + rol ecx, $5 + movdqa xmm10, xmm4 + add ebx, edi + xor esi, ebp + movdqa oword [rsp + $30], xmm8 + ror edx, $7 + add ebx, ecx + add eax, dword [rsp + $8] + pslld xmm4, $2 + xor esi, edx + mov edi, ebx + psrld xmm10, $1E + rol ebx, $5 + add eax, esi + xor edi, edx + ror ecx, $7 + por xmm4, xmm10 + add eax, ebx + add ebp, dword [rsp + $C] + pshufd xmm8, xmm3, $EE + xor edi, ecx + mov esi, eax + rol eax, $5 + add ebp, edi + xor esi, ecx + ror ebx, $7 + add ebp, eax + pxor xmm5, xmm1 + add edx, dword [rsp + $10] + xor esi, ebx + punpcklqdq xmm8, xmm4 + mov edi, ebp + rol ebp, $5 + pxor xmm5, xmm6 + add edx, esi + xor edi, ebx + movdqa xmm10, xmm9 + ror eax, $7 + paddd xmm9, xmm4 + add edx, ebp + pxor xmm5, xmm8 + add ecx, dword [rsp + $14] + xor edi, eax + mov esi, edx + rol edx, $5 + movdqa xmm8, xmm5 + add ecx, edi + xor esi, eax + movdqa oword [rsp], xmm9 + ror ebp, $7 + add ecx, edx + add ebx, dword [rsp + $18] + pslld xmm5, $2 + xor esi, ebp + mov edi, ecx + psrld xmm8, $1E + rol ecx, $5 + add ebx, esi + xor edi, ebp + ror edx, $7 + por xmm5, xmm8 + add ebx, ecx + add eax, dword [rsp + $1C] + pshufd xmm9, xmm4, $EE + ror ecx, $7 + mov esi, ebx + xor edi, edx + rol ebx, $5 + add eax, edi + xor esi, ecx + xor ecx, edx + add eax, ebx + pxor xmm6, xmm2 + add ebp, dword [rsp + $20] + and esi, ecx + xor ecx, edx + ror ebx, $7 + punpcklqdq xmm9, xmm5 + mov edi, eax + xor esi, ecx + pxor xmm6, xmm7 + rol eax, $5 + add ebp, esi + movdqa xmm8, xmm10 + xor edi, ebx + paddd xmm10, xmm5 + xor ebx, ecx + pxor xmm6, xmm9 + add ebp, eax + add edx, dword [rsp + $24] + and edi, ebx + xor ebx, ecx + ror eax, $7 + movdqa xmm9, xmm6 + mov esi, ebp + xor edi, ebx + movdqa oword [rsp + $10], xmm10 + rol ebp, $5 + add edx, edi + xor esi, eax + pslld xmm6, $2 + xor eax, ebx + add edx, ebp + psrld xmm9, $1E + add ecx, dword [rsp + $28] + and esi, eax + xor eax, ebx + por xmm6, xmm9 + ror ebp, $7 + mov edi, edx + xor esi, eax + rol edx, $5 + pshufd xmm10, xmm5, $EE + add ecx, esi + xor edi, ebp + xor ebp, eax + add ecx, edx + add ebx, dword [rsp + $2C] + and edi, ebp + xor ebp, eax + ror edx, $7 + mov esi, ecx + xor edi, ebp + rol ecx, $5 + add ebx, edi + xor esi, edx + xor edx, ebp + add ebx, ecx + pxor xmm7, xmm3 + add eax, dword [rsp + $30] + and esi, edx + xor edx, ebp + ror ecx, $7 + punpcklqdq xmm10, xmm6 + mov edi, ebx + xor esi, edx + pxor xmm7, xmm0 + rol ebx, $5 + add eax, esi + movdqu xmm9, oword [r14 + $20] + xor edi, ecx + paddd xmm8, xmm6 + xor ecx, edx + pxor xmm7, xmm10 + add eax, ebx + add ebp, dword [rsp + $34] + and edi, ecx + xor ecx, edx + ror ebx, $7 + movdqa xmm10, xmm7 + mov esi, eax + xor edi, ecx + movdqa oword [rsp + $20], xmm8 + rol eax, $5 + add ebp, edi + xor esi, ebx + pslld xmm7, $2 + xor ebx, ecx + add ebp, eax + psrld xmm10, $1E + add edx, dword [rsp + $38] + and esi, ebx + xor ebx, ecx + por xmm7, xmm10 + ror eax, $7 + mov edi, ebp + xor esi, ebx + rol ebp, $5 + pshufd xmm8, xmm6, $EE + add edx, esi + xor edi, eax + xor eax, ebx + add edx, ebp + add ecx, dword [rsp + $3C] + and edi, eax + xor eax, ebx + ror ebp, $7 + mov esi, edx + xor edi, eax + rol edx, $5 + add ecx, edi + xor esi, ebp + xor ebp, eax + add ecx, edx + pxor xmm0, xmm4 + add ebx, dword [rsp] + and esi, ebp + xor ebp, eax + ror edx, $7 + punpcklqdq xmm8, xmm7 + mov edi, ecx + xor esi, ebp + pxor xmm0, xmm1 + rol ecx, $5 + add ebx, esi + movdqa xmm10, xmm9 + xor edi, edx + paddd xmm9, xmm7 + xor edx, ebp + pxor xmm0, xmm8 + add ebx, ecx + add eax, dword [rsp + $4] + and edi, edx + xor edx, ebp + ror ecx, $7 + movdqa xmm8, xmm0 + mov esi, ebx + xor edi, edx + movdqa oword [rsp + $30], xmm9 + rol ebx, $5 + add eax, edi + xor esi, ecx + pslld xmm0, $2 + xor ecx, edx + add eax, ebx + psrld xmm8, $1E + add ebp, dword [rsp + $8] + and esi, ecx + xor ecx, edx + por xmm0, xmm8 + ror ebx, $7 + mov edi, eax + xor esi, ecx + rol eax, $5 + pshufd xmm9, xmm7, $EE + add ebp, esi + xor edi, ebx + xor ebx, ecx + add ebp, eax + add edx, dword [rsp + $C] + and edi, ebx + xor ebx, ecx + ror eax, $7 + mov esi, ebp + xor edi, ebx + rol ebp, $5 + add edx, edi + xor esi, eax + xor eax, ebx + add edx, ebp + pxor xmm1, xmm5 + add ecx, dword [rsp + $10] + and esi, eax + xor eax, ebx + ror ebp, $7 + punpcklqdq xmm9, xmm0 + mov edi, edx + xor esi, eax + pxor xmm1, xmm2 + rol edx, $5 + add ecx, esi + movdqa xmm8, xmm10 + xor edi, ebp + paddd xmm10, xmm0 + xor ebp, eax + pxor xmm1, xmm9 + add ecx, edx + add ebx, dword [rsp + $14] + and edi, ebp + xor ebp, eax + ror edx, $7 + movdqa xmm9, xmm1 + mov esi, ecx + xor edi, ebp + movdqa oword [rsp], xmm10 + rol ecx, $5 + add ebx, edi + xor esi, edx + pslld xmm1, $2 + xor edx, ebp + add ebx, ecx + psrld xmm9, $1E + add eax, dword [rsp + $18] + and esi, edx + xor edx, ebp + por xmm1, xmm9 + ror ecx, $7 + mov edi, ebx + xor esi, edx + rol ebx, $5 + pshufd xmm10, xmm0, $EE + add eax, esi + xor edi, ecx + xor ecx, edx + add eax, ebx + add ebp, dword [rsp + $1C] + and edi, ecx + xor ecx, edx + ror ebx, $7 + mov esi, eax + xor edi, ecx + rol eax, $5 + add ebp, edi + xor esi, ebx + xor ebx, ecx + add ebp, eax + pxor xmm2, xmm6 + add edx, dword [rsp + $20] + and esi, ebx + xor ebx, ecx + ror eax, $7 + punpcklqdq xmm10, xmm1 + mov edi, ebp + xor esi, ebx + pxor xmm2, xmm3 + rol ebp, $5 + add edx, esi + movdqa xmm9, xmm8 + xor edi, eax + paddd xmm8, xmm1 + xor eax, ebx + pxor xmm2, xmm10 + add edx, ebp + add ecx, dword [rsp + $24] + and edi, eax + xor eax, ebx + ror ebp, $7 + movdqa xmm10, xmm2 + mov esi, edx + xor edi, eax + movdqa oword [rsp + $10], xmm8 + rol edx, $5 + add ecx, edi + xor esi, ebp + pslld xmm2, $2 + xor ebp, eax + add ecx, edx + psrld xmm10, $1E + add ebx, dword [rsp + $28] + and esi, ebp + xor ebp, eax + por xmm2, xmm10 + ror edx, $7 + mov edi, ecx + xor esi, ebp + rol ecx, $5 + pshufd xmm8, xmm1, $EE + add ebx, esi + xor edi, edx + xor edx, ebp + add ebx, ecx + add eax, dword [rsp + $2C] + and edi, edx + xor edx, ebp + ror ecx, $7 + mov esi, ebx + xor edi, edx + rol ebx, $5 + add eax, edi + xor esi, edx + add eax, ebx + pxor xmm3, xmm7 + add ebp, dword [rsp + $30] + xor esi, ecx + punpcklqdq xmm8, xmm2 + mov edi, eax + rol eax, $5 + pxor xmm3, xmm4 + add ebp, esi + xor edi, ecx + movdqa xmm10, xmm9 + ror ebx, $7 + paddd xmm9, xmm2 + add ebp, eax + pxor xmm3, xmm8 + add edx, dword [rsp + $34] + xor edi, ebx + mov esi, ebp + rol ebp, $5 + movdqa xmm8, xmm3 + add edx, edi + xor esi, ebx + movdqa oword [rsp + $20], xmm9 + ror eax, $7 + add edx, ebp + add ecx, dword [rsp + $38] + pslld xmm3, $2 + xor esi, eax + mov edi, edx + psrld xmm8, $1E + rol edx, $5 + add ecx, esi + xor edi, eax + ror ebp, $7 + por xmm3, xmm8 + add ecx, edx + add ebx, dword [rsp + $3C] + xor edi, ebp + mov esi, ecx + rol ecx, $5 + add ebx, edi + xor esi, ebp + ror edx, $7 + add ebx, ecx + add eax, dword [rsp] + xor esi, edx + mov edi, ebx + rol ebx, $5 + paddd xmm10, xmm3 + add eax, esi + xor edi, edx + movdqa oword [rsp + $30], xmm10 + ror ecx, $7 + add eax, ebx + add ebp, dword [rsp + $4] + xor edi, ecx + mov esi, eax + rol eax, $5 + add ebp, edi + xor esi, ecx + ror ebx, $7 + add ebp, eax + add edx, dword [rsp + $8] + xor esi, ebx + mov edi, ebp + rol ebp, $5 + add edx, esi + xor edi, ebx + ror eax, $7 + add edx, ebp + add ecx, dword [rsp + $C] + xor edi, eax + mov esi, edx + rol edx, $5 + add ecx, edi + xor esi, eax + ror ebp, $7 + add ecx, edx + cmp r9, r10 + je @done + movdqu xmm6, oword [r14 + $40] + movdqu xmm9, oword [r14 - $40] + movdqu xmm0, oword [r9] + movdqu xmm1, oword [r9 + $10] + movdqu xmm2, oword [r9 + $20] + movdqu xmm3, oword [r9 + $30] + db $66, $41, $0F, $38, $00, $C4 // pshufb xmm0, xmm12 + add r9, $40 + add ebx, dword [rsp + $10] + xor esi, ebp + mov edi, ecx + db $66, $41, $0F, $38, $00, $CC // pshufb xmm1, xmm12 + rol ecx, $5 + add ebx, esi + xor edi, ebp + ror edx, $7 + paddd xmm0, xmm9 + add ebx, ecx + add eax, dword [rsp + $14] + xor edi, edx + mov esi, ebx + movdqa oword [rsp], xmm0 + rol ebx, $5 + add eax, edi + xor esi, edx + ror ecx, $7 + psubd xmm0, xmm9 + add eax, ebx + add ebp, dword [rsp + $18] + xor esi, ecx + mov edi, eax + rol eax, $5 + add ebp, esi + xor edi, ecx + ror ebx, $7 + add ebp, eax + add edx, dword [rsp + $1C] + xor edi, ebx + mov esi, ebp + rol ebp, $5 + add edx, edi + xor esi, ebx + ror eax, $7 + add edx, ebp + add ecx, dword [rsp + $20] + xor esi, eax + mov edi, edx + db $66, $41, $0F, $38, $00, $D4 // pshufb xmm2, xmm12 + rol edx, $5 + add ecx, esi + xor edi, eax + ror ebp, $7 + paddd xmm1, xmm9 + add ecx, edx + add ebx, dword [rsp + $24] + xor edi, ebp + mov esi, ecx + movdqa oword [rsp + $10], xmm1 + rol ecx, $5 + add ebx, edi + xor esi, ebp + ror edx, $7 + psubd xmm1, xmm9 + add ebx, ecx + add eax, dword [rsp + $28] + xor esi, edx + mov edi, ebx + rol ebx, $5 + add eax, esi + xor edi, edx + ror ecx, $7 + add eax, ebx + add ebp, dword [rsp + $2C] + xor edi, ecx + mov esi, eax + rol eax, $5 + add ebp, edi + xor esi, ecx + ror ebx, $7 + add ebp, eax + add edx, dword [rsp + $30] + xor esi, ebx + mov edi, ebp + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + rol ebp, $5 + add edx, esi + xor edi, ebx + ror eax, $7 + paddd xmm2, xmm9 + add edx, ebp + add ecx, dword [rsp + $34] + xor edi, eax + mov esi, edx + movdqa oword [rsp + $20], xmm2 + rol edx, $5 + add ecx, edi + xor esi, eax + ror ebp, $7 + psubd xmm2, xmm9 + add ecx, edx + add ebx, dword [rsp + $38] + xor esi, ebp + mov edi, ecx + rol ecx, $5 + add ebx, esi + xor edi, ebp + ror edx, $7 + add ebx, ecx + add eax, dword [rsp + $3C] + xor edi, edx + mov esi, ebx + rol ebx, $5 + add eax, edi + ror ecx, $7 + add eax, ebx + add eax, dword [r8] + add esi, dword [r8 + $4] + add ecx, dword [r8 + $8] + add edx, dword [r8 + $C] + mov dword [r8], eax + add ebp, dword [r8 + $10] + mov dword [r8 + $4], esi + mov ebx, esi + mov dword [r8 + $8], ecx + mov edi, ecx + mov dword [r8 + $C], edx + xor edi, edx + mov dword [r8 + $10], ebp + and esi, edi + jmp @block_loop + +@done: + add ebx, dword [rsp + $10] + xor esi, ebp + mov edi, ecx + rol ecx, $5 + add ebx, esi + xor edi, ebp + ror edx, $7 + add ebx, ecx + add eax, dword [rsp + $14] + xor edi, edx + mov esi, ebx + rol ebx, $5 + add eax, edi + xor esi, edx + ror ecx, $7 + add eax, ebx + add ebp, dword [rsp + $18] + xor esi, ecx + mov edi, eax + rol eax, $5 + add ebp, esi + xor edi, ecx + ror ebx, $7 + add ebp, eax + add edx, dword [rsp + $1C] + xor edi, ebx + mov esi, ebp + rol ebp, $5 + add edx, edi + xor esi, ebx + ror eax, $7 + add edx, ebp + add ecx, dword [rsp + $20] + xor esi, eax + mov edi, edx + rol edx, $5 + add ecx, esi + xor edi, eax + ror ebp, $7 + add ecx, edx + add ebx, dword [rsp + $24] + xor edi, ebp + mov esi, ecx + rol ecx, $5 + add ebx, edi + xor esi, ebp + ror edx, $7 + add ebx, ecx + add eax, dword [rsp + $28] + xor esi, edx + mov edi, ebx + rol ebx, $5 + add eax, esi + xor edi, edx + ror ecx, $7 + add eax, ebx + add ebp, dword [rsp + $2C] + xor edi, ecx + mov esi, eax + rol eax, $5 + add ebp, edi + xor esi, ecx + ror ebx, $7 + add ebp, eax + add edx, dword [rsp + $30] + xor esi, ebx + mov edi, ebp + rol ebp, $5 + add edx, esi + xor edi, ebx + ror eax, $7 + add edx, ebp + add ecx, dword [rsp + $34] + xor edi, eax + mov esi, edx + rol edx, $5 + add ecx, edi + xor esi, eax + ror ebp, $7 + add ecx, edx + add ebx, dword [rsp + $38] + xor esi, ebp + mov edi, ecx + rol ecx, $5 + add ebx, esi + xor edi, ebp + ror edx, $7 + add ebx, ecx + add eax, dword [rsp + $3C] + xor edi, edx + mov esi, ebx + rol ebx, $5 + add eax, edi + ror ecx, $7 + add eax, ebx + add eax, dword [r8] + add esi, dword [r8 + $4] + add ecx, dword [r8 + $8] + mov dword [r8], eax + add edx, dword [r8 + $C] + mov dword [r8 + $4], esi + add ebp, dword [r8 + $10] + mov dword [r8 + $8], ecx + mov dword [r8 + $C], edx + mov dword [r8 + $10], ebp + mov rsi, qword [r11 - $40] + mov rdi, qword [r11 - $38] + mov r14, qword [r11 - $30] + mov r13, qword [r11 - $28] + mov r12, qword [r11 - $20] + mov rbp, qword [r11 - $18] + mov rbx, qword [r11 - $10] + lea rsp, [r11] + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc index 0eb5c276..1ab6c893 100644 --- a/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc @@ -1,20 +1,20 @@ -// SHA-256 AVX2 two-block implementation (VEX-256), ported from OpenSSL's -// sha512-x86_64.pl (CRYPTOGAMS; that generator emits both the SHA-256 and the -// SHA-512 kernels). Two message blocks are scheduled together in 256-bit lanes; -// the compression rounds stay serial (as SHA-256 requires), so scheduling two -// blocks at once is what makes this faster than the single-block AVX path. AVX2 -// and BMI2 (rorx/andn) instructions are db-encoded for broad assembler +// AVX2 implementation of SHA-256 compress (VEX-256, two-block). Two message +// blocks are scheduled together in 256-bit lanes; the compression rounds +// stay serial (as SHA-256 requires), so scheduling two blocks at once is +// what makes this faster than the single-block AVX path. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K256 ptr (each 128-bit K256 quadruple +// stored twice so one table feeds both 256-bit lanes, with the byte-swap +// and two message-schedule masks appended; see K256_Doubled; the masks +// are read unaligned, so the Pascal const needs no special alignment). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// AVX2 and BMI2 (rorx/andn) instructions are db-encoded for broad assembler // compatibility (every AVX2 CPU also provides BMI2). -// Expects MS x64 ABI (after HlpSimdProc4Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = doubled-K256 ptr (each 128-bit K256 quadruple stored -// twice so one table feeds both 256-bit lanes, with the byte-swap and two -// message-schedule masks appended; see K256_Doubled). The masks are read -// unaligned, so the Pascal const needs no special alignment. -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// Reference: OpenSSL CRYPTOGAMS sha512-x86_64.pl (AVX2 kernel; that +// generator emits both the SHA-256 and the SHA-512 kernels). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} mov rax, rsp push r9 @@ -1252,4 +1252,4 @@ lea rsp, [rsi] mov rsi, rax - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx_i386.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx_i386.inc new file mode 100644 index 00000000..346c68c2 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx_i386.inc @@ -0,0 +1,1208 @@ +// AVX (VEX-128) implementation of SHA-256 compress (SIMD schedule; IA-32); +// the Avx name states the ISA requirement: VEX-128 encodings only - there +// is no i386 AVX2 SHA-256 upstream. The 3-operand VEX forms eliminate the +// SSSE3 kernel's register-copy movdqa traffic; the byte-swap mask stays +// resident in xmm7. K256 is read from K256_Doubled at a 32-byte stride +// (the walk and offsets are doubled relative to the original; VEX memory +// operands are read unaligned, so the Pascal const needs no special +// alignment). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = K256_Doubled ptr. +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K walk +// pointer. Frame and slot layout (ctx@96/data@100/end@104/savedsp@108) +// mirror the SSE2/SSSE3 siblings. +// AVX instructions are db-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha256-586.pl (AVX section). + + push ebp + mov ebp, eax + mov eax, edi + mov edi, esi + mov esi, ebx + mov ebx, esp + sub esp, $10 + and esp, $FFFFFFC0 + shl eax, $6 + add eax, edi + mov dword ptr [esp], esi + mov dword ptr [esp + $4], edi + mov dword ptr [esp + $8], eax + mov dword ptr [esp + $C], ebx + sub esp, $60 + db $C5, $FC, $77 // vzeroall + mov eax, dword ptr [esi] + mov ebx, dword ptr [esi + $4] + mov ecx, dword ptr [esi + $8] + mov edi, dword ptr [esi + $C] + mov dword ptr [esp + $4], ebx + xor ebx, ecx + mov dword ptr [esp + $8], ecx + mov dword ptr [esp + $C], edi + mov edx, dword ptr [esi + $10] + mov edi, dword ptr [esi + $14] + mov ecx, dword ptr [esi + $18] + mov esi, dword ptr [esi + $1C] + mov dword ptr [esp + $14], edi + mov edi, dword ptr [esp + $64] + mov dword ptr [esp + $18], ecx + mov dword ptr [esp + $1C], esi + db $C5, $FA, $6F, $BD, $00, $02, $00, $00 // vmovdqu xmm7, oword [ebp + $200] + jmp @grand_avx_014 + +@grand_avx_014: + db $C5, $FA, $6F, $07 // vmovdqu xmm0, oword [edi] + db $C5, $FA, $6F, $4F, $10 // vmovdqu xmm1, oword [edi + $10] + db $C5, $FA, $6F, $57, $20 // vmovdqu xmm2, oword [edi + $20] + db $C5, $FA, $6F, $5F, $30 // vmovdqu xmm3, oword [edi + $30] + add edi, $40 + db $C4, $E2, $79, $00, $C7 // vpshufb xmm0, xmm0, xmm7 + mov dword ptr [esp + $64], edi + db $C4, $E2, $71, $00, $CF // vpshufb xmm1, xmm1, xmm7 + db $C4, $E2, $69, $00, $D7 // vpshufb xmm2, xmm2, xmm7 + db $C5, $F9, $FE, $65, $00 // vpaddd xmm4, xmm0, oword [ebp + $0] + db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7 + db $C5, $F1, $FE, $6D, $20 // vpaddd xmm5, xmm1, oword [ebp + $20] + db $C5, $E9, $FE, $75, $40 // vpaddd xmm6, xmm2, oword [ebp + $40] + db $C5, $E1, $FE, $7D, $60 // vpaddd xmm7, xmm3, oword [ebp + $60] + db $C5, $F9, $7F, $64, $24, $20 // vmovdqa oword [esp + $20], xmm4 + db $C5, $F9, $7F, $6C, $24, $30 // vmovdqa oword [esp + $30], xmm5 + db $C5, $F9, $7F, $74, $24, $40 // vmovdqa oword [esp + $40], xmm6 + db $C5, $F9, $7F, $7C, $24, $50 // vmovdqa oword [esp + $50], xmm7 + jmp @avx_00_47_015 + +@avx_00_47_015: + add ebp, $80 + db $C4, $E3, $71, $0F, $E0, $04 // vpalignr xmm4, xmm1, xmm0, $4 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $14] + db $C4, $E3, $61, $0F, $FA, $04 // vpalignr xmm7, xmm3, xmm2, $4 + xor edx, ecx + mov edi, dword ptr [esp + $18] + xor esi, edi + db $C5, $C9, $72, $D4, $07 // vpsrld xmm6, xmm4, $7 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $10], ecx + db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $C1, $72, $D4, $03 // vpsrld xmm7, xmm4, $3 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $4] + db $C5, $D1, $72, $F4, $0E // vpslld xmm5, xmm4, $E + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp], eax + db $C5, $C1, $EF, $E6 // vpxor xmm4, xmm7, xmm6 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $1C] + db $C5, $F9, $70, $FB, $FA // vpshufd xmm7, xmm3, $FA + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $C9, $72, $D6, $0B // vpsrld xmm6, xmm6, $B + add edx, dword ptr [esp + $20] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + add ebx, edx + add edx, dword ptr [esp + $C] + add ebx, ecx + db $C5, $D1, $72, $F5, $0B // vpslld xmm5, xmm5, $B + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $10] + db $C5, $D9, $EF, $E6 // vpxor xmm4, xmm4, xmm6 + xor edx, ecx + mov edi, dword ptr [esp + $14] + xor esi, edi + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $C], ecx + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp] + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $1C], ebx + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $18] + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + add edx, dword ptr [esp + $24] + xor eax, edi + shrd ecx, ecx, $2 + db $C5, $F9, $70, $FE, $84 // vpshufd xmm7, xmm6, $84 + add eax, edx + add edx, dword ptr [esp + $8] + add eax, ecx + db $C5, $C1, $73, $DF, $08 // vpsrldq xmm7, xmm7, $8 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $C] + db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 + xor edx, ecx + mov edi, dword ptr [esp + $10] + xor esi, edi + db $C5, $F9, $70, $F8, $50 // vpshufd xmm7, xmm0, $50 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $8], ecx + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $1C] + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $18], eax + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $14] + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $F9, $70, $FE, $E8 // vpshufd xmm7, xmm6, $E8 + add edx, dword ptr [esp + $28] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $C1, $73, $FF, $08 // vpslldq xmm7, xmm7, $8 + add ebx, edx + add edx, dword ptr [esp + $4] + add ebx, ecx + db $C5, $F9, $FE, $C7 // vpaddd xmm0, xmm0, xmm7 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $8] + db $C5, $F9, $FE, $75, $00 // vpaddd xmm6, xmm0, oword [ebp + $0] + xor edx, ecx + mov edi, dword ptr [esp + $C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $4], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $18] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $10] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $2C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp] + add eax, ecx + db $C5, $F9, $7F, $74, $24, $20 // vmovdqa oword [esp + $20], xmm6 + db $C4, $E3, $69, $0F, $E1, $04 // vpalignr xmm4, xmm2, xmm1, $4 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $4] + db $C4, $E3, $79, $0F, $FB, $04 // vpalignr xmm7, xmm0, xmm3, $4 + xor edx, ecx + mov edi, dword ptr [esp + $8] + xor esi, edi + db $C5, $C9, $72, $D4, $07 // vpsrld xmm6, xmm4, $7 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp], ecx + db $C5, $F1, $FE, $CF // vpaddd xmm1, xmm1, xmm7 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $C1, $72, $D4, $03 // vpsrld xmm7, xmm4, $3 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $14] + db $C5, $D1, $72, $F4, $0E // vpslld xmm5, xmm4, $E + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $10], eax + db $C5, $C1, $EF, $E6 // vpxor xmm4, xmm7, xmm6 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $C] + db $C5, $F9, $70, $F8, $FA // vpshufd xmm7, xmm0, $FA + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $C9, $72, $D6, $0B // vpsrld xmm6, xmm6, $B + add edx, dword ptr [esp + $30] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + add ebx, edx + add edx, dword ptr [esp + $1C] + add ebx, ecx + db $C5, $D1, $72, $F5, $0B // vpslld xmm5, xmm5, $B + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp] + db $C5, $D9, $EF, $E6 // vpxor xmm4, xmm4, xmm6 + xor edx, ecx + mov edi, dword ptr [esp + $4] + xor esi, edi + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $1C], ecx + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $10] + db $C5, $F1, $FE, $CC // vpaddd xmm1, xmm1, xmm4 + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $C], ebx + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $8] + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + add edx, dword ptr [esp + $34] + xor eax, edi + shrd ecx, ecx, $2 + db $C5, $F9, $70, $FE, $84 // vpshufd xmm7, xmm6, $84 + add eax, edx + add edx, dword ptr [esp + $18] + add eax, ecx + db $C5, $C1, $73, $DF, $08 // vpsrldq xmm7, xmm7, $8 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $1C] + db $C5, $F1, $FE, $CF // vpaddd xmm1, xmm1, xmm7 + xor edx, ecx + mov edi, dword ptr [esp] + xor esi, edi + db $C5, $F9, $70, $F9, $50 // vpshufd xmm7, xmm1, $50 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $18], ecx + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $C] + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $8], eax + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $4] + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $F9, $70, $FE, $E8 // vpshufd xmm7, xmm6, $E8 + add edx, dword ptr [esp + $38] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $C1, $73, $FF, $08 // vpslldq xmm7, xmm7, $8 + add ebx, edx + add edx, dword ptr [esp + $14] + add ebx, ecx + db $C5, $F1, $FE, $CF // vpaddd xmm1, xmm1, xmm7 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $18] + db $C5, $F1, $FE, $75, $20 // vpaddd xmm6, xmm1, oword [ebp + $20] + xor edx, ecx + mov edi, dword ptr [esp + $1C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $14], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $8] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $3C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $10] + add eax, ecx + db $C5, $F9, $7F, $74, $24, $30 // vmovdqa oword [esp + $30], xmm6 + db $C4, $E3, $61, $0F, $E2, $04 // vpalignr xmm4, xmm3, xmm2, $4 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $14] + db $C4, $E3, $71, $0F, $F8, $04 // vpalignr xmm7, xmm1, xmm0, $4 + xor edx, ecx + mov edi, dword ptr [esp + $18] + xor esi, edi + db $C5, $C9, $72, $D4, $07 // vpsrld xmm6, xmm4, $7 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $10], ecx + db $C5, $E9, $FE, $D7 // vpaddd xmm2, xmm2, xmm7 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $C1, $72, $D4, $03 // vpsrld xmm7, xmm4, $3 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $4] + db $C5, $D1, $72, $F4, $0E // vpslld xmm5, xmm4, $E + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp], eax + db $C5, $C1, $EF, $E6 // vpxor xmm4, xmm7, xmm6 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $1C] + db $C5, $F9, $70, $F9, $FA // vpshufd xmm7, xmm1, $FA + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $C9, $72, $D6, $0B // vpsrld xmm6, xmm6, $B + add edx, dword ptr [esp + $40] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + add ebx, edx + add edx, dword ptr [esp + $C] + add ebx, ecx + db $C5, $D1, $72, $F5, $0B // vpslld xmm5, xmm5, $B + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $10] + db $C5, $D9, $EF, $E6 // vpxor xmm4, xmm4, xmm6 + xor edx, ecx + mov edi, dword ptr [esp + $14] + xor esi, edi + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $C], ecx + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp] + db $C5, $E9, $FE, $D4 // vpaddd xmm2, xmm2, xmm4 + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $1C], ebx + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $18] + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + add edx, dword ptr [esp + $44] + xor eax, edi + shrd ecx, ecx, $2 + db $C5, $F9, $70, $FE, $84 // vpshufd xmm7, xmm6, $84 + add eax, edx + add edx, dword ptr [esp + $8] + add eax, ecx + db $C5, $C1, $73, $DF, $08 // vpsrldq xmm7, xmm7, $8 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $C] + db $C5, $E9, $FE, $D7 // vpaddd xmm2, xmm2, xmm7 + xor edx, ecx + mov edi, dword ptr [esp + $10] + xor esi, edi + db $C5, $F9, $70, $FA, $50 // vpshufd xmm7, xmm2, $50 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $8], ecx + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $1C] + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $18], eax + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $14] + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $F9, $70, $FE, $E8 // vpshufd xmm7, xmm6, $E8 + add edx, dword ptr [esp + $48] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $C1, $73, $FF, $08 // vpslldq xmm7, xmm7, $8 + add ebx, edx + add edx, dword ptr [esp + $4] + add ebx, ecx + db $C5, $E9, $FE, $D7 // vpaddd xmm2, xmm2, xmm7 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $8] + db $C5, $E9, $FE, $75, $40 // vpaddd xmm6, xmm2, oword [ebp + $40] + xor edx, ecx + mov edi, dword ptr [esp + $C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $4], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $18] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $10] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $4C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp] + add eax, ecx + db $C5, $F9, $7F, $74, $24, $40 // vmovdqa oword [esp + $40], xmm6 + db $C4, $E3, $79, $0F, $E3, $04 // vpalignr xmm4, xmm0, xmm3, $4 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $4] + db $C4, $E3, $69, $0F, $F9, $04 // vpalignr xmm7, xmm2, xmm1, $4 + xor edx, ecx + mov edi, dword ptr [esp + $8] + xor esi, edi + db $C5, $C9, $72, $D4, $07 // vpsrld xmm6, xmm4, $7 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp], ecx + db $C5, $E1, $FE, $DF // vpaddd xmm3, xmm3, xmm7 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $C1, $72, $D4, $03 // vpsrld xmm7, xmm4, $3 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $14] + db $C5, $D1, $72, $F4, $0E // vpslld xmm5, xmm4, $E + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $10], eax + db $C5, $C1, $EF, $E6 // vpxor xmm4, xmm7, xmm6 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $C] + db $C5, $F9, $70, $FA, $FA // vpshufd xmm7, xmm2, $FA + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $C9, $72, $D6, $0B // vpsrld xmm6, xmm6, $B + add edx, dword ptr [esp + $50] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + add ebx, edx + add edx, dword ptr [esp + $1C] + add ebx, ecx + db $C5, $D1, $72, $F5, $0B // vpslld xmm5, xmm5, $B + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp] + db $C5, $D9, $EF, $E6 // vpxor xmm4, xmm4, xmm6 + xor edx, ecx + mov edi, dword ptr [esp + $4] + xor esi, edi + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $1C], ecx + db $C5, $D9, $EF, $E5 // vpxor xmm4, xmm4, xmm5 + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $10] + db $C5, $E1, $FE, $DC // vpaddd xmm3, xmm3, xmm4 + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $C], ebx + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $8] + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + add edx, dword ptr [esp + $54] + xor eax, edi + shrd ecx, ecx, $2 + db $C5, $F9, $70, $FE, $84 // vpshufd xmm7, xmm6, $84 + add eax, edx + add edx, dword ptr [esp + $18] + add eax, ecx + db $C5, $C1, $73, $DF, $08 // vpsrldq xmm7, xmm7, $8 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $1C] + db $C5, $E1, $FE, $DF // vpaddd xmm3, xmm3, xmm7 + xor edx, ecx + mov edi, dword ptr [esp] + xor esi, edi + db $C5, $F9, $70, $FB, $50 // vpshufd xmm7, xmm3, $50 + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $18], ecx + db $C5, $C9, $72, $D7, $0A // vpsrld xmm6, xmm7, $A + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + db $C5, $D1, $73, $D7, $11 // vpsrlq xmm5, xmm7, $11 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $C] + db $C5, $C9, $EF, $F5 // vpxor xmm6, xmm6, xmm5 + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $8], eax + db $C5, $C1, $73, $D7, $13 // vpsrlq xmm7, xmm7, $13 + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $4] + db $C5, $C9, $EF, $F7 // vpxor xmm6, xmm6, xmm7 + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + db $C5, $F9, $70, $FE, $E8 // vpshufd xmm7, xmm6, $E8 + add edx, dword ptr [esp + $58] + xor ebx, edi + shrd ecx, ecx, $2 + db $C5, $C1, $73, $FF, $08 // vpslldq xmm7, xmm7, $8 + add ebx, edx + add edx, dword ptr [esp + $14] + add ebx, ecx + db $C5, $E1, $FE, $DF // vpaddd xmm3, xmm3, xmm7 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $18] + db $C5, $E1, $FE, $75, $60 // vpaddd xmm6, xmm3, oword [ebp + $60] + xor edx, ecx + mov edi, dword ptr [esp + $1C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $14], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $8] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $5C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $10] + add eax, ecx + db $C5, $F9, $7F, $74, $24, $50 // vmovdqa oword [esp + $50], xmm6 + cmp dword ptr [ebp + $80], $10203 + jne @avx_00_47_015 + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $14] + xor edx, ecx + mov edi, dword ptr [esp + $18] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $10], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $4] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $1C] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $20] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $C] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $10] + xor edx, ecx + mov edi, dword ptr [esp + $14] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $C], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $1C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $18] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $24] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $8] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $C] + xor edx, ecx + mov edi, dword ptr [esp + $10] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $8], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $1C] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $18], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $14] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $28] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $4] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $8] + xor edx, ecx + mov edi, dword ptr [esp + $C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $4], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $18] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $10] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $2C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $4] + xor edx, ecx + mov edi, dword ptr [esp + $8] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $14] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $10], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $C] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $30] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $1C] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp] + xor edx, ecx + mov edi, dword ptr [esp + $4] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $1C], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $10] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $8] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $34] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $18] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $1C] + xor edx, ecx + mov edi, dword ptr [esp] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $18], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $C] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $8], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $4] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $38] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $14] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $18] + xor edx, ecx + mov edi, dword ptr [esp + $1C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $14], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $8] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $3C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $10] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $14] + xor edx, ecx + mov edi, dword ptr [esp + $18] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $10], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $4] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $1C] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $40] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $C] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $10] + xor edx, ecx + mov edi, dword ptr [esp + $14] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $C], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $1C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $18] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $44] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $8] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $C] + xor edx, ecx + mov edi, dword ptr [esp + $10] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $8], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $1C] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $18], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $14] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $48] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $4] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $8] + xor edx, ecx + mov edi, dword ptr [esp + $C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $4], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $18] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $10] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $4C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $4] + xor edx, ecx + mov edi, dword ptr [esp + $8] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $14] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $10], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $C] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $50] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $1C] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp] + xor edx, ecx + mov edi, dword ptr [esp + $4] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $1C], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $10] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp + $8] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $54] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $18] + add eax, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $1C] + xor edx, ecx + mov edi, dword ptr [esp] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $18], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword ptr [esp + $C] + mov esi, eax + shrd ecx, ecx, $9 + mov dword ptr [esp + $8], eax + xor ecx, eax + xor eax, edi + add edx, dword ptr [esp + $4] + shrd ecx, ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword ptr [esp + $58] + xor ebx, edi + shrd ecx, ecx, $2 + add ebx, edx + add edx, dword ptr [esp + $14] + add ebx, ecx + mov ecx, edx + shrd edx, edx, $E + mov esi, dword ptr [esp + $18] + xor edx, ecx + mov edi, dword ptr [esp + $1C] + xor esi, edi + shrd edx, edx, $5 + and esi, ecx + mov dword ptr [esp + $14], ecx + xor edx, ecx + xor edi, esi + shrd edx, edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword ptr [esp + $8] + mov esi, ebx + shrd ecx, ecx, $9 + mov dword ptr [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword ptr [esp] + shrd ecx, ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword ptr [esp + $5C] + xor eax, edi + shrd ecx, ecx, $2 + add eax, edx + add edx, dword ptr [esp + $10] + add eax, ecx + mov esi, dword ptr [esp + $60] + xor ebx, edi + mov ecx, dword ptr [esp + $C] + add eax, dword ptr [esi] + add ebx, dword ptr [esi + $4] + add edi, dword ptr [esi + $8] + add ecx, dword ptr [esi + $C] + mov dword ptr [esi], eax + mov dword ptr [esi + $4], ebx + mov dword ptr [esi + $8], edi + mov dword ptr [esi + $C], ecx + mov dword ptr [esp + $4], ebx + xor ebx, edi + mov dword ptr [esp + $8], edi + mov dword ptr [esp + $C], ecx + mov edi, dword ptr [esp + $14] + mov ecx, dword ptr [esp + $18] + add edx, dword ptr [esi + $10] + add edi, dword ptr [esi + $14] + add ecx, dword ptr [esi + $18] + mov dword ptr [esi + $10], edx + mov dword ptr [esi + $14], edi + mov dword ptr [esp + $14], edi + mov edi, dword ptr [esp + $1C] + mov dword ptr [esi + $18], ecx + add edi, dword ptr [esi + $1C] + mov dword ptr [esp + $18], ecx + mov dword ptr [esi + $1C], edi + mov dword ptr [esp + $1C], edi + mov edi, dword ptr [esp + $64] + db $C5, $FA, $6F, $BD, $80, $00, $00, $00 // vmovdqu xmm7, oword [ebp + $80] + sub ebp, $180 + cmp edi, dword ptr [esp + $68] + jb @grand_avx_014 + mov esp, dword ptr [esp + $6C] + db $C5, $FC, $77 // vzeroall + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressCryptoExt_aarch64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressCryptoExt_aarch64.inc index 4148f1fe..5f910332 100644 --- a/HashLib/src/Include/Simd/SHA256/SHA256CompressCryptoExt_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressCryptoExt_aarch64.inc @@ -1,16 +1,16 @@ -// SHA-256 AArch64 FEAT_SHA256 implementation. -// Expects AAPCS64: x0 = state ptr (8 x UInt32), x1 = data ptr, -// w2 = numblocks, x3 = K256 ptr (64 UInt32 round constants). -// Leaf nostackframe; caller-saved GPR/vector only. -// K pointer rewound each block (sub x3, #240) after post-index loads. -// Reference: OpenSSL CRYPTOGAMS sha256-armv8.S. -// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// CryptoExt (FEAT_SHA256) implementation of SHA-256 compress. +// The K pointer is rewound each block (sub x3, #240) after post-index loads. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (8 x UInt32), +// x1 = data ptr, w2 = numblocks, x3 = K256 ptr (64 UInt32 round constants). // Register map: // v0, v1 = working state (ABCD, EFGH) // v2 = sha256h2 temp (copy of v0 before each h/h2 pair) // v4-v7 = message schedule (W) // v16, v17 = W+K for sha256h / sha256h2 (ping-pong) // v18, v19 = saved state (ABCD, EFGH) for the final add +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha256-armv8.S. cbz w2, .Lcryptoext_sha256_done .long 0xad400400 // ldp q0, q1, [x0] diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressGpr_aarch64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressGpr_aarch64.inc new file mode 100644 index 00000000..2a111fff --- /dev/null +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressGpr_aarch64.inc @@ -0,0 +1,949 @@ +// Gpr implementation of SHA-256 compress (AArch64) for cores without the +// SHA-256 crypto extension; the Gpr name states the ISA requirement: none +// beyond base AArch64. Serial SHA maps better onto the 31 GPRs than onto +// NEON lanes (single dependency chain, 1-op ror, rotate-folded eor +// operands). +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (8 x UInt32), +// x1 = data ptr, w2 = numblocks, x3 = K256_Gpr ptr. +// K256_Gpr carries the 64 round constants PLUS a zero terminator word - +// the message-schedule loop ends when the loaded K word is zero (cbnz), +// exactly like the original's sentinel-terminated table; a plain K256 +// const cannot drive this loop. +// Frame: 128-byte AAPCS64 frame (x29/x30 + x19-x28 callee-saved) + 16 +// bytes schedule scratch; x30 (LR) is repurposed as the K walk pointer +// between the save and the restore, as in the original. +// AArch64 instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha512-armv8.pl (sha256_block_data_order, +// the plain path; the generator emits both SHA-256 and SHA-512). + + .long 0x2a0203e2 // mov w2,w2 // zero-extend ANumBlocks (AAPCS64 leaves x2's top half undefined) + .long 0xa9b87bfd // stp x29,x30,[sp,#-128]! + .long 0x910003fd // add x29,sp,#0 + + .long 0xa90153f3 // stp x19,x20,[sp,#16] + .long 0xa9025bf5 // stp x21,x22,[sp,#32] + .long 0xa90363f7 // stp x23,x24,[sp,#48] + .long 0xa9046bf9 // stp x25,x26,[sp,#64] + .long 0xa90573fb // stp x27,x28,[sp,#80] + .long 0xd10043ff // sub sp,sp,#4*4 + + .long 0x29405414 // ldp w20,w21,[x0] // load context + .long 0x29415c16 // ldp w22,w23,[x0,#2*4] + .long 0x29426418 // ldp w24,w25,[x0,#4*4] + .long 0x8b021822 // add x2,x1,x2, lsl #6 // end of input + .long 0x29436c1a // ldp w26,w27,[x0,#6*4] + .long 0xaa0303fe // mov x30,x3 // K256_Gpr (param 4; x3 is reused for message words below) + .long 0xa9060ba0 // stp x0,x2,[x29,#96] + + +.Lsha256_gpr_loop: + .long 0x28c11023 // ldp w3,w4,[x1],#2*4 + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++ + .long 0x4a1602bc // eor w28,w21,w22 // magic seed + .long 0xf9003ba1 // str x1,[x29,#112] + .long 0x5ac00863 // rev w3,w3 // 0 + .long 0x13981b10 // ror w16,w24,#6 + .long 0x0b13037b // add w27,w27,w19 // h+=K[i] + .long 0x4ad83b06 // eor w6,w24,w24, ror #14 + .long 0x0a180331 // and w17,w25,w24 + .long 0x0a380353 // bic w19,w26,w24 + .long 0x0b03037b // add w27,w27,w3 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a150293 // eor w19,w20,w21 // a^b, b^c in next round + .long 0x4ac62e10 // eor w16,w16,w6, ror #11 // Sigma1(e) + .long 0x13940a86 // ror w6,w20,#2 + .long 0x0b11037b // add w27,w27,w17 // h+=Ch(e,f,g) + .long 0x4ad42691 // eor w17,w20,w20, ror #9 + .long 0x0b10037b // add w27,w27,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b1b02f7 // add w23,w23,w27 // d+=h + .long 0x4a15039c // eor w28,w28,w21 // Maj(a,b,c) + .long 0x4ad134d1 // eor w17,w6,w17, ror #13 // Sigma0(a) + .long 0x0b1c037b // add w27,w27,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w27,w27,w17 // h+=Sigma0(a) + .long 0x5ac00884 // rev w4,w4 // 1 + .long 0x28c11825 // ldp w5,w6,[x1],#2*4 + .long 0x0b11037b // add w27,w27,w17 // h+=Sigma0(a) + .long 0x13971af0 // ror w16,w23,#6 + .long 0x0b1c035a // add w26,w26,w28 // h+=K[i] + .long 0x4ad73ae7 // eor w7,w23,w23, ror #14 + .long 0x0a170311 // and w17,w24,w23 + .long 0x0a37033c // bic w28,w25,w23 + .long 0x0b04035a // add w26,w26,w4 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a14037c // eor w28,w27,w20 // a^b, b^c in next round + .long 0x4ac72e10 // eor w16,w16,w7, ror #11 // Sigma1(e) + .long 0x139b0b67 // ror w7,w27,#2 + .long 0x0b11035a // add w26,w26,w17 // h+=Ch(e,f,g) + .long 0x4adb2771 // eor w17,w27,w27, ror #9 + .long 0x0b10035a // add w26,w26,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b1a02d6 // add w22,w22,w26 // d+=h + .long 0x4a140273 // eor w19,w19,w20 // Maj(a,b,c) + .long 0x4ad134f1 // eor w17,w7,w17, ror #13 // Sigma0(a) + .long 0x0b13035a // add w26,w26,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w26,w26,w17 // h+=Sigma0(a) + .long 0x5ac008a5 // rev w5,w5 // 2 + .long 0x0b11035a // add w26,w26,w17 // h+=Sigma0(a) + .long 0x13961ad0 // ror w16,w22,#6 + .long 0x0b130339 // add w25,w25,w19 // h+=K[i] + .long 0x4ad63ac8 // eor w8,w22,w22, ror #14 + .long 0x0a1602f1 // and w17,w23,w22 + .long 0x0a360313 // bic w19,w24,w22 + .long 0x0b050339 // add w25,w25,w5 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1b0353 // eor w19,w26,w27 // a^b, b^c in next round + .long 0x4ac82e10 // eor w16,w16,w8, ror #11 // Sigma1(e) + .long 0x139a0b48 // ror w8,w26,#2 + .long 0x0b110339 // add w25,w25,w17 // h+=Ch(e,f,g) + .long 0x4ada2751 // eor w17,w26,w26, ror #9 + .long 0x0b100339 // add w25,w25,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b1902b5 // add w21,w21,w25 // d+=h + .long 0x4a1b039c // eor w28,w28,w27 // Maj(a,b,c) + .long 0x4ad13511 // eor w17,w8,w17, ror #13 // Sigma0(a) + .long 0x0b1c0339 // add w25,w25,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w25,w25,w17 // h+=Sigma0(a) + .long 0x5ac008c6 // rev w6,w6 // 3 + .long 0x28c12027 // ldp w7,w8,[x1],#2*4 + .long 0x0b110339 // add w25,w25,w17 // h+=Sigma0(a) + .long 0x13951ab0 // ror w16,w21,#6 + .long 0x0b1c0318 // add w24,w24,w28 // h+=K[i] + .long 0x4ad53aa9 // eor w9,w21,w21, ror #14 + .long 0x0a1502d1 // and w17,w22,w21 + .long 0x0a3502fc // bic w28,w23,w21 + .long 0x0b060318 // add w24,w24,w6 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1a033c // eor w28,w25,w26 // a^b, b^c in next round + .long 0x4ac92e10 // eor w16,w16,w9, ror #11 // Sigma1(e) + .long 0x13990b29 // ror w9,w25,#2 + .long 0x0b110318 // add w24,w24,w17 // h+=Ch(e,f,g) + .long 0x4ad92731 // eor w17,w25,w25, ror #9 + .long 0x0b100318 // add w24,w24,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b180294 // add w20,w20,w24 // d+=h + .long 0x4a1a0273 // eor w19,w19,w26 // Maj(a,b,c) + .long 0x4ad13531 // eor w17,w9,w17, ror #13 // Sigma0(a) + .long 0x0b130318 // add w24,w24,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w24,w24,w17 // h+=Sigma0(a) + .long 0x5ac008e7 // rev w7,w7 // 4 + .long 0x0b110318 // add w24,w24,w17 // h+=Sigma0(a) + .long 0x13941a90 // ror w16,w20,#6 + .long 0x0b1302f7 // add w23,w23,w19 // h+=K[i] + .long 0x4ad43a8a // eor w10,w20,w20, ror #14 + .long 0x0a1402b1 // and w17,w21,w20 + .long 0x0a3402d3 // bic w19,w22,w20 + .long 0x0b0702f7 // add w23,w23,w7 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a190313 // eor w19,w24,w25 // a^b, b^c in next round + .long 0x4aca2e10 // eor w16,w16,w10, ror #11 // Sigma1(e) + .long 0x13980b0a // ror w10,w24,#2 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Ch(e,f,g) + .long 0x4ad82711 // eor w17,w24,w24, ror #9 + .long 0x0b1002f7 // add w23,w23,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b17037b // add w27,w27,w23 // d+=h + .long 0x4a19039c // eor w28,w28,w25 // Maj(a,b,c) + .long 0x4ad13551 // eor w17,w10,w17, ror #13 // Sigma0(a) + .long 0x0b1c02f7 // add w23,w23,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w23,w23,w17 // h+=Sigma0(a) + .long 0x5ac00908 // rev w8,w8 // 5 + .long 0x28c12829 // ldp w9,w10,[x1],#2*4 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Sigma0(a) + .long 0x139b1b70 // ror w16,w27,#6 + .long 0x0b1c02d6 // add w22,w22,w28 // h+=K[i] + .long 0x4adb3b6b // eor w11,w27,w27, ror #14 + .long 0x0a1b0291 // and w17,w20,w27 + .long 0x0a3b02bc // bic w28,w21,w27 + .long 0x0b0802d6 // add w22,w22,w8 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1802fc // eor w28,w23,w24 // a^b, b^c in next round + .long 0x4acb2e10 // eor w16,w16,w11, ror #11 // Sigma1(e) + .long 0x13970aeb // ror w11,w23,#2 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Ch(e,f,g) + .long 0x4ad726f1 // eor w17,w23,w23, ror #9 + .long 0x0b1002d6 // add w22,w22,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b16035a // add w26,w26,w22 // d+=h + .long 0x4a180273 // eor w19,w19,w24 // Maj(a,b,c) + .long 0x4ad13571 // eor w17,w11,w17, ror #13 // Sigma0(a) + .long 0x0b1302d6 // add w22,w22,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w22,w22,w17 // h+=Sigma0(a) + .long 0x5ac00929 // rev w9,w9 // 6 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Sigma0(a) + .long 0x139a1b50 // ror w16,w26,#6 + .long 0x0b1302b5 // add w21,w21,w19 // h+=K[i] + .long 0x4ada3b4c // eor w12,w26,w26, ror #14 + .long 0x0a1a0371 // and w17,w27,w26 + .long 0x0a3a0293 // bic w19,w20,w26 + .long 0x0b0902b5 // add w21,w21,w9 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1702d3 // eor w19,w22,w23 // a^b, b^c in next round + .long 0x4acc2e10 // eor w16,w16,w12, ror #11 // Sigma1(e) + .long 0x13960acc // ror w12,w22,#2 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Ch(e,f,g) + .long 0x4ad626d1 // eor w17,w22,w22, ror #9 + .long 0x0b1002b5 // add w21,w21,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b150339 // add w25,w25,w21 // d+=h + .long 0x4a17039c // eor w28,w28,w23 // Maj(a,b,c) + .long 0x4ad13591 // eor w17,w12,w17, ror #13 // Sigma0(a) + .long 0x0b1c02b5 // add w21,w21,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w21,w21,w17 // h+=Sigma0(a) + .long 0x5ac0094a // rev w10,w10 // 7 + .long 0x28c1302b // ldp w11,w12,[x1],#2*4 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Sigma0(a) + .long 0x13991b30 // ror w16,w25,#6 + .long 0x0b1c0294 // add w20,w20,w28 // h+=K[i] + .long 0x4ad93b2d // eor w13,w25,w25, ror #14 + .long 0x0a190351 // and w17,w26,w25 + .long 0x0a39037c // bic w28,w27,w25 + .long 0x0b0a0294 // add w20,w20,w10 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1602bc // eor w28,w21,w22 // a^b, b^c in next round + .long 0x4acd2e10 // eor w16,w16,w13, ror #11 // Sigma1(e) + .long 0x13950aad // ror w13,w21,#2 + .long 0x0b110294 // add w20,w20,w17 // h+=Ch(e,f,g) + .long 0x4ad526b1 // eor w17,w21,w21, ror #9 + .long 0x0b100294 // add w20,w20,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b140318 // add w24,w24,w20 // d+=h + .long 0x4a160273 // eor w19,w19,w22 // Maj(a,b,c) + .long 0x4ad135b1 // eor w17,w13,w17, ror #13 // Sigma0(a) + .long 0x0b130294 // add w20,w20,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w20,w20,w17 // h+=Sigma0(a) + .long 0x5ac0096b // rev w11,w11 // 8 + .long 0x0b110294 // add w20,w20,w17 // h+=Sigma0(a) + .long 0x13981b10 // ror w16,w24,#6 + .long 0x0b13037b // add w27,w27,w19 // h+=K[i] + .long 0x4ad83b0e // eor w14,w24,w24, ror #14 + .long 0x0a180331 // and w17,w25,w24 + .long 0x0a380353 // bic w19,w26,w24 + .long 0x0b0b037b // add w27,w27,w11 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a150293 // eor w19,w20,w21 // a^b, b^c in next round + .long 0x4ace2e10 // eor w16,w16,w14, ror #11 // Sigma1(e) + .long 0x13940a8e // ror w14,w20,#2 + .long 0x0b11037b // add w27,w27,w17 // h+=Ch(e,f,g) + .long 0x4ad42691 // eor w17,w20,w20, ror #9 + .long 0x0b10037b // add w27,w27,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b1b02f7 // add w23,w23,w27 // d+=h + .long 0x4a15039c // eor w28,w28,w21 // Maj(a,b,c) + .long 0x4ad135d1 // eor w17,w14,w17, ror #13 // Sigma0(a) + .long 0x0b1c037b // add w27,w27,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w27,w27,w17 // h+=Sigma0(a) + .long 0x5ac0098c // rev w12,w12 // 9 + .long 0x28c1382d // ldp w13,w14,[x1],#2*4 + .long 0x0b11037b // add w27,w27,w17 // h+=Sigma0(a) + .long 0x13971af0 // ror w16,w23,#6 + .long 0x0b1c035a // add w26,w26,w28 // h+=K[i] + .long 0x4ad73aef // eor w15,w23,w23, ror #14 + .long 0x0a170311 // and w17,w24,w23 + .long 0x0a37033c // bic w28,w25,w23 + .long 0x0b0c035a // add w26,w26,w12 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a14037c // eor w28,w27,w20 // a^b, b^c in next round + .long 0x4acf2e10 // eor w16,w16,w15, ror #11 // Sigma1(e) + .long 0x139b0b6f // ror w15,w27,#2 + .long 0x0b11035a // add w26,w26,w17 // h+=Ch(e,f,g) + .long 0x4adb2771 // eor w17,w27,w27, ror #9 + .long 0x0b10035a // add w26,w26,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b1a02d6 // add w22,w22,w26 // d+=h + .long 0x4a140273 // eor w19,w19,w20 // Maj(a,b,c) + .long 0x4ad135f1 // eor w17,w15,w17, ror #13 // Sigma0(a) + .long 0x0b13035a // add w26,w26,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w26,w26,w17 // h+=Sigma0(a) + .long 0x5ac009ad // rev w13,w13 // 10 + .long 0x0b11035a // add w26,w26,w17 // h+=Sigma0(a) + .long 0x13961ad0 // ror w16,w22,#6 + .long 0x0b130339 // add w25,w25,w19 // h+=K[i] + .long 0x4ad63ac0 // eor w0,w22,w22, ror #14 + .long 0x0a1602f1 // and w17,w23,w22 + .long 0x0a360313 // bic w19,w24,w22 + .long 0x0b0d0339 // add w25,w25,w13 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1b0353 // eor w19,w26,w27 // a^b, b^c in next round + .long 0x4ac02e10 // eor w16,w16,w0, ror #11 // Sigma1(e) + .long 0x139a0b40 // ror w0,w26,#2 + .long 0x0b110339 // add w25,w25,w17 // h+=Ch(e,f,g) + .long 0x4ada2751 // eor w17,w26,w26, ror #9 + .long 0x0b100339 // add w25,w25,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b1902b5 // add w21,w21,w25 // d+=h + .long 0x4a1b039c // eor w28,w28,w27 // Maj(a,b,c) + .long 0x4ad13411 // eor w17,w0,w17, ror #13 // Sigma0(a) + .long 0x0b1c0339 // add w25,w25,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w25,w25,w17 // h+=Sigma0(a) + .long 0x5ac009ce // rev w14,w14 // 11 + .long 0x28c1002f // ldp w15,w0,[x1],#2*4 + .long 0x0b110339 // add w25,w25,w17 // h+=Sigma0(a) + .long 0xb9000fe6 // str w6,[sp,#12] + .long 0x13951ab0 // ror w16,w21,#6 + .long 0x0b1c0318 // add w24,w24,w28 // h+=K[i] + .long 0x4ad53aa6 // eor w6,w21,w21, ror #14 + .long 0x0a1502d1 // and w17,w22,w21 + .long 0x0a3502fc // bic w28,w23,w21 + .long 0x0b0e0318 // add w24,w24,w14 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1a033c // eor w28,w25,w26 // a^b, b^c in next round + .long 0x4ac62e10 // eor w16,w16,w6, ror #11 // Sigma1(e) + .long 0x13990b26 // ror w6,w25,#2 + .long 0x0b110318 // add w24,w24,w17 // h+=Ch(e,f,g) + .long 0x4ad92731 // eor w17,w25,w25, ror #9 + .long 0x0b100318 // add w24,w24,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b180294 // add w20,w20,w24 // d+=h + .long 0x4a1a0273 // eor w19,w19,w26 // Maj(a,b,c) + .long 0x4ad134d1 // eor w17,w6,w17, ror #13 // Sigma0(a) + .long 0x0b130318 // add w24,w24,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w24,w24,w17 // h+=Sigma0(a) + .long 0x5ac009ef // rev w15,w15 // 12 + .long 0x0b110318 // add w24,w24,w17 // h+=Sigma0(a) + .long 0xb90003e7 // str w7,[sp,#0] + .long 0x13941a90 // ror w16,w20,#6 + .long 0x0b1302f7 // add w23,w23,w19 // h+=K[i] + .long 0x4ad43a87 // eor w7,w20,w20, ror #14 + .long 0x0a1402b1 // and w17,w21,w20 + .long 0x0a3402d3 // bic w19,w22,w20 + .long 0x0b0f02f7 // add w23,w23,w15 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a190313 // eor w19,w24,w25 // a^b, b^c in next round + .long 0x4ac72e10 // eor w16,w16,w7, ror #11 // Sigma1(e) + .long 0x13980b07 // ror w7,w24,#2 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Ch(e,f,g) + .long 0x4ad82711 // eor w17,w24,w24, ror #9 + .long 0x0b1002f7 // add w23,w23,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b17037b // add w27,w27,w23 // d+=h + .long 0x4a19039c // eor w28,w28,w25 // Maj(a,b,c) + .long 0x4ad134f1 // eor w17,w7,w17, ror #13 // Sigma0(a) + .long 0x0b1c02f7 // add w23,w23,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w23,w23,w17 // h+=Sigma0(a) + .long 0x5ac00800 // rev w0,w0 // 13 + .long 0x29400821 // ldp w1,w2,[x1] + .long 0x0b1102f7 // add w23,w23,w17 // h+=Sigma0(a) + .long 0xb90007e8 // str w8,[sp,#4] + .long 0x139b1b70 // ror w16,w27,#6 + .long 0x0b1c02d6 // add w22,w22,w28 // h+=K[i] + .long 0x4adb3b68 // eor w8,w27,w27, ror #14 + .long 0x0a1b0291 // and w17,w20,w27 + .long 0x0a3b02bc // bic w28,w21,w27 + .long 0x0b0002d6 // add w22,w22,w0 // h+=X[i] + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1802fc // eor w28,w23,w24 // a^b, b^c in next round + .long 0x4ac82e10 // eor w16,w16,w8, ror #11 // Sigma1(e) + .long 0x13970ae8 // ror w8,w23,#2 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Ch(e,f,g) + .long 0x4ad726f1 // eor w17,w23,w23, ror #9 + .long 0x0b1002d6 // add w22,w22,w16 // h+=Sigma1(e) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x0b16035a // add w26,w26,w22 // d+=h + .long 0x4a180273 // eor w19,w19,w24 // Maj(a,b,c) + .long 0x4ad13511 // eor w17,w8,w17, ror #13 // Sigma0(a) + .long 0x0b1302d6 // add w22,w22,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + //add w22,w22,w17 // h+=Sigma0(a) + .long 0x5ac00821 // rev w1,w1 // 14 + .long 0xb9400fe6 // ldr w6,[sp,#12] + .long 0x0b1102d6 // add w22,w22,w17 // h+=Sigma0(a) + .long 0xb9000be9 // str w9,[sp,#8] + .long 0x139a1b50 // ror w16,w26,#6 + .long 0x0b1302b5 // add w21,w21,w19 // h+=K[i] + .long 0x4ada3b49 // eor w9,w26,w26, ror #14 + .long 0x0a1a0371 // and w17,w27,w26 + .long 0x0a3a0293 // bic w19,w20,w26 + .long 0x0b0102b5 // add w21,w21,w1 // h+=X[i] + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1702d3 // eor w19,w22,w23 // a^b, b^c in next round + .long 0x4ac92e10 // eor w16,w16,w9, ror #11 // Sigma1(e) + .long 0x13960ac9 // ror w9,w22,#2 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Ch(e,f,g) + .long 0x4ad626d1 // eor w17,w22,w22, ror #9 + .long 0x0b1002b5 // add w21,w21,w16 // h+=Sigma1(e) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x0b150339 // add w25,w25,w21 // d+=h + .long 0x4a17039c // eor w28,w28,w23 // Maj(a,b,c) + .long 0x4ad13531 // eor w17,w9,w17, ror #13 // Sigma0(a) + .long 0x0b1c02b5 // add w21,w21,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + //add w21,w21,w17 // h+=Sigma0(a) + .long 0x5ac00842 // rev w2,w2 // 15 + .long 0xb94003e7 // ldr w7,[sp,#0] + .long 0x0b1102b5 // add w21,w21,w17 // h+=Sigma0(a) + .long 0xb9000fea // str w10,[sp,#12] + .long 0x13991b30 // ror w16,w25,#6 + .long 0x0b1c0294 // add w20,w20,w28 // h+=K[i] + .long 0x13841c89 // ror w9,w4,#7 + .long 0x0a190351 // and w17,w26,w25 + .long 0x13814428 // ror w8,w1,#17 + .long 0x0a39037c // bic w28,w27,w25 + .long 0x13950aaa // ror w10,w21,#2 + .long 0x0b020294 // add w20,w20,w2 // h+=X[i] + .long 0x4ad92e10 // eor w16,w16,w25, ror #11 + .long 0x4ac44929 // eor w9,w9,w4, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1602bc // eor w28,w21,w22 // a^b, b^c in next round + .long 0x4ad96610 // eor w16,w16,w25, ror #25 // Sigma1(e) + .long 0x4ad5354a // eor w10,w10,w21, ror #13 + .long 0x0b110294 // add w20,w20,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4ac14d08 // eor w8,w8,w1, ror #19 + .long 0x4a440d29 // eor w9,w9,w4, lsr #3 // sigma0(X[i+1]) + .long 0x0b100294 // add w20,w20,w16 // h+=Sigma1(e) + .long 0x4a160273 // eor w19,w19,w22 // Maj(a,b,c) + .long 0x4ad55951 // eor w17,w10,w21, ror #22 // Sigma0(a) + .long 0x4a412908 // eor w8,w8,w1, lsr #10 // sigma1(X[i+14]) + .long 0x0b0c0063 // add w3,w3,w12 + .long 0x0b140318 // add w24,w24,w20 // d+=h + .long 0x0b130294 // add w20,w20,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b090063 // add w3,w3,w9 + .long 0x0b110294 // add w20,w20,w17 // h+=Sigma0(a) + .long 0x0b080063 // add w3,w3,w8 + +.Lsha256_gpr_loop_16_xx: + .long 0xb94007e8 // ldr w8,[sp,#4] + .long 0xb90003eb // str w11,[sp,#0] + .long 0x13981b10 // ror w16,w24,#6 + .long 0x0b13037b // add w27,w27,w19 // h+=K[i] + .long 0x13851caa // ror w10,w5,#7 + .long 0x0a180331 // and w17,w25,w24 + .long 0x13824449 // ror w9,w2,#17 + .long 0x0a380353 // bic w19,w26,w24 + .long 0x13940a8b // ror w11,w20,#2 + .long 0x0b03037b // add w27,w27,w3 // h+=X[i] + .long 0x4ad82e10 // eor w16,w16,w24, ror #11 + .long 0x4ac5494a // eor w10,w10,w5, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a150293 // eor w19,w20,w21 // a^b, b^c in next round + .long 0x4ad86610 // eor w16,w16,w24, ror #25 // Sigma1(e) + .long 0x4ad4356b // eor w11,w11,w20, ror #13 + .long 0x0b11037b // add w27,w27,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4ac24d29 // eor w9,w9,w2, ror #19 + .long 0x4a450d4a // eor w10,w10,w5, lsr #3 // sigma0(X[i+1]) + .long 0x0b10037b // add w27,w27,w16 // h+=Sigma1(e) + .long 0x4a15039c // eor w28,w28,w21 // Maj(a,b,c) + .long 0x4ad45971 // eor w17,w11,w20, ror #22 // Sigma0(a) + .long 0x4a422929 // eor w9,w9,w2, lsr #10 // sigma1(X[i+14]) + .long 0x0b0d0084 // add w4,w4,w13 + .long 0x0b1b02f7 // add w23,w23,w27 // d+=h + .long 0x0b1c037b // add w27,w27,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b0a0084 // add w4,w4,w10 + .long 0x0b11037b // add w27,w27,w17 // h+=Sigma0(a) + .long 0x0b090084 // add w4,w4,w9 + .long 0xb9400be9 // ldr w9,[sp,#8] + .long 0xb90007ec // str w12,[sp,#4] + .long 0x13971af0 // ror w16,w23,#6 + .long 0x0b1c035a // add w26,w26,w28 // h+=K[i] + .long 0x13861ccb // ror w11,w6,#7 + .long 0x0a170311 // and w17,w24,w23 + .long 0x1383446a // ror w10,w3,#17 + .long 0x0a37033c // bic w28,w25,w23 + .long 0x139b0b6c // ror w12,w27,#2 + .long 0x0b04035a // add w26,w26,w4 // h+=X[i] + .long 0x4ad72e10 // eor w16,w16,w23, ror #11 + .long 0x4ac6496b // eor w11,w11,w6, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a14037c // eor w28,w27,w20 // a^b, b^c in next round + .long 0x4ad76610 // eor w16,w16,w23, ror #25 // Sigma1(e) + .long 0x4adb358c // eor w12,w12,w27, ror #13 + .long 0x0b11035a // add w26,w26,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4ac34d4a // eor w10,w10,w3, ror #19 + .long 0x4a460d6b // eor w11,w11,w6, lsr #3 // sigma0(X[i+1]) + .long 0x0b10035a // add w26,w26,w16 // h+=Sigma1(e) + .long 0x4a140273 // eor w19,w19,w20 // Maj(a,b,c) + .long 0x4adb5991 // eor w17,w12,w27, ror #22 // Sigma0(a) + .long 0x4a43294a // eor w10,w10,w3, lsr #10 // sigma1(X[i+14]) + .long 0x0b0e00a5 // add w5,w5,w14 + .long 0x0b1a02d6 // add w22,w22,w26 // d+=h + .long 0x0b13035a // add w26,w26,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b0b00a5 // add w5,w5,w11 + .long 0x0b11035a // add w26,w26,w17 // h+=Sigma0(a) + .long 0x0b0a00a5 // add w5,w5,w10 + .long 0xb9400fea // ldr w10,[sp,#12] + .long 0xb9000bed // str w13,[sp,#8] + .long 0x13961ad0 // ror w16,w22,#6 + .long 0x0b130339 // add w25,w25,w19 // h+=K[i] + .long 0x13871cec // ror w12,w7,#7 + .long 0x0a1602f1 // and w17,w23,w22 + .long 0x1384448b // ror w11,w4,#17 + .long 0x0a360313 // bic w19,w24,w22 + .long 0x139a0b4d // ror w13,w26,#2 + .long 0x0b050339 // add w25,w25,w5 // h+=X[i] + .long 0x4ad62e10 // eor w16,w16,w22, ror #11 + .long 0x4ac7498c // eor w12,w12,w7, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1b0353 // eor w19,w26,w27 // a^b, b^c in next round + .long 0x4ad66610 // eor w16,w16,w22, ror #25 // Sigma1(e) + .long 0x4ada35ad // eor w13,w13,w26, ror #13 + .long 0x0b110339 // add w25,w25,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4ac44d6b // eor w11,w11,w4, ror #19 + .long 0x4a470d8c // eor w12,w12,w7, lsr #3 // sigma0(X[i+1]) + .long 0x0b100339 // add w25,w25,w16 // h+=Sigma1(e) + .long 0x4a1b039c // eor w28,w28,w27 // Maj(a,b,c) + .long 0x4ada59b1 // eor w17,w13,w26, ror #22 // Sigma0(a) + .long 0x4a44296b // eor w11,w11,w4, lsr #10 // sigma1(X[i+14]) + .long 0x0b0f00c6 // add w6,w6,w15 + .long 0x0b1902b5 // add w21,w21,w25 // d+=h + .long 0x0b1c0339 // add w25,w25,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b0c00c6 // add w6,w6,w12 + .long 0x0b110339 // add w25,w25,w17 // h+=Sigma0(a) + .long 0x0b0b00c6 // add w6,w6,w11 + .long 0xb94003eb // ldr w11,[sp,#0] + .long 0xb9000fee // str w14,[sp,#12] + .long 0x13951ab0 // ror w16,w21,#6 + .long 0x0b1c0318 // add w24,w24,w28 // h+=K[i] + .long 0x13881d0d // ror w13,w8,#7 + .long 0x0a1502d1 // and w17,w22,w21 + .long 0x138544ac // ror w12,w5,#17 + .long 0x0a3502fc // bic w28,w23,w21 + .long 0x13990b2e // ror w14,w25,#2 + .long 0x0b060318 // add w24,w24,w6 // h+=X[i] + .long 0x4ad52e10 // eor w16,w16,w21, ror #11 + .long 0x4ac849ad // eor w13,w13,w8, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1a033c // eor w28,w25,w26 // a^b, b^c in next round + .long 0x4ad56610 // eor w16,w16,w21, ror #25 // Sigma1(e) + .long 0x4ad935ce // eor w14,w14,w25, ror #13 + .long 0x0b110318 // add w24,w24,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4ac54d8c // eor w12,w12,w5, ror #19 + .long 0x4a480dad // eor w13,w13,w8, lsr #3 // sigma0(X[i+1]) + .long 0x0b100318 // add w24,w24,w16 // h+=Sigma1(e) + .long 0x4a1a0273 // eor w19,w19,w26 // Maj(a,b,c) + .long 0x4ad959d1 // eor w17,w14,w25, ror #22 // Sigma0(a) + .long 0x4a45298c // eor w12,w12,w5, lsr #10 // sigma1(X[i+14]) + .long 0x0b0000e7 // add w7,w7,w0 + .long 0x0b180294 // add w20,w20,w24 // d+=h + .long 0x0b130318 // add w24,w24,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b0d00e7 // add w7,w7,w13 + .long 0x0b110318 // add w24,w24,w17 // h+=Sigma0(a) + .long 0x0b0c00e7 // add w7,w7,w12 + .long 0xb94007ec // ldr w12,[sp,#4] + .long 0xb90003ef // str w15,[sp,#0] + .long 0x13941a90 // ror w16,w20,#6 + .long 0x0b1302f7 // add w23,w23,w19 // h+=K[i] + .long 0x13891d2e // ror w14,w9,#7 + .long 0x0a1402b1 // and w17,w21,w20 + .long 0x138644cd // ror w13,w6,#17 + .long 0x0a3402d3 // bic w19,w22,w20 + .long 0x13980b0f // ror w15,w24,#2 + .long 0x0b0702f7 // add w23,w23,w7 // h+=X[i] + .long 0x4ad42e10 // eor w16,w16,w20, ror #11 + .long 0x4ac949ce // eor w14,w14,w9, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a190313 // eor w19,w24,w25 // a^b, b^c in next round + .long 0x4ad46610 // eor w16,w16,w20, ror #25 // Sigma1(e) + .long 0x4ad835ef // eor w15,w15,w24, ror #13 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4ac64dad // eor w13,w13,w6, ror #19 + .long 0x4a490dce // eor w14,w14,w9, lsr #3 // sigma0(X[i+1]) + .long 0x0b1002f7 // add w23,w23,w16 // h+=Sigma1(e) + .long 0x4a19039c // eor w28,w28,w25 // Maj(a,b,c) + .long 0x4ad859f1 // eor w17,w15,w24, ror #22 // Sigma0(a) + .long 0x4a4629ad // eor w13,w13,w6, lsr #10 // sigma1(X[i+14]) + .long 0x0b010108 // add w8,w8,w1 + .long 0x0b17037b // add w27,w27,w23 // d+=h + .long 0x0b1c02f7 // add w23,w23,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b0e0108 // add w8,w8,w14 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Sigma0(a) + .long 0x0b0d0108 // add w8,w8,w13 + .long 0xb9400bed // ldr w13,[sp,#8] + .long 0xb90007e0 // str w0,[sp,#4] + .long 0x139b1b70 // ror w16,w27,#6 + .long 0x0b1c02d6 // add w22,w22,w28 // h+=K[i] + .long 0x138a1d4f // ror w15,w10,#7 + .long 0x0a1b0291 // and w17,w20,w27 + .long 0x138744ee // ror w14,w7,#17 + .long 0x0a3b02bc // bic w28,w21,w27 + .long 0x13970ae0 // ror w0,w23,#2 + .long 0x0b0802d6 // add w22,w22,w8 // h+=X[i] + .long 0x4adb2e10 // eor w16,w16,w27, ror #11 + .long 0x4aca49ef // eor w15,w15,w10, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1802fc // eor w28,w23,w24 // a^b, b^c in next round + .long 0x4adb6610 // eor w16,w16,w27, ror #25 // Sigma1(e) + .long 0x4ad73400 // eor w0,w0,w23, ror #13 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4ac74dce // eor w14,w14,w7, ror #19 + .long 0x4a4a0def // eor w15,w15,w10, lsr #3 // sigma0(X[i+1]) + .long 0x0b1002d6 // add w22,w22,w16 // h+=Sigma1(e) + .long 0x4a180273 // eor w19,w19,w24 // Maj(a,b,c) + .long 0x4ad75811 // eor w17,w0,w23, ror #22 // Sigma0(a) + .long 0x4a4729ce // eor w14,w14,w7, lsr #10 // sigma1(X[i+14]) + .long 0x0b020129 // add w9,w9,w2 + .long 0x0b16035a // add w26,w26,w22 // d+=h + .long 0x0b1302d6 // add w22,w22,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b0f0129 // add w9,w9,w15 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Sigma0(a) + .long 0x0b0e0129 // add w9,w9,w14 + .long 0xb9400fee // ldr w14,[sp,#12] + .long 0xb9000be1 // str w1,[sp,#8] + .long 0x139a1b50 // ror w16,w26,#6 + .long 0x0b1302b5 // add w21,w21,w19 // h+=K[i] + .long 0x138b1d60 // ror w0,w11,#7 + .long 0x0a1a0371 // and w17,w27,w26 + .long 0x1388450f // ror w15,w8,#17 + .long 0x0a3a0293 // bic w19,w20,w26 + .long 0x13960ac1 // ror w1,w22,#2 + .long 0x0b0902b5 // add w21,w21,w9 // h+=X[i] + .long 0x4ada2e10 // eor w16,w16,w26, ror #11 + .long 0x4acb4800 // eor w0,w0,w11, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1702d3 // eor w19,w22,w23 // a^b, b^c in next round + .long 0x4ada6610 // eor w16,w16,w26, ror #25 // Sigma1(e) + .long 0x4ad63421 // eor w1,w1,w22, ror #13 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4ac84def // eor w15,w15,w8, ror #19 + .long 0x4a4b0c00 // eor w0,w0,w11, lsr #3 // sigma0(X[i+1]) + .long 0x0b1002b5 // add w21,w21,w16 // h+=Sigma1(e) + .long 0x4a17039c // eor w28,w28,w23 // Maj(a,b,c) + .long 0x4ad65831 // eor w17,w1,w22, ror #22 // Sigma0(a) + .long 0x4a4829ef // eor w15,w15,w8, lsr #10 // sigma1(X[i+14]) + .long 0x0b03014a // add w10,w10,w3 + .long 0x0b150339 // add w25,w25,w21 // d+=h + .long 0x0b1c02b5 // add w21,w21,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b00014a // add w10,w10,w0 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Sigma0(a) + .long 0x0b0f014a // add w10,w10,w15 + .long 0xb94003ef // ldr w15,[sp,#0] + .long 0xb9000fe2 // str w2,[sp,#12] + .long 0x13991b30 // ror w16,w25,#6 + .long 0x0b1c0294 // add w20,w20,w28 // h+=K[i] + .long 0x138c1d81 // ror w1,w12,#7 + .long 0x0a190351 // and w17,w26,w25 + .long 0x13894520 // ror w0,w9,#17 + .long 0x0a39037c // bic w28,w27,w25 + .long 0x13950aa2 // ror w2,w21,#2 + .long 0x0b0a0294 // add w20,w20,w10 // h+=X[i] + .long 0x4ad92e10 // eor w16,w16,w25, ror #11 + .long 0x4acc4821 // eor w1,w1,w12, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1602bc // eor w28,w21,w22 // a^b, b^c in next round + .long 0x4ad96610 // eor w16,w16,w25, ror #25 // Sigma1(e) + .long 0x4ad53442 // eor w2,w2,w21, ror #13 + .long 0x0b110294 // add w20,w20,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4ac94c00 // eor w0,w0,w9, ror #19 + .long 0x4a4c0c21 // eor w1,w1,w12, lsr #3 // sigma0(X[i+1]) + .long 0x0b100294 // add w20,w20,w16 // h+=Sigma1(e) + .long 0x4a160273 // eor w19,w19,w22 // Maj(a,b,c) + .long 0x4ad55851 // eor w17,w2,w21, ror #22 // Sigma0(a) + .long 0x4a492800 // eor w0,w0,w9, lsr #10 // sigma1(X[i+14]) + .long 0x0b04016b // add w11,w11,w4 + .long 0x0b140318 // add w24,w24,w20 // d+=h + .long 0x0b130294 // add w20,w20,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b01016b // add w11,w11,w1 + .long 0x0b110294 // add w20,w20,w17 // h+=Sigma0(a) + .long 0x0b00016b // add w11,w11,w0 + .long 0xb94007e0 // ldr w0,[sp,#4] + .long 0xb90003e3 // str w3,[sp,#0] + .long 0x13981b10 // ror w16,w24,#6 + .long 0x0b13037b // add w27,w27,w19 // h+=K[i] + .long 0x138d1da2 // ror w2,w13,#7 + .long 0x0a180331 // and w17,w25,w24 + .long 0x138a4541 // ror w1,w10,#17 + .long 0x0a380353 // bic w19,w26,w24 + .long 0x13940a83 // ror w3,w20,#2 + .long 0x0b0b037b // add w27,w27,w11 // h+=X[i] + .long 0x4ad82e10 // eor w16,w16,w24, ror #11 + .long 0x4acd4842 // eor w2,w2,w13, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a150293 // eor w19,w20,w21 // a^b, b^c in next round + .long 0x4ad86610 // eor w16,w16,w24, ror #25 // Sigma1(e) + .long 0x4ad43463 // eor w3,w3,w20, ror #13 + .long 0x0b11037b // add w27,w27,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4aca4c21 // eor w1,w1,w10, ror #19 + .long 0x4a4d0c42 // eor w2,w2,w13, lsr #3 // sigma0(X[i+1]) + .long 0x0b10037b // add w27,w27,w16 // h+=Sigma1(e) + .long 0x4a15039c // eor w28,w28,w21 // Maj(a,b,c) + .long 0x4ad45871 // eor w17,w3,w20, ror #22 // Sigma0(a) + .long 0x4a4a2821 // eor w1,w1,w10, lsr #10 // sigma1(X[i+14]) + .long 0x0b05018c // add w12,w12,w5 + .long 0x0b1b02f7 // add w23,w23,w27 // d+=h + .long 0x0b1c037b // add w27,w27,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b02018c // add w12,w12,w2 + .long 0x0b11037b // add w27,w27,w17 // h+=Sigma0(a) + .long 0x0b01018c // add w12,w12,w1 + .long 0xb9400be1 // ldr w1,[sp,#8] + .long 0xb90007e4 // str w4,[sp,#4] + .long 0x13971af0 // ror w16,w23,#6 + .long 0x0b1c035a // add w26,w26,w28 // h+=K[i] + .long 0x138e1dc3 // ror w3,w14,#7 + .long 0x0a170311 // and w17,w24,w23 + .long 0x138b4562 // ror w2,w11,#17 + .long 0x0a37033c // bic w28,w25,w23 + .long 0x139b0b64 // ror w4,w27,#2 + .long 0x0b0c035a // add w26,w26,w12 // h+=X[i] + .long 0x4ad72e10 // eor w16,w16,w23, ror #11 + .long 0x4ace4863 // eor w3,w3,w14, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a14037c // eor w28,w27,w20 // a^b, b^c in next round + .long 0x4ad76610 // eor w16,w16,w23, ror #25 // Sigma1(e) + .long 0x4adb3484 // eor w4,w4,w27, ror #13 + .long 0x0b11035a // add w26,w26,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4acb4c42 // eor w2,w2,w11, ror #19 + .long 0x4a4e0c63 // eor w3,w3,w14, lsr #3 // sigma0(X[i+1]) + .long 0x0b10035a // add w26,w26,w16 // h+=Sigma1(e) + .long 0x4a140273 // eor w19,w19,w20 // Maj(a,b,c) + .long 0x4adb5891 // eor w17,w4,w27, ror #22 // Sigma0(a) + .long 0x4a4b2842 // eor w2,w2,w11, lsr #10 // sigma1(X[i+14]) + .long 0x0b0601ad // add w13,w13,w6 + .long 0x0b1a02d6 // add w22,w22,w26 // d+=h + .long 0x0b13035a // add w26,w26,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b0301ad // add w13,w13,w3 + .long 0x0b11035a // add w26,w26,w17 // h+=Sigma0(a) + .long 0x0b0201ad // add w13,w13,w2 + .long 0xb9400fe2 // ldr w2,[sp,#12] + .long 0xb9000be5 // str w5,[sp,#8] + .long 0x13961ad0 // ror w16,w22,#6 + .long 0x0b130339 // add w25,w25,w19 // h+=K[i] + .long 0x138f1de4 // ror w4,w15,#7 + .long 0x0a1602f1 // and w17,w23,w22 + .long 0x138c4583 // ror w3,w12,#17 + .long 0x0a360313 // bic w19,w24,w22 + .long 0x139a0b45 // ror w5,w26,#2 + .long 0x0b0d0339 // add w25,w25,w13 // h+=X[i] + .long 0x4ad62e10 // eor w16,w16,w22, ror #11 + .long 0x4acf4884 // eor w4,w4,w15, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1b0353 // eor w19,w26,w27 // a^b, b^c in next round + .long 0x4ad66610 // eor w16,w16,w22, ror #25 // Sigma1(e) + .long 0x4ada34a5 // eor w5,w5,w26, ror #13 + .long 0x0b110339 // add w25,w25,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4acc4c63 // eor w3,w3,w12, ror #19 + .long 0x4a4f0c84 // eor w4,w4,w15, lsr #3 // sigma0(X[i+1]) + .long 0x0b100339 // add w25,w25,w16 // h+=Sigma1(e) + .long 0x4a1b039c // eor w28,w28,w27 // Maj(a,b,c) + .long 0x4ada58b1 // eor w17,w5,w26, ror #22 // Sigma0(a) + .long 0x4a4c2863 // eor w3,w3,w12, lsr #10 // sigma1(X[i+14]) + .long 0x0b0701ce // add w14,w14,w7 + .long 0x0b1902b5 // add w21,w21,w25 // d+=h + .long 0x0b1c0339 // add w25,w25,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b0401ce // add w14,w14,w4 + .long 0x0b110339 // add w25,w25,w17 // h+=Sigma0(a) + .long 0x0b0301ce // add w14,w14,w3 + .long 0xb94003e3 // ldr w3,[sp,#0] + .long 0xb9000fe6 // str w6,[sp,#12] + .long 0x13951ab0 // ror w16,w21,#6 + .long 0x0b1c0318 // add w24,w24,w28 // h+=K[i] + .long 0x13801c05 // ror w5,w0,#7 + .long 0x0a1502d1 // and w17,w22,w21 + .long 0x138d45a4 // ror w4,w13,#17 + .long 0x0a3502fc // bic w28,w23,w21 + .long 0x13990b26 // ror w6,w25,#2 + .long 0x0b0e0318 // add w24,w24,w14 // h+=X[i] + .long 0x4ad52e10 // eor w16,w16,w21, ror #11 + .long 0x4ac048a5 // eor w5,w5,w0, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1a033c // eor w28,w25,w26 // a^b, b^c in next round + .long 0x4ad56610 // eor w16,w16,w21, ror #25 // Sigma1(e) + .long 0x4ad934c6 // eor w6,w6,w25, ror #13 + .long 0x0b110318 // add w24,w24,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4acd4c84 // eor w4,w4,w13, ror #19 + .long 0x4a400ca5 // eor w5,w5,w0, lsr #3 // sigma0(X[i+1]) + .long 0x0b100318 // add w24,w24,w16 // h+=Sigma1(e) + .long 0x4a1a0273 // eor w19,w19,w26 // Maj(a,b,c) + .long 0x4ad958d1 // eor w17,w6,w25, ror #22 // Sigma0(a) + .long 0x4a4d2884 // eor w4,w4,w13, lsr #10 // sigma1(X[i+14]) + .long 0x0b0801ef // add w15,w15,w8 + .long 0x0b180294 // add w20,w20,w24 // d+=h + .long 0x0b130318 // add w24,w24,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b0501ef // add w15,w15,w5 + .long 0x0b110318 // add w24,w24,w17 // h+=Sigma0(a) + .long 0x0b0401ef // add w15,w15,w4 + .long 0xb94007e4 // ldr w4,[sp,#4] + .long 0xb90003e7 // str w7,[sp,#0] + .long 0x13941a90 // ror w16,w20,#6 + .long 0x0b1302f7 // add w23,w23,w19 // h+=K[i] + .long 0x13811c26 // ror w6,w1,#7 + .long 0x0a1402b1 // and w17,w21,w20 + .long 0x138e45c5 // ror w5,w14,#17 + .long 0x0a3402d3 // bic w19,w22,w20 + .long 0x13980b07 // ror w7,w24,#2 + .long 0x0b0f02f7 // add w23,w23,w15 // h+=X[i] + .long 0x4ad42e10 // eor w16,w16,w20, ror #11 + .long 0x4ac148c6 // eor w6,w6,w1, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a190313 // eor w19,w24,w25 // a^b, b^c in next round + .long 0x4ad46610 // eor w16,w16,w20, ror #25 // Sigma1(e) + .long 0x4ad834e7 // eor w7,w7,w24, ror #13 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4ace4ca5 // eor w5,w5,w14, ror #19 + .long 0x4a410cc6 // eor w6,w6,w1, lsr #3 // sigma0(X[i+1]) + .long 0x0b1002f7 // add w23,w23,w16 // h+=Sigma1(e) + .long 0x4a19039c // eor w28,w28,w25 // Maj(a,b,c) + .long 0x4ad858f1 // eor w17,w7,w24, ror #22 // Sigma0(a) + .long 0x4a4e28a5 // eor w5,w5,w14, lsr #10 // sigma1(X[i+14]) + .long 0x0b090000 // add w0,w0,w9 + .long 0x0b17037b // add w27,w27,w23 // d+=h + .long 0x0b1c02f7 // add w23,w23,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b060000 // add w0,w0,w6 + .long 0x0b1102f7 // add w23,w23,w17 // h+=Sigma0(a) + .long 0x0b050000 // add w0,w0,w5 + .long 0xb9400be5 // ldr w5,[sp,#8] + .long 0xb90007e8 // str w8,[sp,#4] + .long 0x139b1b70 // ror w16,w27,#6 + .long 0x0b1c02d6 // add w22,w22,w28 // h+=K[i] + .long 0x13821c47 // ror w7,w2,#7 + .long 0x0a1b0291 // and w17,w20,w27 + .long 0x138f45e6 // ror w6,w15,#17 + .long 0x0a3b02bc // bic w28,w21,w27 + .long 0x13970ae8 // ror w8,w23,#2 + .long 0x0b0002d6 // add w22,w22,w0 // h+=X[i] + .long 0x4adb2e10 // eor w16,w16,w27, ror #11 + .long 0x4ac248e7 // eor w7,w7,w2, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1802fc // eor w28,w23,w24 // a^b, b^c in next round + .long 0x4adb6610 // eor w16,w16,w27, ror #25 // Sigma1(e) + .long 0x4ad73508 // eor w8,w8,w23, ror #13 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4acf4cc6 // eor w6,w6,w15, ror #19 + .long 0x4a420ce7 // eor w7,w7,w2, lsr #3 // sigma0(X[i+1]) + .long 0x0b1002d6 // add w22,w22,w16 // h+=Sigma1(e) + .long 0x4a180273 // eor w19,w19,w24 // Maj(a,b,c) + .long 0x4ad75911 // eor w17,w8,w23, ror #22 // Sigma0(a) + .long 0x4a4f28c6 // eor w6,w6,w15, lsr #10 // sigma1(X[i+14]) + .long 0x0b0a0021 // add w1,w1,w10 + .long 0x0b16035a // add w26,w26,w22 // d+=h + .long 0x0b1302d6 // add w22,w22,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b070021 // add w1,w1,w7 + .long 0x0b1102d6 // add w22,w22,w17 // h+=Sigma0(a) + .long 0x0b060021 // add w1,w1,w6 + .long 0xb9400fe6 // ldr w6,[sp,#12] + .long 0xb9000be9 // str w9,[sp,#8] + .long 0x139a1b50 // ror w16,w26,#6 + .long 0x0b1302b5 // add w21,w21,w19 // h+=K[i] + .long 0x13831c68 // ror w8,w3,#7 + .long 0x0a1a0371 // and w17,w27,w26 + .long 0x13804407 // ror w7,w0,#17 + .long 0x0a3a0293 // bic w19,w20,w26 + .long 0x13960ac9 // ror w9,w22,#2 + .long 0x0b0102b5 // add w21,w21,w1 // h+=X[i] + .long 0x4ada2e10 // eor w16,w16,w26, ror #11 + .long 0x4ac34908 // eor w8,w8,w3, ror #18 + .long 0x2a130231 // orr w17,w17,w19 // Ch(e,f,g) + .long 0x4a1702d3 // eor w19,w22,w23 // a^b, b^c in next round + .long 0x4ada6610 // eor w16,w16,w26, ror #25 // Sigma1(e) + .long 0x4ad63529 // eor w9,w9,w22, ror #13 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Ch(e,f,g) + .long 0x0a13039c // and w28,w28,w19 // (b^c)&=(a^b) + .long 0x4ac04ce7 // eor w7,w7,w0, ror #19 + .long 0x4a430d08 // eor w8,w8,w3, lsr #3 // sigma0(X[i+1]) + .long 0x0b1002b5 // add w21,w21,w16 // h+=Sigma1(e) + .long 0x4a17039c // eor w28,w28,w23 // Maj(a,b,c) + .long 0x4ad65931 // eor w17,w9,w22, ror #22 // Sigma0(a) + .long 0x4a4028e7 // eor w7,w7,w0, lsr #10 // sigma1(X[i+14]) + .long 0x0b0b0042 // add w2,w2,w11 + .long 0x0b150339 // add w25,w25,w21 // d+=h + .long 0x0b1c02b5 // add w21,w21,w28 // h+=Maj(a,b,c) + .long 0xb84047dc // ldr w28,[x30],#4 // *K++, w19 in next round + .long 0x0b080042 // add w2,w2,w8 + .long 0x0b1102b5 // add w21,w21,w17 // h+=Sigma0(a) + .long 0x0b070042 // add w2,w2,w7 + .long 0xb94003e7 // ldr w7,[sp,#0] + .long 0xb9000fea // str w10,[sp,#12] + .long 0x13991b30 // ror w16,w25,#6 + .long 0x0b1c0294 // add w20,w20,w28 // h+=K[i] + .long 0x13841c89 // ror w9,w4,#7 + .long 0x0a190351 // and w17,w26,w25 + .long 0x13814428 // ror w8,w1,#17 + .long 0x0a39037c // bic w28,w27,w25 + .long 0x13950aaa // ror w10,w21,#2 + .long 0x0b020294 // add w20,w20,w2 // h+=X[i] + .long 0x4ad92e10 // eor w16,w16,w25, ror #11 + .long 0x4ac44929 // eor w9,w9,w4, ror #18 + .long 0x2a1c0231 // orr w17,w17,w28 // Ch(e,f,g) + .long 0x4a1602bc // eor w28,w21,w22 // a^b, b^c in next round + .long 0x4ad96610 // eor w16,w16,w25, ror #25 // Sigma1(e) + .long 0x4ad5354a // eor w10,w10,w21, ror #13 + .long 0x0b110294 // add w20,w20,w17 // h+=Ch(e,f,g) + .long 0x0a1c0273 // and w19,w19,w28 // (b^c)&=(a^b) + .long 0x4ac14d08 // eor w8,w8,w1, ror #19 + .long 0x4a440d29 // eor w9,w9,w4, lsr #3 // sigma0(X[i+1]) + .long 0x0b100294 // add w20,w20,w16 // h+=Sigma1(e) + .long 0x4a160273 // eor w19,w19,w22 // Maj(a,b,c) + .long 0x4ad55951 // eor w17,w10,w21, ror #22 // Sigma0(a) + .long 0x4a412908 // eor w8,w8,w1, lsr #10 // sigma1(X[i+14]) + .long 0x0b0c0063 // add w3,w3,w12 + .long 0x0b140318 // add w24,w24,w20 // d+=h + .long 0x0b130294 // add w20,w20,w19 // h+=Maj(a,b,c) + .long 0xb84047d3 // ldr w19,[x30],#4 // *K++, w28 in next round + .long 0x0b090063 // add w3,w3,w9 + .long 0x0b110294 // add w20,w20,w17 // h+=Sigma0(a) + .long 0x0b080063 // add w3,w3,w8 + cbnz w19,.Lsha256_gpr_loop_16_xx + + .long 0xa9460ba0 // ldp x0,x2,[x29,#96] + .long 0xf9403ba1 // ldr x1,[x29,#112] + .long 0xd10413de // sub x30,x30,#260 // rewind + + .long 0x29401003 // ldp w3,w4,[x0] + .long 0x29411805 // ldp w5,w6,[x0,#2*4] + .long 0x9100e021 // add x1,x1,#14*4 // advance input pointer + .long 0x29422007 // ldp w7,w8,[x0,#4*4] + .long 0x0b030294 // add w20,w20,w3 + .long 0x29432809 // ldp w9,w10,[x0,#6*4] + .long 0x0b0402b5 // add w21,w21,w4 + .long 0x0b0502d6 // add w22,w22,w5 + .long 0x0b0602f7 // add w23,w23,w6 + .long 0x29005414 // stp w20,w21,[x0] + .long 0x0b070318 // add w24,w24,w7 + .long 0x0b080339 // add w25,w25,w8 + .long 0x29015c16 // stp w22,w23,[x0,#2*4] + .long 0x0b09035a // add w26,w26,w9 + .long 0x0b0a037b // add w27,w27,w10 + .long 0xeb02003f // cmp x1,x2 + .long 0x29026418 // stp w24,w25,[x0,#4*4] + .long 0x29036c1a // stp w26,w27,[x0,#6*4] + b.ne .Lsha256_gpr_loop + + .long 0xa94153b3 // ldp x19,x20,[x29,#16] + .long 0x910043ff // add sp,sp,#4*4 + .long 0xa9425bb5 // ldp x21,x22,[x29,#32] + .long 0xa94363b7 // ldp x23,x24,[x29,#48] + .long 0xa9446bb9 // ldp x25,x26,[x29,#64] + .long 0xa94573bb // ldp x27,x28,[x29,#80] + .long 0xa8c87bfd // ldp x29,x30,[sp],#128 + ret diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_i386.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_i386.inc new file mode 100644 index 00000000..507a9892 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_i386.inc @@ -0,0 +1,224 @@ +// SHA-NI implementation of SHA-256 compress (IA-32; Intel SHA Extensions +// work in 32-bit mode), with the unified function's common prologue +// reproduced by the same proven head as the SSE2/SSSE3/AVX siblings (slots +// ctx/data/end/savedsp; the section's sub $32 shifts them to 32/36/40/44). +// K256 is read from K256_Doubled at a 32-byte stride (bias and offsets +// doubled; read unaligned - Pascal consts have no alignment guarantee); the +// byte-swap mask sits at the doubled +512 (= biased +256). +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = K256_Doubled ptr. +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K +// pointer. +// SHA-NI and shuffle instructions are db-encoded for broad assembler +// compatibility. +// Reference: OpenSSL CRYPTOGAMS sha256-586.pl (shaext section). + + push ebp + mov ebp, eax + mov eax, edi + mov edi, esi + mov esi, ebx + mov ebx, esp + sub esp, $10 + and esp, $FFFFFFC0 + shl eax, $6 + add eax, edi + mov dword ptr [esp], esi + mov dword ptr [esp + $4], edi + mov dword ptr [esp + $8], eax + mov dword ptr [esp + $C], ebx + sub esp, $20 + movdqu xmm1, oword ptr [esi] + lea ebp, [ebp + $100] + movdqu xmm2, oword ptr [esi + $10] + movdqu xmm7, oword ptr [ebp + $100] + pshufd xmm0, xmm1, $1B + pshufd xmm1, xmm1, $B1 + pshufd xmm2, xmm2, $1B + db $66, $0F, $3A, $0F, $CA, $08 // palignr xmm1, xmm2, $8 + db $66, $0F, $6C, $D0 // punpcklqdq xmm2, xmm0 + jmp @loop_shaext_011 + +@loop_shaext_011: + movdqu xmm3, oword ptr [edi] + movdqu xmm4, oword ptr [edi + $10] + movdqu xmm5, oword ptr [edi + $20] + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + movdqu xmm6, oword ptr [edi + $30] + movdqa oword ptr [esp + $10], xmm2 + movdqu xmm0, oword ptr [ebp - $100] + paddd xmm0, xmm3 + db $66, $0F, $38, $00, $E7 // pshufb xmm4, xmm7 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + nop + movdqa oword ptr [esp], xmm1 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $E0] + paddd xmm0, xmm4 + db $66, $0F, $38, $00, $EF // pshufb xmm5, xmm7 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + lea edi, [edi + $40] + db $0F, $38, $CC, $DC // sha256msg1 xmm3, xmm4 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $C0] + paddd xmm0, xmm5 + db $66, $0F, $38, $00, $F7 // pshufb xmm6, xmm7 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm6 + db $66, $0F, $3A, $0F, $FD, $04 // palignr xmm7, xmm5, $4 + nop + paddd xmm3, xmm7 + db $0F, $38, $CC, $E5 // sha256msg1 xmm4, xmm5 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $A0] + paddd xmm0, xmm6 + db $0F, $38, $CD, $DE // sha256msg2 xmm3, xmm6 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm3 + db $66, $0F, $3A, $0F, $FE, $04 // palignr xmm7, xmm6, $4 + nop + paddd xmm4, xmm7 + db $0F, $38, $CC, $EE // sha256msg1 xmm5, xmm6 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $80] + paddd xmm0, xmm3 + db $0F, $38, $CD, $E3 // sha256msg2 xmm4, xmm3 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm4 + db $66, $0F, $3A, $0F, $FB, $04 // palignr xmm7, xmm3, $4 + nop + paddd xmm5, xmm7 + db $0F, $38, $CC, $F3 // sha256msg1 xmm6, xmm3 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $60] + paddd xmm0, xmm4 + db $0F, $38, $CD, $EC // sha256msg2 xmm5, xmm4 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm5 + db $66, $0F, $3A, $0F, $FC, $04 // palignr xmm7, xmm4, $4 + nop + paddd xmm6, xmm7 + db $0F, $38, $CC, $DC // sha256msg1 xmm3, xmm4 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $40] + paddd xmm0, xmm5 + db $0F, $38, $CD, $F5 // sha256msg2 xmm6, xmm5 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm6 + db $66, $0F, $3A, $0F, $FD, $04 // palignr xmm7, xmm5, $4 + nop + paddd xmm3, xmm7 + db $0F, $38, $CC, $E5 // sha256msg1 xmm4, xmm5 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp - $20] + paddd xmm0, xmm6 + db $0F, $38, $CD, $DE // sha256msg2 xmm3, xmm6 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm3 + db $66, $0F, $3A, $0F, $FE, $04 // palignr xmm7, xmm6, $4 + nop + paddd xmm4, xmm7 + db $0F, $38, $CC, $EE // sha256msg1 xmm5, xmm6 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $0] + paddd xmm0, xmm3 + db $0F, $38, $CD, $E3 // sha256msg2 xmm4, xmm3 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm4 + db $66, $0F, $3A, $0F, $FB, $04 // palignr xmm7, xmm3, $4 + nop + paddd xmm5, xmm7 + db $0F, $38, $CC, $F3 // sha256msg1 xmm6, xmm3 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $20] + paddd xmm0, xmm4 + db $0F, $38, $CD, $EC // sha256msg2 xmm5, xmm4 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm5 + db $66, $0F, $3A, $0F, $FC, $04 // palignr xmm7, xmm4, $4 + nop + paddd xmm6, xmm7 + db $0F, $38, $CC, $DC // sha256msg1 xmm3, xmm4 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $40] + paddd xmm0, xmm5 + db $0F, $38, $CD, $F5 // sha256msg2 xmm6, xmm5 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm6 + db $66, $0F, $3A, $0F, $FD, $04 // palignr xmm7, xmm5, $4 + nop + paddd xmm3, xmm7 + db $0F, $38, $CC, $E5 // sha256msg1 xmm4, xmm5 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $60] + paddd xmm0, xmm6 + db $0F, $38, $CD, $DE // sha256msg2 xmm3, xmm6 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm3 + db $66, $0F, $3A, $0F, $FE, $04 // palignr xmm7, xmm6, $4 + nop + paddd xmm4, xmm7 + db $0F, $38, $CC, $EE // sha256msg1 xmm5, xmm6 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $80] + paddd xmm0, xmm3 + db $0F, $38, $CD, $E3 // sha256msg2 xmm4, xmm3 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm4 + db $66, $0F, $3A, $0F, $FB, $04 // palignr xmm7, xmm3, $4 + nop + paddd xmm5, xmm7 + db $0F, $38, $CC, $F3 // sha256msg1 xmm6, xmm3 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $A0] + paddd xmm0, xmm4 + db $0F, $38, $CD, $EC // sha256msg2 xmm5, xmm4 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + movdqa xmm7, xmm5 + db $66, $0F, $3A, $0F, $FC, $04 // palignr xmm7, xmm4, $4 + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + paddd xmm6, xmm7 + movdqu xmm0, oword ptr [ebp + $C0] + paddd xmm0, xmm5 + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + db $0F, $38, $CD, $F5 // sha256msg2 xmm6, xmm5 + movdqu xmm7, oword ptr [ebp + $100] + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + movdqu xmm0, oword ptr [ebp + $E0] + paddd xmm0, xmm6 + nop + db $0F, $38, $CB, $D1 // sha256rnds2 xmm2, xmm1, xmm0 + pshufd xmm0, xmm0, $E + cmp eax, edi + nop + db $0F, $38, $CB, $CA // sha256rnds2 xmm1, xmm2, xmm0 + paddd xmm2, oword ptr [esp + $10] + paddd xmm1, oword ptr [esp] + jnz @loop_shaext_011 + pshufd xmm2, xmm2, $B1 + pshufd xmm7, xmm1, $1B + pshufd xmm1, xmm1, $B1 + db $66, $0F, $6D, $CA // punpckhqdq xmm1, xmm2 + db $66, $0F, $3A, $0F, $D7, $08 // palignr xmm2, xmm7, $8 + mov esp, dword ptr [esp + $2C] + movdqu oword ptr [esi], xmm1 + movdqu oword ptr [esi + $10], xmm2 + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc index 640ed16e..7fc621b4 100644 --- a/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc @@ -1,36 +1,27 @@ -// SHA-256 SHA-NI implementation (Intel SHA Extensions), the same design as -// OpenSSL's x86_64 sha256_block_data_order_shaext (CRYPTOGAMS). Each 4-round -// group is two sha256rnds2 with the message schedule (sha256msg1/sha256msg2 + -// palignr) interleaved. The input is byte-swapped with pshufb against -// BSWAP32_MASK (a plain per-dword swap - SHA-256 consumes its words in natural -// order, so no dword reversal is needed). -// -// Expects MS x64 ABI (after HlpSimdProc5Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = K256 ptr (64 UInt32 round constants), r10 = byte-swap -// mask ptr (16 bytes; see BSWAP32_MASK). -// -// SHA-NI instructions, db-encoded for assembler compatibility: -// sha256rnds2 dst, src = $0F $38 $CB (implicit xmm0) -// sha256msg1 dst, src = $0F $38 $CC -// sha256msg2 dst, src = $0F $38 $CD -// pshufb dst, src = $66 $0F $38 $00 -// -// Register map: -// xmm0 = MSG+K temp for sha256rnds2 (implicit operand) -// xmm1 = STATE0 (ABEF in SHA-NI format) -// xmm2 = STATE1 (CDGH in SHA-NI format) -// xmm3 = MSG0 -// xmm4 = MSG1 -// xmm5 = MSG2 -// xmm6 = MSG3 -// xmm7 = palignr scratch / byte-swap mask during load -// xmm8 = ABEF_SAVE -// xmm9 = CDGH_SAVE -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); the kernel itself uses only volatile GPRs (rcx/rdx/r8-r10). - - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} +// SHA-NI implementation of SHA-256 compress (Intel SHA Extensions). Each +// 4-round group is two sha256rnds2 with the message schedule (sha256msg1/ +// sha256msg2 + palignr) interleaved. The input is byte-swapped with pshufb +// against BSWAP32_MASK (a plain per-dword swap - SHA-256 consumes its words +// in natural order, so no dword reversal is needed). +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = K256 ptr (64 UInt32 round constants), +// r10 = byte-swap mask ptr (16 bytes; see BSWAP32_MASK). +// Register map: xmm0 = MSG+K temp for sha256rnds2 (implicit operand), +// xmm1 = STATE0 (ABEF in SHA-NI format), xmm2 = STATE1 (CDGH), +// xmm3-xmm6 = MSG0-MSG3, xmm7 = palignr scratch / byte-swap mask during +// load, xmm8 = ABEF_SAVE, xmm9 = CDGH_SAVE. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); the kernel itself uses only volatile GPRs +// (rcx/rdx/r8-r10). +// SHA-NI instructions are db-encoded for broad assembler compatibility. +// Encoding notes: +// sha256rnds2 dst, src = $0F $38 $CB (implicit xmm0) +// sha256msg1 dst, src = $0F $38 $CC +// sha256msg2 dst, src = $0F $38 $CD +// pshufb dst, src = $66 $0F $38 $00 +// Reference: OpenSSL CRYPTOGAMS x86_64 sha256_block_data_order_shaext. + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} test r8d, r8d jz @shani_done @@ -255,4 +246,4 @@ @shani_done: - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_i386.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_i386.inc index e994d133..7632cfbb 100644 --- a/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_i386.inc +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_i386.inc @@ -1,14 +1,16 @@ -// SHA-256 SSE2 SIMD-schedule implementation (IA-32), derived from OpenSSL's -// sha256-586 (CRYPTOGAMS) SSSE3 kernel. The message schedule runs in SSE2 -// (paddd/psrld/pslld/pxor/pshufd/psrldq/pslldq) interleaved with the GPR -// compression rounds; the 32-bit state lives in eax/ebx/ecx plus the local frame -// (IA-32 has too few GPRs to keep all eight in registers). OpenSSL's two SSSE3-only -// ops are emulated in SSE2: the pshufb dword byte-swap becomes psrlw/psllw/por + +// SSE2 implementation of SHA-256 compress (SIMD schedule; IA-32). The +// message schedule runs in SSE2 (paddd/psrld/pslld/pxor/pshufd/psrldq/ +// pslldq) interleaved with the GPR compression rounds; the 32-bit state +// lives in eax/ebx/ecx plus the local frame (IA-32 has too few GPRs to keep +// all eight in registers). The original's two SSSE3-only ops are emulated +// in SSE2: the pshufb dword byte-swap becomes psrlw/psllw/por + // pshuflw/pshufhw, and palignr becomes psrldq/pslldq/por. -// Expects (after HlpSimdProc4Begin_i386): ebx = state, esi = data, edi = numblocks, -// eax = doubled-K256 ptr (read at a 32-byte stride so the duplicated halves are -// skipped; see K256_Doubled). HlpSimdProc4Begin pushes ebx/esi/edi; ebp is pushed -// here as the K256 walk pointer; state loads into eax/ebx/ecx and the frame. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = doubled-K256 ptr (read at a 32-byte stride so +// the duplicated halves are skipped; see K256_Doubled). +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K256 +// walk pointer; state loads into eax/ebx/ecx and the frame. +// Reference: OpenSSL CRYPTOGAMS sha256-586.pl (SSSE3 kernel). push ebp mov ebp, eax @@ -24,7 +26,7 @@ mov dword [esp + $4], edi mov dword [esp + $8], eax mov dword [esp + $C], ebx - lea esp, [esp - $60] + sub esp, $60 mov eax, dword [esi] mov ebx, dword [esi + $4] mov ecx, dword [esi + $8] diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc index 11fcdc13..01ce2863 100644 --- a/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc @@ -1,19 +1,18 @@ -// SHA-256 SSE2 SIMD-schedule implementation, derived from OpenSSL's SHA-256 -// SSSE3 kernel in sha512-x86_64.pl (CRYPTOGAMS). The message schedule runs in -// SSE2 (paddd/psrld/pslld/pxor/pshufd) interleaved with the GPR compression -// rounds, so the vector and integer units run in parallel. OpenSSL's two -// SSSE3-only ops are emulated in SSE2: the pshufb dword byte-swap becomes -// psrlw/psllw/por + pshuflw/pshufhw, and palignr becomes psrldq/pslldq/por -// (xmm8 is a free scratch). -// Expects MS x64 ABI (after HlpSimdProc4Begin): rcx = state ptr, rdx = data ptr, +// SSE2 implementation of SHA-256 compress (SIMD schedule). The message +// schedule runs in SSE2 (paddd/psrld/pslld/pxor/pshufd) interleaved with +// the GPR compression rounds, so the vector and integer units run in +// parallel. The original's two SSSE3-only ops are emulated in SSE2: the +// pshufb dword byte-swap becomes psrlw/psllw/por + pshuflw/pshufhw, and +// palignr becomes psrldq/pslldq/por (xmm8 is a free scratch). +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, // r8d = numblocks, r9 = doubled-K256 ptr (read at a 32-byte stride so the -// duplicated halves are skipped; see K256_Doubled). No byte-swap mask table is -// needed - the swap is computed, not shuffled. -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// duplicated halves are skipped; see K256_Doubled). No byte-swap mask +// table is needed - the swap is computed, not shuffled. +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// Reference: OpenSSL CRYPTOGAMS sha512-x86_64.pl (SHA-256 SSSE3 kernel). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} mov rax, rsp push r9 @@ -1150,4 +1149,4 @@ lea rsp, [rsi] mov rsi, rax - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_i386.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_i386.inc new file mode 100644 index 00000000..d1b2b506 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_i386.inc @@ -0,0 +1,1237 @@ +// SSSE3 implementation of SHA-256 compress (SIMD schedule; IA-32); SSE2 +// sibling: SHA256CompressSse2_i386.inc. The message schedule runs in SIMD +// interleaved with the GPR compression rounds; the 32-bit state lives in +// eax/ebx/ecx plus the local frame (IA-32 has too few GPRs to keep all +// eight in registers). Unlike the SSE2 sibling, the two SSSE3 ops are the +// real instructions: pshufb dword byte-swap against the mask kept in xmm7 +// for the block head, and palignr $4 in the schedule. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = doubled-K256 ptr (read at a 32-byte stride so +// the duplicated halves are skipped; see K256_Doubled). +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K256 +// walk pointer; state loads into eax/ebx/ecx and the frame. +// pshufb/palignr instructions are db-encoded for broad assembler +// compatibility. +// Reference: OpenSSL CRYPTOGAMS sha256-586.pl (SSSE3 kernel). + + push ebp + mov ebp, eax + mov eax, edi + mov edi, esi + mov esi, ebx + mov ebx, esp + sub esp, $10 + and esp, $FFFFFFC0 + shl eax, $6 + add eax, edi + mov dword [esp], esi + mov dword [esp + $4], edi + mov dword [esp + $8], eax + mov dword [esp + $C], ebx + sub esp, $60 + mov eax, dword [esi] + mov ebx, dword [esi + $4] + mov ecx, dword [esi + $8] + mov edi, dword [esi + $C] + mov dword [esp + $4], ebx + xor ebx, ecx + mov dword [esp + $8], ecx + mov dword [esp + $C], edi + mov edx, dword [esi + $10] + mov edi, dword [esi + $14] + mov ecx, dword [esi + $18] + mov esi, dword [esi + $1C] + mov dword [esp + $14], edi + mov edi, dword [esp + $64] + mov dword [esp + $18], ecx + mov dword [esp + $1C], esi + jmp @block_loop + +@block_loop: + movdqu xmm0, oword [edi] + movdqu xmm1, oword [edi + $10] + movdqu xmm2, oword [edi + $20] + movdqu xmm3, oword [edi + $30] + add edi, $40 + movdqu xmm7, oword [ebp + $200] // dword byte-swap mask (K256_Doubled + 512) + db $66, $0F, $38, $00, $C7 // pshufb xmm0, xmm7 + mov dword [esp + $64], edi + db $66, $0F, $38, $00, $CF // pshufb xmm1, xmm7 + movdqu xmm4, oword [ebp + $0] + db $66, $0F, $38, $00, $D7 // pshufb xmm2, xmm7 + movdqu xmm5, oword [ebp + $20] + paddd xmm4, xmm0 + db $66, $0F, $38, $00, $DF // pshufb xmm3, xmm7 + movdqu xmm6, oword [ebp + $40] + paddd xmm5, xmm1 + movdqu xmm7, oword [ebp + $60] + movdqa oword [esp + $20], xmm4 + paddd xmm6, xmm2 + movdqa oword [esp + $30], xmm5 + paddd xmm7, xmm3 + movdqa oword [esp + $40], xmm6 + movdqa oword [esp + $50], xmm7 + jmp @sched_loop + +@sched_loop: + add ebp, $80 + mov ecx, edx + movdqa xmm4, xmm1 + ror edx, $E + mov esi, dword [esp + $14] + movdqa xmm7, xmm3 + xor edx, ecx + mov edi, dword [esp + $18] + db $66, $0F, $3A, $0F, $E0, $04 // palignr xmm4, xmm0, 4 + xor esi, edi + ror edx, $5 + and esi, ecx + db $66, $0F, $3A, $0F, $FA, $04 // palignr xmm7, xmm2, 4 + mov dword [esp + $10], ecx + xor edx, ecx + xor edi, esi + movdqa xmm5, xmm4 + ror edx, $6 + mov ecx, eax + movdqa xmm6, xmm4 + add edx, edi + mov edi, dword [esp + $4] + psrld xmm4, $3 + mov esi, eax + ror ecx, $9 + paddd xmm0, xmm7 + mov dword [esp], eax + xor ecx, eax + psrld xmm6, $7 + xor eax, edi + add edx, dword [esp + $1C] + ror ecx, $B + and ebx, eax + pshufd xmm7, xmm3, $FA + xor ecx, esi + add edx, dword [esp + $20] + pslld xmm5, $E + xor ebx, edi + ror ecx, $2 + pxor xmm4, xmm6 + add ebx, edx + add edx, dword [esp + $C] + psrld xmm6, $B + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm4, xmm5 + mov esi, dword [esp + $10] + xor edx, ecx + pslld xmm5, $B + mov edi, dword [esp + $14] + xor esi, edi + ror edx, $5 + pxor xmm4, xmm6 + and esi, ecx + mov dword [esp + $C], ecx + movdqa xmm6, xmm7 + xor edx, ecx + xor edi, esi + ror edx, $6 + pxor xmm4, xmm5 + mov ecx, ebx + add edx, edi + psrld xmm7, $A + mov edi, dword [esp] + mov esi, ebx + ror ecx, $9 + paddd xmm0, xmm4 + mov dword [esp + $1C], ebx + xor ecx, ebx + psrlq xmm6, $11 + xor ebx, edi + add edx, dword [esp + $18] + ror ecx, $B + pxor xmm7, xmm6 + and eax, ebx + xor ecx, esi + psrlq xmm6, $2 + add edx, dword [esp + $24] + xor eax, edi + ror ecx, $2 + pxor xmm7, xmm6 + add eax, edx + add edx, dword [esp + $8] + pshufd xmm7, xmm7, $80 + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $C] + xor edx, ecx + mov edi, dword [esp + $10] + xor esi, edi + ror edx, $5 + and esi, ecx + psrldq xmm7, $8 + mov dword [esp + $8], ecx + xor edx, ecx + xor edi, esi + paddd xmm0, xmm7 + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $1C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $18], eax + pshufd xmm7, xmm0, $50 + xor ecx, eax + xor eax, edi + add edx, dword [esp + $14] + movdqa xmm6, xmm7 + ror ecx, $B + psrld xmm7, $A + and ebx, eax + psrlq xmm6, $11 + xor ecx, esi + add edx, dword [esp + $28] + xor ebx, edi + ror ecx, $2 + pxor xmm7, xmm6 + add ebx, edx + add edx, dword [esp + $4] + psrlq xmm6, $2 + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm7, xmm6 + mov esi, dword [esp + $8] + xor edx, ecx + mov edi, dword [esp + $C] + pshufd xmm7, xmm7, $8 + xor esi, edi + ror edx, $5 + movdqu xmm6, oword [ebp + $0] + and esi, ecx + mov dword [esp + $4], ecx + pslldq xmm7, $8 + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $18] + mov esi, ebx + ror ecx, $9 + paddd xmm0, xmm7 + mov dword [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $10] + paddd xmm6, xmm0 + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $2C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp] + add eax, ecx + movdqa oword [esp + $20], xmm6 + mov ecx, edx + movdqa xmm4, xmm2 + ror edx, $E + mov esi, dword [esp + $4] + movdqa xmm7, xmm0 + xor edx, ecx + mov edi, dword [esp + $8] + db $66, $0F, $3A, $0F, $E1, $04 // palignr xmm4, xmm1, 4 + xor esi, edi + ror edx, $5 + and esi, ecx + db $66, $0F, $3A, $0F, $FB, $04 // palignr xmm7, xmm3, 4 + mov dword [esp], ecx + xor edx, ecx + xor edi, esi + movdqa xmm5, xmm4 + ror edx, $6 + mov ecx, eax + movdqa xmm6, xmm4 + add edx, edi + mov edi, dword [esp + $14] + psrld xmm4, $3 + mov esi, eax + ror ecx, $9 + paddd xmm1, xmm7 + mov dword [esp + $10], eax + xor ecx, eax + psrld xmm6, $7 + xor eax, edi + add edx, dword [esp + $C] + ror ecx, $B + and ebx, eax + pshufd xmm7, xmm0, $FA + xor ecx, esi + add edx, dword [esp + $30] + pslld xmm5, $E + xor ebx, edi + ror ecx, $2 + pxor xmm4, xmm6 + add ebx, edx + add edx, dword [esp + $1C] + psrld xmm6, $B + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm4, xmm5 + mov esi, dword [esp] + xor edx, ecx + pslld xmm5, $B + mov edi, dword [esp + $4] + xor esi, edi + ror edx, $5 + pxor xmm4, xmm6 + and esi, ecx + mov dword [esp + $1C], ecx + movdqa xmm6, xmm7 + xor edx, ecx + xor edi, esi + ror edx, $6 + pxor xmm4, xmm5 + mov ecx, ebx + add edx, edi + psrld xmm7, $A + mov edi, dword [esp + $10] + mov esi, ebx + ror ecx, $9 + paddd xmm1, xmm4 + mov dword [esp + $C], ebx + xor ecx, ebx + psrlq xmm6, $11 + xor ebx, edi + add edx, dword [esp + $8] + ror ecx, $B + pxor xmm7, xmm6 + and eax, ebx + xor ecx, esi + psrlq xmm6, $2 + add edx, dword [esp + $34] + xor eax, edi + ror ecx, $2 + pxor xmm7, xmm6 + add eax, edx + add edx, dword [esp + $18] + pshufd xmm7, xmm7, $80 + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $1C] + xor edx, ecx + mov edi, dword [esp] + xor esi, edi + ror edx, $5 + and esi, ecx + psrldq xmm7, $8 + mov dword [esp + $18], ecx + xor edx, ecx + xor edi, esi + paddd xmm1, xmm7 + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $8], eax + pshufd xmm7, xmm1, $50 + xor ecx, eax + xor eax, edi + add edx, dword [esp + $4] + movdqa xmm6, xmm7 + ror ecx, $B + psrld xmm7, $A + and ebx, eax + psrlq xmm6, $11 + xor ecx, esi + add edx, dword [esp + $38] + xor ebx, edi + ror ecx, $2 + pxor xmm7, xmm6 + add ebx, edx + add edx, dword [esp + $14] + psrlq xmm6, $2 + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm7, xmm6 + mov esi, dword [esp + $18] + xor edx, ecx + mov edi, dword [esp + $1C] + pshufd xmm7, xmm7, $8 + xor esi, edi + ror edx, $5 + movdqu xmm6, oword [ebp + $20] + and esi, ecx + mov dword [esp + $14], ecx + pslldq xmm7, $8 + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $8] + mov esi, ebx + ror ecx, $9 + paddd xmm1, xmm7 + mov dword [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp] + paddd xmm6, xmm1 + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $3C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $10] + add eax, ecx + movdqa oword [esp + $30], xmm6 + mov ecx, edx + movdqa xmm4, xmm3 + ror edx, $E + mov esi, dword [esp + $14] + movdqa xmm7, xmm1 + xor edx, ecx + mov edi, dword [esp + $18] + db $66, $0F, $3A, $0F, $E2, $04 // palignr xmm4, xmm2, 4 + xor esi, edi + ror edx, $5 + and esi, ecx + db $66, $0F, $3A, $0F, $F8, $04 // palignr xmm7, xmm0, 4 + mov dword [esp + $10], ecx + xor edx, ecx + xor edi, esi + movdqa xmm5, xmm4 + ror edx, $6 + mov ecx, eax + movdqa xmm6, xmm4 + add edx, edi + mov edi, dword [esp + $4] + psrld xmm4, $3 + mov esi, eax + ror ecx, $9 + paddd xmm2, xmm7 + mov dword [esp], eax + xor ecx, eax + psrld xmm6, $7 + xor eax, edi + add edx, dword [esp + $1C] + ror ecx, $B + and ebx, eax + pshufd xmm7, xmm1, $FA + xor ecx, esi + add edx, dword [esp + $40] + pslld xmm5, $E + xor ebx, edi + ror ecx, $2 + pxor xmm4, xmm6 + add ebx, edx + add edx, dword [esp + $C] + psrld xmm6, $B + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm4, xmm5 + mov esi, dword [esp + $10] + xor edx, ecx + pslld xmm5, $B + mov edi, dword [esp + $14] + xor esi, edi + ror edx, $5 + pxor xmm4, xmm6 + and esi, ecx + mov dword [esp + $C], ecx + movdqa xmm6, xmm7 + xor edx, ecx + xor edi, esi + ror edx, $6 + pxor xmm4, xmm5 + mov ecx, ebx + add edx, edi + psrld xmm7, $A + mov edi, dword [esp] + mov esi, ebx + ror ecx, $9 + paddd xmm2, xmm4 + mov dword [esp + $1C], ebx + xor ecx, ebx + psrlq xmm6, $11 + xor ebx, edi + add edx, dword [esp + $18] + ror ecx, $B + pxor xmm7, xmm6 + and eax, ebx + xor ecx, esi + psrlq xmm6, $2 + add edx, dword [esp + $44] + xor eax, edi + ror ecx, $2 + pxor xmm7, xmm6 + add eax, edx + add edx, dword [esp + $8] + pshufd xmm7, xmm7, $80 + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $C] + xor edx, ecx + mov edi, dword [esp + $10] + xor esi, edi + ror edx, $5 + and esi, ecx + psrldq xmm7, $8 + mov dword [esp + $8], ecx + xor edx, ecx + xor edi, esi + paddd xmm2, xmm7 + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $1C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $18], eax + pshufd xmm7, xmm2, $50 + xor ecx, eax + xor eax, edi + add edx, dword [esp + $14] + movdqa xmm6, xmm7 + ror ecx, $B + psrld xmm7, $A + and ebx, eax + psrlq xmm6, $11 + xor ecx, esi + add edx, dword [esp + $48] + xor ebx, edi + ror ecx, $2 + pxor xmm7, xmm6 + add ebx, edx + add edx, dword [esp + $4] + psrlq xmm6, $2 + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm7, xmm6 + mov esi, dword [esp + $8] + xor edx, ecx + mov edi, dword [esp + $C] + pshufd xmm7, xmm7, $8 + xor esi, edi + ror edx, $5 + movdqu xmm6, oword [ebp + $40] + and esi, ecx + mov dword [esp + $4], ecx + pslldq xmm7, $8 + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $18] + mov esi, ebx + ror ecx, $9 + paddd xmm2, xmm7 + mov dword [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $10] + paddd xmm6, xmm2 + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $4C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp] + add eax, ecx + movdqa oword [esp + $40], xmm6 + mov ecx, edx + movdqa xmm4, xmm0 + ror edx, $E + mov esi, dword [esp + $4] + movdqa xmm7, xmm2 + xor edx, ecx + mov edi, dword [esp + $8] + db $66, $0F, $3A, $0F, $E3, $04 // palignr xmm4, xmm3, 4 + xor esi, edi + ror edx, $5 + and esi, ecx + db $66, $0F, $3A, $0F, $F9, $04 // palignr xmm7, xmm1, 4 + mov dword [esp], ecx + xor edx, ecx + xor edi, esi + movdqa xmm5, xmm4 + ror edx, $6 + mov ecx, eax + movdqa xmm6, xmm4 + add edx, edi + mov edi, dword [esp + $14] + psrld xmm4, $3 + mov esi, eax + ror ecx, $9 + paddd xmm3, xmm7 + mov dword [esp + $10], eax + xor ecx, eax + psrld xmm6, $7 + xor eax, edi + add edx, dword [esp + $C] + ror ecx, $B + and ebx, eax + pshufd xmm7, xmm2, $FA + xor ecx, esi + add edx, dword [esp + $50] + pslld xmm5, $E + xor ebx, edi + ror ecx, $2 + pxor xmm4, xmm6 + add ebx, edx + add edx, dword [esp + $1C] + psrld xmm6, $B + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm4, xmm5 + mov esi, dword [esp] + xor edx, ecx + pslld xmm5, $B + mov edi, dword [esp + $4] + xor esi, edi + ror edx, $5 + pxor xmm4, xmm6 + and esi, ecx + mov dword [esp + $1C], ecx + movdqa xmm6, xmm7 + xor edx, ecx + xor edi, esi + ror edx, $6 + pxor xmm4, xmm5 + mov ecx, ebx + add edx, edi + psrld xmm7, $A + mov edi, dword [esp + $10] + mov esi, ebx + ror ecx, $9 + paddd xmm3, xmm4 + mov dword [esp + $C], ebx + xor ecx, ebx + psrlq xmm6, $11 + xor ebx, edi + add edx, dword [esp + $8] + ror ecx, $B + pxor xmm7, xmm6 + and eax, ebx + xor ecx, esi + psrlq xmm6, $2 + add edx, dword [esp + $54] + xor eax, edi + ror ecx, $2 + pxor xmm7, xmm6 + add eax, edx + add edx, dword [esp + $18] + pshufd xmm7, xmm7, $80 + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $1C] + xor edx, ecx + mov edi, dword [esp] + xor esi, edi + ror edx, $5 + and esi, ecx + psrldq xmm7, $8 + mov dword [esp + $18], ecx + xor edx, ecx + xor edi, esi + paddd xmm3, xmm7 + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $8], eax + pshufd xmm7, xmm3, $50 + xor ecx, eax + xor eax, edi + add edx, dword [esp + $4] + movdqa xmm6, xmm7 + ror ecx, $B + psrld xmm7, $A + and ebx, eax + psrlq xmm6, $11 + xor ecx, esi + add edx, dword [esp + $58] + xor ebx, edi + ror ecx, $2 + pxor xmm7, xmm6 + add ebx, edx + add edx, dword [esp + $14] + psrlq xmm6, $2 + add ebx, ecx + mov ecx, edx + ror edx, $E + pxor xmm7, xmm6 + mov esi, dword [esp + $18] + xor edx, ecx + mov edi, dword [esp + $1C] + pshufd xmm7, xmm7, $8 + xor esi, edi + ror edx, $5 + movdqu xmm6, oword [ebp + $60] + and esi, ecx + mov dword [esp + $14], ecx + pslldq xmm7, $8 + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $8] + mov esi, ebx + ror ecx, $9 + paddd xmm3, xmm7 + mov dword [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp] + paddd xmm6, xmm3 + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $5C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $10] + add eax, ecx + movdqa oword [esp + $50], xmm6 + cmp dword [ebp + $80], $10203 + jne @sched_loop + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $14] + xor edx, ecx + mov edi, dword [esp + $18] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $10], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $4] + mov esi, eax + ror ecx, $9 + mov dword [esp], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $1C] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $20] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $C] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $10] + xor edx, ecx + mov edi, dword [esp + $14] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $C], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $1C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $18] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $24] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $8] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $C] + xor edx, ecx + mov edi, dword [esp + $10] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $8], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $1C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $18], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $14] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $28] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $4] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $8] + xor edx, ecx + mov edi, dword [esp + $C] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $4], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $18] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $10] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $2C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $4] + xor edx, ecx + mov edi, dword [esp + $8] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $14] + mov esi, eax + ror ecx, $9 + mov dword [esp + $10], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $C] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $30] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $1C] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp] + xor edx, ecx + mov edi, dword [esp + $4] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $1C], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $10] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $8] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $34] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $18] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $1C] + xor edx, ecx + mov edi, dword [esp] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $18], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $8], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $4] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $38] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $14] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $18] + xor edx, ecx + mov edi, dword [esp + $1C] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $14], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $8] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $3C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $10] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $14] + xor edx, ecx + mov edi, dword [esp + $18] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $10], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $4] + mov esi, eax + ror ecx, $9 + mov dword [esp], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $1C] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $40] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $C] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $10] + xor edx, ecx + mov edi, dword [esp + $14] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $C], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $1C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $18] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $44] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $8] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $C] + xor edx, ecx + mov edi, dword [esp + $10] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $8], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $1C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $18], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $14] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $48] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $4] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $8] + xor edx, ecx + mov edi, dword [esp + $C] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $4], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $18] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $14], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $10] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $4C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $4] + xor edx, ecx + mov edi, dword [esp + $8] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $14] + mov esi, eax + ror ecx, $9 + mov dword [esp + $10], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $C] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $50] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $1C] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp] + xor edx, ecx + mov edi, dword [esp + $4] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $1C], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $10] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $C], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp + $8] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $54] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $18] + add eax, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $1C] + xor edx, ecx + mov edi, dword [esp] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $18], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, eax + add edx, edi + mov edi, dword [esp + $C] + mov esi, eax + ror ecx, $9 + mov dword [esp + $8], eax + xor ecx, eax + xor eax, edi + add edx, dword [esp + $4] + ror ecx, $B + and ebx, eax + xor ecx, esi + add edx, dword [esp + $58] + xor ebx, edi + ror ecx, $2 + add ebx, edx + add edx, dword [esp + $14] + add ebx, ecx + mov ecx, edx + ror edx, $E + mov esi, dword [esp + $18] + xor edx, ecx + mov edi, dword [esp + $1C] + xor esi, edi + ror edx, $5 + and esi, ecx + mov dword [esp + $14], ecx + xor edx, ecx + xor edi, esi + ror edx, $6 + mov ecx, ebx + add edx, edi + mov edi, dword [esp + $8] + mov esi, ebx + ror ecx, $9 + mov dword [esp + $4], ebx + xor ecx, ebx + xor ebx, edi + add edx, dword [esp] + ror ecx, $B + and eax, ebx + xor ecx, esi + add edx, dword [esp + $5C] + xor eax, edi + ror ecx, $2 + add eax, edx + add edx, dword [esp + $10] + add eax, ecx + mov esi, dword [esp + $60] + xor ebx, edi + mov ecx, dword [esp + $C] + add eax, dword [esi] + add ebx, dword [esi + $4] + add edi, dword [esi + $8] + add ecx, dword [esi + $C] + mov dword [esi], eax + mov dword [esi + $4], ebx + mov dword [esi + $8], edi + mov dword [esi + $C], ecx + mov dword [esp + $4], ebx + xor ebx, edi + mov dword [esp + $8], edi + mov dword [esp + $C], ecx + mov edi, dword [esp + $14] + mov ecx, dword [esp + $18] + add edx, dword [esi + $10] + add edi, dword [esi + $14] + add ecx, dword [esi + $18] + mov dword [esi + $10], edx + mov dword [esi + $14], edi + mov dword [esp + $14], edi + mov edi, dword [esp + $1C] + mov dword [esi + $18], ecx + add edi, dword [esi + $1C] + mov dword [esp + $18], ecx + mov dword [esi + $1C], edi + mov dword [esp + $1C], edi + mov edi, dword [esp + $64] + sub ebp, $180 + cmp edi, dword [esp + $68] + jb @block_loop + mov esp, dword [esp + $6C] + pop ebp + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc new file mode 100644 index 00000000..2ce26d6c --- /dev/null +++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc @@ -0,0 +1,1111 @@ +// SSSE3 implementation of SHA-256 compress (SIMD schedule); SSE2 sibling: +// SHA256CompressSse2_x86_64.inc. The message schedule runs in SIMD +// (paddd/psrld/pslld/pxor/pshufd) interleaved with the GPR compression +// rounds, so the vector and integer units run in parallel. Unlike the SSE2 +// sibling, the two SSSE3 ops are the real instructions: pshufb dword +// byte-swap against the mask kept in xmm8 for the whole kernel, and +// palignr $4 in the schedule. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K256 ptr (read at a 32-byte stride so the +// duplicated halves are skipped; the byte-swap mask sits at +512). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// pshufb/palignr instructions are db-encoded for broad assembler +// compatibility. +// Reference: OpenSSL CRYPTOGAMS sha512-x86_64.pl (SHA-256 SSSE3 kernel). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + mov rax, rsp + push r9 + movdqu xmm8, oword [r9 + $200] // dword byte-swap mask (K256_Doubled + 512) + push rbx + push rbp + push rsi + push rdi + push r12 + push r13 + push r14 + push r15 + mov rdi, rcx + mov rsi, rdx + mov edx, r8d + shl rdx, $4 + sub rsp, $A0 + lea rdx, [rsi + rdx*4] + and rsp, $FFFFFFFFFFFFFFC0 + mov qword [rsp + $40], rdi + mov qword [rsp + $48], rsi + mov qword [rsp + $50], rdx + mov qword [rsp + $58], rax + +@prologue: + mov eax, dword [rdi] + mov ebx, dword [rdi + $4] + mov ecx, dword [rdi + $8] + mov edx, dword [rdi + $C] + mov r8d, dword [rdi + $10] + mov r9d, dword [rdi + $14] + mov r10d, dword [rdi + $18] + mov r11d, dword [rdi + $1C] + jmp @block_loop + +@block_loop: + movdqu xmm0, oword [rsi] + movdqu xmm1, oword [rsi + $10] + movdqu xmm2, oword [rsi + $20] + db $66, $41, $0F, $38, $00, $C0 // pshufb xmm0, xmm8 + movdqu xmm3, oword [rsi + $30] + mov rbp, qword [rsp + $58] + mov rbp, qword [rbp - $8] + db $66, $41, $0F, $38, $00, $C8 // pshufb xmm1, xmm8 + movdqu xmm4, oword [rbp + $0] + movdqu xmm5, oword [rbp + $20] + db $66, $41, $0F, $38, $00, $D0 // pshufb xmm2, xmm8 + paddd xmm4, xmm0 + movdqu xmm6, oword [rbp + $40] + db $66, $41, $0F, $38, $00, $D8 // pshufb xmm3, xmm8 + movdqu xmm7, oword [rbp + $60] + paddd xmm5, xmm1 + paddd xmm6, xmm2 + paddd xmm7, xmm3 + movdqa oword [rsp], xmm4 + mov r14d, eax + movdqa oword [rsp + $10], xmm5 + mov edi, ebx + movdqa oword [rsp + $20], xmm6 + xor edi, ecx + movdqa oword [rsp + $30], xmm7 + mov r13d, r8d + jmp @sched_loop + +@sched_loop: + sub rbp, $FFFFFFFFFFFFFF80 + ror r13d, $E + movdqa xmm4, xmm1 + mov eax, r14d + mov r12d, r9d + movdqa xmm7, xmm3 + ror r14d, $9 + xor r13d, r8d + xor r12d, r10d + ror r13d, $5 + xor r14d, eax + db $66, $0F, $3A, $0F, $E0, $04 // palignr xmm4, xmm0, 4 + and r12d, r8d + xor r13d, r8d + db $66, $0F, $3A, $0F, $FA, $04 // palignr xmm7, xmm2, 4 + add r11d, dword [rsp] + mov r15d, eax + xor r12d, r10d + ror r14d, $B + movdqa xmm5, xmm4 + xor r15d, ebx + add r11d, r12d + movdqa xmm6, xmm4 + ror r13d, $6 + and edi, r15d + psrld xmm4, $3 + xor r14d, eax + add r11d, r13d + xor edi, ebx + paddd xmm0, xmm7 + ror r14d, $2 + add edx, r11d + psrld xmm6, $7 + add r11d, edi + mov r13d, edx + pshufd xmm7, xmm3, $FA + add r14d, r11d + ror r13d, $E + pslld xmm5, $E + mov r11d, r14d + mov r12d, r8d + pxor xmm4, xmm6 + ror r14d, $9 + xor r13d, edx + xor r12d, r9d + ror r13d, $5 + psrld xmm6, $B + xor r14d, r11d + pxor xmm4, xmm5 + and r12d, edx + xor r13d, edx + pslld xmm5, $B + add r10d, dword [rsp + $4] + mov edi, r11d + pxor xmm4, xmm6 + xor r12d, r9d + ror r14d, $B + movdqa xmm6, xmm7 + xor edi, eax + add r10d, r12d + pxor xmm4, xmm5 + ror r13d, $6 + and r15d, edi + xor r14d, r11d + psrld xmm7, $A + add r10d, r13d + xor r15d, eax + paddd xmm0, xmm4 + ror r14d, $2 + add ecx, r10d + psrlq xmm6, $11 + add r10d, r15d + mov r13d, ecx + add r14d, r10d + pxor xmm7, xmm6 + ror r13d, $E + mov r10d, r14d + mov r12d, edx + ror r14d, $9 + psrlq xmm6, $2 + xor r13d, ecx + xor r12d, r8d + pxor xmm7, xmm6 + ror r13d, $5 + xor r14d, r10d + and r12d, ecx + pshufd xmm7, xmm7, $80 + xor r13d, ecx + add r9d, dword [rsp + $8] + mov r15d, r10d + psrldq xmm7, $8 + xor r12d, r8d + ror r14d, $B + xor r15d, r11d + add r9d, r12d + ror r13d, $6 + paddd xmm0, xmm7 + and edi, r15d + xor r14d, r10d + add r9d, r13d + pshufd xmm7, xmm0, $50 + xor edi, r11d + ror r14d, $2 + add ebx, r9d + movdqa xmm6, xmm7 + add r9d, edi + mov r13d, ebx + psrld xmm7, $A + add r14d, r9d + ror r13d, $E + psrlq xmm6, $11 + mov r9d, r14d + mov r12d, ecx + pxor xmm7, xmm6 + ror r14d, $9 + xor r13d, ebx + xor r12d, edx + ror r13d, $5 + xor r14d, r9d + psrlq xmm6, $2 + and r12d, ebx + xor r13d, ebx + add r8d, dword [rsp + $C] + pxor xmm7, xmm6 + mov edi, r9d + xor r12d, edx + ror r14d, $B + pshufd xmm7, xmm7, $8 + xor edi, r10d + add r8d, r12d + movdqu xmm6, oword [rbp + $0] + ror r13d, $6 + and r15d, edi + pslldq xmm7, $8 + xor r14d, r9d + add r8d, r13d + xor r15d, r10d + paddd xmm0, xmm7 + ror r14d, $2 + add eax, r8d + add r8d, r15d + paddd xmm6, xmm0 + mov r13d, eax + add r14d, r8d + movdqa oword [rsp], xmm6 + ror r13d, $E + movdqa xmm4, xmm2 + mov r8d, r14d + mov r12d, ebx + movdqa xmm7, xmm0 + ror r14d, $9 + xor r13d, eax + xor r12d, ecx + ror r13d, $5 + xor r14d, r8d + db $66, $0F, $3A, $0F, $E1, $04 // palignr xmm4, xmm1, 4 + and r12d, eax + xor r13d, eax + db $66, $0F, $3A, $0F, $FB, $04 // palignr xmm7, xmm3, 4 + add edx, dword [rsp + $10] + mov r15d, r8d + xor r12d, ecx + ror r14d, $B + movdqa xmm5, xmm4 + xor r15d, r9d + add edx, r12d + movdqa xmm6, xmm4 + ror r13d, $6 + and edi, r15d + psrld xmm4, $3 + xor r14d, r8d + add edx, r13d + xor edi, r9d + paddd xmm1, xmm7 + ror r14d, $2 + add r11d, edx + psrld xmm6, $7 + add edx, edi + mov r13d, r11d + pshufd xmm7, xmm0, $FA + add r14d, edx + ror r13d, $E + pslld xmm5, $E + mov edx, r14d + mov r12d, eax + pxor xmm4, xmm6 + ror r14d, $9 + xor r13d, r11d + xor r12d, ebx + ror r13d, $5 + psrld xmm6, $B + xor r14d, edx + pxor xmm4, xmm5 + and r12d, r11d + xor r13d, r11d + pslld xmm5, $B + add ecx, dword [rsp + $14] + mov edi, edx + pxor xmm4, xmm6 + xor r12d, ebx + ror r14d, $B + movdqa xmm6, xmm7 + xor edi, r8d + add ecx, r12d + pxor xmm4, xmm5 + ror r13d, $6 + and r15d, edi + xor r14d, edx + psrld xmm7, $A + add ecx, r13d + xor r15d, r8d + paddd xmm1, xmm4 + ror r14d, $2 + add r10d, ecx + psrlq xmm6, $11 + add ecx, r15d + mov r13d, r10d + add r14d, ecx + pxor xmm7, xmm6 + ror r13d, $E + mov ecx, r14d + mov r12d, r11d + ror r14d, $9 + psrlq xmm6, $2 + xor r13d, r10d + xor r12d, eax + pxor xmm7, xmm6 + ror r13d, $5 + xor r14d, ecx + and r12d, r10d + pshufd xmm7, xmm7, $80 + xor r13d, r10d + add ebx, dword [rsp + $18] + mov r15d, ecx + psrldq xmm7, $8 + xor r12d, eax + ror r14d, $B + xor r15d, edx + add ebx, r12d + ror r13d, $6 + paddd xmm1, xmm7 + and edi, r15d + xor r14d, ecx + add ebx, r13d + pshufd xmm7, xmm1, $50 + xor edi, edx + ror r14d, $2 + add r9d, ebx + movdqa xmm6, xmm7 + add ebx, edi + mov r13d, r9d + psrld xmm7, $A + add r14d, ebx + ror r13d, $E + psrlq xmm6, $11 + mov ebx, r14d + mov r12d, r10d + pxor xmm7, xmm6 + ror r14d, $9 + xor r13d, r9d + xor r12d, r11d + ror r13d, $5 + xor r14d, ebx + psrlq xmm6, $2 + and r12d, r9d + xor r13d, r9d + add eax, dword [rsp + $1C] + pxor xmm7, xmm6 + mov edi, ebx + xor r12d, r11d + ror r14d, $B + pshufd xmm7, xmm7, $8 + xor edi, ecx + add eax, r12d + movdqu xmm6, oword [rbp + $20] + ror r13d, $6 + and r15d, edi + pslldq xmm7, $8 + xor r14d, ebx + add eax, r13d + xor r15d, ecx + paddd xmm1, xmm7 + ror r14d, $2 + add r8d, eax + add eax, r15d + paddd xmm6, xmm1 + mov r13d, r8d + add r14d, eax + movdqa oword [rsp + $10], xmm6 + ror r13d, $E + movdqa xmm4, xmm3 + mov eax, r14d + mov r12d, r9d + movdqa xmm7, xmm1 + ror r14d, $9 + xor r13d, r8d + xor r12d, r10d + ror r13d, $5 + xor r14d, eax + db $66, $0F, $3A, $0F, $E2, $04 // palignr xmm4, xmm2, 4 + and r12d, r8d + xor r13d, r8d + db $66, $0F, $3A, $0F, $F8, $04 // palignr xmm7, xmm0, 4 + add r11d, dword [rsp + $20] + mov r15d, eax + xor r12d, r10d + ror r14d, $B + movdqa xmm5, xmm4 + xor r15d, ebx + add r11d, r12d + movdqa xmm6, xmm4 + ror r13d, $6 + and edi, r15d + psrld xmm4, $3 + xor r14d, eax + add r11d, r13d + xor edi, ebx + paddd xmm2, xmm7 + ror r14d, $2 + add edx, r11d + psrld xmm6, $7 + add r11d, edi + mov r13d, edx + pshufd xmm7, xmm1, $FA + add r14d, r11d + ror r13d, $E + pslld xmm5, $E + mov r11d, r14d + mov r12d, r8d + pxor xmm4, xmm6 + ror r14d, $9 + xor r13d, edx + xor r12d, r9d + ror r13d, $5 + psrld xmm6, $B + xor r14d, r11d + pxor xmm4, xmm5 + and r12d, edx + xor r13d, edx + pslld xmm5, $B + add r10d, dword [rsp + $24] + mov edi, r11d + pxor xmm4, xmm6 + xor r12d, r9d + ror r14d, $B + movdqa xmm6, xmm7 + xor edi, eax + add r10d, r12d + pxor xmm4, xmm5 + ror r13d, $6 + and r15d, edi + xor r14d, r11d + psrld xmm7, $A + add r10d, r13d + xor r15d, eax + paddd xmm2, xmm4 + ror r14d, $2 + add ecx, r10d + psrlq xmm6, $11 + add r10d, r15d + mov r13d, ecx + add r14d, r10d + pxor xmm7, xmm6 + ror r13d, $E + mov r10d, r14d + mov r12d, edx + ror r14d, $9 + psrlq xmm6, $2 + xor r13d, ecx + xor r12d, r8d + pxor xmm7, xmm6 + ror r13d, $5 + xor r14d, r10d + and r12d, ecx + pshufd xmm7, xmm7, $80 + xor r13d, ecx + add r9d, dword [rsp + $28] + mov r15d, r10d + psrldq xmm7, $8 + xor r12d, r8d + ror r14d, $B + xor r15d, r11d + add r9d, r12d + ror r13d, $6 + paddd xmm2, xmm7 + and edi, r15d + xor r14d, r10d + add r9d, r13d + pshufd xmm7, xmm2, $50 + xor edi, r11d + ror r14d, $2 + add ebx, r9d + movdqa xmm6, xmm7 + add r9d, edi + mov r13d, ebx + psrld xmm7, $A + add r14d, r9d + ror r13d, $E + psrlq xmm6, $11 + mov r9d, r14d + mov r12d, ecx + pxor xmm7, xmm6 + ror r14d, $9 + xor r13d, ebx + xor r12d, edx + ror r13d, $5 + xor r14d, r9d + psrlq xmm6, $2 + and r12d, ebx + xor r13d, ebx + add r8d, dword [rsp + $2C] + pxor xmm7, xmm6 + mov edi, r9d + xor r12d, edx + ror r14d, $B + pshufd xmm7, xmm7, $8 + xor edi, r10d + add r8d, r12d + movdqu xmm6, oword [rbp + $40] + ror r13d, $6 + and r15d, edi + pslldq xmm7, $8 + xor r14d, r9d + add r8d, r13d + xor r15d, r10d + paddd xmm2, xmm7 + ror r14d, $2 + add eax, r8d + add r8d, r15d + paddd xmm6, xmm2 + mov r13d, eax + add r14d, r8d + movdqa oword [rsp + $20], xmm6 + ror r13d, $E + movdqa xmm4, xmm0 + mov r8d, r14d + mov r12d, ebx + movdqa xmm7, xmm2 + ror r14d, $9 + xor r13d, eax + xor r12d, ecx + ror r13d, $5 + xor r14d, r8d + db $66, $0F, $3A, $0F, $E3, $04 // palignr xmm4, xmm3, 4 + and r12d, eax + xor r13d, eax + db $66, $0F, $3A, $0F, $F9, $04 // palignr xmm7, xmm1, 4 + add edx, dword [rsp + $30] + mov r15d, r8d + xor r12d, ecx + ror r14d, $B + movdqa xmm5, xmm4 + xor r15d, r9d + add edx, r12d + movdqa xmm6, xmm4 + ror r13d, $6 + and edi, r15d + psrld xmm4, $3 + xor r14d, r8d + add edx, r13d + xor edi, r9d + paddd xmm3, xmm7 + ror r14d, $2 + add r11d, edx + psrld xmm6, $7 + add edx, edi + mov r13d, r11d + pshufd xmm7, xmm2, $FA + add r14d, edx + ror r13d, $E + pslld xmm5, $E + mov edx, r14d + mov r12d, eax + pxor xmm4, xmm6 + ror r14d, $9 + xor r13d, r11d + xor r12d, ebx + ror r13d, $5 + psrld xmm6, $B + xor r14d, edx + pxor xmm4, xmm5 + and r12d, r11d + xor r13d, r11d + pslld xmm5, $B + add ecx, dword [rsp + $34] + mov edi, edx + pxor xmm4, xmm6 + xor r12d, ebx + ror r14d, $B + movdqa xmm6, xmm7 + xor edi, r8d + add ecx, r12d + pxor xmm4, xmm5 + ror r13d, $6 + and r15d, edi + xor r14d, edx + psrld xmm7, $A + add ecx, r13d + xor r15d, r8d + paddd xmm3, xmm4 + ror r14d, $2 + add r10d, ecx + psrlq xmm6, $11 + add ecx, r15d + mov r13d, r10d + add r14d, ecx + pxor xmm7, xmm6 + ror r13d, $E + mov ecx, r14d + mov r12d, r11d + ror r14d, $9 + psrlq xmm6, $2 + xor r13d, r10d + xor r12d, eax + pxor xmm7, xmm6 + ror r13d, $5 + xor r14d, ecx + and r12d, r10d + pshufd xmm7, xmm7, $80 + xor r13d, r10d + add ebx, dword [rsp + $38] + mov r15d, ecx + psrldq xmm7, $8 + xor r12d, eax + ror r14d, $B + xor r15d, edx + add ebx, r12d + ror r13d, $6 + paddd xmm3, xmm7 + and edi, r15d + xor r14d, ecx + add ebx, r13d + pshufd xmm7, xmm3, $50 + xor edi, edx + ror r14d, $2 + add r9d, ebx + movdqa xmm6, xmm7 + add ebx, edi + mov r13d, r9d + psrld xmm7, $A + add r14d, ebx + ror r13d, $E + psrlq xmm6, $11 + mov ebx, r14d + mov r12d, r10d + pxor xmm7, xmm6 + ror r14d, $9 + xor r13d, r9d + xor r12d, r11d + ror r13d, $5 + xor r14d, ebx + psrlq xmm6, $2 + and r12d, r9d + xor r13d, r9d + add eax, dword [rsp + $3C] + pxor xmm7, xmm6 + mov edi, ebx + xor r12d, r11d + ror r14d, $B + pshufd xmm7, xmm7, $8 + xor edi, ecx + add eax, r12d + movdqu xmm6, oword [rbp + $60] + ror r13d, $6 + and r15d, edi + pslldq xmm7, $8 + xor r14d, ebx + add eax, r13d + xor r15d, ecx + paddd xmm3, xmm7 + ror r14d, $2 + add r8d, eax + add eax, r15d + paddd xmm6, xmm3 + mov r13d, r8d + add r14d, eax + movdqa oword [rsp + $30], xmm6 + cmp byte [rbp + $83], $0 + jne @sched_loop + ror r13d, $E + mov eax, r14d + mov r12d, r9d + ror r14d, $9 + xor r13d, r8d + xor r12d, r10d + ror r13d, $5 + xor r14d, eax + and r12d, r8d + xor r13d, r8d + add r11d, dword [rsp] + mov r15d, eax + xor r12d, r10d + ror r14d, $B + xor r15d, ebx + add r11d, r12d + ror r13d, $6 + and edi, r15d + xor r14d, eax + add r11d, r13d + xor edi, ebx + ror r14d, $2 + add edx, r11d + add r11d, edi + mov r13d, edx + add r14d, r11d + ror r13d, $E + mov r11d, r14d + mov r12d, r8d + ror r14d, $9 + xor r13d, edx + xor r12d, r9d + ror r13d, $5 + xor r14d, r11d + and r12d, edx + xor r13d, edx + add r10d, dword [rsp + $4] + mov edi, r11d + xor r12d, r9d + ror r14d, $B + xor edi, eax + add r10d, r12d + ror r13d, $6 + and r15d, edi + xor r14d, r11d + add r10d, r13d + xor r15d, eax + ror r14d, $2 + add ecx, r10d + add r10d, r15d + mov r13d, ecx + add r14d, r10d + ror r13d, $E + mov r10d, r14d + mov r12d, edx + ror r14d, $9 + xor r13d, ecx + xor r12d, r8d + ror r13d, $5 + xor r14d, r10d + and r12d, ecx + xor r13d, ecx + add r9d, dword [rsp + $8] + mov r15d, r10d + xor r12d, r8d + ror r14d, $B + xor r15d, r11d + add r9d, r12d + ror r13d, $6 + and edi, r15d + xor r14d, r10d + add r9d, r13d + xor edi, r11d + ror r14d, $2 + add ebx, r9d + add r9d, edi + mov r13d, ebx + add r14d, r9d + ror r13d, $E + mov r9d, r14d + mov r12d, ecx + ror r14d, $9 + xor r13d, ebx + xor r12d, edx + ror r13d, $5 + xor r14d, r9d + and r12d, ebx + xor r13d, ebx + add r8d, dword [rsp + $C] + mov edi, r9d + xor r12d, edx + ror r14d, $B + xor edi, r10d + add r8d, r12d + ror r13d, $6 + and r15d, edi + xor r14d, r9d + add r8d, r13d + xor r15d, r10d + ror r14d, $2 + add eax, r8d + add r8d, r15d + mov r13d, eax + add r14d, r8d + ror r13d, $E + mov r8d, r14d + mov r12d, ebx + ror r14d, $9 + xor r13d, eax + xor r12d, ecx + ror r13d, $5 + xor r14d, r8d + and r12d, eax + xor r13d, eax + add edx, dword [rsp + $10] + mov r15d, r8d + xor r12d, ecx + ror r14d, $B + xor r15d, r9d + add edx, r12d + ror r13d, $6 + and edi, r15d + xor r14d, r8d + add edx, r13d + xor edi, r9d + ror r14d, $2 + add r11d, edx + add edx, edi + mov r13d, r11d + add r14d, edx + ror r13d, $E + mov edx, r14d + mov r12d, eax + ror r14d, $9 + xor r13d, r11d + xor r12d, ebx + ror r13d, $5 + xor r14d, edx + and r12d, r11d + xor r13d, r11d + add ecx, dword [rsp + $14] + mov edi, edx + xor r12d, ebx + ror r14d, $B + xor edi, r8d + add ecx, r12d + ror r13d, $6 + and r15d, edi + xor r14d, edx + add ecx, r13d + xor r15d, r8d + ror r14d, $2 + add r10d, ecx + add ecx, r15d + mov r13d, r10d + add r14d, ecx + ror r13d, $E + mov ecx, r14d + mov r12d, r11d + ror r14d, $9 + xor r13d, r10d + xor r12d, eax + ror r13d, $5 + xor r14d, ecx + and r12d, r10d + xor r13d, r10d + add ebx, dword [rsp + $18] + mov r15d, ecx + xor r12d, eax + ror r14d, $B + xor r15d, edx + add ebx, r12d + ror r13d, $6 + and edi, r15d + xor r14d, ecx + add ebx, r13d + xor edi, edx + ror r14d, $2 + add r9d, ebx + add ebx, edi + mov r13d, r9d + add r14d, ebx + ror r13d, $E + mov ebx, r14d + mov r12d, r10d + ror r14d, $9 + xor r13d, r9d + xor r12d, r11d + ror r13d, $5 + xor r14d, ebx + and r12d, r9d + xor r13d, r9d + add eax, dword [rsp + $1C] + mov edi, ebx + xor r12d, r11d + ror r14d, $B + xor edi, ecx + add eax, r12d + ror r13d, $6 + and r15d, edi + xor r14d, ebx + add eax, r13d + xor r15d, ecx + ror r14d, $2 + add r8d, eax + add eax, r15d + mov r13d, r8d + add r14d, eax + ror r13d, $E + mov eax, r14d + mov r12d, r9d + ror r14d, $9 + xor r13d, r8d + xor r12d, r10d + ror r13d, $5 + xor r14d, eax + and r12d, r8d + xor r13d, r8d + add r11d, dword [rsp + $20] + mov r15d, eax + xor r12d, r10d + ror r14d, $B + xor r15d, ebx + add r11d, r12d + ror r13d, $6 + and edi, r15d + xor r14d, eax + add r11d, r13d + xor edi, ebx + ror r14d, $2 + add edx, r11d + add r11d, edi + mov r13d, edx + add r14d, r11d + ror r13d, $E + mov r11d, r14d + mov r12d, r8d + ror r14d, $9 + xor r13d, edx + xor r12d, r9d + ror r13d, $5 + xor r14d, r11d + and r12d, edx + xor r13d, edx + add r10d, dword [rsp + $24] + mov edi, r11d + xor r12d, r9d + ror r14d, $B + xor edi, eax + add r10d, r12d + ror r13d, $6 + and r15d, edi + xor r14d, r11d + add r10d, r13d + xor r15d, eax + ror r14d, $2 + add ecx, r10d + add r10d, r15d + mov r13d, ecx + add r14d, r10d + ror r13d, $E + mov r10d, r14d + mov r12d, edx + ror r14d, $9 + xor r13d, ecx + xor r12d, r8d + ror r13d, $5 + xor r14d, r10d + and r12d, ecx + xor r13d, ecx + add r9d, dword [rsp + $28] + mov r15d, r10d + xor r12d, r8d + ror r14d, $B + xor r15d, r11d + add r9d, r12d + ror r13d, $6 + and edi, r15d + xor r14d, r10d + add r9d, r13d + xor edi, r11d + ror r14d, $2 + add ebx, r9d + add r9d, edi + mov r13d, ebx + add r14d, r9d + ror r13d, $E + mov r9d, r14d + mov r12d, ecx + ror r14d, $9 + xor r13d, ebx + xor r12d, edx + ror r13d, $5 + xor r14d, r9d + and r12d, ebx + xor r13d, ebx + add r8d, dword [rsp + $2C] + mov edi, r9d + xor r12d, edx + ror r14d, $B + xor edi, r10d + add r8d, r12d + ror r13d, $6 + and r15d, edi + xor r14d, r9d + add r8d, r13d + xor r15d, r10d + ror r14d, $2 + add eax, r8d + add r8d, r15d + mov r13d, eax + add r14d, r8d + ror r13d, $E + mov r8d, r14d + mov r12d, ebx + ror r14d, $9 + xor r13d, eax + xor r12d, ecx + ror r13d, $5 + xor r14d, r8d + and r12d, eax + xor r13d, eax + add edx, dword [rsp + $30] + mov r15d, r8d + xor r12d, ecx + ror r14d, $B + xor r15d, r9d + add edx, r12d + ror r13d, $6 + and edi, r15d + xor r14d, r8d + add edx, r13d + xor edi, r9d + ror r14d, $2 + add r11d, edx + add edx, edi + mov r13d, r11d + add r14d, edx + ror r13d, $E + mov edx, r14d + mov r12d, eax + ror r14d, $9 + xor r13d, r11d + xor r12d, ebx + ror r13d, $5 + xor r14d, edx + and r12d, r11d + xor r13d, r11d + add ecx, dword [rsp + $34] + mov edi, edx + xor r12d, ebx + ror r14d, $B + xor edi, r8d + add ecx, r12d + ror r13d, $6 + and r15d, edi + xor r14d, edx + add ecx, r13d + xor r15d, r8d + ror r14d, $2 + add r10d, ecx + add ecx, r15d + mov r13d, r10d + add r14d, ecx + ror r13d, $E + mov ecx, r14d + mov r12d, r11d + ror r14d, $9 + xor r13d, r10d + xor r12d, eax + ror r13d, $5 + xor r14d, ecx + and r12d, r10d + xor r13d, r10d + add ebx, dword [rsp + $38] + mov r15d, ecx + xor r12d, eax + ror r14d, $B + xor r15d, edx + add ebx, r12d + ror r13d, $6 + and edi, r15d + xor r14d, ecx + add ebx, r13d + xor edi, edx + ror r14d, $2 + add r9d, ebx + add ebx, edi + mov r13d, r9d + add r14d, ebx + ror r13d, $E + mov ebx, r14d + mov r12d, r10d + ror r14d, $9 + xor r13d, r9d + xor r12d, r11d + ror r13d, $5 + xor r14d, ebx + and r12d, r9d + xor r13d, r9d + add eax, dword [rsp + $3C] + mov edi, ebx + xor r12d, r11d + ror r14d, $B + xor edi, ecx + add eax, r12d + ror r13d, $6 + and r15d, edi + xor r14d, ebx + add eax, r13d + xor r15d, ecx + ror r14d, $2 + add r8d, eax + add eax, r15d + mov r13d, r8d + add r14d, eax + mov rdi, qword [rsp + $40] + mov eax, r14d + add eax, dword [rdi] + lea rsi, [rsi + $40] + add ebx, dword [rdi + $4] + add ecx, dword [rdi + $8] + add edx, dword [rdi + $C] + add r8d, dword [rdi + $10] + add r9d, dword [rdi + $14] + add r10d, dword [rdi + $18] + add r11d, dword [rdi + $1C] + cmp rsi, qword [rsp + $50] + mov dword [rdi], eax + mov dword [rdi + $4], ebx + mov dword [rdi + $8], ecx + mov dword [rdi + $C], edx + mov dword [rdi + $10], r8d + mov dword [rdi + $14], r9d + mov dword [rdi + $18], r10d + mov dword [rdi + $1C], r11d + jb @block_loop + mov rsi, qword [rsp + $58] + mov r15, qword [rsi - $48] + mov r14, qword [rsi - $40] + mov r13, qword [rsi - $38] + mov r12, qword [rsi - $30] + mov rdi, qword [rsi - $28] + mov rbp, qword [rsi - $18] + mov rbx, qword [rsi - $10] + mov rax, qword [rsi - $20] + lea rsp, [rsi] + mov rsi, rax + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc index 9174094e..c3e018dc 100644 --- a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc @@ -1,21 +1,12 @@ -// Keccak-f[1600] AVX2 multi-block absorb. -// Fuses data-XOR + permutation into a single loop with one gather/scatter. -// Ported from Andy Polyakov's keccak1600-avx2.pl (CRYPTOGAMS project), -// as adopted by XKCP (eXtended Keccak Code Package): -// https://github.com/dot-asm/cryptogams/blob/master/x86_64/keccak1600-avx2.pl -// https://github.com/XKCP/XKCP/blob/master/lib/low/KeccakP-1600/AVX2/KeccakP-1600-AVX2.s -// -// Uses "plane-per-register" technique: 25 Keccak lanes packed into 7 YMM registers. -// All VEX instructions are db-encoded for broad assembler compatibility. -// -// After HlpSimdProc5Begin_x86_64.inc: -// rcx = state ptr (25 x UInt64, standard layout) -// rdx = data ptr (input bytes, little-endian UInt64 words) -// r8d = block count -// r9d = block size in bytes -// r10 = constants ptr (K_KECCAK: rhotates, iotas, A_JAGGED) -// -// YMM register mapping (after gather): +// AVX2 implementation of the Keccak-f[1600] multi-block absorb. Fuses +// data-XOR + permutation into a single loop with one gather/scatter. Uses +// the "plane-per-register" technique: 25 Keccak lanes packed into 7 YMM +// registers. +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = state ptr (25 x UInt64, +// standard layout), rdx = data ptr (little-endian UInt64 words), +// r8d = block count, r9d = block size in bytes, r10 = constants ptr +// (K_KECCAK: rhotates, iotas, A_JAGGED). +// Register map (after gather): // ymm0 = broadcast(state[0]) // ymm1 = { state[1], state[2], state[3], state[4] } // ymm2 = { state[10], state[20], state[5], state[15] } @@ -23,24 +14,20 @@ // ymm4 = { state[11], state[22], state[8], state[19] } // ymm5 = { state[21], state[17], state[13], state[9] } // ymm6 = { state[6], state[12], state[18], state[24] } -// -// MS x64 non-volatile saves: xmm6-xmm15, rbx, rbp, r12-r15, rdi, rsi. -// rdi/rsi are also pushed under non-MS ABIs because the absorb loop uses rsi -// as the A_JAGGED base pointer for the duration of the routine. -// -// Stack layout (sub rsp, 424): -// [rsp+ 0..223]: jagged buffer (7 x 32 bytes) -// [rsp+224..255]: scratch -// [rsp+256..415]: xmm6-xmm15 save area -// -// Register allocation across absorb loop: -// r12 = state pointer (for final scatter) -// r13 = data pointer (advances per block) -// r14d = block count (decremented per block) -// r15d = block size in bytes (constant) -// rbx = constants pointer (constant) -// ebp = block size in qwords (constant) -// rsi = A_JAGGED base pointer (constant) +// Absorb-loop GPRs: r12 = state ptr (final scatter), r13 = data ptr +// (advances per block), r14d = block count (decremented), r15d = block +// size in bytes, rbx = constants ptr, ebp = block size in qwords, +// rsi = A_JAGGED base pointer. +// Frame (sub rsp, 424): +// [rsp+ 0..223] = jagged buffer (7 x 32 bytes) +// [rsp+224..255] = scratch +// [rsp+256..415] = xmm6-xmm15 save area +// Saves: xmm6-xmm15, rbx, rbp, r12-r15, rdi, rsi. rdi/rsi are also pushed +// under non-MS ABIs because the absorb loop uses rsi as the A_JAGGED +// base pointer for the duration of the routine. +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: Andy Polyakov's keccak1600-avx2.pl (CRYPTOGAMS), as adopted +// by XKCP (KeccakP-1600-AVX2.s). // ===== Prologue: save callee-saved GPRs ===== push rbx @@ -52,7 +39,7 @@ push rdi push rsi - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} sub rsp, 264 @@ -393,7 +380,7 @@ add rsp, 264 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} pop rsi pop rdi diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc index e40f3091..a642d31d 100644 --- a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc @@ -1,14 +1,10 @@ -// Keccak-f[1600] AVX2 permutation. -// Ported from Andy Polyakov's keccak1600-avx2.pl (CRYPTOGAMS project), -// as adopted by XKCP (eXtended Keccak Code Package): -// https://github.com/dot-asm/cryptogams/blob/master/x86_64/keccak1600-avx2.pl -// https://github.com/XKCP/XKCP/blob/master/lib/low/KeccakP-1600/AVX2/KeccakP-1600-AVX2.s -// Uses "plane-per-register" technique: 25 Keccak lanes packed into 7 YMM registers. -// All VEX instructions are db-encoded for broad assembler compatibility. -// Expects MS x64 ABI: rcx = state ptr (25 x UInt64, standard layout), -// rdx = constants ptr (rhotates_left[24], rhotates_right[24], iotas[96]). -// -// YMM register mapping (after gather): +// AVX2 implementation of the Keccak-f[1600] permutation. Uses the +// "plane-per-register" technique: 25 Keccak lanes packed into 7 YMM +// registers. +// ABI (after HlpSimdProc2Begin_x86_64.inc): rcx = state ptr (25 x UInt64, +// standard layout), rdx = constants ptr (rhotates_left[24], +// rhotates_right[24], iotas[96]). +// Register map (after gather): // ymm0 = broadcast(state[0]) // ymm1 = { state[1], state[2], state[3], state[4] } // ymm2 = { state[10], state[20], state[5], state[15] } @@ -16,15 +12,16 @@ // ymm4 = { state[11], state[22], state[8], state[19] } // ymm5 = { state[21], state[17], state[13], state[9] } // ymm6 = { state[6], state[12], state[18], state[24] } -// -// MS x64 non-volatile saves: xmm6-xmm15. -// -// Stack layout (sub rsp, 224): -// [rsp+ 0.. 31]: 32-byte temp buffer for gather/scatter -// [rsp+ 32..191]: xmm6-xmm15 save area -// [rsp+192..223]: padding +// Frame (sub rsp, 224): +// [rsp+ 0.. 31] = 32-byte temp buffer for gather/scatter +// [rsp+ 32..191] = xmm6-xmm15 save area +// [rsp+192..223] = padding +// Saves: xmm6-xmm15 (MS x64 non-volatile) in the frame. +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: Andy Polyakov's keccak1600-avx2.pl (CRYPTOGAMS), as adopted +// by XKCP (KeccakP-1600-AVX2.s). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} sub rsp, 64 @@ -274,4 +271,4 @@ add rsp, 64 - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc index fc514f01..62e86f7f 100644 --- a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExtAbsorb_aarch64.inc @@ -1,15 +1,18 @@ -// Keccak-F1600 absorb AArch64 FEAT_SHA3 implementation. -// Expects AAPCS64: x0 = state ptr (25 x UInt64), x1 = data ptr, -// w2 = numblocks, w3 = blocksize (bytes, = rate), x4 = iotas ptr. -// Saves/restores d8-d15 on 64-byte frame; XORs blocksize bytes per block then permutes. -// Reference: OpenSSL CRYPTOGAMS keccak1600-armv8.S. -// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// CryptoExt (FEAT_SHA3) implementation of the Keccak-f[1600] multi-block +// absorb: XORs blocksize bytes per block into the state, then permutes. +// ABI (after HlpSimdProc5Begin_aarch64.inc): x0 = state ptr (25 x UInt64), +// x1 = data ptr, w2 = numblocks, w3 = blocksize (bytes, = rate), +// x4 = iotas ptr. // Register map: // v0-v24 = Keccak state lanes (data XORed lane-by-lane into v0..) // v25-v30 = permutation scratch // v31 = loaded input lane +// Saves: d8-d15 via the shared HlpSimdNonVolatileSave/Restore include (the +// state uses v0-v24). +// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS keccak1600-armv8.S. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} .long 0x6d400400 // ldp d0, d1, [x0] .long 0x6d410c02 // ldp d2, d3, [x0, #16] .long 0x6d421404 // ldp d4, d5, [x0, #32] @@ -200,5 +203,5 @@ .long 0x6d0a5414 // stp d20, d21, [x0, #160] .long 0x6d0b5c16 // stp d22, d23, [x0, #176] .long 0xfd006018 // str d24, [x0, #192] - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc index 7728a65e..badbc559 100644 --- a/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600CryptoExt_aarch64.inc @@ -1,17 +1,18 @@ -// Keccak-F1600 permute AArch64 FEAT_SHA3 implementation. -// Expects AAPCS64: x0 = state ptr (25 x UInt64, little-endian lanes), -// x1 = iotas ptr (24 x UInt64 round constants == HlpSHA3 RC table). -// Saves/restores d8-d15 on 64-byte frame; state uses v0-v24 (clobbers d8-d15 lanes). -// Leaf (no calls); x30 untouched. -// Reference: OpenSSL CRYPTOGAMS keccak1600-armv8.S. -// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// CryptoExt (FEAT_SHA3) implementation of the Keccak-f[1600] permutation. +// ABI (after HlpSimdProc2Begin_aarch64.inc): x0 = state ptr (25 x UInt64, +// little-endian lanes), x1 = iotas ptr (24 x UInt64 round constants == +// HlpSHA3 RC table). // Register map: // v0-v24 = Keccak state lanes A[y][x] (lane k = v[k], 64 bits in low half) // v25-v29 = Theta C[]/D[] column scratch // v26 = iota round constant (ld1r) // v30-v31 = Rho/Pi scratch +// Saves: d8-d15 via the shared HlpSimdNonVolatileSave/Restore include (the +// state uses v0-v24). Leaf (no calls); x30 untouched. +// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS keccak1600-armv8.S. - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_aarch64.inc} .long 0x6d400400 // ldp d0, d1, [x0] .long 0x6d410c02 // ldp d2, d3, [x0, #16] .long 0x6d421404 // ldp d4, d5, [x0, #32] @@ -109,5 +110,5 @@ .long 0x6d0a5414 // stp d20, d21, [x0, #160] .long 0x6d0b5c16 // stp d22, d23, [x0, #176] .long 0xfd006018 // str d24, [x0, #192] - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_aarch64.inc} ret diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600GprAbsorb_aarch64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600GprAbsorb_aarch64.inc new file mode 100644 index 00000000..bf32a4c6 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600GprAbsorb_aarch64.inc @@ -0,0 +1,352 @@ +// Gpr implementation of the Keccak-f[1600] multi-block absorb (AArch64) +// for cores without FEAT_SHA3; the Gpr name states the ISA requirement: +// none beyond base AArch64. XORs each rate-sized block into the state +// (unrolled per-lane ladder driven by the block size), then runs the same +// GPR permutation as KeccakF1600Gpr_aarch64.inc - the inner is this file's +// own copy. The reference takes a byte length; it is computed as +// ABlockCount * ABlockSize on entry. +// ABI (after HlpSimdProc5Begin_aarch64.inc): x0 = AState (25 x UInt64, +// standard layout), x1 = AData, w2 = ABlockCount, w3 = ABlockSize +// (bytes), x4 = AConstants (iotas: 24 x UInt64). +// Frame: the reference frame grows 64 -> 80 because its own slots occupy +// [sp,#32..63] - the iotas end pointer for the permutation's end-pointer +// termination (see the permute kernel's header) lives at [sp,#64]. +// AArch64 instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS keccak1600-armv8.pl (SHA3_absorb + +// KeccakF1600_int, the plain path) by Andy Polyakov. + + .long 0xa9b87bfd // stp x29,x30,[sp,#-128]! + .long 0x910003fd // add x29,sp,#0 + .long 0xa90153f3 // stp x19,x20,[sp,#16] + .long 0xa9025bf5 // stp x21,x22,[sp,#32] + .long 0xa90363f7 // stp x23,x24,[sp,#48] + .long 0xa9046bf9 // stp x25,x26,[sp,#64] + .long 0xa90573fb // stp x27,x28,[sp,#80] + .long 0xd10143ff // sub sp,sp,#80 // reference frame + 16 for the iotas end slot + .long 0x91030084 // add x4,x4,#192 // iotas end = AConstants + 24*8 + .long 0xf90023e4 // str x4,[sp,#64] // stashed for the inner + .long 0x9ba37c42 // umull x2,w2,w3 // len = ABlockCount * ABlockSize bytes + .long 0x2a0303e3 // mov w3,w3 // zero-extend ABlockSize (AAPCS64 leaves x3's top half undefined) + + .long 0xa90207e0 // stp x0,x1,[sp,#32] // offload arguments + .long 0xa9030fe2 // stp x2,x3,[sp,#48] + + .long 0xaa0003fa // mov x26,x0 // uint64_t A[5][5] + .long 0xaa0103fb // mov x27,x1 // const void *inp + .long 0xaa0203fc // mov x28,x2 // size_t len + .long 0xaa0303fe // mov x30,x3 // size_t bsz + .long 0xa9400740 // ldp x0,x1,[x26,#16*0] + .long 0xa9410f42 // ldp x2,x3,[x26,#16*1] + .long 0xa9421744 // ldp x4,x5,[x26,#16*2] + .long 0xa9431f46 // ldp x6,x7,[x26,#16*3] + .long 0xa9442748 // ldp x8,x9,[x26,#16*4] + .long 0xa9452f4a // ldp x10,x11,[x26,#16*5] + .long 0xa946374c // ldp x12,x13,[x26,#16*6] + .long 0xa9473f4e // ldp x14,x15,[x26,#16*7] + .long 0xa9484750 // ldp x16,x17,[x26,#16*8] + .long 0xa9494f59 // ldp x25,x19,[x26,#16*9] + .long 0xa94a5754 // ldp x20,x21,[x26,#16*10] + .long 0xa94b5f56 // ldp x22,x23,[x26,#16*11] + .long 0xf9406358 // ldr x24,[x26,#16*12] + b .Lkeccak_gpr_absorb_loop_absorb + + +.Lkeccak_gpr_absorb_loop_absorb: + .long 0xeb1e039a // subs x26,x28,x30 // len - bsz + b.lo .Lkeccak_gpr_absorb_absorbed + + .long 0xf9001bfa // str x26,[sp,#48] // save len - bsz + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0000 // eor x0,x0,x26 + .long 0xf10043df // cmp x30,#8*(0+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0021 // eor x1,x1,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0042 // eor x2,x2,x26 + .long 0xf10083df // cmp x30,#8*(2+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0063 // eor x3,x3,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0084 // eor x4,x4,x26 + .long 0xf100c3df // cmp x30,#8*(4+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a00a5 // eor x5,x5,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a00c6 // eor x6,x6,x26 + .long 0xf10103df // cmp x30,#8*(6+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a00e7 // eor x7,x7,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0108 // eor x8,x8,x26 + .long 0xf10143df // cmp x30,#8*(8+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0129 // eor x9,x9,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a014a // eor x10,x10,x26 + .long 0xf10183df // cmp x30,#8*(10+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a016b // eor x11,x11,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a018c // eor x12,x12,x26 + .long 0xf101c3df // cmp x30,#8*(12+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a01ad // eor x13,x13,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a01ce // eor x14,x14,x26 + .long 0xf10203df // cmp x30,#8*(14+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a01ef // eor x15,x15,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0210 // eor x16,x16,x26 + .long 0xf10243df // cmp x30,#8*(16+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0231 // eor x17,x17,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0339 // eor x25,x25,x26 + .long 0xf10283df // cmp x30,#8*(18+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0273 // eor x19,x19,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0294 // eor x20,x20,x26 + .long 0xf102c3df // cmp x30,#8*(20+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a02b5 // eor x21,x21,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a02d6 // eor x22,x22,x26 + .long 0xf10303df // cmp x30,#8*(22+2) + b.lo .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a02f7 // eor x23,x23,x26 + b.eq .Lkeccak_gpr_absorb_process_block + .long 0xf840877a // ldr x26,[x27],#8 // *inp++ + .long 0xca1a0318 // eor x24,x24,x26 + + +.Lkeccak_gpr_absorb_process_block: + .long 0xf90017fb // str x27,[sp,#40] // save inp + + bl .Lkeccak_gpr_absorb_int + + .long 0xf94017fb // ldr x27,[sp,#40] // restore arguments + .long 0xa9437bfc // ldp x28,x30,[sp,#48] + b .Lkeccak_gpr_absorb_loop_absorb + + +.Lkeccak_gpr_absorb_absorbed: + .long 0xf94013fb // ldr x27,[sp,#32] + .long 0xa9000760 // stp x0,x1,[x27,#16*0] + .long 0xa9010f62 // stp x2,x3,[x27,#16*1] + .long 0xa9021764 // stp x4,x5,[x27,#16*2] + .long 0xa9031f66 // stp x6,x7,[x27,#16*3] + .long 0xa9042768 // stp x8,x9,[x27,#16*4] + .long 0xa9052f6a // stp x10,x11,[x27,#16*5] + .long 0xa906376c // stp x12,x13,[x27,#16*6] + .long 0xa9073f6e // stp x14,x15,[x27,#16*7] + .long 0xa9084770 // stp x16,x17,[x27,#16*8] + .long 0xa9094f79 // stp x25,x19,[x27,#16*9] + .long 0xa90a5774 // stp x20,x21,[x27,#16*10] + .long 0xa90b5f76 // stp x22,x23,[x27,#16*11] + .long 0xf9006378 // str x24,[x27,#16*12] + + .long 0xaa1c03e0 // mov x0,x28 // return value + .long 0xa94153b3 // ldp x19,x20,[x29,#16] + .long 0x910143ff // add sp,sp,#80 + .long 0xa9425bb5 // ldp x21,x22,[x29,#32] + .long 0xa94363b7 // ldp x23,x24,[x29,#48] + .long 0xa9446bb9 // ldp x25,x26,[x29,#64] + .long 0xa94573bb // ldp x27,x28,[x29,#80] + .long 0xa8c87bfd // ldp x29,x30,[sp],#128 + ret + + +.Lkeccak_gpr_absorb_int: + .long 0xf94023fc // ldr x28,[sp,#64] // iotas end (stashed by the absorb) + .long 0xd103039c // sub x28,x28,#192 // iotas base (24 x 8) + .long 0xa9017bfc // stp x28,x30,[sp,#16] // 32 bytes on top are mine + b .Lkeccak_gpr_absorb_loop + +.Lkeccak_gpr_absorb_loop: + ////////////////////////////////////////// Theta + .long 0xca05001a // eor x26,x0,x5 + .long 0xa90027e4 // stp x4,x9,[sp,#0] // offload pair... + .long 0xca06003b // eor x27,x1,x6 + .long 0xca07005c // eor x28,x2,x7 + .long 0xca08007e // eor x30,x3,x8 + .long 0xca090084 // eor x4,x4,x9 + .long 0xca0a035a // eor x26,x26,x10 + .long 0xca0b037b // eor x27,x27,x11 + .long 0xca0c039c // eor x28,x28,x12 + .long 0xca0d03de // eor x30,x30,x13 + .long 0xca0e0084 // eor x4,x4,x14 + .long 0xca0f035a // eor x26,x26,x15 + .long 0xca10037b // eor x27,x27,x16 + .long 0xca11039c // eor x28,x28,x17 + .long 0xca1903de // eor x30,x30,x25 + .long 0xca130084 // eor x4,x4,x19 + .long 0xca14035a // eor x26,x26,x20 + .long 0xca16039c // eor x28,x28,x22 + .long 0xca15037b // eor x27,x27,x21 + .long 0xca1703de // eor x30,x30,x23 + .long 0xca180084 // eor x4,x4,x24 + + .long 0xcadcff49 // eor x9,x26,x28, ror #63 + + .long 0xca090021 // eor x1,x1,x9 + .long 0xca0900c6 // eor x6,x6,x9 + .long 0xca09016b // eor x11,x11,x9 + .long 0xca090210 // eor x16,x16,x9 + .long 0xca0902b5 // eor x21,x21,x9 + + .long 0xcadeff69 // eor x9,x27,x30, ror #63 + .long 0xcac4ff9c // eor x28,x28,x4, ror #63 + .long 0xcadaffde // eor x30,x30,x26, ror #63 + .long 0xcadbfc84 // eor x4,x4,x27, ror #63 + + .long 0xca09005b // eor x27, x2,x9 // mov x27,x2 + .long 0xca0900e7 // eor x7,x7,x9 + .long 0xca09018c // eor x12,x12,x9 + .long 0xca090231 // eor x17,x17,x9 + .long 0xca0902d6 // eor x22,x22,x9 + + .long 0xca040000 // eor x0,x0,x4 + .long 0xca0400a5 // eor x5,x5,x4 + .long 0xca04014a // eor x10,x10,x4 + .long 0xca0401ef // eor x15,x15,x4 + .long 0xca040294 // eor x20,x20,x4 + .long 0xa94027e4 // ldp x4,x9,[sp,#0] // re-load offloaded data + .long 0xca1c007a // eor x26, x3,x28 // mov x26,x3 + .long 0xca1c0108 // eor x8,x8,x28 + .long 0xca1c01ad // eor x13,x13,x28 + .long 0xca1c0339 // eor x25,x25,x28 + .long 0xca1c02f7 // eor x23,x23,x28 + + .long 0xca1e009c // eor x28, x4,x30 // mov x28,x4 + .long 0xca1e0129 // eor x9,x9,x30 + .long 0xca1e01ce // eor x14,x14,x30 + .long 0xca1e0273 // eor x19,x19,x30 + .long 0xca1e0318 // eor x24,x24,x30 + + ////////////////////////////////////////// Rho+Pi + .long 0xaa0103fe // mov x30,x1 + .long 0x93c650c1 // ror x1,x6,#64-44 + //mov x27,x2 + .long 0x93cc5582 // ror x2,x12,#64-43 + //mov x26,x3 + .long 0x93d9af23 // ror x3,x25,#64-21 + //mov x28,x4 + .long 0x93d8cb04 // ror x4,x24,#64-14 + + .long 0x93c9b126 // ror x6,x9,#64-20 + .long 0x93cd9dac // ror x12,x13,#64-25 + .long 0x93d1c639 // ror x25,x17,#64-15 + .long 0x93d5fab8 // ror x24,x21,#64-2 + + .long 0x93d60ec9 // ror x9,x22,#64-61 + .long 0x93d3e26d // ror x13,x19,#64-8 + .long 0x93cbd971 // ror x17,x11,#64-10 + .long 0x93c82515 // ror x21,x8,#64-55 + + .long 0x93ce65d6 // ror x22,x14,#64-39 + .long 0x93d722f3 // ror x19,x23,#64-56 + .long 0x93c7e8eb // ror x11,x7,#64-6 + .long 0x93d04e08 // ror x8,x16,#64-45 + + .long 0x93d4ba8e // ror x14,x20,#64-18 + .long 0x93cf5df7 // ror x23,x15,#64-41 + .long 0x93caf547 // ror x7,x10,#64-3 + .long 0x93c570b0 // ror x16,x5,#64-36 + + .long 0x93da9345 // ror x5,x26,#64-28 + .long 0x93deffca // ror x10,x30,#64-1 + .long 0x93dc978f // ror x15,x28,#64-27 + .long 0x93db0b74 // ror x20,x27,#64-62 + + ////////////////////////////////////////// Chi+Iota + .long 0x8a21005a // bic x26,x2,x1 + .long 0x8a22007b // bic x27,x3,x2 + .long 0x8a24001c // bic x28,x0,x4 + .long 0x8a20003e // bic x30,x1,x0 + .long 0xca1a0000 // eor x0,x0,x26 + .long 0x8a23009a // bic x26,x4,x3 + .long 0xca1b0021 // eor x1,x1,x27 + .long 0xf9400bfb // ldr x27,[sp,#16] + .long 0xca1c0063 // eor x3,x3,x28 + .long 0xca1e0084 // eor x4,x4,x30 + .long 0xca1a0042 // eor x2,x2,x26 + .long 0xf840877e // ldr x30,[x27],#8 // Iota[i++] + .long 0xca1e0000 // eor x0,x0,x30 // A[0][0] ^= Iota (hoisted - frees x30 for the end test) + + .long 0x8a2600fa // bic x26,x7,x6 + .long 0xf94023fe // ldr x30,[sp,#64] // iotas end + .long 0xeb1e037f // cmp x27,x30 // are we done? (end-pointer test; a Pascal const + // cannot provide the original's 256-aligned table) + .long 0xf9000bfb // str x27,[sp,#16] + .long 0x8a27011b // bic x27,x8,x7 + .long 0x8a2900bc // bic x28,x5,x9 + .long 0x8a2500de // bic x30,x6,x5 + .long 0xca1a00a5 // eor x5,x5,x26 + .long 0x8a28013a // bic x26,x9,x8 + .long 0xca1b00c6 // eor x6,x6,x27 + .long 0xca1c0108 // eor x8,x8,x28 + .long 0xca1e0129 // eor x9,x9,x30 + .long 0xca1a00e7 // eor x7,x7,x26 + + .long 0x8a2b019a // bic x26,x12,x11 + .long 0x8a2c01bb // bic x27,x13,x12 + .long 0x8a2e015c // bic x28,x10,x14 + .long 0x8a2a017e // bic x30,x11,x10 + .long 0xca1a014a // eor x10,x10,x26 + .long 0x8a2d01da // bic x26,x14,x13 + .long 0xca1b016b // eor x11,x11,x27 + .long 0xca1c01ad // eor x13,x13,x28 + .long 0xca1e01ce // eor x14,x14,x30 + .long 0xca1a018c // eor x12,x12,x26 + + .long 0x8a30023a // bic x26,x17,x16 + .long 0x8a31033b // bic x27,x25,x17 + .long 0x8a3301fc // bic x28,x15,x19 + .long 0x8a2f021e // bic x30,x16,x15 + .long 0xca1a01ef // eor x15,x15,x26 + .long 0x8a39027a // bic x26,x19,x25 + .long 0xca1b0210 // eor x16,x16,x27 + .long 0xca1c0339 // eor x25,x25,x28 + .long 0xca1e0273 // eor x19,x19,x30 + .long 0xca1a0231 // eor x17,x17,x26 + + .long 0x8a3502da // bic x26,x22,x21 + .long 0x8a3602fb // bic x27,x23,x22 + .long 0x8a38029c // bic x28,x20,x24 + .long 0x8a3402be // bic x30,x21,x20 + .long 0xca1a0294 // eor x20,x20,x26 + .long 0x8a37031a // bic x26,x24,x23 + .long 0xca1b02b5 // eor x21,x21,x27 + .long 0xca1c02f7 // eor x23,x23,x28 + .long 0xca1e0318 // eor x24,x24,x30 + .long 0xca1a02d6 // eor x22,x22,x26 + + b.ne .Lkeccak_gpr_absorb_loop + + .long 0xf9400ffe // ldr x30,[sp,#24] + ret diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600GprAbsorb_x86_64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600GprAbsorb_x86_64.inc new file mode 100644 index 00000000..c0c4f2d1 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600GprAbsorb_x86_64.inc @@ -0,0 +1,306 @@ +// Gpr implementation of the Keccak-f[1600] multi-block absorb (pure 64-bit +// GPR code) - the asm twin of the AVX2 absorb: XORs whole blocks into the +// state and permutes, keeping the state in the lane-complemented domain +// across the entire batch (the NOT conversion is hoisted out of the block +// loop - the standalone permute pays it per call). The permutation body is +// the same inner as KeccakF1600Gpr_x86_64.inc (embedded again because +// local labels cannot cross include files), with the same end-pointer loop +// termination via the sentinel at [rsp]. +// ABI (after HlpSimdProc5Begin_x86_64.inc): rcx = AState (25 x UInt64, +// standard layout), rdx = AData, r8 = ABlockCount, r9 = ABlockSize (both +// Int32 - upper halves undefined, zero-extended here), r10 = AConstants +// (iotas: 24 x UInt64). +// Frame: 200-byte scratch plane at [rsp+16] (rsi biased +100), sentinel at +// [rsp], spills at [rsi+100..116]. +// Saves: every callee-saved GPR is pushed here (clobbers all GPRs). +// Reference: OpenSSL CRYPTOGAMS keccak1600-x86_64.pl (SHA3_absorb) by +// Andy Polyakov. + + push rbx + push rbp + push rsi + push rdi + push r12 + push r13 + push r14 + push r15 + + mov r11d, r9d // ABlockSize, zero-extended + mov eax, r8d // ABlockCount, zero-extended + imul rax, r11 // total length in bytes + + lea rdi, [rcx + 100] // biased state pointer + mov r9, rdx // input pointer + mov rdx, rax // remaining length + mov rcx, r11 // block size in bytes + mov r15, r10 // iotas + + sub rsp, 240 + lea rsi, [rsp + 116] // scratch plane, biased by +100 + lea rax, [r15 + 192] + mov qword [rsp], rax // iotas end sentinel (inner: [rsp + 8]) + mov qword [rsi + 116], rcx // block size, reloaded after each permute + + // enter the complemented-lane domain once for the whole batch + not qword [rdi - 92] + not qword [rdi - 84] + not qword [rdi - 36] + not qword [rdi - 4] + not qword [rdi + 36] + not qword [rdi + 60] + +@absorb_loop: + cmp rdx, rcx + jb @absorb_done + + shr rcx, 3 + lea r8, [rdi - 100] + +@absorb_block: + mov rax, qword [r9] + lea r9, [r9 + 8] + xor rax, qword [r8] + lea r8, [r8 + 8] + sub rdx, 8 + mov qword [r8 - 8], rax + sub rcx, 1 + jnz @absorb_block + + mov qword [rsi + 100], r9 + mov qword [rsi + 108], rdx + call @keccak_inner + mov r9, qword [rsi + 100] + mov rdx, qword [rsi + 108] + mov rcx, qword [rsi + 116] + jmp @absorb_loop + +@absorb_done: + // leave the complemented-lane domain + not qword [rdi - 92] + not qword [rdi - 84] + not qword [rdi - 36] + not qword [rdi - 4] + not qword [rdi + 36] + not qword [rdi + 60] + + add rsp, 240 + pop r15 + pop r14 + pop r13 + pop r12 + pop rdi + pop rsi + pop rbp + pop rbx + jmp @absorb_end + +@keccak_inner: + mov rax, qword [rdi + $3C] + mov rbx, qword [rdi + $44] + mov rcx, qword [rdi + $4C] + mov rdx, qword [rdi + $54] + mov rbp, qword [rdi + $5C] + jmp @keccak_round_loop + +@keccak_round_loop: + mov r8, qword [rdi - $64] + mov r9, qword [rdi - $34] + mov r10, qword [rdi - $4] + mov r11, qword [rdi + $2C] + xor rcx, qword [rdi - $54] + xor rdx, qword [rdi - $4C] + xor rax, r8 + xor rbx, qword [rdi - $5C] + xor rcx, qword [rdi - $2C] + xor rax, qword [rdi - $3C] + mov r12, rbp + xor rbp, qword [rdi - $44] + xor rcx, r10 + xor rax, qword [rdi - $14] + xor rdx, qword [rdi - $24] + xor rbx, r9 + xor rbp, qword [rdi - $1C] + xor rcx, qword [rdi + $24] + xor rax, qword [rdi + $14] + xor rdx, qword [rdi + $4] + xor rbx, qword [rdi - $C] + xor rbp, qword [rdi + $C] + mov r13, rcx + rol rcx, 1 + xor rcx, rax + xor rdx, r11 + rol rax, 1 + xor rax, rdx + xor rbx, qword [rdi + $1C] + rol rdx, 1 + xor rdx, rbx + xor rbp, qword [rdi + $34] + rol rbx, 1 + xor rbx, rbp + rol rbp, 1 + xor rbp, r13 + xor r9, rcx + xor r10, rdx + rol r9, $2C + xor r11, rbp + xor r12, rax + rol r10, $2B + xor r8, rbx + mov r13, r9 + rol r11, $15 + or r9, r10 + xor r9, r8 + rol r12, $E + xor r9, qword [r15] + lea r15, [r15 + $8] + mov r14, r12 + and r12, r11 + mov qword [rsi - $64], r9 + xor r12, r10 + not r10 + mov qword [rsi - $54], r12 + or r10, r11 + mov r12, qword [rdi + $4C] + xor r10, r13 + mov qword [rsi - $5C], r10 + and r13, r8 + mov r9, qword [rdi - $1C] + xor r13, r14 + mov r10, qword [rdi - $14] + mov qword [rsi - $44], r13 + or r14, r8 + mov r8, qword [rdi - $4C] + xor r14, r11 + mov r11, qword [rdi + $1C] + mov qword [rsi - $4C], r14 + xor r8, rbp + xor r12, rdx + rol r8, $1C + xor r11, rcx + xor r9, rax + rol r12, $3D + rol r11, $2D + xor r10, rbx + rol r9, $14 + mov r13, r8 + or r8, r12 + rol r10, $3 + xor r8, r11 + mov qword [rsi - $24], r8 + mov r14, r9 + and r9, r13 + mov r8, qword [rdi - $5C] + xor r9, r12 + not r12 + mov qword [rsi - $1C], r9 + or r12, r11 + mov r9, qword [rdi - $2C] + xor r12, r10 + mov qword [rsi - $2C], r12 + and r11, r10 + mov r12, qword [rdi + $3C] + xor r11, r14 + mov qword [rsi - $34], r11 + or r14, r10 + mov r10, qword [rdi + $4] + xor r14, r13 + mov r11, qword [rdi + $34] + mov qword [rsi - $3C], r14 + xor r10, rbp + xor r11, rax + rol r10, $19 + xor r9, rdx + rol r11, $8 + xor r12, rbx + rol r9, $6 + xor r8, rcx + rol r12, $12 + mov r13, r10 + and r10, r11 + rol r8, 1 + not r11 + xor r10, r9 + mov qword [rsi - $C], r10 + mov r14, r12 + and r12, r11 + mov r10, qword [rdi - $C] + xor r12, r13 + mov qword [rsi - $4], r12 + or r13, r9 + mov r12, qword [rdi + $54] + xor r13, r8 + mov qword [rsi - $14], r13 + and r9, r8 + xor r9, r14 + mov qword [rsi + $C], r9 + or r14, r8 + mov r9, qword [rdi - $3C] + xor r14, r11 + mov r11, qword [rdi + $24] + mov qword [rsi + $4], r14 + mov r8, qword [rdi - $44] + xor r10, rcx + xor r11, rdx + rol r10, $A + xor r9, rbx + rol r11, $F + xor r12, rbp + rol r9, $24 + xor r8, rax + rol r12, $38 + mov r13, r10 + or r10, r11 + rol r8, $1B + not r11 + xor r10, r9 + mov qword [rsi + $1C], r10 + mov r14, r12 + or r12, r11 + xor r12, r13 + mov qword [rsi + $24], r12 + and r13, r9 + xor r13, r8 + mov qword [rsi + $14], r13 + or r9, r8 + xor r9, r14 + mov qword [rsi + $34], r9 + and r8, r14 + xor r8, r11 + mov qword [rsi + $2C], r8 + xor rdx, qword [rdi - $54] + xor rbp, qword [rdi - $24] + rol rdx, $3E + xor rcx, qword [rdi + $44] + rol rbp, $37 + xor rax, qword [rdi + $C] + rol rcx, $2 + xor rbx, qword [rdi + $14] + xchg rdi, rsi + rol rax, $27 + rol rbx, $29 + mov r13, rdx + and rdx, rbp + not rbp + xor rdx, rcx + mov qword [rdi + $5C], rdx + mov r14, rax + and rax, rbp + xor rax, r13 + mov qword [rdi + $3C], rax + or r13, rcx + xor r13, rbx + mov qword [rdi + $54], r13 + and rcx, rbx + xor rcx, r14 + mov qword [rdi + $4C], rcx + or rbx, r14 + xor rbx, rbp + mov qword [rdi + $44], rbx + mov rbp, rdx + mov rdx, r13 + cmp r15, qword [rsp + $8] + jb @keccak_round_loop + lea r15, [r15 - $C0] + ret + +@absorb_end: diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Gpr_aarch64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Gpr_aarch64.inc new file mode 100644 index 00000000..f3bbdc8b --- /dev/null +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Gpr_aarch64.inc @@ -0,0 +1,240 @@ +// Gpr implementation of the Keccak-f[1600] permutation (AArch64) for cores +// without FEAT_SHA3; the Gpr name states the ISA requirement: none beyond +// base AArch64. The 25 lanes live in x0-x17/x19-x25 with x26-x28/x30 as +// temps; chi is computed directly with bic, so no lane complementing +// exists in this lineage. The original round loop ended on a 256-ALIGNED +// iotas table (tst #255), which a Pascal const cannot guarantee - the +// termination is patched to an end-pointer compare (iotas end stashed at +// [sp,#40]; the Iota xor is hoisted to free x30 for the test; the +// intervening bic/eor stream writes no flags, so the b.ne polarity is +// unchanged). The inner keeps the original called shape (bl/ret); x30 (LR) +// doubles as a temp inside the rounds, as in the original. +// ABI (after HlpSimdProc2Begin_aarch64.inc): x0 = AState (25 x UInt64, +// standard layout), x1 = AConstants (iotas: 24 x UInt64). +// AArch64 instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS keccak1600-armv8.pl (KeccakF1600 + +// KeccakF1600_int, the plain path) by Andy Polyakov. + + .long 0xa9b87bfd // stp x29,x30,[sp,#-128]! + .long 0x910003fd // add x29,sp,#0 + .long 0xa90153f3 // stp x19,x20,[sp,#16] + .long 0xa9025bf5 // stp x21,x22,[sp,#32] + .long 0xa90363f7 // stp x23,x24,[sp,#48] + .long 0xa9046bf9 // stp x25,x26,[sp,#64] + .long 0xa90573fb // stp x27,x28,[sp,#80] + .long 0xd100c3ff // sub sp,sp,#48 + .long 0x91030021 // add x1,x1,#192 // iotas end = AConstants + 24*8 + .long 0xf90017e1 // str x1,[sp,#40] // stashed for the inner (frame slot is free) + + .long 0xf90013e0 // str x0,[sp,#32] // offload argument + .long 0xaa0003fa // mov x26,x0 + .long 0xa9400400 // ldp x0,x1,[x0,#16*0] + .long 0xa9410f42 // ldp x2,x3,[x26,#16*1] + .long 0xa9421744 // ldp x4,x5,[x26,#16*2] + .long 0xa9431f46 // ldp x6,x7,[x26,#16*3] + .long 0xa9442748 // ldp x8,x9,[x26,#16*4] + .long 0xa9452f4a // ldp x10,x11,[x26,#16*5] + .long 0xa946374c // ldp x12,x13,[x26,#16*6] + .long 0xa9473f4e // ldp x14,x15,[x26,#16*7] + .long 0xa9484750 // ldp x16,x17,[x26,#16*8] + .long 0xa9494f59 // ldp x25,x19,[x26,#16*9] + .long 0xa94a5754 // ldp x20,x21,[x26,#16*10] + .long 0xa94b5f56 // ldp x22,x23,[x26,#16*11] + .long 0xf9406358 // ldr x24,[x26,#16*12] + + bl .Lkeccak_gpr_int + + .long 0xf94013fa // ldr x26,[sp,#32] + .long 0xa9000740 // stp x0,x1,[x26,#16*0] + .long 0xa9010f42 // stp x2,x3,[x26,#16*1] + .long 0xa9021744 // stp x4,x5,[x26,#16*2] + .long 0xa9031f46 // stp x6,x7,[x26,#16*3] + .long 0xa9042748 // stp x8,x9,[x26,#16*4] + .long 0xa9052f4a // stp x10,x11,[x26,#16*5] + .long 0xa906374c // stp x12,x13,[x26,#16*6] + .long 0xa9073f4e // stp x14,x15,[x26,#16*7] + .long 0xa9084750 // stp x16,x17,[x26,#16*8] + .long 0xa9094f59 // stp x25,x19,[x26,#16*9] + .long 0xa90a5754 // stp x20,x21,[x26,#16*10] + .long 0xa90b5f56 // stp x22,x23,[x26,#16*11] + .long 0xf9006358 // str x24,[x26,#16*12] + + .long 0xa94153b3 // ldp x19,x20,[x29,#16] + .long 0x9100c3ff // add sp,sp,#48 + .long 0xa9425bb5 // ldp x21,x22,[x29,#32] + .long 0xa94363b7 // ldp x23,x24,[x29,#48] + .long 0xa9446bb9 // ldp x25,x26,[x29,#64] + .long 0xa94573bb // ldp x27,x28,[x29,#80] + .long 0xa8c87bfd // ldp x29,x30,[sp],#128 + ret + + +.Lkeccak_gpr_int: + .long 0xf94017fc // ldr x28,[sp,#40] // iotas end (stashed by the wrapper) + .long 0xd103039c // sub x28,x28,#192 // iotas base (24 x 8) + .long 0xa9017bfc // stp x28,x30,[sp,#16] // 32 bytes on top are mine + b .Lkeccak_gpr_loop + +.Lkeccak_gpr_loop: + ////////////////////////////////////////// Theta + .long 0xca05001a // eor x26,x0,x5 + .long 0xa90027e4 // stp x4,x9,[sp,#0] // offload pair... + .long 0xca06003b // eor x27,x1,x6 + .long 0xca07005c // eor x28,x2,x7 + .long 0xca08007e // eor x30,x3,x8 + .long 0xca090084 // eor x4,x4,x9 + .long 0xca0a035a // eor x26,x26,x10 + .long 0xca0b037b // eor x27,x27,x11 + .long 0xca0c039c // eor x28,x28,x12 + .long 0xca0d03de // eor x30,x30,x13 + .long 0xca0e0084 // eor x4,x4,x14 + .long 0xca0f035a // eor x26,x26,x15 + .long 0xca10037b // eor x27,x27,x16 + .long 0xca11039c // eor x28,x28,x17 + .long 0xca1903de // eor x30,x30,x25 + .long 0xca130084 // eor x4,x4,x19 + .long 0xca14035a // eor x26,x26,x20 + .long 0xca16039c // eor x28,x28,x22 + .long 0xca15037b // eor x27,x27,x21 + .long 0xca1703de // eor x30,x30,x23 + .long 0xca180084 // eor x4,x4,x24 + + .long 0xcadcff49 // eor x9,x26,x28, ror #63 + + .long 0xca090021 // eor x1,x1,x9 + .long 0xca0900c6 // eor x6,x6,x9 + .long 0xca09016b // eor x11,x11,x9 + .long 0xca090210 // eor x16,x16,x9 + .long 0xca0902b5 // eor x21,x21,x9 + + .long 0xcadeff69 // eor x9,x27,x30, ror #63 + .long 0xcac4ff9c // eor x28,x28,x4, ror #63 + .long 0xcadaffde // eor x30,x30,x26, ror #63 + .long 0xcadbfc84 // eor x4,x4,x27, ror #63 + + .long 0xca09005b // eor x27, x2,x9 // mov x27,x2 + .long 0xca0900e7 // eor x7,x7,x9 + .long 0xca09018c // eor x12,x12,x9 + .long 0xca090231 // eor x17,x17,x9 + .long 0xca0902d6 // eor x22,x22,x9 + + .long 0xca040000 // eor x0,x0,x4 + .long 0xca0400a5 // eor x5,x5,x4 + .long 0xca04014a // eor x10,x10,x4 + .long 0xca0401ef // eor x15,x15,x4 + .long 0xca040294 // eor x20,x20,x4 + .long 0xa94027e4 // ldp x4,x9,[sp,#0] // re-load offloaded data + .long 0xca1c007a // eor x26, x3,x28 // mov x26,x3 + .long 0xca1c0108 // eor x8,x8,x28 + .long 0xca1c01ad // eor x13,x13,x28 + .long 0xca1c0339 // eor x25,x25,x28 + .long 0xca1c02f7 // eor x23,x23,x28 + + .long 0xca1e009c // eor x28, x4,x30 // mov x28,x4 + .long 0xca1e0129 // eor x9,x9,x30 + .long 0xca1e01ce // eor x14,x14,x30 + .long 0xca1e0273 // eor x19,x19,x30 + .long 0xca1e0318 // eor x24,x24,x30 + + ////////////////////////////////////////// Rho+Pi + .long 0xaa0103fe // mov x30,x1 + .long 0x93c650c1 // ror x1,x6,#64-44 + //mov x27,x2 + .long 0x93cc5582 // ror x2,x12,#64-43 + //mov x26,x3 + .long 0x93d9af23 // ror x3,x25,#64-21 + //mov x28,x4 + .long 0x93d8cb04 // ror x4,x24,#64-14 + + .long 0x93c9b126 // ror x6,x9,#64-20 + .long 0x93cd9dac // ror x12,x13,#64-25 + .long 0x93d1c639 // ror x25,x17,#64-15 + .long 0x93d5fab8 // ror x24,x21,#64-2 + + .long 0x93d60ec9 // ror x9,x22,#64-61 + .long 0x93d3e26d // ror x13,x19,#64-8 + .long 0x93cbd971 // ror x17,x11,#64-10 + .long 0x93c82515 // ror x21,x8,#64-55 + + .long 0x93ce65d6 // ror x22,x14,#64-39 + .long 0x93d722f3 // ror x19,x23,#64-56 + .long 0x93c7e8eb // ror x11,x7,#64-6 + .long 0x93d04e08 // ror x8,x16,#64-45 + + .long 0x93d4ba8e // ror x14,x20,#64-18 + .long 0x93cf5df7 // ror x23,x15,#64-41 + .long 0x93caf547 // ror x7,x10,#64-3 + .long 0x93c570b0 // ror x16,x5,#64-36 + + .long 0x93da9345 // ror x5,x26,#64-28 + .long 0x93deffca // ror x10,x30,#64-1 + .long 0x93dc978f // ror x15,x28,#64-27 + .long 0x93db0b74 // ror x20,x27,#64-62 + + ////////////////////////////////////////// Chi+Iota + .long 0x8a21005a // bic x26,x2,x1 + .long 0x8a22007b // bic x27,x3,x2 + .long 0x8a24001c // bic x28,x0,x4 + .long 0x8a20003e // bic x30,x1,x0 + .long 0xca1a0000 // eor x0,x0,x26 + .long 0x8a23009a // bic x26,x4,x3 + .long 0xca1b0021 // eor x1,x1,x27 + .long 0xf9400bfb // ldr x27,[sp,#16] + .long 0xca1c0063 // eor x3,x3,x28 + .long 0xca1e0084 // eor x4,x4,x30 + .long 0xca1a0042 // eor x2,x2,x26 + .long 0xf840877e // ldr x30,[x27],#8 // Iota[i++] + .long 0xca1e0000 // eor x0,x0,x30 // A[0][0] ^= Iota (hoisted - frees x30 for the end test) + + .long 0x8a2600fa // bic x26,x7,x6 + .long 0xf94017fe // ldr x30,[sp,#40] // iotas end + .long 0xeb1e037f // cmp x27,x30 // are we done? (end-pointer test; a Pascal const + // cannot provide the original's 256-aligned table) + .long 0xf9000bfb // str x27,[sp,#16] + .long 0x8a27011b // bic x27,x8,x7 + .long 0x8a2900bc // bic x28,x5,x9 + .long 0x8a2500de // bic x30,x6,x5 + .long 0xca1a00a5 // eor x5,x5,x26 + .long 0x8a28013a // bic x26,x9,x8 + .long 0xca1b00c6 // eor x6,x6,x27 + .long 0xca1c0108 // eor x8,x8,x28 + .long 0xca1e0129 // eor x9,x9,x30 + .long 0xca1a00e7 // eor x7,x7,x26 + + .long 0x8a2b019a // bic x26,x12,x11 + .long 0x8a2c01bb // bic x27,x13,x12 + .long 0x8a2e015c // bic x28,x10,x14 + .long 0x8a2a017e // bic x30,x11,x10 + .long 0xca1a014a // eor x10,x10,x26 + .long 0x8a2d01da // bic x26,x14,x13 + .long 0xca1b016b // eor x11,x11,x27 + .long 0xca1c01ad // eor x13,x13,x28 + .long 0xca1e01ce // eor x14,x14,x30 + .long 0xca1a018c // eor x12,x12,x26 + + .long 0x8a30023a // bic x26,x17,x16 + .long 0x8a31033b // bic x27,x25,x17 + .long 0x8a3301fc // bic x28,x15,x19 + .long 0x8a2f021e // bic x30,x16,x15 + .long 0xca1a01ef // eor x15,x15,x26 + .long 0x8a39027a // bic x26,x19,x25 + .long 0xca1b0210 // eor x16,x16,x27 + .long 0xca1c0339 // eor x25,x25,x28 + .long 0xca1e0273 // eor x19,x19,x30 + .long 0xca1a0231 // eor x17,x17,x26 + + .long 0x8a3502da // bic x26,x22,x21 + .long 0x8a3602fb // bic x27,x23,x22 + .long 0x8a38029c // bic x28,x20,x24 + .long 0x8a3402be // bic x30,x21,x20 + .long 0xca1a0294 // eor x20,x20,x26 + .long 0x8a37031a // bic x26,x24,x23 + .long 0xca1b02b5 // eor x21,x21,x27 + .long 0xca1c02f7 // eor x23,x23,x28 + .long 0xca1e0318 // eor x24,x24,x30 + .long 0xca1a02d6 // eor x22,x22,x26 + + b.ne .Lkeccak_gpr_loop + + .long 0xf9400ffe // ldr x30,[sp,#24] + ret diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Gpr_x86_64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Gpr_x86_64.inc new file mode 100644 index 00000000..28597967 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Gpr_x86_64.inc @@ -0,0 +1,273 @@ +// Gpr implementation of the Keccak-f[1600] permutation (pure 64-bit GPR +// code, no SIMD registers at all) - the fastest non-AVX2 form of Keccak on +// x86-64; the Gpr name states the ISA requirement: none beyond base +// x86-64. The inner routine works in the LANE-COMPLEMENTED domain (lanes +// 1, 2, 8, 12, 17 and 20 stored inverted to save NOTs in chi); this +// wrapper converts in and out around the call. The original loop +// terminated on a 256-aligned iotas table (test r15, 255) which a Pascal +// const cannot guarantee - the termination is patched to compare against +// an end pointer stashed at [rsp] (the inner sees it at [rsp + 8] across +// the call). +// ABI (after HlpSimdProc2Begin_x86_64.inc): rcx = AState (25 x UInt64, +// standard layout), rdx = AConstants (iotas: 24 x UInt64). +// Frame: mirrors the OpenSSL wrapper - 200-byte scratch plane (+16 for the +// sentinel slot), rdi/rsi biased by +100, r15 = iotas. The inner keeps +// the original called shape. +// Saves: every callee-saved GPR is pushed here (clobbers all GPRs). +// Reference: OpenSSL CRYPTOGAMS keccak1600-x86_64.pl by Andy Polyakov. + + push rbx + push rbp + push rsi + push rdi + push r12 + push r13 + push r14 + push r15 + + lea rdi, [rcx + 100] // biased state pointer + mov r15, rdx // iotas + sub rsp, 216 + lea rax, [rdx + 192] + mov qword [rsp], rax // iotas end sentinel (inner: [rsp + 8]) + lea rsi, [rsp + 116] // scratch plane, biased by +100 + + // enter the complemented-lane domain (lanes 1, 2, 8, 12, 17, 20) + not qword [rdi - 92] + not qword [rdi - 84] + not qword [rdi - 36] + not qword [rdi - 4] + not qword [rdi + 36] + not qword [rdi + 60] + + call @keccak_inner + + // leave the complemented-lane domain + not qword [rdi - 92] + not qword [rdi - 84] + not qword [rdi - 36] + not qword [rdi - 4] + not qword [rdi + 36] + not qword [rdi + 60] + + add rsp, 216 + pop r15 + pop r14 + pop r13 + pop r12 + pop rdi + pop rsi + pop rbp + pop rbx + jmp @keccak_done + +@keccak_inner: + mov rax, qword [rdi + $3C] + mov rbx, qword [rdi + $44] + mov rcx, qword [rdi + $4C] + mov rdx, qword [rdi + $54] + mov rbp, qword [rdi + $5C] + jmp @keccak_round_loop + +@keccak_round_loop: + mov r8, qword [rdi - $64] + mov r9, qword [rdi - $34] + mov r10, qword [rdi - $4] + mov r11, qword [rdi + $2C] + xor rcx, qword [rdi - $54] + xor rdx, qword [rdi - $4C] + xor rax, r8 + xor rbx, qword [rdi - $5C] + xor rcx, qword [rdi - $2C] + xor rax, qword [rdi - $3C] + mov r12, rbp + xor rbp, qword [rdi - $44] + xor rcx, r10 + xor rax, qword [rdi - $14] + xor rdx, qword [rdi - $24] + xor rbx, r9 + xor rbp, qword [rdi - $1C] + xor rcx, qword [rdi + $24] + xor rax, qword [rdi + $14] + xor rdx, qword [rdi + $4] + xor rbx, qword [rdi - $C] + xor rbp, qword [rdi + $C] + mov r13, rcx + rol rcx, 1 + xor rcx, rax + xor rdx, r11 + rol rax, 1 + xor rax, rdx + xor rbx, qword [rdi + $1C] + rol rdx, 1 + xor rdx, rbx + xor rbp, qword [rdi + $34] + rol rbx, 1 + xor rbx, rbp + rol rbp, 1 + xor rbp, r13 + xor r9, rcx + xor r10, rdx + rol r9, $2C + xor r11, rbp + xor r12, rax + rol r10, $2B + xor r8, rbx + mov r13, r9 + rol r11, $15 + or r9, r10 + xor r9, r8 + rol r12, $E + xor r9, qword [r15] + lea r15, [r15 + $8] + mov r14, r12 + and r12, r11 + mov qword [rsi - $64], r9 + xor r12, r10 + not r10 + mov qword [rsi - $54], r12 + or r10, r11 + mov r12, qword [rdi + $4C] + xor r10, r13 + mov qword [rsi - $5C], r10 + and r13, r8 + mov r9, qword [rdi - $1C] + xor r13, r14 + mov r10, qword [rdi - $14] + mov qword [rsi - $44], r13 + or r14, r8 + mov r8, qword [rdi - $4C] + xor r14, r11 + mov r11, qword [rdi + $1C] + mov qword [rsi - $4C], r14 + xor r8, rbp + xor r12, rdx + rol r8, $1C + xor r11, rcx + xor r9, rax + rol r12, $3D + rol r11, $2D + xor r10, rbx + rol r9, $14 + mov r13, r8 + or r8, r12 + rol r10, $3 + xor r8, r11 + mov qword [rsi - $24], r8 + mov r14, r9 + and r9, r13 + mov r8, qword [rdi - $5C] + xor r9, r12 + not r12 + mov qword [rsi - $1C], r9 + or r12, r11 + mov r9, qword [rdi - $2C] + xor r12, r10 + mov qword [rsi - $2C], r12 + and r11, r10 + mov r12, qword [rdi + $3C] + xor r11, r14 + mov qword [rsi - $34], r11 + or r14, r10 + mov r10, qword [rdi + $4] + xor r14, r13 + mov r11, qword [rdi + $34] + mov qword [rsi - $3C], r14 + xor r10, rbp + xor r11, rax + rol r10, $19 + xor r9, rdx + rol r11, $8 + xor r12, rbx + rol r9, $6 + xor r8, rcx + rol r12, $12 + mov r13, r10 + and r10, r11 + rol r8, 1 + not r11 + xor r10, r9 + mov qword [rsi - $C], r10 + mov r14, r12 + and r12, r11 + mov r10, qword [rdi - $C] + xor r12, r13 + mov qword [rsi - $4], r12 + or r13, r9 + mov r12, qword [rdi + $54] + xor r13, r8 + mov qword [rsi - $14], r13 + and r9, r8 + xor r9, r14 + mov qword [rsi + $C], r9 + or r14, r8 + mov r9, qword [rdi - $3C] + xor r14, r11 + mov r11, qword [rdi + $24] + mov qword [rsi + $4], r14 + mov r8, qword [rdi - $44] + xor r10, rcx + xor r11, rdx + rol r10, $A + xor r9, rbx + rol r11, $F + xor r12, rbp + rol r9, $24 + xor r8, rax + rol r12, $38 + mov r13, r10 + or r10, r11 + rol r8, $1B + not r11 + xor r10, r9 + mov qword [rsi + $1C], r10 + mov r14, r12 + or r12, r11 + xor r12, r13 + mov qword [rsi + $24], r12 + and r13, r9 + xor r13, r8 + mov qword [rsi + $14], r13 + or r9, r8 + xor r9, r14 + mov qword [rsi + $34], r9 + and r8, r14 + xor r8, r11 + mov qword [rsi + $2C], r8 + xor rdx, qword [rdi - $54] + xor rbp, qword [rdi - $24] + rol rdx, $3E + xor rcx, qword [rdi + $44] + rol rbp, $37 + xor rax, qword [rdi + $C] + rol rcx, $2 + xor rbx, qword [rdi + $14] + xchg rdi, rsi + rol rax, $27 + rol rbx, $29 + mov r13, rdx + and rdx, rbp + not rbp + xor rdx, rcx + mov qword [rdi + $5C], rdx + mov r14, rax + and rax, rbp + xor rax, r13 + mov qword [rdi + $3C], rax + or r13, rcx + xor r13, rbx + mov qword [rdi + $54], r13 + and rcx, rbx + xor rcx, r14 + mov qword [rdi + $4C], rcx + or rbx, r14 + xor rbx, rbp + mov qword [rdi + $44], rbx + mov rbp, rdx + mov rdx, r13 + cmp r15, qword [rsp + $8] + jb @keccak_round_loop + lea r15, [r15 - $C0] + ret + +@keccak_done: diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Sse2Absorb_i386.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Sse2Absorb_i386.inc new file mode 100644 index 00000000..c0e4a39e --- /dev/null +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Sse2Absorb_i386.inc @@ -0,0 +1,427 @@ +// SSE2 implementation of the Keccak-f[1600] multi-block absorb (IA-32) - +// the asm twin of the x86-64 absorb: XORs whole blocks into the state and +// permutes. MMX converted to SSE2 like the permute. The permutation body +// is the same inner as KeccakF1600Sse2_i386.inc (embedded again because +// local labels cannot cross include files). +// ABI (after HlpSimdProc5Begin_i386.inc): ebx = AState (25 x UInt64, +// standard layout), esi = AData, edi = ABlockCount, eax = ABlockSize, +// ecx = AConstants (iotas: 24 x UInt64). +// Frame: mirrors the OpenSSL wrapper - ebp = saved esp with spills at +// [ebp-4/-8], 248-byte scratch, esp 8-aligned, inner scratch plane at +// esp+140, state biased by +100. +// Saves: ebp saved here (the shared prologue saves ebx/esi/edi); clobbers +// eax, ecx, edx, xmm0-xmm7. +// Reference: OpenSSL CRYPTOGAMS keccak1600-mmx.pl (SHA3_absorb) by +// Andy Polyakov. + + push ebp + mov ebp, esp + + // remap to the OpenSSL absorb roles: + // esi = state+100, edi = xor walk, eax = input, ecx = remaining length, + // edx = block size, ebx = iotas + imul edi, eax // total length in bytes (count * size) + mov edx, eax // block size + mov eax, esi // input pointer + mov esi, ebx // state + mov ebx, ecx // iotas + mov ecx, edi // remaining length + mov edi, esi // xor walk = state base + lea esi, [esi + 100] // biased state pointer + + mov dword ptr [ebp - 4], edx // block size, reloaded after each permute + sub esp, 248 + and esp, $FFFFFFF8 + +@absorb_loop: + cmp ecx, edx + jb @absorb_done + + shr edx, 3 + +@absorb_block: + movq xmm0, qword ptr [eax] + lea eax, [eax + 8] + movq xmm1, qword ptr [edi] + pxor xmm0, xmm1 + lea edi, [edi + 8] + sub ecx, 8 + movq qword ptr [edi - 8], xmm0 + dec edx + jnz @absorb_block + + lea edi, [esp + 140] // scratch plane for the inner routine + mov dword ptr [ebp - 8], ecx + call @keccak_inner + mov ecx, dword ptr [ebp - 8] + mov edx, dword ptr [ebp - 4] + lea edi, [esi - 100] // reset the xor walk to the state base + jmp @absorb_loop + +@absorb_done: + mov esp, ebp + pop ebp + pop edi + pop esi + pop ebx + jmp @absorb_end + +@keccak_inner: + movq xmm0, qword ptr [esi + 60] + movq xmm1, qword ptr [esi + 68] + movq xmm2, qword ptr [esi + 76] + movq xmm3, qword ptr [esi + 84] + movq xmm4, qword ptr [esi + 92] + mov ecx, 24 + jmp @keccak_round_loop + +@keccak_round_loop: + movq xmm5, qword ptr [esi - 100] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi - 92] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi - 84] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi - 76] + pxor xmm3, xmm5 + movq xmm5, qword ptr [esi - 68] + pxor xmm4, xmm5 + movq xmm5, qword ptr [esi - 60] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi - 52] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi - 44] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi - 36] + pxor xmm3, xmm5 + movq xmm5, qword ptr [esi - 28] + pxor xmm4, xmm5 + movq xmm5, qword ptr [esi - 20] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi - 12] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi - 4] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi + 4] + pxor xmm3, xmm5 + movq xmm5, qword ptr [esi + 12] + pxor xmm4, xmm5 + movq xmm5, qword ptr [esi + 36] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi + 20] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi + 28] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi + 44] + pxor xmm3, xmm5 + movq xmm5, xmm2 + movq xmm6, qword ptr [esi + 52] + pxor xmm4, xmm6 + movq xmm7, xmm2 + psrlq xmm5, 63 + movq xmm6, xmm0 + psllq xmm7, 1 + pxor xmm5, xmm0 + psrlq xmm0, 63 + pxor xmm5, xmm7 + psllq xmm6, 1 + movq xmm7, xmm1 + movq qword ptr [esp + 12], xmm5 + pxor xmm6, xmm0 + psrlq xmm7, 63 + pxor xmm6, xmm3 + movq xmm0, xmm1 + movq qword ptr [esp + 36], xmm6 + psllq xmm0, 1 + pxor xmm7, xmm4 + pxor xmm0, xmm7 + movq xmm7, xmm3 + psrlq xmm3, 63 + movq qword ptr [esp + 4], xmm0 + psllq xmm7, 1 + movq xmm5, xmm4 + psrlq xmm4, 63 + pxor xmm1, xmm3 + psllq xmm5, 1 + pxor xmm1, xmm7 + pxor xmm2, xmm4 + movq qword ptr [esp + 20], xmm1 + pxor xmm2, xmm5 + movq xmm3, qword ptr [esi + 44] + movq qword ptr [esp + 28], xmm2 + pxor xmm3, xmm2 + movq xmm4, qword ptr [esi + 92] + movq xmm7, xmm3 + psrlq xmm3, 43 + pxor xmm4, xmm6 + psllq xmm7, 21 + movq xmm6, xmm4 + psrlq xmm4, 50 + por xmm3, xmm7 + psllq xmm6, 14 + movq xmm2, qword ptr [esi - 4] + por xmm4, xmm6 + pxor xmm2, xmm1 + movq xmm1, qword ptr [esi - 52] + movq xmm6, xmm2 + psrlq xmm2, 21 + movq xmm5, qword ptr [esp + 12] + pxor xmm1, xmm5 + psllq xmm6, 43 + movq xmm7, xmm1 + psrlq xmm1, 20 + por xmm2, xmm6 + psllq xmm7, 44 + movq xmm5, qword ptr [esi - 100] + pxor xmm0, xmm5 + por xmm1, xmm7 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, qword ptr [ebx] + pxor xmm5, xmm7 + lea ebx, [ebx + 8] + movq xmm7, xmm3 + movq qword ptr [edi - 100], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi - 92], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 76] + movq qword ptr [edi - 84], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi - 76], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 28] + pxor xmm7, xmm1 + movq qword ptr [edi - 68], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 36 + movq xmm1, qword ptr [esi - 28] + psllq xmm0, 28 + movq xmm2, qword ptr [esp + 36] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 44 + movq xmm2, qword ptr [esi - 20] + psllq xmm6, 20 + movq xmm3, qword ptr [esp + 4] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 61 + movq xmm3, qword ptr [esi + 28] + psllq xmm7, 3 + movq xmm4, qword ptr [esp + 12] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 19 + movq xmm4, qword ptr [esi + 76] + psllq xmm5, 45 + movq xmm6, qword ptr [esp + 20] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 3 + psllq xmm6, 61 + por xmm4, xmm6 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, xmm3 + movq qword ptr [edi - 60], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi - 52], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 92] + movq qword ptr [edi - 44], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi - 36], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 12] + pxor xmm7, xmm1 + movq qword ptr [edi - 28], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 63 + movq xmm1, qword ptr [esi - 44] + psllq xmm0, 1 + movq xmm2, qword ptr [esp + 20] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 58 + movq xmm2, qword ptr [esi + 4] + psllq xmm6, 6 + movq xmm3, qword ptr [esp + 28] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 39 + movq xmm3, qword ptr [esi + 52] + psllq xmm7, 25 + movq xmm4, qword ptr [esp + 36] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 56 + movq xmm4, qword ptr [esi + 60] + psllq xmm5, 8 + movq xmm6, qword ptr [esp + 4] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 46 + psllq xmm6, 18 + por xmm4, xmm6 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, xmm3 + movq qword ptr [edi - 20], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi - 12], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 68] + movq qword ptr [edi - 4], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi + 4], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 36] + pxor xmm7, xmm1 + movq qword ptr [edi + 12], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 37 + movq xmm1, qword ptr [esi - 60] + psllq xmm0, 27 + movq xmm2, qword ptr [esp + 4] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 28 + movq xmm2, qword ptr [esi - 12] + psllq xmm6, 36 + movq xmm3, qword ptr [esp + 12] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 54 + movq xmm3, qword ptr [esi + 36] + psllq xmm7, 10 + movq xmm4, qword ptr [esp + 20] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 49 + movq xmm4, qword ptr [esi + 84] + psllq xmm5, 15 + movq xmm6, qword ptr [esp + 28] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 8 + psllq xmm6, 56 + por xmm4, xmm6 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, xmm3 + movq qword ptr [edi + 20], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi + 28], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 84] + movq qword ptr [edi + 36], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi + 44], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 20] + pxor xmm7, xmm1 + movq qword ptr [edi + 52], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 2 + movq xmm1, qword ptr [esi - 36] + psllq xmm0, 62 + movq xmm2, qword ptr [esp + 28] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 9 + movq xmm2, qword ptr [esi + 12] + psllq xmm6, 55 + movq xmm3, qword ptr [esp + 36] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 25 + movq xmm3, qword ptr [esi + 20] + psllq xmm7, 39 + movq xmm4, qword ptr [esp + 4] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 23 + movq xmm4, qword ptr [esi + 68] + psllq xmm5, 41 + movq xmm6, qword ptr [esp + 12] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 62 + psllq xmm6, 2 + por xmm4, xmm6 + movq xmm5, xmm0 + xor edi, esi + movq qword ptr [esp + 12], xmm1 + xor esi, edi + xor edi, esi + movq xmm6, xmm1 + movq xmm7, xmm2 + pandn xmm6, xmm2 + pandn xmm7, xmm3 + pxor xmm0, xmm6 + pxor xmm1, xmm7 + movq xmm6, xmm3 + movq qword ptr [esi + 60], xmm0 + pandn xmm6, xmm4 + movq qword ptr [esi + 68], xmm1 + pxor xmm2, xmm6 + movq xmm7, xmm4 + movq qword ptr [esi + 76], xmm2 + pandn xmm7, xmm5 + movq xmm6, qword ptr [esp + 12] + pandn xmm5, xmm6 + pxor xmm3, xmm7 + pxor xmm4, xmm5 + movq qword ptr [esi + 84], xmm3 + sub ecx, 1 + movq qword ptr [esi + 92], xmm4 + jnz @keccak_round_loop + lea ebx, [ebx - 192] + ret + +@absorb_end: diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Sse2_i386.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Sse2_i386.inc new file mode 100644 index 00000000..23d73123 --- /dev/null +++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Sse2_i386.inc @@ -0,0 +1,398 @@ +// SSE2 implementation of the Keccak-f[1600] permutation (IA-32; 64-bit +// lanes in the low qwords of xmm0-xmm7, upper halves carry garbage that is +// never stored). The original MMX inner function is converted to SSE2: +// mm0-7 -> xmm0-7, packed memory-source ops rewritten through a +// liveness-proven dead scratch register (SSE2 has no m64 forms), emms +// dropped. The round loop XOR-swaps esi/edi to ping-pong between the state +// and the stack scratch plane each round (24 rounds = even, so pointers +// end where they started). +// ABI (after HlpSimdProc2Begin_i386.inc): ebx = AState (25 x UInt64, +// standard layout), esi = AConstants (iotas: 24 x UInt64). +// Frame: mirrors the OpenSSL wrapper - 240-byte scratch, esp 8-aligned, +// edi = esp + 140, esi biased by +100. The inner routine keeps the +// original calling shape (call/ret) so its esp-relative spill slots stay +// valid. +// Saves: edi/ebp saved here (the shared prologue only saves ebx/esi); +// clobbers eax, ecx, edx, xmm0-xmm7. +// Reference: OpenSSL CRYPTOGAMS keccak1600-mmx.pl by Andy Polyakov. + + push edi + push ebp + + mov eax, ebx // AState + mov ebx, esi // iotas -> ebx (inner's table pointer) + lea esi, [eax + 100] // biased state pointer (lanes at esi - 100 .. + 96) + + mov ebp, esp + sub esp, 240 + and esp, $FFFFFFF8 + lea edi, [esp + 140] + + call @keccak_inner + + mov esp, ebp + pop ebp + pop edi + jmp @keccak_done + +@keccak_inner: + movq xmm0, qword ptr [esi + 60] + movq xmm1, qword ptr [esi + 68] + movq xmm2, qword ptr [esi + 76] + movq xmm3, qword ptr [esi + 84] + movq xmm4, qword ptr [esi + 92] + mov ecx, 24 + jmp @keccak_round_loop + +@keccak_round_loop: + movq xmm5, qword ptr [esi - 100] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi - 92] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi - 84] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi - 76] + pxor xmm3, xmm5 + movq xmm5, qword ptr [esi - 68] + pxor xmm4, xmm5 + movq xmm5, qword ptr [esi - 60] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi - 52] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi - 44] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi - 36] + pxor xmm3, xmm5 + movq xmm5, qword ptr [esi - 28] + pxor xmm4, xmm5 + movq xmm5, qword ptr [esi - 20] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi - 12] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi - 4] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi + 4] + pxor xmm3, xmm5 + movq xmm5, qword ptr [esi + 12] + pxor xmm4, xmm5 + movq xmm5, qword ptr [esi + 36] + pxor xmm2, xmm5 + movq xmm5, qword ptr [esi + 20] + pxor xmm0, xmm5 + movq xmm5, qword ptr [esi + 28] + pxor xmm1, xmm5 + movq xmm5, qword ptr [esi + 44] + pxor xmm3, xmm5 + movq xmm5, xmm2 + movq xmm6, qword ptr [esi + 52] + pxor xmm4, xmm6 + movq xmm7, xmm2 + psrlq xmm5, 63 + movq xmm6, xmm0 + psllq xmm7, 1 + pxor xmm5, xmm0 + psrlq xmm0, 63 + pxor xmm5, xmm7 + psllq xmm6, 1 + movq xmm7, xmm1 + movq qword ptr [esp + 12], xmm5 + pxor xmm6, xmm0 + psrlq xmm7, 63 + pxor xmm6, xmm3 + movq xmm0, xmm1 + movq qword ptr [esp + 36], xmm6 + psllq xmm0, 1 + pxor xmm7, xmm4 + pxor xmm0, xmm7 + movq xmm7, xmm3 + psrlq xmm3, 63 + movq qword ptr [esp + 4], xmm0 + psllq xmm7, 1 + movq xmm5, xmm4 + psrlq xmm4, 63 + pxor xmm1, xmm3 + psllq xmm5, 1 + pxor xmm1, xmm7 + pxor xmm2, xmm4 + movq qword ptr [esp + 20], xmm1 + pxor xmm2, xmm5 + movq xmm3, qword ptr [esi + 44] + movq qword ptr [esp + 28], xmm2 + pxor xmm3, xmm2 + movq xmm4, qword ptr [esi + 92] + movq xmm7, xmm3 + psrlq xmm3, 43 + pxor xmm4, xmm6 + psllq xmm7, 21 + movq xmm6, xmm4 + psrlq xmm4, 50 + por xmm3, xmm7 + psllq xmm6, 14 + movq xmm2, qword ptr [esi - 4] + por xmm4, xmm6 + pxor xmm2, xmm1 + movq xmm1, qword ptr [esi - 52] + movq xmm6, xmm2 + psrlq xmm2, 21 + movq xmm5, qword ptr [esp + 12] + pxor xmm1, xmm5 + psllq xmm6, 43 + movq xmm7, xmm1 + psrlq xmm1, 20 + por xmm2, xmm6 + psllq xmm7, 44 + movq xmm5, qword ptr [esi - 100] + pxor xmm0, xmm5 + por xmm1, xmm7 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, qword ptr [ebx] + pxor xmm5, xmm7 + lea ebx, [ebx + 8] + movq xmm7, xmm3 + movq qword ptr [edi - 100], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi - 92], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 76] + movq qword ptr [edi - 84], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi - 76], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 28] + pxor xmm7, xmm1 + movq qword ptr [edi - 68], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 36 + movq xmm1, qword ptr [esi - 28] + psllq xmm0, 28 + movq xmm2, qword ptr [esp + 36] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 44 + movq xmm2, qword ptr [esi - 20] + psllq xmm6, 20 + movq xmm3, qword ptr [esp + 4] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 61 + movq xmm3, qword ptr [esi + 28] + psllq xmm7, 3 + movq xmm4, qword ptr [esp + 12] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 19 + movq xmm4, qword ptr [esi + 76] + psllq xmm5, 45 + movq xmm6, qword ptr [esp + 20] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 3 + psllq xmm6, 61 + por xmm4, xmm6 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, xmm3 + movq qword ptr [edi - 60], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi - 52], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 92] + movq qword ptr [edi - 44], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi - 36], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 12] + pxor xmm7, xmm1 + movq qword ptr [edi - 28], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 63 + movq xmm1, qword ptr [esi - 44] + psllq xmm0, 1 + movq xmm2, qword ptr [esp + 20] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 58 + movq xmm2, qword ptr [esi + 4] + psllq xmm6, 6 + movq xmm3, qword ptr [esp + 28] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 39 + movq xmm3, qword ptr [esi + 52] + psllq xmm7, 25 + movq xmm4, qword ptr [esp + 36] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 56 + movq xmm4, qword ptr [esi + 60] + psllq xmm5, 8 + movq xmm6, qword ptr [esp + 4] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 46 + psllq xmm6, 18 + por xmm4, xmm6 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, xmm3 + movq qword ptr [edi - 20], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi - 12], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 68] + movq qword ptr [edi - 4], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi + 4], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 36] + pxor xmm7, xmm1 + movq qword ptr [edi + 12], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 37 + movq xmm1, qword ptr [esi - 60] + psllq xmm0, 27 + movq xmm2, qword ptr [esp + 4] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 28 + movq xmm2, qword ptr [esi - 12] + psllq xmm6, 36 + movq xmm3, qword ptr [esp + 12] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 54 + movq xmm3, qword ptr [esi + 36] + psllq xmm7, 10 + movq xmm4, qword ptr [esp + 20] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 49 + movq xmm4, qword ptr [esi + 84] + psllq xmm5, 15 + movq xmm6, qword ptr [esp + 28] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 8 + psllq xmm6, 56 + por xmm4, xmm6 + movq xmm5, xmm1 + movq xmm6, xmm2 + pandn xmm5, xmm2 + pandn xmm2, xmm3 + pxor xmm5, xmm0 + pxor xmm2, xmm1 + movq xmm7, xmm3 + movq qword ptr [edi + 20], xmm5 + movq xmm5, xmm4 + pandn xmm3, xmm4 + pandn xmm4, xmm0 + pxor xmm3, xmm6 + movq qword ptr [edi + 28], xmm2 + pxor xmm4, xmm7 + movq xmm7, qword ptr [esi - 84] + movq qword ptr [edi + 36], xmm3 + pandn xmm0, xmm1 + movq qword ptr [edi + 44], xmm4 + pxor xmm0, xmm5 + movq xmm1, qword ptr [esp + 20] + pxor xmm7, xmm1 + movq qword ptr [edi + 52], xmm0 + movq xmm0, xmm7 + psrlq xmm7, 2 + movq xmm1, qword ptr [esi - 36] + psllq xmm0, 62 + movq xmm2, qword ptr [esp + 28] + pxor xmm1, xmm2 + por xmm0, xmm7 + movq xmm6, xmm1 + psrlq xmm1, 9 + movq xmm2, qword ptr [esi + 12] + psllq xmm6, 55 + movq xmm3, qword ptr [esp + 36] + pxor xmm2, xmm3 + por xmm1, xmm6 + movq xmm7, xmm2 + psrlq xmm2, 25 + movq xmm3, qword ptr [esi + 20] + psllq xmm7, 39 + movq xmm4, qword ptr [esp + 4] + pxor xmm3, xmm4 + por xmm2, xmm7 + movq xmm5, xmm3 + psrlq xmm3, 23 + movq xmm4, qword ptr [esi + 68] + psllq xmm5, 41 + movq xmm6, qword ptr [esp + 12] + pxor xmm4, xmm6 + por xmm3, xmm5 + movq xmm6, xmm4 + psrlq xmm4, 62 + psllq xmm6, 2 + por xmm4, xmm6 + movq xmm5, xmm0 + xor edi, esi + movq qword ptr [esp + 12], xmm1 + xor esi, edi + xor edi, esi + movq xmm6, xmm1 + movq xmm7, xmm2 + pandn xmm6, xmm2 + pandn xmm7, xmm3 + pxor xmm0, xmm6 + pxor xmm1, xmm7 + movq xmm6, xmm3 + movq qword ptr [esi + 60], xmm0 + pandn xmm6, xmm4 + movq qword ptr [esi + 68], xmm1 + pxor xmm2, xmm6 + movq xmm7, xmm4 + movq qword ptr [esi + 76], xmm2 + pandn xmm7, xmm5 + movq xmm6, qword ptr [esp + 12] + pandn xmm5, xmm6 + pxor xmm3, xmm7 + pxor xmm4, xmm5 + movq qword ptr [esi + 84], xmm3 + sub ecx, 1 + movq qword ptr [esi + 92], xmm4 + jnz @keccak_round_loop + lea ebx, [ebx - 192] + ret + +@keccak_done: + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc index 7195dee9..bfb7a40d 100644 --- a/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc @@ -1,19 +1,19 @@ -// SHA-512 AVX2 two-block implementation (VEX-256), ported from OpenSSL's -// sha512-x86_64.pl (CRYPTOGAMS). Two message blocks are scheduled together in -// 256-bit lanes; the compression rounds stay serial (as SHA-512 requires), so -// scheduling two blocks at once is what makes this faster than the single-block -// AVX path. AVX2 and BMI2 (rorx/andn) instructions are db-encoded for broad -// assembler compatibility (every AVX2 CPU also provides BMI2). -// Expects MS x64 ABI (after HlpSimdProc4Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = doubled-K512 ptr (each 128-bit K512 pair stored twice -// so one table feeds both 256-bit lanes, with the BSWAP64 mask appended; see -// K512_Doubled). The mask is read unaligned, so the Pascal const needs no -// special alignment. -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// AVX2 implementation of SHA-512 compress (VEX-256, two-block). Two message +// blocks are scheduled together in 256-bit lanes; the compression rounds +// stay serial (as SHA-512 requires), so scheduling two blocks at once is +// what makes this faster than the single-block AVX path. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K512 ptr (each 128-bit K512 pair stored +// twice so one table feeds both 256-bit lanes, with the BSWAP64 mask +// appended; see K512_Doubled; the mask is read unaligned, so the Pascal +// const needs no special alignment). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// AVX2 and BMI2 (rorx/andn) instructions are db-encoded for broad assembler +// compatibility (every AVX2 CPU also provides BMI2). +// Reference: OpenSSL CRYPTOGAMS sha512-x86_64.pl (AVX2 kernel). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} mov rax, rsp push r9 @@ -1344,4 +1344,4 @@ lea rsp, [rsi] mov rsi, rax - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressCryptoExt_aarch64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressCryptoExt_aarch64.inc index a6736168..843b0288 100644 --- a/HashLib/src/Include/Simd/SHA512/SHA512CompressCryptoExt_aarch64.inc +++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressCryptoExt_aarch64.inc @@ -1,16 +1,17 @@ -// SHA-512 AArch64 FEAT_SHA512 implementation. -// Expects AAPCS64: x0 = state ptr (8 x UInt64), x1 = data ptr, -// w2 = numblocks, x3 = K512 ptr (80 UInt64 round constants). -// Leaf nostackframe; caller-saved GPR/vector only. -// K pointer rewound each block (sub x3, #640). Two rounds per iteration; csel avoids OOB prefetch. -// Reference: OpenSSL CRYPTOGAMS sha512-armv8.S. -// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// CryptoExt (FEAT_SHA512) implementation of SHA-512 compress. +// The K pointer is rewound each block (sub x3, #640). Two rounds per +// iteration; csel avoids OOB prefetch. +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (8 x UInt64), +// x1 = data ptr, w2 = numblocks, x3 = K512 ptr (80 UInt64 round constants). // Register map: // v0-v3 = working state (a..h, two UInt64 per register) // v5, v6 = ext scratch (fg, de) v7 = ext scratch (m9_10) // v16-v23 = message schedule (W) // v24, v25 = K + W ping-pong pair // v26-v29 = saved state for the final add +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector/crypto instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha512-armv8.S. cbz w2, .Lcryptoext_sha512_done .long 0xacc14430 // ldp q16, q17, [x1], #32 diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressGpr_aarch64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressGpr_aarch64.inc new file mode 100644 index 00000000..1abcbe2f --- /dev/null +++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressGpr_aarch64.inc @@ -0,0 +1,949 @@ +// Gpr implementation of SHA-512 compress (AArch64) for cores without +// FEAT_SHA512 (absent on most consumer Cortex-A7x); the Gpr name states +// the ISA requirement: none beyond base AArch64. Serial SHA maps better +// onto the 31 GPRs than onto NEON lanes (single dependency chain, 1-op +// ror, rotate-folded eor operands). +// ABI (after HlpSimdProc4Begin_aarch64.inc): x0 = state ptr (8 x UInt64), +// x1 = data ptr, w2 = numblocks, x3 = K512_Gpr ptr. +// K512_Gpr carries the 80 round constants PLUS a zero terminator quad - +// the message-schedule loop ends when the loaded K word is zero (cbnz), +// exactly like the original's sentinel-terminated table; a plain K512 +// const cannot drive this loop. +// Frame: 128-byte AAPCS64 frame (x29/x30 + x19-x28 callee-saved) + 32 +// bytes schedule scratch; x30 (LR) is repurposed as the K walk pointer +// between the save and the restore, as in the original. +// AArch64 instructions are .long-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha512-armv8.pl (sha512_block_data_order, +// the plain path). + + .long 0x2a0203e2 // mov w2,w2 // zero-extend ANumBlocks (AAPCS64 leaves x2's top half undefined) + .long 0xa9b87bfd // stp x29,x30,[sp,#-128]! + .long 0x910003fd // add x29,sp,#0 + + .long 0xa90153f3 // stp x19,x20,[sp,#16] + .long 0xa9025bf5 // stp x21,x22,[sp,#32] + .long 0xa90363f7 // stp x23,x24,[sp,#48] + .long 0xa9046bf9 // stp x25,x26,[sp,#64] + .long 0xa90573fb // stp x27,x28,[sp,#80] + .long 0xd10083ff // sub sp,sp,#4*8 + + .long 0xa9405414 // ldp x20,x21,[x0] // load context + .long 0xa9415c16 // ldp x22,x23,[x0,#2*8] + .long 0xa9426418 // ldp x24,x25,[x0,#4*8] + .long 0x8b021c22 // add x2,x1,x2, lsl #7 // end of input + .long 0xa9436c1a // ldp x26,x27,[x0,#6*8] + .long 0xaa0303fe // mov x30,x3 // K512_Gpr (param 4; x3 is reused for message words below) + .long 0xa9060ba0 // stp x0,x2,[x29,#96] + + +.Lsha512_gpr_loop: + .long 0xa8c11023 // ldp x3,x4,[x1],#2*8 + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++ + .long 0xca1602bc // eor x28,x21,x22 // magic seed + .long 0xf9003ba1 // str x1,[x29,#112] + .long 0xdac00c63 // rev x3,x3 // 0 + .long 0x93d83b10 // ror x16,x24,#14 + .long 0x8b13037b // add x27,x27,x19 // h+=K[i] + .long 0xcad85f06 // eor x6,x24,x24, ror #23 + .long 0x8a180331 // and x17,x25,x24 + .long 0x8a380353 // bic x19,x26,x24 + .long 0x8b03037b // add x27,x27,x3 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca150293 // eor x19,x20,x21 // a^b, b^c in next round + .long 0xcac64a10 // eor x16,x16,x6, ror #18 // Sigma1(e) + .long 0x93d47286 // ror x6,x20,#28 + .long 0x8b11037b // add x27,x27,x17 // h+=Ch(e,f,g) + .long 0xcad41691 // eor x17,x20,x20, ror #5 + .long 0x8b10037b // add x27,x27,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b1b02f7 // add x23,x23,x27 // d+=h + .long 0xca15039c // eor x28,x28,x21 // Maj(a,b,c) + .long 0xcad188d1 // eor x17,x6,x17, ror #34 // Sigma0(a) + .long 0x8b1c037b // add x27,x27,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x27,x27,x17 // h+=Sigma0(a) + .long 0xdac00c84 // rev x4,x4 // 1 + .long 0xa8c11825 // ldp x5,x6,[x1],#2*8 + .long 0x8b11037b // add x27,x27,x17 // h+=Sigma0(a) + .long 0x93d73af0 // ror x16,x23,#14 + .long 0x8b1c035a // add x26,x26,x28 // h+=K[i] + .long 0xcad75ee7 // eor x7,x23,x23, ror #23 + .long 0x8a170311 // and x17,x24,x23 + .long 0x8a37033c // bic x28,x25,x23 + .long 0x8b04035a // add x26,x26,x4 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca14037c // eor x28,x27,x20 // a^b, b^c in next round + .long 0xcac74a10 // eor x16,x16,x7, ror #18 // Sigma1(e) + .long 0x93db7367 // ror x7,x27,#28 + .long 0x8b11035a // add x26,x26,x17 // h+=Ch(e,f,g) + .long 0xcadb1771 // eor x17,x27,x27, ror #5 + .long 0x8b10035a // add x26,x26,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b1a02d6 // add x22,x22,x26 // d+=h + .long 0xca140273 // eor x19,x19,x20 // Maj(a,b,c) + .long 0xcad188f1 // eor x17,x7,x17, ror #34 // Sigma0(a) + .long 0x8b13035a // add x26,x26,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x26,x26,x17 // h+=Sigma0(a) + .long 0xdac00ca5 // rev x5,x5 // 2 + .long 0x8b11035a // add x26,x26,x17 // h+=Sigma0(a) + .long 0x93d63ad0 // ror x16,x22,#14 + .long 0x8b130339 // add x25,x25,x19 // h+=K[i] + .long 0xcad65ec8 // eor x8,x22,x22, ror #23 + .long 0x8a1602f1 // and x17,x23,x22 + .long 0x8a360313 // bic x19,x24,x22 + .long 0x8b050339 // add x25,x25,x5 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1b0353 // eor x19,x26,x27 // a^b, b^c in next round + .long 0xcac84a10 // eor x16,x16,x8, ror #18 // Sigma1(e) + .long 0x93da7348 // ror x8,x26,#28 + .long 0x8b110339 // add x25,x25,x17 // h+=Ch(e,f,g) + .long 0xcada1751 // eor x17,x26,x26, ror #5 + .long 0x8b100339 // add x25,x25,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b1902b5 // add x21,x21,x25 // d+=h + .long 0xca1b039c // eor x28,x28,x27 // Maj(a,b,c) + .long 0xcad18911 // eor x17,x8,x17, ror #34 // Sigma0(a) + .long 0x8b1c0339 // add x25,x25,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x25,x25,x17 // h+=Sigma0(a) + .long 0xdac00cc6 // rev x6,x6 // 3 + .long 0xa8c12027 // ldp x7,x8,[x1],#2*8 + .long 0x8b110339 // add x25,x25,x17 // h+=Sigma0(a) + .long 0x93d53ab0 // ror x16,x21,#14 + .long 0x8b1c0318 // add x24,x24,x28 // h+=K[i] + .long 0xcad55ea9 // eor x9,x21,x21, ror #23 + .long 0x8a1502d1 // and x17,x22,x21 + .long 0x8a3502fc // bic x28,x23,x21 + .long 0x8b060318 // add x24,x24,x6 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1a033c // eor x28,x25,x26 // a^b, b^c in next round + .long 0xcac94a10 // eor x16,x16,x9, ror #18 // Sigma1(e) + .long 0x93d97329 // ror x9,x25,#28 + .long 0x8b110318 // add x24,x24,x17 // h+=Ch(e,f,g) + .long 0xcad91731 // eor x17,x25,x25, ror #5 + .long 0x8b100318 // add x24,x24,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b180294 // add x20,x20,x24 // d+=h + .long 0xca1a0273 // eor x19,x19,x26 // Maj(a,b,c) + .long 0xcad18931 // eor x17,x9,x17, ror #34 // Sigma0(a) + .long 0x8b130318 // add x24,x24,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x24,x24,x17 // h+=Sigma0(a) + .long 0xdac00ce7 // rev x7,x7 // 4 + .long 0x8b110318 // add x24,x24,x17 // h+=Sigma0(a) + .long 0x93d43a90 // ror x16,x20,#14 + .long 0x8b1302f7 // add x23,x23,x19 // h+=K[i] + .long 0xcad45e8a // eor x10,x20,x20, ror #23 + .long 0x8a1402b1 // and x17,x21,x20 + .long 0x8a3402d3 // bic x19,x22,x20 + .long 0x8b0702f7 // add x23,x23,x7 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca190313 // eor x19,x24,x25 // a^b, b^c in next round + .long 0xcaca4a10 // eor x16,x16,x10, ror #18 // Sigma1(e) + .long 0x93d8730a // ror x10,x24,#28 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Ch(e,f,g) + .long 0xcad81711 // eor x17,x24,x24, ror #5 + .long 0x8b1002f7 // add x23,x23,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b17037b // add x27,x27,x23 // d+=h + .long 0xca19039c // eor x28,x28,x25 // Maj(a,b,c) + .long 0xcad18951 // eor x17,x10,x17, ror #34 // Sigma0(a) + .long 0x8b1c02f7 // add x23,x23,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x23,x23,x17 // h+=Sigma0(a) + .long 0xdac00d08 // rev x8,x8 // 5 + .long 0xa8c12829 // ldp x9,x10,[x1],#2*8 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Sigma0(a) + .long 0x93db3b70 // ror x16,x27,#14 + .long 0x8b1c02d6 // add x22,x22,x28 // h+=K[i] + .long 0xcadb5f6b // eor x11,x27,x27, ror #23 + .long 0x8a1b0291 // and x17,x20,x27 + .long 0x8a3b02bc // bic x28,x21,x27 + .long 0x8b0802d6 // add x22,x22,x8 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1802fc // eor x28,x23,x24 // a^b, b^c in next round + .long 0xcacb4a10 // eor x16,x16,x11, ror #18 // Sigma1(e) + .long 0x93d772eb // ror x11,x23,#28 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Ch(e,f,g) + .long 0xcad716f1 // eor x17,x23,x23, ror #5 + .long 0x8b1002d6 // add x22,x22,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b16035a // add x26,x26,x22 // d+=h + .long 0xca180273 // eor x19,x19,x24 // Maj(a,b,c) + .long 0xcad18971 // eor x17,x11,x17, ror #34 // Sigma0(a) + .long 0x8b1302d6 // add x22,x22,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x22,x22,x17 // h+=Sigma0(a) + .long 0xdac00d29 // rev x9,x9 // 6 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Sigma0(a) + .long 0x93da3b50 // ror x16,x26,#14 + .long 0x8b1302b5 // add x21,x21,x19 // h+=K[i] + .long 0xcada5f4c // eor x12,x26,x26, ror #23 + .long 0x8a1a0371 // and x17,x27,x26 + .long 0x8a3a0293 // bic x19,x20,x26 + .long 0x8b0902b5 // add x21,x21,x9 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1702d3 // eor x19,x22,x23 // a^b, b^c in next round + .long 0xcacc4a10 // eor x16,x16,x12, ror #18 // Sigma1(e) + .long 0x93d672cc // ror x12,x22,#28 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Ch(e,f,g) + .long 0xcad616d1 // eor x17,x22,x22, ror #5 + .long 0x8b1002b5 // add x21,x21,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b150339 // add x25,x25,x21 // d+=h + .long 0xca17039c // eor x28,x28,x23 // Maj(a,b,c) + .long 0xcad18991 // eor x17,x12,x17, ror #34 // Sigma0(a) + .long 0x8b1c02b5 // add x21,x21,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x21,x21,x17 // h+=Sigma0(a) + .long 0xdac00d4a // rev x10,x10 // 7 + .long 0xa8c1302b // ldp x11,x12,[x1],#2*8 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Sigma0(a) + .long 0x93d93b30 // ror x16,x25,#14 + .long 0x8b1c0294 // add x20,x20,x28 // h+=K[i] + .long 0xcad95f2d // eor x13,x25,x25, ror #23 + .long 0x8a190351 // and x17,x26,x25 + .long 0x8a39037c // bic x28,x27,x25 + .long 0x8b0a0294 // add x20,x20,x10 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1602bc // eor x28,x21,x22 // a^b, b^c in next round + .long 0xcacd4a10 // eor x16,x16,x13, ror #18 // Sigma1(e) + .long 0x93d572ad // ror x13,x21,#28 + .long 0x8b110294 // add x20,x20,x17 // h+=Ch(e,f,g) + .long 0xcad516b1 // eor x17,x21,x21, ror #5 + .long 0x8b100294 // add x20,x20,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b140318 // add x24,x24,x20 // d+=h + .long 0xca160273 // eor x19,x19,x22 // Maj(a,b,c) + .long 0xcad189b1 // eor x17,x13,x17, ror #34 // Sigma0(a) + .long 0x8b130294 // add x20,x20,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x20,x20,x17 // h+=Sigma0(a) + .long 0xdac00d6b // rev x11,x11 // 8 + .long 0x8b110294 // add x20,x20,x17 // h+=Sigma0(a) + .long 0x93d83b10 // ror x16,x24,#14 + .long 0x8b13037b // add x27,x27,x19 // h+=K[i] + .long 0xcad85f0e // eor x14,x24,x24, ror #23 + .long 0x8a180331 // and x17,x25,x24 + .long 0x8a380353 // bic x19,x26,x24 + .long 0x8b0b037b // add x27,x27,x11 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca150293 // eor x19,x20,x21 // a^b, b^c in next round + .long 0xcace4a10 // eor x16,x16,x14, ror #18 // Sigma1(e) + .long 0x93d4728e // ror x14,x20,#28 + .long 0x8b11037b // add x27,x27,x17 // h+=Ch(e,f,g) + .long 0xcad41691 // eor x17,x20,x20, ror #5 + .long 0x8b10037b // add x27,x27,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b1b02f7 // add x23,x23,x27 // d+=h + .long 0xca15039c // eor x28,x28,x21 // Maj(a,b,c) + .long 0xcad189d1 // eor x17,x14,x17, ror #34 // Sigma0(a) + .long 0x8b1c037b // add x27,x27,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x27,x27,x17 // h+=Sigma0(a) + .long 0xdac00d8c // rev x12,x12 // 9 + .long 0xa8c1382d // ldp x13,x14,[x1],#2*8 + .long 0x8b11037b // add x27,x27,x17 // h+=Sigma0(a) + .long 0x93d73af0 // ror x16,x23,#14 + .long 0x8b1c035a // add x26,x26,x28 // h+=K[i] + .long 0xcad75eef // eor x15,x23,x23, ror #23 + .long 0x8a170311 // and x17,x24,x23 + .long 0x8a37033c // bic x28,x25,x23 + .long 0x8b0c035a // add x26,x26,x12 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca14037c // eor x28,x27,x20 // a^b, b^c in next round + .long 0xcacf4a10 // eor x16,x16,x15, ror #18 // Sigma1(e) + .long 0x93db736f // ror x15,x27,#28 + .long 0x8b11035a // add x26,x26,x17 // h+=Ch(e,f,g) + .long 0xcadb1771 // eor x17,x27,x27, ror #5 + .long 0x8b10035a // add x26,x26,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b1a02d6 // add x22,x22,x26 // d+=h + .long 0xca140273 // eor x19,x19,x20 // Maj(a,b,c) + .long 0xcad189f1 // eor x17,x15,x17, ror #34 // Sigma0(a) + .long 0x8b13035a // add x26,x26,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x26,x26,x17 // h+=Sigma0(a) + .long 0xdac00dad // rev x13,x13 // 10 + .long 0x8b11035a // add x26,x26,x17 // h+=Sigma0(a) + .long 0x93d63ad0 // ror x16,x22,#14 + .long 0x8b130339 // add x25,x25,x19 // h+=K[i] + .long 0xcad65ec0 // eor x0,x22,x22, ror #23 + .long 0x8a1602f1 // and x17,x23,x22 + .long 0x8a360313 // bic x19,x24,x22 + .long 0x8b0d0339 // add x25,x25,x13 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1b0353 // eor x19,x26,x27 // a^b, b^c in next round + .long 0xcac04a10 // eor x16,x16,x0, ror #18 // Sigma1(e) + .long 0x93da7340 // ror x0,x26,#28 + .long 0x8b110339 // add x25,x25,x17 // h+=Ch(e,f,g) + .long 0xcada1751 // eor x17,x26,x26, ror #5 + .long 0x8b100339 // add x25,x25,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b1902b5 // add x21,x21,x25 // d+=h + .long 0xca1b039c // eor x28,x28,x27 // Maj(a,b,c) + .long 0xcad18811 // eor x17,x0,x17, ror #34 // Sigma0(a) + .long 0x8b1c0339 // add x25,x25,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x25,x25,x17 // h+=Sigma0(a) + .long 0xdac00dce // rev x14,x14 // 11 + .long 0xa8c1002f // ldp x15,x0,[x1],#2*8 + .long 0x8b110339 // add x25,x25,x17 // h+=Sigma0(a) + .long 0xf9000fe6 // str x6,[sp,#24] + .long 0x93d53ab0 // ror x16,x21,#14 + .long 0x8b1c0318 // add x24,x24,x28 // h+=K[i] + .long 0xcad55ea6 // eor x6,x21,x21, ror #23 + .long 0x8a1502d1 // and x17,x22,x21 + .long 0x8a3502fc // bic x28,x23,x21 + .long 0x8b0e0318 // add x24,x24,x14 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1a033c // eor x28,x25,x26 // a^b, b^c in next round + .long 0xcac64a10 // eor x16,x16,x6, ror #18 // Sigma1(e) + .long 0x93d97326 // ror x6,x25,#28 + .long 0x8b110318 // add x24,x24,x17 // h+=Ch(e,f,g) + .long 0xcad91731 // eor x17,x25,x25, ror #5 + .long 0x8b100318 // add x24,x24,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b180294 // add x20,x20,x24 // d+=h + .long 0xca1a0273 // eor x19,x19,x26 // Maj(a,b,c) + .long 0xcad188d1 // eor x17,x6,x17, ror #34 // Sigma0(a) + .long 0x8b130318 // add x24,x24,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x24,x24,x17 // h+=Sigma0(a) + .long 0xdac00def // rev x15,x15 // 12 + .long 0x8b110318 // add x24,x24,x17 // h+=Sigma0(a) + .long 0xf90003e7 // str x7,[sp,#0] + .long 0x93d43a90 // ror x16,x20,#14 + .long 0x8b1302f7 // add x23,x23,x19 // h+=K[i] + .long 0xcad45e87 // eor x7,x20,x20, ror #23 + .long 0x8a1402b1 // and x17,x21,x20 + .long 0x8a3402d3 // bic x19,x22,x20 + .long 0x8b0f02f7 // add x23,x23,x15 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca190313 // eor x19,x24,x25 // a^b, b^c in next round + .long 0xcac74a10 // eor x16,x16,x7, ror #18 // Sigma1(e) + .long 0x93d87307 // ror x7,x24,#28 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Ch(e,f,g) + .long 0xcad81711 // eor x17,x24,x24, ror #5 + .long 0x8b1002f7 // add x23,x23,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b17037b // add x27,x27,x23 // d+=h + .long 0xca19039c // eor x28,x28,x25 // Maj(a,b,c) + .long 0xcad188f1 // eor x17,x7,x17, ror #34 // Sigma0(a) + .long 0x8b1c02f7 // add x23,x23,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x23,x23,x17 // h+=Sigma0(a) + .long 0xdac00c00 // rev x0,x0 // 13 + .long 0xa9400821 // ldp x1,x2,[x1] + .long 0x8b1102f7 // add x23,x23,x17 // h+=Sigma0(a) + .long 0xf90007e8 // str x8,[sp,#8] + .long 0x93db3b70 // ror x16,x27,#14 + .long 0x8b1c02d6 // add x22,x22,x28 // h+=K[i] + .long 0xcadb5f68 // eor x8,x27,x27, ror #23 + .long 0x8a1b0291 // and x17,x20,x27 + .long 0x8a3b02bc // bic x28,x21,x27 + .long 0x8b0002d6 // add x22,x22,x0 // h+=X[i] + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1802fc // eor x28,x23,x24 // a^b, b^c in next round + .long 0xcac84a10 // eor x16,x16,x8, ror #18 // Sigma1(e) + .long 0x93d772e8 // ror x8,x23,#28 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Ch(e,f,g) + .long 0xcad716f1 // eor x17,x23,x23, ror #5 + .long 0x8b1002d6 // add x22,x22,x16 // h+=Sigma1(e) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0x8b16035a // add x26,x26,x22 // d+=h + .long 0xca180273 // eor x19,x19,x24 // Maj(a,b,c) + .long 0xcad18911 // eor x17,x8,x17, ror #34 // Sigma0(a) + .long 0x8b1302d6 // add x22,x22,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + //add x22,x22,x17 // h+=Sigma0(a) + .long 0xdac00c21 // rev x1,x1 // 14 + .long 0xf9400fe6 // ldr x6,[sp,#24] + .long 0x8b1102d6 // add x22,x22,x17 // h+=Sigma0(a) + .long 0xf9000be9 // str x9,[sp,#16] + .long 0x93da3b50 // ror x16,x26,#14 + .long 0x8b1302b5 // add x21,x21,x19 // h+=K[i] + .long 0xcada5f49 // eor x9,x26,x26, ror #23 + .long 0x8a1a0371 // and x17,x27,x26 + .long 0x8a3a0293 // bic x19,x20,x26 + .long 0x8b0102b5 // add x21,x21,x1 // h+=X[i] + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1702d3 // eor x19,x22,x23 // a^b, b^c in next round + .long 0xcac94a10 // eor x16,x16,x9, ror #18 // Sigma1(e) + .long 0x93d672c9 // ror x9,x22,#28 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Ch(e,f,g) + .long 0xcad616d1 // eor x17,x22,x22, ror #5 + .long 0x8b1002b5 // add x21,x21,x16 // h+=Sigma1(e) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0x8b150339 // add x25,x25,x21 // d+=h + .long 0xca17039c // eor x28,x28,x23 // Maj(a,b,c) + .long 0xcad18931 // eor x17,x9,x17, ror #34 // Sigma0(a) + .long 0x8b1c02b5 // add x21,x21,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + //add x21,x21,x17 // h+=Sigma0(a) + .long 0xdac00c42 // rev x2,x2 // 15 + .long 0xf94003e7 // ldr x7,[sp,#0] + .long 0x8b1102b5 // add x21,x21,x17 // h+=Sigma0(a) + .long 0xf9000fea // str x10,[sp,#24] + .long 0x93d93b30 // ror x16,x25,#14 + .long 0x8b1c0294 // add x20,x20,x28 // h+=K[i] + .long 0x93c40489 // ror x9,x4,#1 + .long 0x8a190351 // and x17,x26,x25 + .long 0x93c14c28 // ror x8,x1,#19 + .long 0x8a39037c // bic x28,x27,x25 + .long 0x93d572aa // ror x10,x21,#28 + .long 0x8b020294 // add x20,x20,x2 // h+=X[i] + .long 0xcad94a10 // eor x16,x16,x25, ror #18 + .long 0xcac42129 // eor x9,x9,x4, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1602bc // eor x28,x21,x22 // a^b, b^c in next round + .long 0xcad9a610 // eor x16,x16,x25, ror #41 // Sigma1(e) + .long 0xcad5894a // eor x10,x10,x21, ror #34 + .long 0x8b110294 // add x20,x20,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcac1f508 // eor x8,x8,x1, ror #61 + .long 0xca441d29 // eor x9,x9,x4, lsr #7 // sigma0(X[i+1]) + .long 0x8b100294 // add x20,x20,x16 // h+=Sigma1(e) + .long 0xca160273 // eor x19,x19,x22 // Maj(a,b,c) + .long 0xcad59d51 // eor x17,x10,x21, ror #39 // Sigma0(a) + .long 0xca411908 // eor x8,x8,x1, lsr #6 // sigma1(X[i+14]) + .long 0x8b0c0063 // add x3,x3,x12 + .long 0x8b140318 // add x24,x24,x20 // d+=h + .long 0x8b130294 // add x20,x20,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b090063 // add x3,x3,x9 + .long 0x8b110294 // add x20,x20,x17 // h+=Sigma0(a) + .long 0x8b080063 // add x3,x3,x8 + +.Lsha512_gpr_loop_16_xx: + .long 0xf94007e8 // ldr x8,[sp,#8] + .long 0xf90003eb // str x11,[sp,#0] + .long 0x93d83b10 // ror x16,x24,#14 + .long 0x8b13037b // add x27,x27,x19 // h+=K[i] + .long 0x93c504aa // ror x10,x5,#1 + .long 0x8a180331 // and x17,x25,x24 + .long 0x93c24c49 // ror x9,x2,#19 + .long 0x8a380353 // bic x19,x26,x24 + .long 0x93d4728b // ror x11,x20,#28 + .long 0x8b03037b // add x27,x27,x3 // h+=X[i] + .long 0xcad84a10 // eor x16,x16,x24, ror #18 + .long 0xcac5214a // eor x10,x10,x5, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca150293 // eor x19,x20,x21 // a^b, b^c in next round + .long 0xcad8a610 // eor x16,x16,x24, ror #41 // Sigma1(e) + .long 0xcad4896b // eor x11,x11,x20, ror #34 + .long 0x8b11037b // add x27,x27,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcac2f529 // eor x9,x9,x2, ror #61 + .long 0xca451d4a // eor x10,x10,x5, lsr #7 // sigma0(X[i+1]) + .long 0x8b10037b // add x27,x27,x16 // h+=Sigma1(e) + .long 0xca15039c // eor x28,x28,x21 // Maj(a,b,c) + .long 0xcad49d71 // eor x17,x11,x20, ror #39 // Sigma0(a) + .long 0xca421929 // eor x9,x9,x2, lsr #6 // sigma1(X[i+14]) + .long 0x8b0d0084 // add x4,x4,x13 + .long 0x8b1b02f7 // add x23,x23,x27 // d+=h + .long 0x8b1c037b // add x27,x27,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b0a0084 // add x4,x4,x10 + .long 0x8b11037b // add x27,x27,x17 // h+=Sigma0(a) + .long 0x8b090084 // add x4,x4,x9 + .long 0xf9400be9 // ldr x9,[sp,#16] + .long 0xf90007ec // str x12,[sp,#8] + .long 0x93d73af0 // ror x16,x23,#14 + .long 0x8b1c035a // add x26,x26,x28 // h+=K[i] + .long 0x93c604cb // ror x11,x6,#1 + .long 0x8a170311 // and x17,x24,x23 + .long 0x93c34c6a // ror x10,x3,#19 + .long 0x8a37033c // bic x28,x25,x23 + .long 0x93db736c // ror x12,x27,#28 + .long 0x8b04035a // add x26,x26,x4 // h+=X[i] + .long 0xcad74a10 // eor x16,x16,x23, ror #18 + .long 0xcac6216b // eor x11,x11,x6, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca14037c // eor x28,x27,x20 // a^b, b^c in next round + .long 0xcad7a610 // eor x16,x16,x23, ror #41 // Sigma1(e) + .long 0xcadb898c // eor x12,x12,x27, ror #34 + .long 0x8b11035a // add x26,x26,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcac3f54a // eor x10,x10,x3, ror #61 + .long 0xca461d6b // eor x11,x11,x6, lsr #7 // sigma0(X[i+1]) + .long 0x8b10035a // add x26,x26,x16 // h+=Sigma1(e) + .long 0xca140273 // eor x19,x19,x20 // Maj(a,b,c) + .long 0xcadb9d91 // eor x17,x12,x27, ror #39 // Sigma0(a) + .long 0xca43194a // eor x10,x10,x3, lsr #6 // sigma1(X[i+14]) + .long 0x8b0e00a5 // add x5,x5,x14 + .long 0x8b1a02d6 // add x22,x22,x26 // d+=h + .long 0x8b13035a // add x26,x26,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b0b00a5 // add x5,x5,x11 + .long 0x8b11035a // add x26,x26,x17 // h+=Sigma0(a) + .long 0x8b0a00a5 // add x5,x5,x10 + .long 0xf9400fea // ldr x10,[sp,#24] + .long 0xf9000bed // str x13,[sp,#16] + .long 0x93d63ad0 // ror x16,x22,#14 + .long 0x8b130339 // add x25,x25,x19 // h+=K[i] + .long 0x93c704ec // ror x12,x7,#1 + .long 0x8a1602f1 // and x17,x23,x22 + .long 0x93c44c8b // ror x11,x4,#19 + .long 0x8a360313 // bic x19,x24,x22 + .long 0x93da734d // ror x13,x26,#28 + .long 0x8b050339 // add x25,x25,x5 // h+=X[i] + .long 0xcad64a10 // eor x16,x16,x22, ror #18 + .long 0xcac7218c // eor x12,x12,x7, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1b0353 // eor x19,x26,x27 // a^b, b^c in next round + .long 0xcad6a610 // eor x16,x16,x22, ror #41 // Sigma1(e) + .long 0xcada89ad // eor x13,x13,x26, ror #34 + .long 0x8b110339 // add x25,x25,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcac4f56b // eor x11,x11,x4, ror #61 + .long 0xca471d8c // eor x12,x12,x7, lsr #7 // sigma0(X[i+1]) + .long 0x8b100339 // add x25,x25,x16 // h+=Sigma1(e) + .long 0xca1b039c // eor x28,x28,x27 // Maj(a,b,c) + .long 0xcada9db1 // eor x17,x13,x26, ror #39 // Sigma0(a) + .long 0xca44196b // eor x11,x11,x4, lsr #6 // sigma1(X[i+14]) + .long 0x8b0f00c6 // add x6,x6,x15 + .long 0x8b1902b5 // add x21,x21,x25 // d+=h + .long 0x8b1c0339 // add x25,x25,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b0c00c6 // add x6,x6,x12 + .long 0x8b110339 // add x25,x25,x17 // h+=Sigma0(a) + .long 0x8b0b00c6 // add x6,x6,x11 + .long 0xf94003eb // ldr x11,[sp,#0] + .long 0xf9000fee // str x14,[sp,#24] + .long 0x93d53ab0 // ror x16,x21,#14 + .long 0x8b1c0318 // add x24,x24,x28 // h+=K[i] + .long 0x93c8050d // ror x13,x8,#1 + .long 0x8a1502d1 // and x17,x22,x21 + .long 0x93c54cac // ror x12,x5,#19 + .long 0x8a3502fc // bic x28,x23,x21 + .long 0x93d9732e // ror x14,x25,#28 + .long 0x8b060318 // add x24,x24,x6 // h+=X[i] + .long 0xcad54a10 // eor x16,x16,x21, ror #18 + .long 0xcac821ad // eor x13,x13,x8, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1a033c // eor x28,x25,x26 // a^b, b^c in next round + .long 0xcad5a610 // eor x16,x16,x21, ror #41 // Sigma1(e) + .long 0xcad989ce // eor x14,x14,x25, ror #34 + .long 0x8b110318 // add x24,x24,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcac5f58c // eor x12,x12,x5, ror #61 + .long 0xca481dad // eor x13,x13,x8, lsr #7 // sigma0(X[i+1]) + .long 0x8b100318 // add x24,x24,x16 // h+=Sigma1(e) + .long 0xca1a0273 // eor x19,x19,x26 // Maj(a,b,c) + .long 0xcad99dd1 // eor x17,x14,x25, ror #39 // Sigma0(a) + .long 0xca45198c // eor x12,x12,x5, lsr #6 // sigma1(X[i+14]) + .long 0x8b0000e7 // add x7,x7,x0 + .long 0x8b180294 // add x20,x20,x24 // d+=h + .long 0x8b130318 // add x24,x24,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b0d00e7 // add x7,x7,x13 + .long 0x8b110318 // add x24,x24,x17 // h+=Sigma0(a) + .long 0x8b0c00e7 // add x7,x7,x12 + .long 0xf94007ec // ldr x12,[sp,#8] + .long 0xf90003ef // str x15,[sp,#0] + .long 0x93d43a90 // ror x16,x20,#14 + .long 0x8b1302f7 // add x23,x23,x19 // h+=K[i] + .long 0x93c9052e // ror x14,x9,#1 + .long 0x8a1402b1 // and x17,x21,x20 + .long 0x93c64ccd // ror x13,x6,#19 + .long 0x8a3402d3 // bic x19,x22,x20 + .long 0x93d8730f // ror x15,x24,#28 + .long 0x8b0702f7 // add x23,x23,x7 // h+=X[i] + .long 0xcad44a10 // eor x16,x16,x20, ror #18 + .long 0xcac921ce // eor x14,x14,x9, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca190313 // eor x19,x24,x25 // a^b, b^c in next round + .long 0xcad4a610 // eor x16,x16,x20, ror #41 // Sigma1(e) + .long 0xcad889ef // eor x15,x15,x24, ror #34 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcac6f5ad // eor x13,x13,x6, ror #61 + .long 0xca491dce // eor x14,x14,x9, lsr #7 // sigma0(X[i+1]) + .long 0x8b1002f7 // add x23,x23,x16 // h+=Sigma1(e) + .long 0xca19039c // eor x28,x28,x25 // Maj(a,b,c) + .long 0xcad89df1 // eor x17,x15,x24, ror #39 // Sigma0(a) + .long 0xca4619ad // eor x13,x13,x6, lsr #6 // sigma1(X[i+14]) + .long 0x8b010108 // add x8,x8,x1 + .long 0x8b17037b // add x27,x27,x23 // d+=h + .long 0x8b1c02f7 // add x23,x23,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b0e0108 // add x8,x8,x14 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Sigma0(a) + .long 0x8b0d0108 // add x8,x8,x13 + .long 0xf9400bed // ldr x13,[sp,#16] + .long 0xf90007e0 // str x0,[sp,#8] + .long 0x93db3b70 // ror x16,x27,#14 + .long 0x8b1c02d6 // add x22,x22,x28 // h+=K[i] + .long 0x93ca054f // ror x15,x10,#1 + .long 0x8a1b0291 // and x17,x20,x27 + .long 0x93c74cee // ror x14,x7,#19 + .long 0x8a3b02bc // bic x28,x21,x27 + .long 0x93d772e0 // ror x0,x23,#28 + .long 0x8b0802d6 // add x22,x22,x8 // h+=X[i] + .long 0xcadb4a10 // eor x16,x16,x27, ror #18 + .long 0xcaca21ef // eor x15,x15,x10, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1802fc // eor x28,x23,x24 // a^b, b^c in next round + .long 0xcadba610 // eor x16,x16,x27, ror #41 // Sigma1(e) + .long 0xcad78800 // eor x0,x0,x23, ror #34 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcac7f5ce // eor x14,x14,x7, ror #61 + .long 0xca4a1def // eor x15,x15,x10, lsr #7 // sigma0(X[i+1]) + .long 0x8b1002d6 // add x22,x22,x16 // h+=Sigma1(e) + .long 0xca180273 // eor x19,x19,x24 // Maj(a,b,c) + .long 0xcad79c11 // eor x17,x0,x23, ror #39 // Sigma0(a) + .long 0xca4719ce // eor x14,x14,x7, lsr #6 // sigma1(X[i+14]) + .long 0x8b020129 // add x9,x9,x2 + .long 0x8b16035a // add x26,x26,x22 // d+=h + .long 0x8b1302d6 // add x22,x22,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b0f0129 // add x9,x9,x15 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Sigma0(a) + .long 0x8b0e0129 // add x9,x9,x14 + .long 0xf9400fee // ldr x14,[sp,#24] + .long 0xf9000be1 // str x1,[sp,#16] + .long 0x93da3b50 // ror x16,x26,#14 + .long 0x8b1302b5 // add x21,x21,x19 // h+=K[i] + .long 0x93cb0560 // ror x0,x11,#1 + .long 0x8a1a0371 // and x17,x27,x26 + .long 0x93c84d0f // ror x15,x8,#19 + .long 0x8a3a0293 // bic x19,x20,x26 + .long 0x93d672c1 // ror x1,x22,#28 + .long 0x8b0902b5 // add x21,x21,x9 // h+=X[i] + .long 0xcada4a10 // eor x16,x16,x26, ror #18 + .long 0xcacb2000 // eor x0,x0,x11, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1702d3 // eor x19,x22,x23 // a^b, b^c in next round + .long 0xcadaa610 // eor x16,x16,x26, ror #41 // Sigma1(e) + .long 0xcad68821 // eor x1,x1,x22, ror #34 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcac8f5ef // eor x15,x15,x8, ror #61 + .long 0xca4b1c00 // eor x0,x0,x11, lsr #7 // sigma0(X[i+1]) + .long 0x8b1002b5 // add x21,x21,x16 // h+=Sigma1(e) + .long 0xca17039c // eor x28,x28,x23 // Maj(a,b,c) + .long 0xcad69c31 // eor x17,x1,x22, ror #39 // Sigma0(a) + .long 0xca4819ef // eor x15,x15,x8, lsr #6 // sigma1(X[i+14]) + .long 0x8b03014a // add x10,x10,x3 + .long 0x8b150339 // add x25,x25,x21 // d+=h + .long 0x8b1c02b5 // add x21,x21,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b00014a // add x10,x10,x0 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Sigma0(a) + .long 0x8b0f014a // add x10,x10,x15 + .long 0xf94003ef // ldr x15,[sp,#0] + .long 0xf9000fe2 // str x2,[sp,#24] + .long 0x93d93b30 // ror x16,x25,#14 + .long 0x8b1c0294 // add x20,x20,x28 // h+=K[i] + .long 0x93cc0581 // ror x1,x12,#1 + .long 0x8a190351 // and x17,x26,x25 + .long 0x93c94d20 // ror x0,x9,#19 + .long 0x8a39037c // bic x28,x27,x25 + .long 0x93d572a2 // ror x2,x21,#28 + .long 0x8b0a0294 // add x20,x20,x10 // h+=X[i] + .long 0xcad94a10 // eor x16,x16,x25, ror #18 + .long 0xcacc2021 // eor x1,x1,x12, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1602bc // eor x28,x21,x22 // a^b, b^c in next round + .long 0xcad9a610 // eor x16,x16,x25, ror #41 // Sigma1(e) + .long 0xcad58842 // eor x2,x2,x21, ror #34 + .long 0x8b110294 // add x20,x20,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcac9f400 // eor x0,x0,x9, ror #61 + .long 0xca4c1c21 // eor x1,x1,x12, lsr #7 // sigma0(X[i+1]) + .long 0x8b100294 // add x20,x20,x16 // h+=Sigma1(e) + .long 0xca160273 // eor x19,x19,x22 // Maj(a,b,c) + .long 0xcad59c51 // eor x17,x2,x21, ror #39 // Sigma0(a) + .long 0xca491800 // eor x0,x0,x9, lsr #6 // sigma1(X[i+14]) + .long 0x8b04016b // add x11,x11,x4 + .long 0x8b140318 // add x24,x24,x20 // d+=h + .long 0x8b130294 // add x20,x20,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b01016b // add x11,x11,x1 + .long 0x8b110294 // add x20,x20,x17 // h+=Sigma0(a) + .long 0x8b00016b // add x11,x11,x0 + .long 0xf94007e0 // ldr x0,[sp,#8] + .long 0xf90003e3 // str x3,[sp,#0] + .long 0x93d83b10 // ror x16,x24,#14 + .long 0x8b13037b // add x27,x27,x19 // h+=K[i] + .long 0x93cd05a2 // ror x2,x13,#1 + .long 0x8a180331 // and x17,x25,x24 + .long 0x93ca4d41 // ror x1,x10,#19 + .long 0x8a380353 // bic x19,x26,x24 + .long 0x93d47283 // ror x3,x20,#28 + .long 0x8b0b037b // add x27,x27,x11 // h+=X[i] + .long 0xcad84a10 // eor x16,x16,x24, ror #18 + .long 0xcacd2042 // eor x2,x2,x13, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca150293 // eor x19,x20,x21 // a^b, b^c in next round + .long 0xcad8a610 // eor x16,x16,x24, ror #41 // Sigma1(e) + .long 0xcad48863 // eor x3,x3,x20, ror #34 + .long 0x8b11037b // add x27,x27,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcacaf421 // eor x1,x1,x10, ror #61 + .long 0xca4d1c42 // eor x2,x2,x13, lsr #7 // sigma0(X[i+1]) + .long 0x8b10037b // add x27,x27,x16 // h+=Sigma1(e) + .long 0xca15039c // eor x28,x28,x21 // Maj(a,b,c) + .long 0xcad49c71 // eor x17,x3,x20, ror #39 // Sigma0(a) + .long 0xca4a1821 // eor x1,x1,x10, lsr #6 // sigma1(X[i+14]) + .long 0x8b05018c // add x12,x12,x5 + .long 0x8b1b02f7 // add x23,x23,x27 // d+=h + .long 0x8b1c037b // add x27,x27,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b02018c // add x12,x12,x2 + .long 0x8b11037b // add x27,x27,x17 // h+=Sigma0(a) + .long 0x8b01018c // add x12,x12,x1 + .long 0xf9400be1 // ldr x1,[sp,#16] + .long 0xf90007e4 // str x4,[sp,#8] + .long 0x93d73af0 // ror x16,x23,#14 + .long 0x8b1c035a // add x26,x26,x28 // h+=K[i] + .long 0x93ce05c3 // ror x3,x14,#1 + .long 0x8a170311 // and x17,x24,x23 + .long 0x93cb4d62 // ror x2,x11,#19 + .long 0x8a37033c // bic x28,x25,x23 + .long 0x93db7364 // ror x4,x27,#28 + .long 0x8b0c035a // add x26,x26,x12 // h+=X[i] + .long 0xcad74a10 // eor x16,x16,x23, ror #18 + .long 0xcace2063 // eor x3,x3,x14, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca14037c // eor x28,x27,x20 // a^b, b^c in next round + .long 0xcad7a610 // eor x16,x16,x23, ror #41 // Sigma1(e) + .long 0xcadb8884 // eor x4,x4,x27, ror #34 + .long 0x8b11035a // add x26,x26,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcacbf442 // eor x2,x2,x11, ror #61 + .long 0xca4e1c63 // eor x3,x3,x14, lsr #7 // sigma0(X[i+1]) + .long 0x8b10035a // add x26,x26,x16 // h+=Sigma1(e) + .long 0xca140273 // eor x19,x19,x20 // Maj(a,b,c) + .long 0xcadb9c91 // eor x17,x4,x27, ror #39 // Sigma0(a) + .long 0xca4b1842 // eor x2,x2,x11, lsr #6 // sigma1(X[i+14]) + .long 0x8b0601ad // add x13,x13,x6 + .long 0x8b1a02d6 // add x22,x22,x26 // d+=h + .long 0x8b13035a // add x26,x26,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b0301ad // add x13,x13,x3 + .long 0x8b11035a // add x26,x26,x17 // h+=Sigma0(a) + .long 0x8b0201ad // add x13,x13,x2 + .long 0xf9400fe2 // ldr x2,[sp,#24] + .long 0xf9000be5 // str x5,[sp,#16] + .long 0x93d63ad0 // ror x16,x22,#14 + .long 0x8b130339 // add x25,x25,x19 // h+=K[i] + .long 0x93cf05e4 // ror x4,x15,#1 + .long 0x8a1602f1 // and x17,x23,x22 + .long 0x93cc4d83 // ror x3,x12,#19 + .long 0x8a360313 // bic x19,x24,x22 + .long 0x93da7345 // ror x5,x26,#28 + .long 0x8b0d0339 // add x25,x25,x13 // h+=X[i] + .long 0xcad64a10 // eor x16,x16,x22, ror #18 + .long 0xcacf2084 // eor x4,x4,x15, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1b0353 // eor x19,x26,x27 // a^b, b^c in next round + .long 0xcad6a610 // eor x16,x16,x22, ror #41 // Sigma1(e) + .long 0xcada88a5 // eor x5,x5,x26, ror #34 + .long 0x8b110339 // add x25,x25,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcaccf463 // eor x3,x3,x12, ror #61 + .long 0xca4f1c84 // eor x4,x4,x15, lsr #7 // sigma0(X[i+1]) + .long 0x8b100339 // add x25,x25,x16 // h+=Sigma1(e) + .long 0xca1b039c // eor x28,x28,x27 // Maj(a,b,c) + .long 0xcada9cb1 // eor x17,x5,x26, ror #39 // Sigma0(a) + .long 0xca4c1863 // eor x3,x3,x12, lsr #6 // sigma1(X[i+14]) + .long 0x8b0701ce // add x14,x14,x7 + .long 0x8b1902b5 // add x21,x21,x25 // d+=h + .long 0x8b1c0339 // add x25,x25,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b0401ce // add x14,x14,x4 + .long 0x8b110339 // add x25,x25,x17 // h+=Sigma0(a) + .long 0x8b0301ce // add x14,x14,x3 + .long 0xf94003e3 // ldr x3,[sp,#0] + .long 0xf9000fe6 // str x6,[sp,#24] + .long 0x93d53ab0 // ror x16,x21,#14 + .long 0x8b1c0318 // add x24,x24,x28 // h+=K[i] + .long 0x93c00405 // ror x5,x0,#1 + .long 0x8a1502d1 // and x17,x22,x21 + .long 0x93cd4da4 // ror x4,x13,#19 + .long 0x8a3502fc // bic x28,x23,x21 + .long 0x93d97326 // ror x6,x25,#28 + .long 0x8b0e0318 // add x24,x24,x14 // h+=X[i] + .long 0xcad54a10 // eor x16,x16,x21, ror #18 + .long 0xcac020a5 // eor x5,x5,x0, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1a033c // eor x28,x25,x26 // a^b, b^c in next round + .long 0xcad5a610 // eor x16,x16,x21, ror #41 // Sigma1(e) + .long 0xcad988c6 // eor x6,x6,x25, ror #34 + .long 0x8b110318 // add x24,x24,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcacdf484 // eor x4,x4,x13, ror #61 + .long 0xca401ca5 // eor x5,x5,x0, lsr #7 // sigma0(X[i+1]) + .long 0x8b100318 // add x24,x24,x16 // h+=Sigma1(e) + .long 0xca1a0273 // eor x19,x19,x26 // Maj(a,b,c) + .long 0xcad99cd1 // eor x17,x6,x25, ror #39 // Sigma0(a) + .long 0xca4d1884 // eor x4,x4,x13, lsr #6 // sigma1(X[i+14]) + .long 0x8b0801ef // add x15,x15,x8 + .long 0x8b180294 // add x20,x20,x24 // d+=h + .long 0x8b130318 // add x24,x24,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b0501ef // add x15,x15,x5 + .long 0x8b110318 // add x24,x24,x17 // h+=Sigma0(a) + .long 0x8b0401ef // add x15,x15,x4 + .long 0xf94007e4 // ldr x4,[sp,#8] + .long 0xf90003e7 // str x7,[sp,#0] + .long 0x93d43a90 // ror x16,x20,#14 + .long 0x8b1302f7 // add x23,x23,x19 // h+=K[i] + .long 0x93c10426 // ror x6,x1,#1 + .long 0x8a1402b1 // and x17,x21,x20 + .long 0x93ce4dc5 // ror x5,x14,#19 + .long 0x8a3402d3 // bic x19,x22,x20 + .long 0x93d87307 // ror x7,x24,#28 + .long 0x8b0f02f7 // add x23,x23,x15 // h+=X[i] + .long 0xcad44a10 // eor x16,x16,x20, ror #18 + .long 0xcac120c6 // eor x6,x6,x1, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca190313 // eor x19,x24,x25 // a^b, b^c in next round + .long 0xcad4a610 // eor x16,x16,x20, ror #41 // Sigma1(e) + .long 0xcad888e7 // eor x7,x7,x24, ror #34 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcacef4a5 // eor x5,x5,x14, ror #61 + .long 0xca411cc6 // eor x6,x6,x1, lsr #7 // sigma0(X[i+1]) + .long 0x8b1002f7 // add x23,x23,x16 // h+=Sigma1(e) + .long 0xca19039c // eor x28,x28,x25 // Maj(a,b,c) + .long 0xcad89cf1 // eor x17,x7,x24, ror #39 // Sigma0(a) + .long 0xca4e18a5 // eor x5,x5,x14, lsr #6 // sigma1(X[i+14]) + .long 0x8b090000 // add x0,x0,x9 + .long 0x8b17037b // add x27,x27,x23 // d+=h + .long 0x8b1c02f7 // add x23,x23,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b060000 // add x0,x0,x6 + .long 0x8b1102f7 // add x23,x23,x17 // h+=Sigma0(a) + .long 0x8b050000 // add x0,x0,x5 + .long 0xf9400be5 // ldr x5,[sp,#16] + .long 0xf90007e8 // str x8,[sp,#8] + .long 0x93db3b70 // ror x16,x27,#14 + .long 0x8b1c02d6 // add x22,x22,x28 // h+=K[i] + .long 0x93c20447 // ror x7,x2,#1 + .long 0x8a1b0291 // and x17,x20,x27 + .long 0x93cf4de6 // ror x6,x15,#19 + .long 0x8a3b02bc // bic x28,x21,x27 + .long 0x93d772e8 // ror x8,x23,#28 + .long 0x8b0002d6 // add x22,x22,x0 // h+=X[i] + .long 0xcadb4a10 // eor x16,x16,x27, ror #18 + .long 0xcac220e7 // eor x7,x7,x2, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1802fc // eor x28,x23,x24 // a^b, b^c in next round + .long 0xcadba610 // eor x16,x16,x27, ror #41 // Sigma1(e) + .long 0xcad78908 // eor x8,x8,x23, ror #34 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcacff4c6 // eor x6,x6,x15, ror #61 + .long 0xca421ce7 // eor x7,x7,x2, lsr #7 // sigma0(X[i+1]) + .long 0x8b1002d6 // add x22,x22,x16 // h+=Sigma1(e) + .long 0xca180273 // eor x19,x19,x24 // Maj(a,b,c) + .long 0xcad79d11 // eor x17,x8,x23, ror #39 // Sigma0(a) + .long 0xca4f18c6 // eor x6,x6,x15, lsr #6 // sigma1(X[i+14]) + .long 0x8b0a0021 // add x1,x1,x10 + .long 0x8b16035a // add x26,x26,x22 // d+=h + .long 0x8b1302d6 // add x22,x22,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b070021 // add x1,x1,x7 + .long 0x8b1102d6 // add x22,x22,x17 // h+=Sigma0(a) + .long 0x8b060021 // add x1,x1,x6 + .long 0xf9400fe6 // ldr x6,[sp,#24] + .long 0xf9000be9 // str x9,[sp,#16] + .long 0x93da3b50 // ror x16,x26,#14 + .long 0x8b1302b5 // add x21,x21,x19 // h+=K[i] + .long 0x93c30468 // ror x8,x3,#1 + .long 0x8a1a0371 // and x17,x27,x26 + .long 0x93c04c07 // ror x7,x0,#19 + .long 0x8a3a0293 // bic x19,x20,x26 + .long 0x93d672c9 // ror x9,x22,#28 + .long 0x8b0102b5 // add x21,x21,x1 // h+=X[i] + .long 0xcada4a10 // eor x16,x16,x26, ror #18 + .long 0xcac32108 // eor x8,x8,x3, ror #8 + .long 0xaa130231 // orr x17,x17,x19 // Ch(e,f,g) + .long 0xca1702d3 // eor x19,x22,x23 // a^b, b^c in next round + .long 0xcadaa610 // eor x16,x16,x26, ror #41 // Sigma1(e) + .long 0xcad68929 // eor x9,x9,x22, ror #34 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Ch(e,f,g) + .long 0x8a13039c // and x28,x28,x19 // (b^c)&=(a^b) + .long 0xcac0f4e7 // eor x7,x7,x0, ror #61 + .long 0xca431d08 // eor x8,x8,x3, lsr #7 // sigma0(X[i+1]) + .long 0x8b1002b5 // add x21,x21,x16 // h+=Sigma1(e) + .long 0xca17039c // eor x28,x28,x23 // Maj(a,b,c) + .long 0xcad69d31 // eor x17,x9,x22, ror #39 // Sigma0(a) + .long 0xca4018e7 // eor x7,x7,x0, lsr #6 // sigma1(X[i+14]) + .long 0x8b0b0042 // add x2,x2,x11 + .long 0x8b150339 // add x25,x25,x21 // d+=h + .long 0x8b1c02b5 // add x21,x21,x28 // h+=Maj(a,b,c) + .long 0xf84087dc // ldr x28,[x30],#8 // *K++, x19 in next round + .long 0x8b080042 // add x2,x2,x8 + .long 0x8b1102b5 // add x21,x21,x17 // h+=Sigma0(a) + .long 0x8b070042 // add x2,x2,x7 + .long 0xf94003e7 // ldr x7,[sp,#0] + .long 0xf9000fea // str x10,[sp,#24] + .long 0x93d93b30 // ror x16,x25,#14 + .long 0x8b1c0294 // add x20,x20,x28 // h+=K[i] + .long 0x93c40489 // ror x9,x4,#1 + .long 0x8a190351 // and x17,x26,x25 + .long 0x93c14c28 // ror x8,x1,#19 + .long 0x8a39037c // bic x28,x27,x25 + .long 0x93d572aa // ror x10,x21,#28 + .long 0x8b020294 // add x20,x20,x2 // h+=X[i] + .long 0xcad94a10 // eor x16,x16,x25, ror #18 + .long 0xcac42129 // eor x9,x9,x4, ror #8 + .long 0xaa1c0231 // orr x17,x17,x28 // Ch(e,f,g) + .long 0xca1602bc // eor x28,x21,x22 // a^b, b^c in next round + .long 0xcad9a610 // eor x16,x16,x25, ror #41 // Sigma1(e) + .long 0xcad5894a // eor x10,x10,x21, ror #34 + .long 0x8b110294 // add x20,x20,x17 // h+=Ch(e,f,g) + .long 0x8a1c0273 // and x19,x19,x28 // (b^c)&=(a^b) + .long 0xcac1f508 // eor x8,x8,x1, ror #61 + .long 0xca441d29 // eor x9,x9,x4, lsr #7 // sigma0(X[i+1]) + .long 0x8b100294 // add x20,x20,x16 // h+=Sigma1(e) + .long 0xca160273 // eor x19,x19,x22 // Maj(a,b,c) + .long 0xcad59d51 // eor x17,x10,x21, ror #39 // Sigma0(a) + .long 0xca411908 // eor x8,x8,x1, lsr #6 // sigma1(X[i+14]) + .long 0x8b0c0063 // add x3,x3,x12 + .long 0x8b140318 // add x24,x24,x20 // d+=h + .long 0x8b130294 // add x20,x20,x19 // h+=Maj(a,b,c) + .long 0xf84087d3 // ldr x19,[x30],#8 // *K++, x28 in next round + .long 0x8b090063 // add x3,x3,x9 + .long 0x8b110294 // add x20,x20,x17 // h+=Sigma0(a) + .long 0x8b080063 // add x3,x3,x8 + cbnz x19,.Lsha512_gpr_loop_16_xx + + .long 0xa9460ba0 // ldp x0,x2,[x29,#96] + .long 0xf9403ba1 // ldr x1,[x29,#112] + .long 0xd10a23de // sub x30,x30,#648 // rewind + + .long 0xa9401003 // ldp x3,x4,[x0] + .long 0xa9411805 // ldp x5,x6,[x0,#2*8] + .long 0x9101c021 // add x1,x1,#14*8 // advance input pointer + .long 0xa9422007 // ldp x7,x8,[x0,#4*8] + .long 0x8b030294 // add x20,x20,x3 + .long 0xa9432809 // ldp x9,x10,[x0,#6*8] + .long 0x8b0402b5 // add x21,x21,x4 + .long 0x8b0502d6 // add x22,x22,x5 + .long 0x8b0602f7 // add x23,x23,x6 + .long 0xa9005414 // stp x20,x21,[x0] + .long 0x8b070318 // add x24,x24,x7 + .long 0x8b080339 // add x25,x25,x8 + .long 0xa9015c16 // stp x22,x23,[x0,#2*8] + .long 0x8b09035a // add x26,x26,x9 + .long 0x8b0a037b // add x27,x27,x10 + .long 0xeb02003f // cmp x1,x2 + .long 0xa9026418 // stp x24,x25,[x0,#4*8] + .long 0xa9036c1a // stp x26,x27,[x0,#6*8] + b.ne .Lsha512_gpr_loop + + .long 0xa94153b3 // ldp x19,x20,[x29,#16] + .long 0x910083ff // add sp,sp,#4*8 + .long 0xa9425bb5 // ldp x21,x22,[x29,#32] + .long 0xa94363b7 // ldp x23,x24,[x29,#48] + .long 0xa9446bb9 // ldp x25,x26,[x29,#64] + .long 0xa94573bb // ldp x27,x28,[x29,#80] + .long 0xa8c87bfd // ldp x29,x30,[sp],#128 + ret diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_i386.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_i386.inc index 5af39e55..bfb60323 100644 --- a/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_i386.inc +++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_i386.inc @@ -1,10 +1,12 @@ -// SHA-512 SSE2 implementation (IA-32), derived from OpenSSL's sha512-586 -// (CRYPTOGAMS). The 64-bit state and message schedule are held in xmm0-xmm7 (low -// 64 bits) and computed entirely in SSE2 (paddq/psrlq/psllq/pxor), so the whole -// hash runs in the vector unit - avoiding IA-32's 32-bit-GPR bottleneck. -// Expects (after HlpSimdProc4Begin_i386): ebx = state, esi = data, edi = numblocks, -// eax = K512 ptr (80 UInt64). HlpSimdProc4Begin pushes ebx/esi/edi; ebp is pushed -// here as the K512 walk pointer; state is loaded into xmm0-xmm7. +// SSE2 implementation of SHA-512 compress (IA-32). The 64-bit state and +// message schedule are held in xmm0-xmm7 (low 64 bits) and computed +// entirely in SSE2 (paddq/psrlq/psllq/pxor), so the whole hash runs in the +// vector unit - avoiding IA-32's 32-bit-GPR bottleneck. +// ABI (after HlpSimdProc4Begin_i386.inc): ebx = state ptr, esi = data ptr, +// edi = numblocks, eax = K512 ptr (80 UInt64). +// Frame: the prologue pushes ebx/esi/edi; ebp is pushed here as the K512 +// walk pointer; state is loaded into xmm0-xmm7. +// Reference: OpenSSL CRYPTOGAMS sha512-586.pl. push ebp mov ebp, eax diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_x86_64.inc index c16de395..200599a7 100644 --- a/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressSse2_x86_64.inc @@ -1,18 +1,17 @@ -// SHA-512 SSE2 implementation with a SIMD message schedule, downported from the -// AVX single-block kernel (OpenSSL / CRYPTOGAMS design): the sigma schedule runs -// in 128-bit SSE2 and is interleaved with the 64-bit GPR compression rounds, so -// the vector and integer units run in parallel. This is much faster than a scalar -// message schedule. -// Expects MS x64 ABI (after HlpSimdProc4Begin): rcx = state ptr, rdx = data ptr, -// r8d = numblocks, r9 = doubled-K512 ptr (K512_Doubled: each 128-bit K512 pair -// stored twice, read at a 32-byte stride). The big-endian input byte-swap is -// emulated with pshuflw/pshufhw/psrlw/psllw/por (SSE2 has no pshufb), so no mask -// table is needed. -// -// MS x64 non-volatile saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore -// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// SSE2 implementation of SHA-512 compress (SIMD schedule): the sigma +// schedule runs in 128-bit SSE2 and is interleaved with the 64-bit GPR +// compression rounds, so the vector and integer units run in parallel - +// much faster than a scalar message schedule. The big-endian input +// byte-swap is emulated with pshuflw/pshufhw/psrlw/psllw/por (SSE2 has no +// pshufb), so no mask table is needed. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K512 ptr (K512_Doubled: each 128-bit K512 +// pair stored twice, read at a 32-byte stride). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// Reference: OpenSSL CRYPTOGAMS sha512-x86_64.pl (AVX single-block kernel). - {$I ..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} push rbx push rbp @@ -1295,4 +1294,4 @@ pop rbp pop rbx - {$I ..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc new file mode 100644 index 00000000..75e77d3a --- /dev/null +++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc @@ -0,0 +1,1260 @@ +// SSSE3 implementation of SHA-512 compress (SIMD schedule); SSE2 sibling: +// SHA512CompressSse2_x86_64.inc. The sigma schedule runs in 128-bit SIMD +// and is interleaved with the 64-bit GPR compression rounds, so the vector +// and integer units run in parallel. Unlike the SSE2 sibling the big-endian +// input byte-swaps are real pshufb against the BSWAP64 mask kept resident +// in xmm12 for the whole kernel. +// ABI (after HlpSimdProc4Begin_x86_64.inc): rcx = state ptr, rdx = data ptr, +// r8d = numblocks, r9 = doubled-K512 ptr (K512_Doubled: each 128-bit K512 +// pair stored twice, read at a 32-byte stride; the BSWAP64 mask sits at +// +1280). +// Saves: xmm6-xmm15 via the shared HlpSimdNonVolatileSave/Restore include +// (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame. +// pshufb instructions are db-encoded for broad assembler compatibility. +// Reference: OpenSSL CRYPTOGAMS sha512-x86_64.pl (AVX single-block kernel). + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileSave_x86_64.inc} + + push rbx + push rbp + push rsi + push rdi + push r12 + push r13 + push r14 + push r15 + mov rax, rsp + sub rsp, $100 + and rsp, $FFFFFFFFFFFFFFC0 + mov qword [rsp + $98], rax + mov qword [rsp + $A0], r9 + movdqu xmm12, oword [r9 + $500] // BSWAP64 mask (K512_Doubled + 1280) + mov qword [rsp + $80], rcx + mov r11d, r8d + shl r11, $7 + lea r11, [rdx + r11*1] + mov qword [rsp + $90], r11 + mov rsi, rdx + mov rdi, qword [rsp + $80] + mov rax, qword [rdi] + mov rbx, qword [rdi + $8] + mov rcx, qword [rdi + $10] + mov rdx, qword [rdi + $18] + mov r8, qword [rdi + $20] + mov r9, qword [rdi + $28] + mov r10, qword [rdi + $30] + mov r11, qword [rdi + $38] + +@block_loop: + movdqu xmm0, oword [rsi] + mov rbp, qword [rsp + $A0] + add rbp, $80 + movdqu xmm1, oword [rsi + $10] + movdqu xmm2, oword [rsi + $20] + db $66, $41, $0F, $38, $00, $C4 // pshufb xmm0, xmm12 + movdqu xmm3, oword [rsi + $30] + db $66, $41, $0F, $38, $00, $CC // pshufb xmm1, xmm12 + movdqu xmm4, oword [rsi + $40] + db $66, $41, $0F, $38, $00, $D4 // pshufb xmm2, xmm12 + movdqu xmm5, oword [rsi + $50] + db $66, $41, $0F, $38, $00, $DC // pshufb xmm3, xmm12 + movdqu xmm6, oword [rsi + $60] + db $66, $41, $0F, $38, $00, $E4 // pshufb xmm4, xmm12 + movdqu xmm7, oword [rsi + $70] + db $66, $41, $0F, $38, $00, $EC // pshufb xmm5, xmm12 + movdqa xmm8, xmm0 + movdqu xmm13, oword [rbp - $80] + paddq xmm8, xmm13 + db $66, $41, $0F, $38, $00, $F4 // pshufb xmm6, xmm12 + movdqa xmm9, xmm1 + movdqu xmm13, oword [rbp - $60] + paddq xmm9, xmm13 + db $66, $41, $0F, $38, $00, $FC // pshufb xmm7, xmm12 + movdqa xmm10, xmm2 + movdqu xmm13, oword [rbp - $40] + paddq xmm10, xmm13 + movdqa xmm11, xmm3 + movdqu xmm13, oword [rbp - $20] + paddq xmm11, xmm13 + movdqa oword [rsp], xmm8 + movdqa xmm8, xmm4 + movdqu xmm13, oword [rbp + $0] + paddq xmm8, xmm13 + movdqa oword [rsp + $10], xmm9 + movdqa xmm9, xmm5 + movdqu xmm13, oword [rbp + $20] + paddq xmm9, xmm13 + movdqa oword [rsp + $20], xmm10 + movdqa xmm10, xmm6 + movdqu xmm13, oword [rbp + $40] + paddq xmm10, xmm13 + movdqa oword [rsp + $30], xmm11 + movdqa xmm11, xmm7 + movdqu xmm13, oword [rbp + $60] + paddq xmm11, xmm13 + movdqa oword [rsp + $40], xmm8 + mov r14, rax + movdqa oword [rsp + $50], xmm9 + mov rdi, rbx + movdqa oword [rsp + $60], xmm10 + xor rdi, rcx + movdqa oword [rsp + $70], xmm11 + mov r13, r8 + jmp @sched_loop + +@sched_loop: + add rbp, $100 + movdqa xmm8, xmm0 + shufpd xmm8, xmm1, $1 + shrd r13, r13, $17 + mov rax, r14 + movdqa xmm11, xmm4 + shufpd xmm11, xmm5, $1 + mov r12, r9 + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, r8 + xor r12, r10 + paddq xmm0, xmm11 + shrd r13, r13, $4 + xor r14, rax + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, r8 + xor r13, r8 + movdqa xmm9, xmm8 + psllq xmm9, $38 + add r11, qword [rsp] + mov r15, rax + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, r10 + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, rbx + add r11, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, rax + add r11, r13 + pxor xmm8, xmm10 + xor rdi, rbx + shrd r14, r14, $1C + movdqa xmm11, xmm7 + psrlq xmm11, $6 + add rdx, r11 + add r11, rdi + pxor xmm8, xmm9 + mov r13, rdx + add r14, r11 + movdqa xmm10, xmm7 + psllq xmm10, $3 + shrd r13, r13, $17 + mov r11, r14 + paddq xmm0, xmm8 + mov r12, r8 + shrd r14, r14, $5 + movdqa xmm9, xmm7 + psrlq xmm9, $13 + xor r13, rdx + xor r12, r9 + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, r11 + psllq xmm10, $2A + and r12, rdx + xor r13, rdx + pxor xmm11, xmm9 + add r10, qword [rsp + $8] + mov rdi, r11 + psrlq xmm9, $2A + xor r12, r9 + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, rax + add r10, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm0, xmm11 + xor r14, r11 + add r10, r13 + movdqa xmm10, xmm0 + movdqu xmm13, oword [rbp - $80] + paddq xmm10, xmm13 + xor r15, rax + shrd r14, r14, $1C + add rcx, r10 + add r10, r15 + mov r13, rcx + add r14, r10 + movdqa oword [rsp], xmm10 + movdqa xmm8, xmm1 + shufpd xmm8, xmm2, $1 + shrd r13, r13, $17 + mov r10, r14 + movdqa xmm11, xmm5 + shufpd xmm11, xmm6, $1 + mov r12, rdx + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, rcx + xor r12, r8 + paddq xmm1, xmm11 + shrd r13, r13, $4 + xor r14, r10 + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, rcx + xor r13, rcx + movdqa xmm9, xmm8 + psllq xmm9, $38 + add r9, qword [rsp + $10] + mov r15, r10 + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, r8 + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, r11 + add r9, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, r10 + add r9, r13 + pxor xmm8, xmm10 + xor rdi, r11 + shrd r14, r14, $1C + movdqa xmm11, xmm0 + psrlq xmm11, $6 + add rbx, r9 + add r9, rdi + pxor xmm8, xmm9 + mov r13, rbx + add r14, r9 + movdqa xmm10, xmm0 + psllq xmm10, $3 + shrd r13, r13, $17 + mov r9, r14 + paddq xmm1, xmm8 + mov r12, rcx + shrd r14, r14, $5 + movdqa xmm9, xmm0 + psrlq xmm9, $13 + xor r13, rbx + xor r12, rdx + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, r9 + psllq xmm10, $2A + and r12, rbx + xor r13, rbx + pxor xmm11, xmm9 + add r8, qword [rsp + $18] + mov rdi, r9 + psrlq xmm9, $2A + xor r12, rdx + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, r10 + add r8, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm1, xmm11 + xor r14, r9 + add r8, r13 + movdqa xmm10, xmm1 + movdqu xmm13, oword [rbp - $60] + paddq xmm10, xmm13 + xor r15, r10 + shrd r14, r14, $1C + add rax, r8 + add r8, r15 + mov r13, rax + add r14, r8 + movdqa oword [rsp + $10], xmm10 + movdqa xmm8, xmm2 + shufpd xmm8, xmm3, $1 + shrd r13, r13, $17 + mov r8, r14 + movdqa xmm11, xmm6 + shufpd xmm11, xmm7, $1 + mov r12, rbx + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, rax + xor r12, rcx + paddq xmm2, xmm11 + shrd r13, r13, $4 + xor r14, r8 + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, rax + xor r13, rax + movdqa xmm9, xmm8 + psllq xmm9, $38 + add rdx, qword [rsp + $20] + mov r15, r8 + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, rcx + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, r9 + add rdx, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, r8 + add rdx, r13 + pxor xmm8, xmm10 + xor rdi, r9 + shrd r14, r14, $1C + movdqa xmm11, xmm1 + psrlq xmm11, $6 + add r11, rdx + add rdx, rdi + pxor xmm8, xmm9 + mov r13, r11 + add r14, rdx + movdqa xmm10, xmm1 + psllq xmm10, $3 + shrd r13, r13, $17 + mov rdx, r14 + paddq xmm2, xmm8 + mov r12, rax + shrd r14, r14, $5 + movdqa xmm9, xmm1 + psrlq xmm9, $13 + xor r13, r11 + xor r12, rbx + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, rdx + psllq xmm10, $2A + and r12, r11 + xor r13, r11 + pxor xmm11, xmm9 + add rcx, qword [rsp + $28] + mov rdi, rdx + psrlq xmm9, $2A + xor r12, rbx + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, r8 + add rcx, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm2, xmm11 + xor r14, rdx + add rcx, r13 + movdqa xmm10, xmm2 + movdqu xmm13, oword [rbp - $40] + paddq xmm10, xmm13 + xor r15, r8 + shrd r14, r14, $1C + add r10, rcx + add rcx, r15 + mov r13, r10 + add r14, rcx + movdqa oword [rsp + $20], xmm10 + movdqa xmm8, xmm3 + shufpd xmm8, xmm4, $1 + shrd r13, r13, $17 + mov rcx, r14 + movdqa xmm11, xmm7 + shufpd xmm11, xmm0, $1 + mov r12, r11 + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, r10 + xor r12, rax + paddq xmm3, xmm11 + shrd r13, r13, $4 + xor r14, rcx + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, r10 + xor r13, r10 + movdqa xmm9, xmm8 + psllq xmm9, $38 + add rbx, qword [rsp + $30] + mov r15, rcx + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, rax + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, rdx + add rbx, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, rcx + add rbx, r13 + pxor xmm8, xmm10 + xor rdi, rdx + shrd r14, r14, $1C + movdqa xmm11, xmm2 + psrlq xmm11, $6 + add r9, rbx + add rbx, rdi + pxor xmm8, xmm9 + mov r13, r9 + add r14, rbx + movdqa xmm10, xmm2 + psllq xmm10, $3 + shrd r13, r13, $17 + mov rbx, r14 + paddq xmm3, xmm8 + mov r12, r10 + shrd r14, r14, $5 + movdqa xmm9, xmm2 + psrlq xmm9, $13 + xor r13, r9 + xor r12, r11 + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, rbx + psllq xmm10, $2A + and r12, r9 + xor r13, r9 + pxor xmm11, xmm9 + add rax, qword [rsp + $38] + mov rdi, rbx + psrlq xmm9, $2A + xor r12, r11 + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, rcx + add rax, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm3, xmm11 + xor r14, rbx + add rax, r13 + movdqa xmm10, xmm3 + movdqu xmm13, oword [rbp - $20] + paddq xmm10, xmm13 + xor r15, rcx + shrd r14, r14, $1C + add r8, rax + add rax, r15 + mov r13, r8 + add r14, rax + movdqa oword [rsp + $30], xmm10 + movdqa xmm8, xmm4 + shufpd xmm8, xmm5, $1 + shrd r13, r13, $17 + mov rax, r14 + movdqa xmm11, xmm0 + shufpd xmm11, xmm1, $1 + mov r12, r9 + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, r8 + xor r12, r10 + paddq xmm4, xmm11 + shrd r13, r13, $4 + xor r14, rax + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, r8 + xor r13, r8 + movdqa xmm9, xmm8 + psllq xmm9, $38 + add r11, qword [rsp + $40] + mov r15, rax + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, r10 + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, rbx + add r11, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, rax + add r11, r13 + pxor xmm8, xmm10 + xor rdi, rbx + shrd r14, r14, $1C + movdqa xmm11, xmm3 + psrlq xmm11, $6 + add rdx, r11 + add r11, rdi + pxor xmm8, xmm9 + mov r13, rdx + add r14, r11 + movdqa xmm10, xmm3 + psllq xmm10, $3 + shrd r13, r13, $17 + mov r11, r14 + paddq xmm4, xmm8 + mov r12, r8 + shrd r14, r14, $5 + movdqa xmm9, xmm3 + psrlq xmm9, $13 + xor r13, rdx + xor r12, r9 + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, r11 + psllq xmm10, $2A + and r12, rdx + xor r13, rdx + pxor xmm11, xmm9 + add r10, qword [rsp + $48] + mov rdi, r11 + psrlq xmm9, $2A + xor r12, r9 + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, rax + add r10, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm4, xmm11 + xor r14, r11 + add r10, r13 + movdqa xmm10, xmm4 + movdqu xmm13, oword [rbp + $0] + paddq xmm10, xmm13 + xor r15, rax + shrd r14, r14, $1C + add rcx, r10 + add r10, r15 + mov r13, rcx + add r14, r10 + movdqa oword [rsp + $40], xmm10 + movdqa xmm8, xmm5 + shufpd xmm8, xmm6, $1 + shrd r13, r13, $17 + mov r10, r14 + movdqa xmm11, xmm1 + shufpd xmm11, xmm2, $1 + mov r12, rdx + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, rcx + xor r12, r8 + paddq xmm5, xmm11 + shrd r13, r13, $4 + xor r14, r10 + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, rcx + xor r13, rcx + movdqa xmm9, xmm8 + psllq xmm9, $38 + add r9, qword [rsp + $50] + mov r15, r10 + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, r8 + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, r11 + add r9, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, r10 + add r9, r13 + pxor xmm8, xmm10 + xor rdi, r11 + shrd r14, r14, $1C + movdqa xmm11, xmm4 + psrlq xmm11, $6 + add rbx, r9 + add r9, rdi + pxor xmm8, xmm9 + mov r13, rbx + add r14, r9 + movdqa xmm10, xmm4 + psllq xmm10, $3 + shrd r13, r13, $17 + mov r9, r14 + paddq xmm5, xmm8 + mov r12, rcx + shrd r14, r14, $5 + movdqa xmm9, xmm4 + psrlq xmm9, $13 + xor r13, rbx + xor r12, rdx + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, r9 + psllq xmm10, $2A + and r12, rbx + xor r13, rbx + pxor xmm11, xmm9 + add r8, qword [rsp + $58] + mov rdi, r9 + psrlq xmm9, $2A + xor r12, rdx + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, r10 + add r8, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm5, xmm11 + xor r14, r9 + add r8, r13 + movdqa xmm10, xmm5 + movdqu xmm13, oword [rbp + $20] + paddq xmm10, xmm13 + xor r15, r10 + shrd r14, r14, $1C + add rax, r8 + add r8, r15 + mov r13, rax + add r14, r8 + movdqa oword [rsp + $50], xmm10 + movdqa xmm8, xmm6 + shufpd xmm8, xmm7, $1 + shrd r13, r13, $17 + mov r8, r14 + movdqa xmm11, xmm2 + shufpd xmm11, xmm3, $1 + mov r12, rbx + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, rax + xor r12, rcx + paddq xmm6, xmm11 + shrd r13, r13, $4 + xor r14, r8 + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, rax + xor r13, rax + movdqa xmm9, xmm8 + psllq xmm9, $38 + add rdx, qword [rsp + $60] + mov r15, r8 + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, rcx + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, r9 + add rdx, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, r8 + add rdx, r13 + pxor xmm8, xmm10 + xor rdi, r9 + shrd r14, r14, $1C + movdqa xmm11, xmm5 + psrlq xmm11, $6 + add r11, rdx + add rdx, rdi + pxor xmm8, xmm9 + mov r13, r11 + add r14, rdx + movdqa xmm10, xmm5 + psllq xmm10, $3 + shrd r13, r13, $17 + mov rdx, r14 + paddq xmm6, xmm8 + mov r12, rax + shrd r14, r14, $5 + movdqa xmm9, xmm5 + psrlq xmm9, $13 + xor r13, r11 + xor r12, rbx + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, rdx + psllq xmm10, $2A + and r12, r11 + xor r13, r11 + pxor xmm11, xmm9 + add rcx, qword [rsp + $68] + mov rdi, rdx + psrlq xmm9, $2A + xor r12, rbx + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, r8 + add rcx, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm6, xmm11 + xor r14, rdx + add rcx, r13 + movdqa xmm10, xmm6 + movdqu xmm13, oword [rbp + $40] + paddq xmm10, xmm13 + xor r15, r8 + shrd r14, r14, $1C + add r10, rcx + add rcx, r15 + mov r13, r10 + add r14, rcx + movdqa oword [rsp + $60], xmm10 + movdqa xmm8, xmm7 + shufpd xmm8, xmm0, $1 + shrd r13, r13, $17 + mov rcx, r14 + movdqa xmm11, xmm3 + shufpd xmm11, xmm4, $1 + mov r12, r11 + shrd r14, r14, $5 + movdqa xmm10, xmm8 + psrlq xmm10, $1 + xor r13, r10 + xor r12, rax + paddq xmm7, xmm11 + shrd r13, r13, $4 + xor r14, rcx + movdqa xmm11, xmm8 + psrlq xmm11, $7 + and r12, r10 + xor r13, r10 + movdqa xmm9, xmm8 + psllq xmm9, $38 + add rbx, qword [rsp + $70] + mov r15, rcx + movdqa xmm8, xmm11 + pxor xmm8, xmm10 + xor r12, rax + shrd r14, r14, $6 + psrlq xmm10, $7 + xor r15, rdx + add rbx, r12 + pxor xmm8, xmm9 + shrd r13, r13, $E + and rdi, r15 + psllq xmm9, $7 + xor r14, rcx + add rbx, r13 + pxor xmm8, xmm10 + xor rdi, rdx + shrd r14, r14, $1C + movdqa xmm11, xmm6 + psrlq xmm11, $6 + add r9, rbx + add rbx, rdi + pxor xmm8, xmm9 + mov r13, r9 + add r14, rbx + movdqa xmm10, xmm6 + psllq xmm10, $3 + shrd r13, r13, $17 + mov rbx, r14 + paddq xmm7, xmm8 + mov r12, r10 + shrd r14, r14, $5 + movdqa xmm9, xmm6 + psrlq xmm9, $13 + xor r13, r9 + xor r12, r11 + pxor xmm11, xmm10 + shrd r13, r13, $4 + xor r14, rbx + psllq xmm10, $2A + and r12, r9 + xor r13, r9 + pxor xmm11, xmm9 + add rax, qword [rsp + $78] + mov rdi, rbx + psrlq xmm9, $2A + xor r12, r11 + shrd r14, r14, $6 + pxor xmm11, xmm10 + xor rdi, rcx + add rax, r12 + pxor xmm11, xmm9 + shrd r13, r13, $E + and r15, rdi + paddq xmm7, xmm11 + xor r14, rbx + add rax, r13 + movdqa xmm10, xmm7 + movdqu xmm13, oword [rbp + $60] + paddq xmm10, xmm13 + xor r15, rcx + shrd r14, r14, $1C + add r8, rax + add rax, r15 + mov r13, r8 + add r14, rax + movdqa oword [rsp + $70], xmm10 + cmp byte [rbp + $87], $0 + jne @sched_loop + shrd r13, r13, $17 + mov rax, r14 + mov r12, r9 + shrd r14, r14, $5 + xor r13, r8 + xor r12, r10 + shrd r13, r13, $4 + xor r14, rax + and r12, r8 + xor r13, r8 + add r11, qword [rsp] + mov r15, rax + xor r12, r10 + shrd r14, r14, $6 + xor r15, rbx + add r11, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, rax + add r11, r13 + xor rdi, rbx + shrd r14, r14, $1C + add rdx, r11 + add r11, rdi + mov r13, rdx + add r14, r11 + shrd r13, r13, $17 + mov r11, r14 + mov r12, r8 + shrd r14, r14, $5 + xor r13, rdx + xor r12, r9 + shrd r13, r13, $4 + xor r14, r11 + and r12, rdx + xor r13, rdx + add r10, qword [rsp + $8] + mov rdi, r11 + xor r12, r9 + shrd r14, r14, $6 + xor rdi, rax + add r10, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, r11 + add r10, r13 + xor r15, rax + shrd r14, r14, $1C + add rcx, r10 + add r10, r15 + mov r13, rcx + add r14, r10 + shrd r13, r13, $17 + mov r10, r14 + mov r12, rdx + shrd r14, r14, $5 + xor r13, rcx + xor r12, r8 + shrd r13, r13, $4 + xor r14, r10 + and r12, rcx + xor r13, rcx + add r9, qword [rsp + $10] + mov r15, r10 + xor r12, r8 + shrd r14, r14, $6 + xor r15, r11 + add r9, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, r10 + add r9, r13 + xor rdi, r11 + shrd r14, r14, $1C + add rbx, r9 + add r9, rdi + mov r13, rbx + add r14, r9 + shrd r13, r13, $17 + mov r9, r14 + mov r12, rcx + shrd r14, r14, $5 + xor r13, rbx + xor r12, rdx + shrd r13, r13, $4 + xor r14, r9 + and r12, rbx + xor r13, rbx + add r8, qword [rsp + $18] + mov rdi, r9 + xor r12, rdx + shrd r14, r14, $6 + xor rdi, r10 + add r8, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, r9 + add r8, r13 + xor r15, r10 + shrd r14, r14, $1C + add rax, r8 + add r8, r15 + mov r13, rax + add r14, r8 + shrd r13, r13, $17 + mov r8, r14 + mov r12, rbx + shrd r14, r14, $5 + xor r13, rax + xor r12, rcx + shrd r13, r13, $4 + xor r14, r8 + and r12, rax + xor r13, rax + add rdx, qword [rsp + $20] + mov r15, r8 + xor r12, rcx + shrd r14, r14, $6 + xor r15, r9 + add rdx, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, r8 + add rdx, r13 + xor rdi, r9 + shrd r14, r14, $1C + add r11, rdx + add rdx, rdi + mov r13, r11 + add r14, rdx + shrd r13, r13, $17 + mov rdx, r14 + mov r12, rax + shrd r14, r14, $5 + xor r13, r11 + xor r12, rbx + shrd r13, r13, $4 + xor r14, rdx + and r12, r11 + xor r13, r11 + add rcx, qword [rsp + $28] + mov rdi, rdx + xor r12, rbx + shrd r14, r14, $6 + xor rdi, r8 + add rcx, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, rdx + add rcx, r13 + xor r15, r8 + shrd r14, r14, $1C + add r10, rcx + add rcx, r15 + mov r13, r10 + add r14, rcx + shrd r13, r13, $17 + mov rcx, r14 + mov r12, r11 + shrd r14, r14, $5 + xor r13, r10 + xor r12, rax + shrd r13, r13, $4 + xor r14, rcx + and r12, r10 + xor r13, r10 + add rbx, qword [rsp + $30] + mov r15, rcx + xor r12, rax + shrd r14, r14, $6 + xor r15, rdx + add rbx, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, rcx + add rbx, r13 + xor rdi, rdx + shrd r14, r14, $1C + add r9, rbx + add rbx, rdi + mov r13, r9 + add r14, rbx + shrd r13, r13, $17 + mov rbx, r14 + mov r12, r10 + shrd r14, r14, $5 + xor r13, r9 + xor r12, r11 + shrd r13, r13, $4 + xor r14, rbx + and r12, r9 + xor r13, r9 + add rax, qword [rsp + $38] + mov rdi, rbx + xor r12, r11 + shrd r14, r14, $6 + xor rdi, rcx + add rax, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, rbx + add rax, r13 + xor r15, rcx + shrd r14, r14, $1C + add r8, rax + add rax, r15 + mov r13, r8 + add r14, rax + shrd r13, r13, $17 + mov rax, r14 + mov r12, r9 + shrd r14, r14, $5 + xor r13, r8 + xor r12, r10 + shrd r13, r13, $4 + xor r14, rax + and r12, r8 + xor r13, r8 + add r11, qword [rsp + $40] + mov r15, rax + xor r12, r10 + shrd r14, r14, $6 + xor r15, rbx + add r11, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, rax + add r11, r13 + xor rdi, rbx + shrd r14, r14, $1C + add rdx, r11 + add r11, rdi + mov r13, rdx + add r14, r11 + shrd r13, r13, $17 + mov r11, r14 + mov r12, r8 + shrd r14, r14, $5 + xor r13, rdx + xor r12, r9 + shrd r13, r13, $4 + xor r14, r11 + and r12, rdx + xor r13, rdx + add r10, qword [rsp + $48] + mov rdi, r11 + xor r12, r9 + shrd r14, r14, $6 + xor rdi, rax + add r10, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, r11 + add r10, r13 + xor r15, rax + shrd r14, r14, $1C + add rcx, r10 + add r10, r15 + mov r13, rcx + add r14, r10 + shrd r13, r13, $17 + mov r10, r14 + mov r12, rdx + shrd r14, r14, $5 + xor r13, rcx + xor r12, r8 + shrd r13, r13, $4 + xor r14, r10 + and r12, rcx + xor r13, rcx + add r9, qword [rsp + $50] + mov r15, r10 + xor r12, r8 + shrd r14, r14, $6 + xor r15, r11 + add r9, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, r10 + add r9, r13 + xor rdi, r11 + shrd r14, r14, $1C + add rbx, r9 + add r9, rdi + mov r13, rbx + add r14, r9 + shrd r13, r13, $17 + mov r9, r14 + mov r12, rcx + shrd r14, r14, $5 + xor r13, rbx + xor r12, rdx + shrd r13, r13, $4 + xor r14, r9 + and r12, rbx + xor r13, rbx + add r8, qword [rsp + $58] + mov rdi, r9 + xor r12, rdx + shrd r14, r14, $6 + xor rdi, r10 + add r8, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, r9 + add r8, r13 + xor r15, r10 + shrd r14, r14, $1C + add rax, r8 + add r8, r15 + mov r13, rax + add r14, r8 + shrd r13, r13, $17 + mov r8, r14 + mov r12, rbx + shrd r14, r14, $5 + xor r13, rax + xor r12, rcx + shrd r13, r13, $4 + xor r14, r8 + and r12, rax + xor r13, rax + add rdx, qword [rsp + $60] + mov r15, r8 + xor r12, rcx + shrd r14, r14, $6 + xor r15, r9 + add rdx, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, r8 + add rdx, r13 + xor rdi, r9 + shrd r14, r14, $1C + add r11, rdx + add rdx, rdi + mov r13, r11 + add r14, rdx + shrd r13, r13, $17 + mov rdx, r14 + mov r12, rax + shrd r14, r14, $5 + xor r13, r11 + xor r12, rbx + shrd r13, r13, $4 + xor r14, rdx + and r12, r11 + xor r13, r11 + add rcx, qword [rsp + $68] + mov rdi, rdx + xor r12, rbx + shrd r14, r14, $6 + xor rdi, r8 + add rcx, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, rdx + add rcx, r13 + xor r15, r8 + shrd r14, r14, $1C + add r10, rcx + add rcx, r15 + mov r13, r10 + add r14, rcx + shrd r13, r13, $17 + mov rcx, r14 + mov r12, r11 + shrd r14, r14, $5 + xor r13, r10 + xor r12, rax + shrd r13, r13, $4 + xor r14, rcx + and r12, r10 + xor r13, r10 + add rbx, qword [rsp + $70] + mov r15, rcx + xor r12, rax + shrd r14, r14, $6 + xor r15, rdx + add rbx, r12 + shrd r13, r13, $E + and rdi, r15 + xor r14, rcx + add rbx, r13 + xor rdi, rdx + shrd r14, r14, $1C + add r9, rbx + add rbx, rdi + mov r13, r9 + add r14, rbx + shrd r13, r13, $17 + mov rbx, r14 + mov r12, r10 + shrd r14, r14, $5 + xor r13, r9 + xor r12, r11 + shrd r13, r13, $4 + xor r14, rbx + and r12, r9 + xor r13, r9 + add rax, qword [rsp + $78] + mov rdi, rbx + xor r12, r11 + shrd r14, r14, $6 + xor rdi, rcx + add rax, r12 + shrd r13, r13, $E + and r15, rdi + xor r14, rbx + add rax, r13 + xor r15, rcx + shrd r14, r14, $1C + add r8, rax + add rax, r15 + mov r13, r8 + add r14, rax + mov rdi, qword [rsp + $80] + mov rax, r14 + add rax, qword [rdi] + lea rsi, [rsi + $80] + add rbx, qword [rdi + $8] + add rcx, qword [rdi + $10] + add rdx, qword [rdi + $18] + add r8, qword [rdi + $20] + add r9, qword [rdi + $28] + add r10, qword [rdi + $30] + add r11, qword [rdi + $38] + cmp rsi, qword [rsp + $90] + mov qword [rdi], rax + mov qword [rdi + $8], rbx + mov qword [rdi + $10], rcx + mov qword [rdi + $18], rdx + mov qword [rdi + $20], r8 + mov qword [rdi + $28], r9 + mov qword [rdi + $30], r10 + mov qword [rdi + $38], r11 + jb @block_loop + mov rsp, qword [rsp + $98] + pop r15 + pop r14 + pop r13 + pop r12 + pop rdi + pop rsi + pop rbp + pop rbx + + {$I ..\..\Include\Simd\Common\HlpSimdNonVolatileRestore_x86_64.inc} diff --git a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx_i386.inc b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx_i386.inc new file mode 100644 index 00000000..be4511ac --- /dev/null +++ b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx_i386.inc @@ -0,0 +1,126 @@ +// AVX (VEX-128) implementation of fused XOR + Salsa20/8 on Percival-permuted +// data (IA-32); the Avx name states the ISA requirement: VEX-128 encodings +// only, no 256-bit ymm. Same structure as ScryptSalsa8Avx_x86_64.inc; the +// state save/reload uses vmovdqu (IA-32 stacks are only 4-aligned), so the +// frame is 64 bytes with no alignment padding. +// The (i*5 mod 16) Percival data permutation arranges each 16-word Salsa20 +// state into role-based diagonal order, enabling lane-parallel SIMD +// processing of column and row quarter-rounds with pshufd-based +// diagonalize/undiagonalize between them. Operation: +// State = Salsa20/8(State XOR Input). +// ABI (after HlpSimdProc2Begin_i386.inc): ebx = State ptr, esi = Input ptr. +// Each pointer addresses 16 UInt32 (64 bytes) in permuted order. +// Register map: xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, +// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}, xmm4-xmm5 = temps. +// Frame: 64 bytes (saved XOR'd state; no alignment padding). +// Saves: none (uses xmm0-xmm5 and eax only; all volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: Colin Percival, "Stronger Key Derivation via Sequential +// Memory-Hard Functions" (2009); Tarsnap crypto_scrypt-sse.c; HashLib +// ScryptSalsa8Avx_x86_64.inc. + + sub esp, $40 + + // Load state, XOR with input, save for final addition + db $C5, $FA, $6F, $03 // vmovdqu xmm0, oword [ebx] + db $C5, $FA, $6F, $26 // vmovdqu xmm4, oword [esi] + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + db $C5, $FA, $6F, $4B, $10 // vmovdqu xmm1, oword [ebx + $10] + db $C5, $FA, $6F, $66, $10 // vmovdqu xmm4, oword [esi + $10] + db $C5, $F1, $EF, $CC // vpxor xmm1, xmm1, xmm4 + db $C5, $FA, $6F, $53, $20 // vmovdqu xmm2, oword [ebx + $20] + db $C5, $FA, $6F, $66, $20 // vmovdqu xmm4, oword [esi + $20] + db $C5, $E9, $EF, $D4 // vpxor xmm2, xmm2, xmm4 + db $C5, $FA, $6F, $5B, $30 // vmovdqu xmm3, oword [ebx + $30] + db $C5, $FA, $6F, $66, $30 // vmovdqu xmm4, oword [esi + $30] + db $C5, $E1, $EF, $DC // vpxor xmm3, xmm3, xmm4 + + db $C5, $FA, $7F, $04, $24 // vmovdqu oword [esp], xmm0 + db $C5, $FA, $7F, $4C, $24, $10 // vmovdqu oword [esp + $10], xmm1 + db $C5, $FA, $7F, $54, $24, $20 // vmovdqu oword [esp + $20], xmm2 + db $C5, $FA, $7F, $5C, $24, $30 // vmovdqu oword [esp + $30], xmm3 + + // 4 double-rounds (= 8 rounds = Salsa20/8) + mov eax, $4 +@double_round: + // --- Column quarter-round --- + // xmm1 ^= rotl(xmm0 + xmm3, 7) + db $C5, $F9, $FE, $E3 // vpaddd xmm4, xmm0, xmm3 + db $C5, $D1, $72, $F4, $07 // vpslld xmm5, xmm4, $7 + db $C5, $D9, $72, $D4, $19 // vpsrld xmm4, xmm4, $19 + db $C5, $F1, $EF, $CD // vpxor xmm1, xmm1, xmm5 + db $C5, $F1, $EF, $CC // vpxor xmm1, xmm1, xmm4 + // xmm2 ^= rotl(xmm1 + xmm0, 9) + db $C5, $F1, $FE, $E0 // vpaddd xmm4, xmm1, xmm0 + db $C5, $D1, $72, $F4, $09 // vpslld xmm5, xmm4, $9 + db $C5, $D9, $72, $D4, $17 // vpsrld xmm4, xmm4, $17 + db $C5, $E9, $EF, $D5 // vpxor xmm2, xmm2, xmm5 + db $C5, $E9, $EF, $D4 // vpxor xmm2, xmm2, xmm4 + // xmm3 ^= rotl(xmm2 + xmm1, 13) + db $C5, $E9, $FE, $E1 // vpaddd xmm4, xmm2, xmm1 + db $C5, $D1, $72, $F4, $0D // vpslld xmm5, xmm4, $D + db $C5, $D9, $72, $D4, $13 // vpsrld xmm4, xmm4, $13 + db $C5, $E1, $EF, $DD // vpxor xmm3, xmm3, xmm5 + db $C5, $E1, $EF, $DC // vpxor xmm3, xmm3, xmm4 + // xmm0 ^= rotl(xmm3 + xmm2, 18) + db $C5, $E1, $FE, $E2 // vpaddd xmm4, xmm3, xmm2 + db $C5, $D1, $72, $F4, $12 // vpslld xmm5, xmm4, $12 + db $C5, $D9, $72, $D4, $0E // vpsrld xmm4, xmm4, $E + db $C5, $F9, $EF, $C5 // vpxor xmm0, xmm0, xmm5 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + // Diagonalize: rotate B right by 1, C by 2, D left by 1 + db $C5, $F9, $70, $C9, $93 // vpshufd xmm1, xmm1, $93 + db $C5, $F9, $70, $D2, $4E // vpshufd xmm2, xmm2, $4E + db $C5, $F9, $70, $DB, $39 // vpshufd xmm3, xmm3, $39 + + // --- Row quarter-round (B/D roles swapped after diagonal shuffle) --- + // xmm3 ^= rotl(xmm0 + xmm1, 7) + db $C5, $F9, $FE, $E1 // vpaddd xmm4, xmm0, xmm1 + db $C5, $D1, $72, $F4, $07 // vpslld xmm5, xmm4, $7 + db $C5, $D9, $72, $D4, $19 // vpsrld xmm4, xmm4, $19 + db $C5, $E1, $EF, $DD // vpxor xmm3, xmm3, xmm5 + db $C5, $E1, $EF, $DC // vpxor xmm3, xmm3, xmm4 + // xmm2 ^= rotl(xmm3 + xmm0, 9) + db $C5, $E1, $FE, $E0 // vpaddd xmm4, xmm3, xmm0 + db $C5, $D1, $72, $F4, $09 // vpslld xmm5, xmm4, $9 + db $C5, $D9, $72, $D4, $17 // vpsrld xmm4, xmm4, $17 + db $C5, $E9, $EF, $D5 // vpxor xmm2, xmm2, xmm5 + db $C5, $E9, $EF, $D4 // vpxor xmm2, xmm2, xmm4 + // xmm1 ^= rotl(xmm2 + xmm3, 13) + db $C5, $E9, $FE, $E3 // vpaddd xmm4, xmm2, xmm3 + db $C5, $D1, $72, $F4, $0D // vpslld xmm5, xmm4, $D + db $C5, $D9, $72, $D4, $13 // vpsrld xmm4, xmm4, $13 + db $C5, $F1, $EF, $CD // vpxor xmm1, xmm1, xmm5 + db $C5, $F1, $EF, $CC // vpxor xmm1, xmm1, xmm4 + // xmm0 ^= rotl(xmm1 + xmm2, 18) + db $C5, $F1, $FE, $E2 // vpaddd xmm4, xmm1, xmm2 + db $C5, $D1, $72, $F4, $12 // vpslld xmm5, xmm4, $12 + db $C5, $D9, $72, $D4, $0E // vpsrld xmm4, xmm4, $E + db $C5, $F9, $EF, $C5 // vpxor xmm0, xmm0, xmm5 + db $C5, $F9, $EF, $C4 // vpxor xmm0, xmm0, xmm4 + // Undiagonalize: reverse the shuffles + db $C5, $F9, $70, $C9, $39 // vpshufd xmm1, xmm1, $39 + db $C5, $F9, $70, $D2, $4E // vpshufd xmm2, xmm2, $4E + db $C5, $F9, $70, $DB, $93 // vpshufd xmm3, xmm3, $93 + + dec eax + jnz @double_round + + // Final addition and store + db $C5, $FA, $6F, $24, $24 // vmovdqu xmm4, oword [esp] + db $C5, $F9, $FE, $C4 // vpaddd xmm0, xmm0, xmm4 + db $C5, $FA, $6F, $64, $24, $10 // vmovdqu xmm4, oword [esp + $10] + db $C5, $F1, $FE, $CC // vpaddd xmm1, xmm1, xmm4 + db $C5, $FA, $6F, $64, $24, $20 // vmovdqu xmm4, oword [esp + $20] + db $C5, $E9, $FE, $D4 // vpaddd xmm2, xmm2, xmm4 + db $C5, $FA, $6F, $64, $24, $30 // vmovdqu xmm4, oword [esp + $30] + db $C5, $E1, $FE, $DC // vpaddd xmm3, xmm3, xmm4 + + db $C5, $FA, $7F, $03 // vmovdqu oword [ebx], xmm0 + db $C5, $FA, $7F, $4B, $10 // vmovdqu oword [ebx + $10], xmm1 + db $C5, $FA, $7F, $53, $20 // vmovdqu oword [ebx + $20], xmm2 + db $C5, $FA, $7F, $5B, $30 // vmovdqu oword [ebx + $30], xmm3 + + add esp, $40 + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx2_x86_64.inc b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx_x86_64.inc similarity index 86% rename from HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx2_x86_64.inc rename to HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx_x86_64.inc index 6ce81abc..b38398f3 100644 --- a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx2_x86_64.inc +++ b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Avx_x86_64.inc @@ -1,24 +1,22 @@ -// AVX2 (VEX-128) implementation of fused XOR + Salsa20/8 on Percival-permuted data. +// AVX (VEX-128) implementation of fused XOR + Salsa20/8 on Percival-permuted +// data; the Avx name states the ISA requirement: VEX-128 encodings only, no +// 256-bit ymm. Identical algorithm to the SSE2 variant but the 3-operand +// VEX encoding eliminates movdqa register copies, reducing each QR step +// from 7 to 5 instructions. +// The (i*5 mod 16) Percival data permutation arranges each 16-word Salsa20 +// state into role-based diagonal order, enabling lane-parallel SIMD +// processing of column and row quarter-rounds with pshufd-based +// diagonalize/undiagonalize between them. Operation: +// State = Salsa20/8(State XOR Input). +// ABI (after HlpSimdProc2Begin_x86_64.inc): rcx = State ptr, rdx = Input ptr. +// Each pointer addresses 16 UInt32 (64 bytes) in permuted order. +// Register map: xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, +// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}, xmm4-xmm5 = temps. +// Frame: 72 bytes (64 for the saved XOR'd state + 8 alignment padding). +// Saves: none (uses xmm0-xmm5 only; volatile under both ABIs). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. -// // Reference: Colin Percival, "Stronger Key Derivation via Sequential -// Memory-Hard Functions" (2009), and the Tarsnap scrypt reference -// implementation (crypto_scrypt-sse.c). The (i*5 mod 16) data permutation -// arranges each 16-word Salsa20 state into role-based diagonal order, -// enabling lane-parallel SIMD processing of column and row quarter-rounds -// with vpshufd-based diagonalize/undiagonalize between them. -// -// Identical algorithm to the SSE2 variant but uses VEX-128 3-operand -// encoding, eliminating movdqa register copies and reducing each QR step -// from 7 to 5 instructions. -// -// Expects MS x64 ABI: rcx = State ptr, rdx = Input ptr. -// Each pointer addresses 16 UInt32 (64 bytes) in permuted order: -// xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, -// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}. -// Operation: State = Salsa20/8(State XOR Input) -// Uses xmm0-xmm5 (all volatile). No spills needed. -// Stack: 72 bytes (64 for saved XOR'd state + 8 alignment padding). +// Memory-Hard Functions" (2009); Tarsnap crypto_scrypt-sse.c. sub rsp, 72 diff --git a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Neon_aarch64.inc b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Neon_aarch64.inc index cc8d05ee..1310aa93 100644 --- a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Neon_aarch64.inc +++ b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Neon_aarch64.inc @@ -1,11 +1,12 @@ -// Scrypt Salsa20/8 XOR AArch64 NEON implementation (Percival-permuted data). -// Expects AAPCS64: x0 = State ptr, x1 = Input ptr. -// Each pointer addresses 16 UInt32 (64 bytes) in permuted order: -// v0 = {w0,w5,w10,w15}, v1 = {w4,w9,w14,w3}, v2 = {w8,w13,w2,w7}, v3 = {w12,w1,w6,w11}. -// Leaf nostackframe; caller-saved GPR/vector only. -// Column/row quarter-rounds via lane-parallel ext-based diagonalize (see Tarsnap scrypt-sse.c). -// Reference: Tarsnap crypto_scrypt-sse.c, HashLib ScryptSalsa8Sse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of fused XOR + Salsa20/8 on Percival-permuted data. +// The (i*5 mod 16) Percival data permutation arranges each 16-word Salsa20 +// state into role-based diagonal order, enabling lane-parallel column and +// row quarter-rounds via ext-based diagonalize (see Tarsnap scrypt-sse.c). +// Operation: State = Salsa20/8(State XOR Input). +// ABI (after HlpSimdProc2Begin_aarch64.inc): x0 = State ptr, x1 = Input ptr. +// Each pointer addresses 16 UInt32 (64 bytes) in permuted order: +// v0 = {w0,w5,w10,w15}, v1 = {w4,w9,w14,w3}, v2 = {w8,w13,w2,w7}, +// v3 = {w12,w1,w6,w11}. // Register map: // w9 = double-round counter (4 iterations) // v0 = A @@ -14,6 +15,10 @@ // v3 = D // v4-v5 = quarter-round temps (add + shl + sri) // v16-v19 = saved XOR'd state (A,B,C,D) for the final addition +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: Tarsnap crypto_scrypt-sse.c, HashLib ScryptSalsa8Sse2_x86_64.inc. + .long 0x3dc00000 // ldr q0, [x0, #0x0] .long 0x3dc00024 // ldr q4, [x1, #0x0] .long 0x6e241c00 // eor v0.16b, v0.16b, v4.16b diff --git a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_i386.inc b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_i386.inc index 8803aca1..ea99cceb 100644 --- a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_i386.inc +++ b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_i386.inc @@ -1,21 +1,20 @@ -// SSE2 implementation of fused XOR + Salsa20/8 on Percival-permuted data. -// +// SSE2 implementation of fused XOR + Salsa20/8 on Percival-permuted data +// (IA-32). Column QR / diagonal / row QR: same structure as +// ScryptSalsa8Sse2_x86_64.inc. +// The (i*5 mod 16) Percival data permutation arranges each 16-word Salsa20 +// state into role-based diagonal order, enabling lane-parallel SIMD +// processing of column and row quarter-rounds with pshufd-based +// diagonalize/undiagonalize between them. Operation: +// State = Salsa20/8(State XOR Input). +// ABI (after HlpSimdProc2Begin_i386.inc): ebx = State ptr, esi = Input ptr. +// Each pointer addresses 16 UInt32 (64 bytes) in permuted order. +// Register map: xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, +// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}, xmm4-xmm5 = temps. +// Frame (sub esp, 72): 64 for the saved XOR'd state + 8 alignment padding. +// Saves: none (uses xmm0-xmm5 only; all volatile on IA-32). // Reference: Colin Percival, "Stronger Key Derivation via Sequential -// Memory-Hard Functions" (2009), and the Tarsnap scrypt reference -// implementation (crypto_scrypt-sse.c). The (i*5 mod 16) data permutation -// arranges each 16-word Salsa20 state into role-based diagonal order, -// enabling lane-parallel SIMD processing of column and row quarter-rounds -// with pshufd-based diagonalize/undiagonalize between them. -// -// IA-32: after HlpSimdProc2Begin_i386 — ebx = State ptr, esi = Input ptr -// (parallel to MS x64 ABI: rcx, rdx). -// Each pointer addresses 16 UInt32 (64 bytes) in permuted order: -// xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, -// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}. -// Operation: State = Salsa20/8(State XOR Input) -// Uses xmm0–xmm5. Stack: sub esp, 72 (64 for saved XOR'd state + 8 alignment padding). -// -// Column QR / diagonal / row QR: same structure as ScryptSalsa8Sse2_x86_64.inc header on x86-64. +// Memory-Hard Functions" (2009); Tarsnap crypto_scrypt-sse.c; HashLib +// ScryptSalsa8Sse2_x86_64.inc. sub esp, 72 diff --git a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_x86_64.inc b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_x86_64.inc index 50911a58..ca348482 100644 --- a/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_x86_64.inc +++ b/HashLib/src/Include/Simd/Scrypt/ScryptSalsa8Sse2_x86_64.inc @@ -1,26 +1,23 @@ // SSE2 implementation of fused XOR + Salsa20/8 on Percival-permuted data. -// -// Reference: Colin Percival, "Stronger Key Derivation via Sequential -// Memory-Hard Functions" (2009), and the Tarsnap scrypt reference -// implementation (crypto_scrypt-sse.c). The (i*5 mod 16) data permutation -// arranges each 16-word Salsa20 state into role-based diagonal order, -// enabling lane-parallel SIMD processing of column and row quarter-rounds -// with pshufd-based diagonalize/undiagonalize between them. -// -// Expects MS x64 ABI: rcx = State ptr, rdx = Input ptr. -// Each pointer addresses 16 UInt32 (64 bytes) in permuted order: -// xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, -// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}. -// Operation: State = Salsa20/8(State XOR Input) -// Uses xmm0-xmm5 (volatile under both ABIs; no saves needed). -// Stack: 72 bytes (64 for saved XOR'd state + 8 alignment padding). -// +// The (i*5 mod 16) Percival data permutation arranges each 16-word Salsa20 +// state into role-based diagonal order, enabling lane-parallel SIMD +// processing of column and row quarter-rounds with pshufd-based +// diagonalize/undiagonalize between them. Operation: +// State = Salsa20/8(State XOR Input). // Column QR (lane-parallel): // B ^= rotl(A+D,7); C ^= rotl(B+A,9); D ^= rotl(C+B,13); A ^= rotl(D+C,18) // Diag: pshufd B,$93; pshufd C,$4E; pshufd D,$39 // Row QR (lane-parallel, swapped B/D roles): // D' ^= rotl(A+B',7); C' ^= rotl(D'+A,9); B' ^= rotl(C'+D',13); A ^= rotl(B'+C',18) // Undiag: pshufd B,$39; pshufd C,$4E; pshufd D,$93 +// ABI (after HlpSimdProc2Begin_x86_64.inc): rcx = State ptr, rdx = Input ptr. +// Each pointer addresses 16 UInt32 (64 bytes) in permuted order. +// Register map: xmm0 = A = {w0,w5,w10,w15}, xmm1 = B = {w4,w9,w14,w3}, +// xmm2 = C = {w8,w13,w2,w7}, xmm3 = D = {w12,w1,w6,w11}, xmm4-xmm5 = temps. +// Frame: 72 bytes (64 for the saved XOR'd state + 8 alignment padding). +// Saves: none (uses xmm0-xmm5 only; volatile under both ABIs). +// Reference: Colin Percival, "Stronger Key Derivation via Sequential +// Memory-Hard Functions" (2009); Tarsnap crypto_scrypt-sse.c. sub rsp, 72 diff --git a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_i386.inc b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_i386.inc new file mode 100644 index 00000000..232af573 --- /dev/null +++ b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_i386.inc @@ -0,0 +1,37 @@ +// AVX2 implementation of XXH3_accumulate_512 (fully unrolled, 2 x 32-byte +// chunks; IA-32). Same structure as XXH3Acc512Avx2_x86_64.inc. +// ABI (after HlpSimdProc3Begin_i386.inc): ebx = acc ptr, esi = input ptr, +// edi = secret ptr. +// Saves: none (uses ymm0-ymm3 only; all volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: official xxHash C - XXH3_accumulate_512_avx2 in xxhash.h, +// HashLib XXH3Acc512Avx2_x86_64.inc. + + // --- Chunk 0: acc[0..3], input[0..31], secret[0..31] --- + db $C5, $FE, $6F, $03 // vmovdqu ymm0, yword [ebx] + db $C5, $FE, $6F, $0E // vmovdqu ymm1, yword [esi] + db $C5, $FE, $6F, $17 // vmovdqu ymm2, yword [edi] + db $C5, $ED, $EF, $D1 // vpxor ymm2, ymm2, ymm1 + db $C5, $E5, $73, $D2, $20 // vpsrlq ymm3, ymm2, $20 + db $C5, $ED, $F4, $D3 // vpmuludq ymm2, ymm2, ymm3 + db $C5, $FD, $70, $C9, $4E // vpshufd ymm1, ymm1, $4E + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C2 // vpaddq ymm0, ymm0, ymm2 + db $C5, $FE, $7F, $03 // vmovdqu yword [ebx], ymm0 + + // --- Chunk 1: acc[4..7], input[32..63], secret[32..63] --- + db $C5, $FE, $6F, $43, $20 // vmovdqu ymm0, yword [ebx + $20] + db $C5, $FE, $6F, $4E, $20 // vmovdqu ymm1, yword [esi + $20] + db $C5, $FE, $6F, $57, $20 // vmovdqu ymm2, yword [edi + $20] + db $C5, $ED, $EF, $D1 // vpxor ymm2, ymm2, ymm1 + db $C5, $E5, $73, $D2, $20 // vpsrlq ymm3, ymm2, $20 + db $C5, $ED, $F4, $D3 // vpmuludq ymm2, ymm2, ymm3 + db $C5, $FD, $70, $C9, $4E // vpshufd ymm1, ymm1, $4E + db $C5, $FD, $D4, $C1 // vpaddq ymm0, ymm0, ymm1 + db $C5, $FD, $D4, $C2 // vpaddq ymm0, ymm0, ymm2 + db $C5, $FE, $7F, $43, $20 // vmovdqu yword [ebx + $20], ymm0 + + db $C5, $F8, $77 // vzeroupper + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_x86_64.inc b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_x86_64.inc index 10cc79fb..29a80e32 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_x86_64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Avx2_x86_64.inc @@ -1,8 +1,10 @@ -// AVX2 implementation of XXH3_accumulate_512 (fully unrolled, 2 x 32-byte chunks). +// AVX2 implementation of XXH3_accumulate_512 (fully unrolled, 2 x 32-byte +// chunks). +// ABI (after HlpSimdProc3Begin_x86_64.inc): rcx = acc ptr, rdx = input ptr, +// r8 = secret ptr. +// Saves: none (uses ymm0-ymm4 only; volatile under both ABIs). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. -// Expects MS x64 ABI: rcx = acc ptr, rdx = input ptr, r8 = secret ptr. -// Uses only volatile registers: ymm0-ymm4. -// Reference: official xxHash C - XXH3_accumulate_512_avx2 in xxhash.h +// Reference: official xxHash C - XXH3_accumulate_512_avx2 in xxhash.h. // --- Chunk 0: acc[0..3], input[0..31], secret[0..31] --- db $C5, $FE, $6F, $01 // vmovdqu ymm0, yword [rcx] diff --git a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Neon_aarch64.inc b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Neon_aarch64.inc index fdd58f61..3f223b7d 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Neon_aarch64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Neon_aarch64.inc @@ -1,15 +1,17 @@ -// XXH3 accumulate_512 AArch64 NEON implementation (one 64-byte stripe). -// Expects AAPCS64: x0 = acc ptr (8 x UInt64, 16-byte aligned), x1 = input ptr, -// x2 = secret ptr. -// Leaf nostackframe; caller-saved GPR/vector only. -// Reference: Cyan4973/xxHash xxhash.h XXH3_accumulate_512_neon, HashLib XXH3Acc512Sse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of XXH3_accumulate_512 (one 64-byte stripe). +// ABI (after HlpSimdProc3Begin_aarch64.inc): x0 = acc ptr (8 x UInt64, +// 16-byte aligned), x1 = input ptr, x2 = secret ptr. // Register map: // v0-v3 = acc[0..3] (4 x uint64x2) // v4-v5 = data_vec_1/2 (input) // v6-v7 = key_vec / uzp temps (data_key_lo, data_key_hi) // v16-v17 = data_swap_1/2 (vext input by 8 bytes) // v18-v19 = sum_1/sum_2 (umlal / umlal2 widen-multiply-add into swap) +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: Cyan4973/xxHash xxhash.h XXH3_accumulate_512_neon, HashLib +// XXH3Acc512Sse2_x86_64.inc. + // --- batch i=0: acc[0..1], input[0x0..], secret[0x0..] --- .long 0x3dc00024 // ldr q4, [x1, #0x0] .long 0x3dc00425 // ldr q5, [x1, #0x10] @@ -19,8 +21,8 @@ .long 0x6e0540b1 // ext v17.16b, v5.16b, v5.16b, #8 .long 0x6e261c84 // eor v4.16b, v4.16b, v6.16b .long 0x6e271ca5 // eor v5.16b, v5.16b, v7.16b - .long 0x4e871886 // uzp1 v6.4s, v4.4s, v5.4s - .long 0x4e875887 // uzp2 v7.4s, v4.4s, v5.4s + .long 0x4e851886 // uzp1 v6.4s, v4.4s, v5.4s + .long 0x4e855887 // uzp2 v7.4s, v4.4s, v5.4s .long 0x2ea780d0 // umlal v16.2d, v6.2s, v7.2s .long 0x6ea780d1 // umlal2 v17.2d, v6.4s, v7.4s .long 0x3dc00000 // ldr q0, [x0, #0x0] @@ -38,8 +40,8 @@ .long 0x6e0540b1 // ext v17.16b, v5.16b, v5.16b, #8 .long 0x6e261c84 // eor v4.16b, v4.16b, v6.16b .long 0x6e271ca5 // eor v5.16b, v5.16b, v7.16b - .long 0x4e871886 // uzp1 v6.4s, v4.4s, v5.4s - .long 0x4e875887 // uzp2 v7.4s, v4.4s, v5.4s + .long 0x4e851886 // uzp1 v6.4s, v4.4s, v5.4s + .long 0x4e855887 // uzp2 v7.4s, v4.4s, v5.4s .long 0x2ea780d0 // umlal v16.2d, v6.2s, v7.2s .long 0x6ea780d1 // umlal2 v17.2d, v6.4s, v7.4s .long 0x3dc00802 // ldr q2, [x0, #0x20] diff --git a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_i386.inc b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_i386.inc index ab7d5b61..b29e02e3 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_i386.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_i386.inc @@ -1,8 +1,9 @@ -// SSE2 implementation of XXH3_accumulate_512 (fully unrolled, 4 x 16-byte chunks). -// IA-32: after HlpSimdProc3Begin_i386 — ebx = acc, esi = input, edi = secret -// (parallel to MS x64 ABI: rcx, rdx, r8). -// Uses only volatile XMM: xmm0–xmm5. -// Reference: official xxHash C - XXH3_accumulate_512_sse2 in xxhash.h +// SSE2 implementation of XXH3_accumulate_512 (fully unrolled, 4 x 16-byte +// chunks; IA-32). +// ABI (after HlpSimdProc3Begin_i386.inc): ebx = acc ptr, esi = input ptr, +// edi = secret ptr. +// Saves: none (uses xmm0-xmm5 only; all volatile on IA-32). +// Reference: official xxHash C - XXH3_accumulate_512_sse2 in xxhash.h. movdqu xmm0, oword [ebx] movdqu xmm1, oword [esi] diff --git a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_x86_64.inc b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_x86_64.inc index e1ecd3af..e6abb6b4 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_x86_64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3Acc512Sse2_x86_64.inc @@ -1,7 +1,9 @@ -// SSE2 implementation of XXH3_accumulate_512 (fully unrolled, 4 x 16-byte chunks). -// Expects MS x64 ABI: rcx = acc ptr, rdx = input ptr, r8 = secret ptr. -// Uses only volatile registers: xmm0-xmm5. -// Reference: official xxHash C - XXH3_accumulate_512_sse2 in xxhash.h +// SSE2 implementation of XXH3_accumulate_512 (fully unrolled, 4 x 16-byte +// chunks). +// ABI (after HlpSimdProc3Begin_x86_64.inc): rcx = acc ptr, rdx = input ptr, +// r8 = secret ptr. +// Saves: none (uses xmm0-xmm5 only; volatile under both ABIs). +// Reference: official xxHash C - XXH3_accumulate_512_sse2 in xxhash.h. // --- Chunk 0: acc[0..1], input[0..15], secret[0..15] --- movdqu xmm0, oword [rcx] diff --git a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_i386.inc b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_i386.inc new file mode 100644 index 00000000..5376d140 --- /dev/null +++ b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_i386.inc @@ -0,0 +1,70 @@ +// AVX2 implementation of XXH3_initCustomSecret (fully unrolled, 6 x +// 32-byte chunks; IA-32). Same structure as XXH3InitSecretAvx2_x86_64.inc; +// the x86_64 version's vmovq xmm, r64 seed transfers (not encodable in +// 32-bit mode) become the SSE2 i386 sibling's push/vmovq-from-stack +// pattern, with the negation done in 32-bit halves (not/not + add/adc +// carry). +// ABI (after HlpSimdProc3Begin_i386.inc): ebx = customSecret ptr, +// esi = defaultSecret ptr, edi = seed ptr (PUInt64). +// Saves: none (uses ymm0-ymm1, eax, ecx, edx; edi reused; all volatile on +// IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: official xxHash C - XXH3_initCustomSecret_avx2 in xxhash.h, +// HashLib XXH3InitSecretAvx2_x86_64.inc. + + // Build seed vector ymm0 = [seed, -seed, seed, -seed] + mov eax, dword ptr [edi] + mov edx, dword ptr [edi + $4] + + push edx + push eax + db $C5, $FA, $7E, $04, $24 // vmovq xmm0, qword [esp] + add esp, $8 + + mov ecx, eax + mov edi, edx + not ecx + not edi + add ecx, $1 + adc edi, $0 + push edi + push ecx + db $C5, $FA, $7E, $0C, $24 // vmovq xmm1, qword [esp] + add esp, $8 + db $C5, $F9, $6C, $C1 // vpunpcklqdq xmm0, xmm0, xmm1 + db $C4, $E3, $7D, $38, $C0, $01 // vinserti128 ymm0, ymm0, xmm0, $1 + + // Block 0 + db $C5, $FE, $6F, $0E // vmovdqu ymm1, yword [esi] + db $C5, $F5, $D4, $C8 // vpaddq ymm1, ymm1, ymm0 + db $C5, $FE, $7F, $0B // vmovdqu yword [ebx], ymm1 + + // Block 1 + db $C5, $FE, $6F, $4E, $20 // vmovdqu ymm1, yword [esi + $20] + db $C5, $F5, $D4, $C8 // vpaddq ymm1, ymm1, ymm0 + db $C5, $FE, $7F, $4B, $20 // vmovdqu yword [ebx + $20], ymm1 + + // Block 2 + db $C5, $FE, $6F, $4E, $40 // vmovdqu ymm1, yword [esi + $40] + db $C5, $F5, $D4, $C8 // vpaddq ymm1, ymm1, ymm0 + db $C5, $FE, $7F, $4B, $40 // vmovdqu yword [ebx + $40], ymm1 + + // Block 3 + db $C5, $FE, $6F, $4E, $60 // vmovdqu ymm1, yword [esi + $60] + db $C5, $F5, $D4, $C8 // vpaddq ymm1, ymm1, ymm0 + db $C5, $FE, $7F, $4B, $60 // vmovdqu yword [ebx + $60], ymm1 + + // Block 4 + db $C5, $FE, $6F, $8E, $80, $00, $00, $00 // vmovdqu ymm1, yword [esi + $80] + db $C5, $F5, $D4, $C8 // vpaddq ymm1, ymm1, ymm0 + db $C5, $FE, $7F, $8B, $80, $00, $00, $00 // vmovdqu yword [ebx + $80], ymm1 + + // Block 5 + db $C5, $FE, $6F, $8E, $A0, $00, $00, $00 // vmovdqu ymm1, yword [esi + $A0] + db $C5, $F5, $D4, $C8 // vpaddq ymm1, ymm1, ymm0 + db $C5, $FE, $7F, $8B, $A0, $00, $00, $00 // vmovdqu yword [ebx + $A0], ymm1 + + db $C5, $F8, $77 // vzeroupper + pop edi + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_x86_64.inc b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_x86_64.inc index a8bde0bf..6ea31860 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretAvx2_x86_64.inc @@ -1,10 +1,13 @@ -// AVX2 implementation of XXH3_initCustomSecret (fully unrolled, 6 x 32-byte chunks). +// AVX2 implementation of XXH3_initCustomSecret (fully unrolled, 6 x +// 32-byte chunks). +// ABI (after HlpSimdProc3Begin_x86_64.inc): rcx = customSecret ptr, +// rdx = defaultSecret ptr, r8 = seed ptr (PUInt64). +// Saves: none (uses ymm0-ymm2 and rax only; volatile under both ABIs). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. -// Expects MS x64 ABI: rcx = customSecret ptr, rdx = defaultSecret ptr, r8 = seed (UInt64). -// Uses only volatile registers: ymm0-ymm2, rax. -// Reference: official xxHash C - XXH3_initCustomSecret_avx2 in xxhash.h +// Reference: official xxHash C - XXH3_initCustomSecret_avx2 in xxhash.h. // Build seed vector ymm0 = [seed, -seed, seed, -seed] + mov r8, qword [r8] db $C4, $C1, $F9, $6E, $C0 // vmovq xmm0, r8 mov rax, r8 neg rax diff --git a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretNeon_aarch64.inc b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretNeon_aarch64.inc index 5be27288..9d3446f1 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretNeon_aarch64.inc @@ -1,13 +1,16 @@ -// XXH3 initCustomSecret AArch64 NEON implementation (192-byte custom secret). -// Expects AAPCS64: x0 = customSecret ptr, x1 = defaultSecret ptr, x2 = seed (UInt64). -// Leaf nostackframe; caller-saved GPR/vector only. -// Reference: HashLib XXH3InitSecretSse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of XXH3_initCustomSecret (192-byte custom secret). +// ABI (after HlpSimdProc3Begin_aarch64.inc): x0 = customSecret ptr, +// x1 = defaultSecret ptr, x2 = seed ptr (PUInt64). // Register map: // x3 = negated seed (-seed) // v0 = vSeed broadcast vector [seed, -seed] // v16-v17 = fmov staging for zip1 pack // v2 = defaultSecret block load / add temp +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: HashLib XXH3InitSecretSse2_x86_64.inc. + + ldr x2, [x2] neg x3, x2 .long 0x9e670050 // fmov d16, x2 .long 0x9e670071 // fmov d17, x3 diff --git a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_i386.inc b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_i386.inc index cad3d2e7..18cbbaad 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_i386.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_i386.inc @@ -1,15 +1,16 @@ -// SSE2 implementation of XXH3_initCustomSecret (fully unrolled, 12 x 16-byte chunks). -// IA-32: after HlpSimdProc3Begin_i386 — ebx = customSecret, esi = defaultSecret, edi = seed.lo32, -// dword [esp+16] = seed.hi32 (UInt64 third parameter; high dword on stack). -// (parallel to MS x64 ABI: rcx, rdx, r8). -// Uses only volatile XMM: xmm0–xmm2. -// Reference: official xxHash C - XXH3_initCustomSecret_sse2 in xxhash.h +// SSE2 implementation of XXH3_initCustomSecret (fully unrolled, 12 x +// 16-byte chunks; IA-32). For each 16-byte block: lo_qword += seed, +// hi_qword -= seed. +// ABI (after HlpSimdProc3Begin_i386.inc): ebx = customSecret ptr, +// esi = defaultSecret ptr, edi = seed ptr (PUInt64). +// Saves: none (uses xmm0-xmm2 only; all volatile on IA-32). +// Reference: official xxHash C - XXH3_initCustomSecret_sse2 in xxhash.h. - mov eax, edi - mov edx, dword ptr [esp + 16] + mov eax, dword ptr [edi] + mov edx, dword ptr [edi + 4] - push eax push edx + push eax movq xmm0, qword ptr [esp] add esp, 8 @@ -19,8 +20,8 @@ not edi add ecx, 1 adc edi, 0 - push ecx push edi + push ecx movq xmm1, qword ptr [esp] add esp, 8 punpcklqdq xmm0, xmm1 diff --git a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_x86_64.inc b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_x86_64.inc index 8b1306fc..d323b92d 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3InitSecretSse2_x86_64.inc @@ -1,10 +1,13 @@ -// SSE2 implementation of XXH3_initCustomSecret (fully unrolled, 12 x 16-byte chunks). -// Expects MS x64 ABI: rcx = customSecret ptr, rdx = defaultSecret ptr, r8 = seed (UInt64). -// Uses only volatile registers: xmm0-xmm2. -// Algorithm: for each 16-byte block, lo_qword += seed, hi_qword -= seed. -// Reference: official xxHash C - XXH3_initCustomSecret_sse2 in xxhash.h +// SSE2 implementation of XXH3_initCustomSecret (fully unrolled, 12 x +// 16-byte chunks). For each 16-byte block: lo_qword += seed, +// hi_qword -= seed. +// ABI (after HlpSimdProc3Begin_x86_64.inc): rcx = customSecret ptr, +// rdx = defaultSecret ptr, r8 = seed ptr (PUInt64). +// Saves: none (uses xmm0-xmm2 only; volatile under both ABIs). +// Reference: official xxHash C - XXH3_initCustomSecret_sse2 in xxhash.h. // Build seed vector xmm0 = [seed, -seed] + mov r8, qword [r8] movq xmm0, r8 mov rax, r8 neg rax diff --git a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_i386.inc b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_i386.inc new file mode 100644 index 00000000..a890624a --- /dev/null +++ b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_i386.inc @@ -0,0 +1,43 @@ +// AVX2 implementation of XXH3_scrambleAcc (fully unrolled, 2 x 32-byte +// chunks; IA-32). Same structure as XXH3ScrambleAvx2_x86_64.inc. +// ABI (after HlpSimdProc2Begin_i386.inc): ebx = acc ptr, esi = secret ptr. +// Saves: none (uses ymm0-ymm4 and eax only; all volatile on IA-32). +// AVX/AVX2 instructions are db-encoded for broad assembler compatibility. +// Reference: official xxHash C - XXH3_scrambleAcc_avx2 in xxhash.h, +// HashLib XXH3ScrambleAvx2_x86_64.inc. + + // Broadcast XXH_PRIME32_1 ($9E3779B1) to all dword lanes of ymm4 + mov eax, $9E3779B1 + db $C5, $F9, $6E, $E0 // vmovd xmm4, eax + db $C4, $E2, $7D, $58, $E4 // vpbroadcastd ymm4, xmm4 + + // --- Chunk 0: acc[0..3], secret[0..31] --- + db $C5, $FE, $6F, $03 // vmovdqu ymm0, yword [ebx] + db $C5, $F5, $73, $D0, $2F // vpsrlq ymm1, ymm0, $2F + db $C5, $FD, $EF, $C1 // vpxor ymm0, ymm0, ymm1 + db $C5, $FE, $6F, $0E // vmovdqu ymm1, yword [esi] + db $C5, $FD, $EF, $C1 // vpxor ymm0, ymm0, ymm1 + // Multiply 64-bit lanes by PRIME32_1 + db $C5, $F5, $73, $D0, $20 // vpsrlq ymm1, ymm0, $20 + db $C5, $FD, $F4, $D4 // vpmuludq ymm2, ymm0, ymm4 + db $C5, $F5, $F4, $CC // vpmuludq ymm1, ymm1, ymm4 + db $C5, $F5, $73, $F1, $20 // vpsllq ymm1, ymm1, $20 + db $C5, $ED, $D4, $C1 // vpaddq ymm0, ymm2, ymm1 + db $C5, $FE, $7F, $03 // vmovdqu yword [ebx], ymm0 + + // --- Chunk 1: acc[4..7], secret[32..63] --- + db $C5, $FE, $6F, $43, $20 // vmovdqu ymm0, yword [ebx + $20] + db $C5, $F5, $73, $D0, $2F // vpsrlq ymm1, ymm0, $2F + db $C5, $FD, $EF, $C1 // vpxor ymm0, ymm0, ymm1 + db $C5, $FE, $6F, $4E, $20 // vmovdqu ymm1, yword [esi + $20] + db $C5, $FD, $EF, $C1 // vpxor ymm0, ymm0, ymm1 + db $C5, $F5, $73, $D0, $20 // vpsrlq ymm1, ymm0, $20 + db $C5, $FD, $F4, $D4 // vpmuludq ymm2, ymm0, ymm4 + db $C5, $F5, $F4, $CC // vpmuludq ymm1, ymm1, ymm4 + db $C5, $F5, $73, $F1, $20 // vpsllq ymm1, ymm1, $20 + db $C5, $ED, $D4, $C1 // vpaddq ymm0, ymm2, ymm1 + db $C5, $FE, $7F, $43, $20 // vmovdqu yword [ebx + $20], ymm0 + + db $C5, $F8, $77 // vzeroupper + pop esi + pop ebx diff --git a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_x86_64.inc b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_x86_64.inc index c58e5bbe..0dd9b130 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_x86_64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleAvx2_x86_64.inc @@ -1,8 +1,9 @@ -// AVX2 implementation of XXH3_scrambleAcc (fully unrolled, 2 x 32-byte chunks). +// AVX2 implementation of XXH3_scrambleAcc (fully unrolled, 2 x 32-byte +// chunks). +// ABI (after HlpSimdProc2Begin_x86_64.inc): rcx = acc ptr, rdx = secret ptr. +// Saves: none (uses ymm0-ymm4 and eax only; volatile under both ABIs). // AVX/AVX2 instructions are db-encoded for broad assembler compatibility. -// Expects MS x64 ABI: rcx = acc ptr, rdx = secret ptr. -// Uses only volatile registers: ymm0-ymm4, eax. -// Reference: official xxHash C - XXH3_scrambleAcc_avx2 in xxhash.h +// Reference: official xxHash C - XXH3_scrambleAcc_avx2 in xxhash.h. // Broadcast XXH_PRIME32_1 ($9E3779B1) to all dword lanes of ymm4 mov eax, $9E3779B1 diff --git a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleNeon_aarch64.inc b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleNeon_aarch64.inc index 9dfc0e40..4825e364 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleNeon_aarch64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleNeon_aarch64.inc @@ -1,14 +1,17 @@ -// XXH3 scrambleAcc AArch64 NEON implementation (64-byte secret slice). -// Expects AAPCS64: x0 = acc ptr (8 x UInt64, 16-byte aligned), x1 = secret ptr. -// Leaf nostackframe; caller-saved GPR/vector only. -// Reference: Cyan4973/xxHash xxhash.h XXH3_scrambleAcc_neon, HashLib XXH3ScrambleSse2_x86_64.inc. -// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// NEON implementation of XXH3_scrambleAcc (64-byte secret slice). +// ABI (after HlpSimdProc2Begin_aarch64.inc): x0 = acc ptr (8 x UInt64, +// 16-byte aligned), x1 = secret ptr. // Register map: // w10 = XXH_PRIME32_1 constant loader // v4 = kPrimeHi ({0, PRIME32_1, 0, PRIME32_1} as u32x4) // v5 = kPrimeLo (PRIME32_1 broadcast to u32x2) // v3 = prod_hi (vmul u32) / final chunk result // v1 = shifted acc, secret key, or data_key_lo (xtn) +// Saves: none (leaf nostackframe; caller-saved GPR/vector only). +// AArch64 vector instructions are .long-encoded for broad assembler compatibility. +// Reference: Cyan4973/xxHash xxhash.h XXH3_scrambleAcc_neon, HashLib +// XXH3ScrambleSse2_x86_64.inc. + .long 0x528f362a // movz w10, #0x79b1, lsl #0 .long 0x72b3c6ea // movk w10, #0x9e37, lsl #16 .long 0x0e040d45 // dup v5.2s, w10 @@ -21,9 +24,9 @@ .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b .long 0x3dc00021 // ldr q1, [x1, #0x0] .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b - .long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s + .long 0x4ea49c03 // mul v3.4s, v0.4s, v4.4s .long 0x0ea12801 // xtn v1.2s, v0.2d - .long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s + .long 0x2ea58023 // umlal v3.2d, v1.2s, v5.2s .long 0x3d800003 // str q3, [x0, #0x0] // --- chunk 1: acc[2..3], secret[0x10..] --- .long 0x3dc00400 // ldr q0, [x0, #0x10] @@ -31,9 +34,9 @@ .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b .long 0x3dc00421 // ldr q1, [x1, #0x10] .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b - .long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s + .long 0x4ea49c03 // mul v3.4s, v0.4s, v4.4s .long 0x0ea12801 // xtn v1.2s, v0.2d - .long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s + .long 0x2ea58023 // umlal v3.2d, v1.2s, v5.2s .long 0x3d800403 // str q3, [x0, #0x10] // --- chunk 2: acc[4..5], secret[0x20..] --- .long 0x3dc00800 // ldr q0, [x0, #0x20] @@ -41,9 +44,9 @@ .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b .long 0x3dc00821 // ldr q1, [x1, #0x20] .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b - .long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s + .long 0x4ea49c03 // mul v3.4s, v0.4s, v4.4s .long 0x0ea12801 // xtn v1.2s, v0.2d - .long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s + .long 0x2ea58023 // umlal v3.2d, v1.2s, v5.2s .long 0x3d800803 // str q3, [x0, #0x20] // --- chunk 3: acc[6..7], secret[0x30..] --- .long 0x3dc00c00 // ldr q0, [x0, #0x30] @@ -51,8 +54,8 @@ .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b .long 0x3dc00c21 // ldr q1, [x1, #0x30] .long 0x6e211c00 // eor v0.16b, v0.16b, v1.16b - .long 0x4ea69c03 // mul v3.4s, v0.4s, v4.4s + .long 0x4ea49c03 // mul v3.4s, v0.4s, v4.4s .long 0x0ea12801 // xtn v1.2s, v0.2d - .long 0x2ea78023 // umlal v3.2d, v1.2s, v5.2s + .long 0x2ea58023 // umlal v3.2d, v1.2s, v5.2s .long 0x3d800c03 // str q3, [x0, #0x30] ret diff --git a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_i386.inc b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_i386.inc index 16ed7be8..354c9b67 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_i386.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_i386.inc @@ -1,12 +1,9 @@ -// SSE2 implementation of XXH3_scrambleAcc (fully unrolled, 4 x 16-byte chunks). -// IA-32: after HlpSimdProc2Begin_i386 — ebx = acc, esi = secret -// (parallel to MS x64 ABI: rcx, rdx). -// Uses only volatile registers: xmm0–xmm5, eax. -// Algorithm per 128-bit chunk: -// acc ^= (acc >> 47) -// acc ^= secret -// acc *= XXH_PRIME32_1 -// Reference: official xxHash C - XXH3_scrambleAcc_sse2 in xxhash.h +// SSE2 implementation of XXH3_scrambleAcc (fully unrolled, 4 x 16-byte +// chunks; IA-32). Algorithm per 128-bit chunk: acc ^= (acc >> 47); +// acc ^= secret; acc *= XXH_PRIME32_1. +// ABI (after HlpSimdProc2Begin_i386.inc): ebx = acc ptr, esi = secret ptr. +// Saves: none (uses xmm0-xmm5 and eax only; all volatile on IA-32). +// Reference: official xxHash C - XXH3_scrambleAcc_sse2 in xxhash.h. mov eax, $9E3779B1 movd xmm5, eax diff --git a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_x86_64.inc b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_x86_64.inc index 50d8218e..9a2532b8 100644 --- a/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_x86_64.inc +++ b/HashLib/src/Include/Simd/XXH3/XXH3ScrambleSse2_x86_64.inc @@ -1,11 +1,9 @@ -// SSE2 implementation of XXH3_scrambleAcc (fully unrolled, 4 x 16-byte chunks). -// Expects MS x64 ABI: rcx = acc ptr, rdx = secret ptr. -// Uses only volatile registers: xmm0-xmm5, eax. -// Algorithm per 128-bit chunk: -// acc ^= (acc >> 47) -// acc ^= secret -// acc *= XXH_PRIME32_1 -// Reference: official xxHash C - XXH3_scrambleAcc_sse2 in xxhash.h +// SSE2 implementation of XXH3_scrambleAcc (fully unrolled, 4 x 16-byte +// chunks). Algorithm per 128-bit chunk: acc ^= (acc >> 47); acc ^= secret; +// acc *= XXH_PRIME32_1. +// ABI (after HlpSimdProc2Begin_x86_64.inc): rcx = acc ptr, rdx = secret ptr. +// Saves: none (uses xmm0-xmm5 and eax only; volatile under both ABIs). +// Reference: official xxHash C - XXH3_scrambleAcc_sse2 in xxhash.h. // Load XXH_PRIME32_1 ($9E3779B1) into xmm5 as broadcast dword mov eax, $9E3779B1 diff --git a/HashLib/src/KDF/HlpArgon2Dispatch.pas b/HashLib/src/KDF/HlpArgon2Dispatch.pas deleted file mode 100644 index 260a6ff6..00000000 --- a/HashLib/src/KDF/HlpArgon2Dispatch.pas +++ /dev/null @@ -1,185 +0,0 @@ -unit HlpArgon2Dispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TArgon2FillBlockProc = procedure(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); - -var - Argon2_FillBlock: TArgon2FillBlockProc; - -implementation - -uses - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -// ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure Argon2_FillBlock_Scalar(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); -var - LR, LZ: array [0 .. 127] of UInt64; - LPLeft, LPRight, LPCurrent: PUInt64; - LIdx, LRoundIdx, LColBase, LRowBase: Int32; - - procedure G(AA, AB, AC, AD: Int32); - var - LProd: UInt64; - begin - LProd := UInt64(UInt32(LZ[AA])) * UInt64(UInt32(LZ[AB])); - LZ[AA] := LZ[AA] + LZ[AB] + (2 * LProd); - LZ[AD] := TBitOperations.RotateRight64(LZ[AD] xor LZ[AA], 32); - - LProd := UInt64(UInt32(LZ[AC])) * UInt64(UInt32(LZ[AD])); - LZ[AC] := LZ[AC] + LZ[AD] + (2 * LProd); - LZ[AB] := TBitOperations.RotateRight64(LZ[AB] xor LZ[AC], 24); - - LProd := UInt64(UInt32(LZ[AA])) * UInt64(UInt32(LZ[AB])); - LZ[AA] := LZ[AA] + LZ[AB] + (2 * LProd); - LZ[AD] := TBitOperations.RotateRight64(LZ[AD] xor LZ[AA], 16); - - LProd := UInt64(UInt32(LZ[AC])) * UInt64(UInt32(LZ[AD])); - LZ[AC] := LZ[AC] + LZ[AD] + (2 * LProd); - LZ[AB] := TBitOperations.RotateRight64(LZ[AB] xor LZ[AC], 63); - end; - - procedure RoundFunction(AV0, AV1, AV2, AV3, AV4, AV5, AV6, AV7, - AV8, AV9, AV10, AV11, AV12, AV13, AV14, AV15: Int32); - begin - G(AV0, AV4, AV8, AV12); - G(AV1, AV5, AV9, AV13); - G(AV2, AV6, AV10, AV14); - G(AV3, AV7, AV11, AV15); - - G(AV0, AV5, AV10, AV15); - G(AV1, AV6, AV11, AV12); - G(AV2, AV7, AV8, AV13); - G(AV3, AV4, AV9, AV14); - end; - -begin - LPLeft := PUInt64(ALeft); - LPRight := PUInt64(ARight); - LPCurrent := PUInt64(ACurrent); - - for LIdx := 0 to 127 do - LR[LIdx] := LPLeft[LIdx] xor LPRight[LIdx]; - - System.Move(LR, LZ, SizeOf(LR)); - - for LRoundIdx := 0 to 7 do - begin - LColBase := 16 * LRoundIdx; - RoundFunction(LColBase, LColBase + 1, LColBase + 2, LColBase + 3, - LColBase + 4, LColBase + 5, LColBase + 6, LColBase + 7, - LColBase + 8, LColBase + 9, LColBase + 10, LColBase + 11, - LColBase + 12, LColBase + 13, LColBase + 14, LColBase + 15); - end; - - for LRoundIdx := 0 to 7 do - begin - LRowBase := 2 * LRoundIdx; - RoundFunction(LRowBase, LRowBase + 1, LRowBase + 16, LRowBase + 17, - LRowBase + 32, LRowBase + 33, LRowBase + 48, LRowBase + 49, - LRowBase + 64, LRowBase + 65, LRowBase + 80, LRowBase + 81, - LRowBase + 96, LRowBase + 97, LRowBase + 112, LRowBase + 113); - end; - - if AWithXor <> 0 then - begin - for LIdx := 0 to 127 do - LPCurrent[LIdx] := LR[LIdx] xor LZ[LIdx] xor LPCurrent[LIdx]; - end - else - begin - for LIdx := 0 to 127 do - LPCurrent[LIdx] := LR[LIdx] xor LZ[LIdx]; - end; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: NEON -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure Argon2_FillBlock_Sse2(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\Argon2\Argon2FillBlockSse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure Argon2_FillBlock_Sse2(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Argon2\Argon2FillBlockSse2_x86_64.inc} -end; - -procedure Argon2_FillBlock_Avx2(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\Argon2\Argon2FillBlockAvx2_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure Argon2_FillBlock_Neon(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\Argon2\Argon2FillBlockNeon_aarch64.inc} -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - Argon2_FillBlock := @Argon2_FillBlock_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - Argon2_FillBlock := @Argon2_FillBlock_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - Argon2_FillBlock := @Argon2_FillBlock_Avx2; - end; - TX86SimdLevel.SSE2: - begin - Argon2_FillBlock := @Argon2_FillBlock_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - TArmSimdLevel.NEON: - begin - Argon2_FillBlock := @Argon2_FillBlock_Neon; - end; - end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/KDF/HlpPBKDF_Argon2NotBuildInAdapter.pas b/HashLib/src/KDF/HlpPBKDF_Argon2NotBuildInAdapter.pas index 284bfeae..016f891b 100644 --- a/HashLib/src/KDF/HlpPBKDF_Argon2NotBuildInAdapter.pas +++ b/HashLib/src/KDF/HlpPBKDF_Argon2NotBuildInAdapter.pas @@ -18,10 +18,15 @@ interface HlpConverters, HlpBinaryPrimitives, HlpArgon2TypeAndVersion, - HlpArgon2Dispatch, HlpArrayUtils, HlpHashLibTypes; +type + TArgon2FillBlockProc = procedure(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); + +var + Argon2_FillBlock: TArgon2FillBlockProc; + resourcestring SInvalidOutputByteCount = '"(AByteCount)" Argument Less Than "%d".'; SBlockInstanceNotInitialized = 'Block Instance not Initialized'; @@ -355,6 +360,95 @@ TFillBlock = record implementation +uses + HlpBitOperations, + HlpArgon2Simd; + +// ============================================================================= +// Scalar reference implementation +// ============================================================================= + +procedure Argon2_FillBlock_Scalar(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); +var + LR, LZ: array [0 .. 127] of UInt64; + LPLeft, LPRight, LPCurrent: PUInt64; + LIdx, LRoundIdx, LColBase, LRowBase: Int32; + + procedure G(AA, AB, AC, AD: Int32); + var + LProd: UInt64; + begin + LProd := UInt64(UInt32(LZ[AA])) * UInt64(UInt32(LZ[AB])); + LZ[AA] := LZ[AA] + LZ[AB] + (2 * LProd); + LZ[AD] := TBitOperations.RotateRight64(LZ[AD] xor LZ[AA], 32); + + LProd := UInt64(UInt32(LZ[AC])) * UInt64(UInt32(LZ[AD])); + LZ[AC] := LZ[AC] + LZ[AD] + (2 * LProd); + LZ[AB] := TBitOperations.RotateRight64(LZ[AB] xor LZ[AC], 24); + + LProd := UInt64(UInt32(LZ[AA])) * UInt64(UInt32(LZ[AB])); + LZ[AA] := LZ[AA] + LZ[AB] + (2 * LProd); + LZ[AD] := TBitOperations.RotateRight64(LZ[AD] xor LZ[AA], 16); + + LProd := UInt64(UInt32(LZ[AC])) * UInt64(UInt32(LZ[AD])); + LZ[AC] := LZ[AC] + LZ[AD] + (2 * LProd); + LZ[AB] := TBitOperations.RotateRight64(LZ[AB] xor LZ[AC], 63); + end; + + procedure RoundFunction(AV0, AV1, AV2, AV3, AV4, AV5, AV6, AV7, + AV8, AV9, AV10, AV11, AV12, AV13, AV14, AV15: Int32); + begin + G(AV0, AV4, AV8, AV12); + G(AV1, AV5, AV9, AV13); + G(AV2, AV6, AV10, AV14); + G(AV3, AV7, AV11, AV15); + + G(AV0, AV5, AV10, AV15); + G(AV1, AV6, AV11, AV12); + G(AV2, AV7, AV8, AV13); + G(AV3, AV4, AV9, AV14); + end; + +begin + LPLeft := PUInt64(ALeft); + LPRight := PUInt64(ARight); + LPCurrent := PUInt64(ACurrent); + + for LIdx := 0 to 127 do + LR[LIdx] := LPLeft[LIdx] xor LPRight[LIdx]; + + System.Move(LR, LZ, SizeOf(LR)); + + for LRoundIdx := 0 to 7 do + begin + LColBase := 16 * LRoundIdx; + RoundFunction(LColBase, LColBase + 1, LColBase + 2, LColBase + 3, + LColBase + 4, LColBase + 5, LColBase + 6, LColBase + 7, + LColBase + 8, LColBase + 9, LColBase + 10, LColBase + 11, + LColBase + 12, LColBase + 13, LColBase + 14, LColBase + 15); + end; + + for LRoundIdx := 0 to 7 do + begin + LRowBase := 2 * LRoundIdx; + RoundFunction(LRowBase, LRowBase + 1, LRowBase + 16, LRowBase + 17, + LRowBase + 32, LRowBase + 33, LRowBase + 48, LRowBase + 49, + LRowBase + 64, LRowBase + 65, LRowBase + 80, LRowBase + 81, + LRowBase + 96, LRowBase + 97, LRowBase + 112, LRowBase + 113); + end; + + if AWithXor <> 0 then + begin + for LIdx := 0 to 127 do + LPCurrent[LIdx] := LR[LIdx] xor LZ[LIdx] xor LPCurrent[LIdx]; + end + else + begin + for LIdx := 0 to 127 do + LPCurrent[LIdx] := LR[LIdx] xor LZ[LIdx]; + end; +end; + { TPBKDF_Argon2NotBuildInAdapter.TBlock } class function TPBKDF_Argon2NotBuildInAdapter.TBlock.CreateBlock: TBlock; @@ -1315,5 +1409,8 @@ destructor TPBKDF_Argon2NotBuildInAdapter.Destroy; inherited Destroy; end; +initialization + Argon2_FillBlock := TArgon2Simd.Select(@Argon2_FillBlock_Scalar); + end. diff --git a/HashLib/src/KDF/HlpPBKDF_ScryptNotBuildInAdapter.pas b/HashLib/src/KDF/HlpPBKDF_ScryptNotBuildInAdapter.pas index 45d69797..997cda31 100644 --- a/HashLib/src/KDF/HlpPBKDF_ScryptNotBuildInAdapter.pas +++ b/HashLib/src/KDF/HlpPBKDF_ScryptNotBuildInAdapter.pas @@ -15,10 +15,18 @@ interface HlpIHashInfo, HlpBinaryPrimitives, HlpPBKDF2_HMACNotBuildInAdapter, - HlpScryptDispatch, HlpArrayUtils, HlpHashLibTypes; +type + TScryptSalsaXorProc = procedure(AState, AInput: Pointer); + +var + Scrypt_SalsaXor: TScryptSalsaXorProc; + +procedure Scrypt_Permute(ABlock: PCardinal; AChunkCount: Int32); +procedure Scrypt_Unpermute(ABlock: PCardinal; AChunkCount: Int32); + resourcestring SInvalidByteCount = '"(AByteCount)" Argument must be a value greater than zero.'; @@ -115,6 +123,157 @@ TPBKDF_ScryptNotBuildInAdapter = class sealed(TKDF, IPBKDF_Scrypt, implementation +uses + HlpBitOperations, + HlpScryptSimd; + +// ============================================================================= +// Percival's (i*5 mod 16) permutation rearranges each 16-word Salsa20 state +// from natural order into role-based diagonal order so that column and row +// quarter-rounds map to lane-parallel SIMD operations. Applied once at +// SMixLane entry/exit; all intermediate data stays in permuted order. +// Reference: Colin Percival, crypto_scrypt-sse.c (Tarsnap). +// ============================================================================= + +procedure Scrypt_Permute(ABlock: PCardinal; AChunkCount: Int32); +var + LTemp: array[0..15] of UInt32; + LIdx: Int32; +begin + while AChunkCount > 0 do + begin + for LIdx := 0 to 15 do + LTemp[LIdx] := ABlock[(LIdx * 5) and 15]; + System.Move(LTemp, ABlock^, 64); + Inc(ABlock, 16); + Dec(AChunkCount); + end; +end; + +procedure Scrypt_Unpermute(ABlock: PCardinal; AChunkCount: Int32); +var + LTemp: array[0..15] of UInt32; + LIdx: Int32; +begin + while AChunkCount > 0 do + begin + for LIdx := 0 to 15 do + LTemp[LIdx] := ABlock[(LIdx * 13) and 15]; + System.Move(LTemp, ABlock^, 64); + Inc(ABlock, 16); + Dec(AChunkCount); + end; +end; + +// ============================================================================= +// Scalar fallback: fused XOR + Salsa20/8 on Percival-permuted data. +// Loads from permuted positions into natural-named locals, performs standard +// Salsa20/8 column+row rounds, stores back to permuted positions. +// ============================================================================= + +procedure Scrypt_SalsaXor_Scalar(AState, AInput: Pointer); +var + LW0, LW1, LW2, LW3, LW4, LW5, LW6, LW7, + LW8, LW9, LW10, LW11, LW12, LW13, LW14, LW15: UInt32; + LPS, LPI: PCardinal; + LIdx: Int32; +begin + LPS := PCardinal(AState); + LPI := PCardinal(AInput); + + // Permuted layout: {w0,w5,w10,w15, w4,w9,w14,w3, w8,w13,w2,w7, w12,w1,w6,w11} + LW0 := LPS[0] xor LPI[0]; + LW5 := LPS[1] xor LPI[1]; + LW10 := LPS[2] xor LPI[2]; + LW15 := LPS[3] xor LPI[3]; + LW4 := LPS[4] xor LPI[4]; + LW9 := LPS[5] xor LPI[5]; + LW14 := LPS[6] xor LPI[6]; + LW3 := LPS[7] xor LPI[7]; + LW8 := LPS[8] xor LPI[8]; + LW13 := LPS[9] xor LPI[9]; + LW2 := LPS[10] xor LPI[10]; + LW7 := LPS[11] xor LPI[11]; + LW12 := LPS[12] xor LPI[12]; + LW1 := LPS[13] xor LPI[13]; + LW6 := LPS[14] xor LPI[14]; + LW11 := LPS[15] xor LPI[15]; + + LPS[0] := LW0; + LPS[1] := LW5; + LPS[2] := LW10; + LPS[3] := LW15; + LPS[4] := LW4; + LPS[5] := LW9; + LPS[6] := LW14; + LPS[7] := LW3; + LPS[8] := LW8; + LPS[9] := LW13; + LPS[10] := LW2; + LPS[11] := LW7; + LPS[12] := LW12; + LPS[13] := LW1; + LPS[14] := LW6; + LPS[15] := LW11; + + LIdx := 4; + while LIdx > 0 do + begin + LW4 := LW4 xor TBitOperations.RotateLeft32(LW0 + LW12, 7); + LW8 := LW8 xor TBitOperations.RotateLeft32(LW4 + LW0, 9); + LW12 := LW12 xor TBitOperations.RotateLeft32(LW8 + LW4, 13); + LW0 := LW0 xor TBitOperations.RotateLeft32(LW12 + LW8, 18); + LW9 := LW9 xor TBitOperations.RotateLeft32(LW5 + LW1, 7); + LW13 := LW13 xor TBitOperations.RotateLeft32(LW9 + LW5, 9); + LW1 := LW1 xor TBitOperations.RotateLeft32(LW13 + LW9, 13); + LW5 := LW5 xor TBitOperations.RotateLeft32(LW1 + LW13, 18); + LW14 := LW14 xor TBitOperations.RotateLeft32(LW10 + LW6, 7); + LW2 := LW2 xor TBitOperations.RotateLeft32(LW14 + LW10, 9); + LW6 := LW6 xor TBitOperations.RotateLeft32(LW2 + LW14, 13); + LW10 := LW10 xor TBitOperations.RotateLeft32(LW6 + LW2, 18); + LW3 := LW3 xor TBitOperations.RotateLeft32(LW15 + LW11, 7); + LW7 := LW7 xor TBitOperations.RotateLeft32(LW3 + LW15, 9); + LW11 := LW11 xor TBitOperations.RotateLeft32(LW7 + LW3, 13); + LW15 := LW15 xor TBitOperations.RotateLeft32(LW11 + LW7, 18); + + LW1 := LW1 xor TBitOperations.RotateLeft32(LW0 + LW3, 7); + LW2 := LW2 xor TBitOperations.RotateLeft32(LW1 + LW0, 9); + LW3 := LW3 xor TBitOperations.RotateLeft32(LW2 + LW1, 13); + LW0 := LW0 xor TBitOperations.RotateLeft32(LW3 + LW2, 18); + LW6 := LW6 xor TBitOperations.RotateLeft32(LW5 + LW4, 7); + LW7 := LW7 xor TBitOperations.RotateLeft32(LW6 + LW5, 9); + LW4 := LW4 xor TBitOperations.RotateLeft32(LW7 + LW6, 13); + LW5 := LW5 xor TBitOperations.RotateLeft32(LW4 + LW7, 18); + LW11 := LW11 xor TBitOperations.RotateLeft32(LW10 + LW9, 7); + LW8 := LW8 xor TBitOperations.RotateLeft32(LW11 + LW10, 9); + LW9 := LW9 xor TBitOperations.RotateLeft32(LW8 + LW11, 13); + LW10 := LW10 xor TBitOperations.RotateLeft32(LW9 + LW8, 18); + LW12 := LW12 xor TBitOperations.RotateLeft32(LW15 + LW14, 7); + LW13 := LW13 xor TBitOperations.RotateLeft32(LW12 + LW15, 9); + LW14 := LW14 xor TBitOperations.RotateLeft32(LW13 + LW12, 13); + LW15 := LW15 xor TBitOperations.RotateLeft32(LW14 + LW13, 18); + + System.Dec(LIdx); + end; + + LPS[0] := LPS[0] + LW0; + LPS[1] := LPS[1] + LW5; + LPS[2] := LPS[2] + LW10; + LPS[3] := LPS[3] + LW15; + LPS[4] := LPS[4] + LW4; + LPS[5] := LPS[5] + LW9; + LPS[6] := LPS[6] + LW14; + LPS[7] := LPS[7] + LW3; + LPS[8] := LPS[8] + LW8; + LPS[9] := LPS[9] + LW13; + LPS[10] := LPS[10] + LW2; + LPS[11] := LPS[11] + LW7; + LPS[12] := LPS[12] + LW12; + LPS[13] := LPS[13] + LW1; + LPS[14] := LPS[14] + LW6; + LPS[15] := LPS[15] + LW11; +end; + { TPBKDF_ScryptNotBuildInAdapter } class function TPBKDF_ScryptNotBuildInAdapter.IsPowerOf2(AValue: Int32): Boolean; @@ -346,4 +505,7 @@ function TPBKDF_ScryptNotBuildInAdapter.GetBytes(AByteCount: Int32) AByteCount); end; +initialization + Scrypt_SalsaXor := TScryptSimd.Select(@Scrypt_SalsaXor_Scalar); + end. diff --git a/HashLib/src/KDF/HlpScryptDispatch.pas b/HashLib/src/KDF/HlpScryptDispatch.pas deleted file mode 100644 index 8d0450c1..00000000 --- a/HashLib/src/KDF/HlpScryptDispatch.pas +++ /dev/null @@ -1,264 +0,0 @@ -unit HlpScryptDispatch; - -{$I ..\Include\HashLib.inc} - -interface - -type - TScryptSalsaXorProc = procedure(AState, AInput: Pointer); - -var - Scrypt_SalsaXor: TScryptSalsaXorProc; - -procedure Scrypt_Permute(ABlock: PCardinal; AChunkCount: Int32); -procedure Scrypt_Unpermute(ABlock: PCardinal; AChunkCount: Int32); - -implementation - -uses - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; - -// ============================================================================= -// Percival's (i*5 mod 16) permutation rearranges each 16-word Salsa20 state -// from natural order into role-based diagonal order so that column and row -// quarter-rounds map to lane-parallel SIMD operations. Applied once at -// SMixLane entry/exit; all intermediate data stays in permuted order. -// Reference: Colin Percival, crypto_scrypt-sse.c (Tarsnap). -// ============================================================================= - -procedure Scrypt_Permute(ABlock: PCardinal; AChunkCount: Int32); -var - LTemp: array[0..15] of UInt32; - LIdx: Int32; -begin - while AChunkCount > 0 do - begin - for LIdx := 0 to 15 do - LTemp[LIdx] := ABlock[(LIdx * 5) and 15]; - System.Move(LTemp, ABlock^, 64); - Inc(ABlock, 16); - Dec(AChunkCount); - end; -end; - -procedure Scrypt_Unpermute(ABlock: PCardinal; AChunkCount: Int32); -var - LTemp: array[0..15] of UInt32; - LIdx: Int32; -begin - while AChunkCount > 0 do - begin - for LIdx := 0 to 15 do - LTemp[LIdx] := ABlock[(LIdx * 13) and 15]; - System.Move(LTemp, ABlock^, 64); - Inc(ABlock, 16); - Dec(AChunkCount); - end; -end; - -// ============================================================================= -// Scalar fallback: fused XOR + Salsa20/8 on Percival-permuted data. -// Loads from permuted positions into natural-named locals, performs standard -// Salsa20/8 column+row rounds, stores back to permuted positions. -// ============================================================================= - -procedure Scrypt_SalsaXor_Scalar(AState, AInput: Pointer); -var - LW0, LW1, LW2, LW3, LW4, LW5, LW6, LW7, - LW8, LW9, LW10, LW11, LW12, LW13, LW14, LW15: UInt32; - LPS, LPI: PCardinal; - LIdx: Int32; -begin - LPS := PCardinal(AState); - LPI := PCardinal(AInput); - - // Permuted layout: {w0,w5,w10,w15, w4,w9,w14,w3, w8,w13,w2,w7, w12,w1,w6,w11} - LW0 := LPS[0] xor LPI[0]; - LW5 := LPS[1] xor LPI[1]; - LW10 := LPS[2] xor LPI[2]; - LW15 := LPS[3] xor LPI[3]; - LW4 := LPS[4] xor LPI[4]; - LW9 := LPS[5] xor LPI[5]; - LW14 := LPS[6] xor LPI[6]; - LW3 := LPS[7] xor LPI[7]; - LW8 := LPS[8] xor LPI[8]; - LW13 := LPS[9] xor LPI[9]; - LW2 := LPS[10] xor LPI[10]; - LW7 := LPS[11] xor LPI[11]; - LW12 := LPS[12] xor LPI[12]; - LW1 := LPS[13] xor LPI[13]; - LW6 := LPS[14] xor LPI[14]; - LW11 := LPS[15] xor LPI[15]; - - LPS[0] := LW0; - LPS[1] := LW5; - LPS[2] := LW10; - LPS[3] := LW15; - LPS[4] := LW4; - LPS[5] := LW9; - LPS[6] := LW14; - LPS[7] := LW3; - LPS[8] := LW8; - LPS[9] := LW13; - LPS[10] := LW2; - LPS[11] := LW7; - LPS[12] := LW12; - LPS[13] := LW1; - LPS[14] := LW6; - LPS[15] := LW11; - - LIdx := 4; - while LIdx > 0 do - begin - LW4 := LW4 xor TBitOperations.RotateLeft32(LW0 + LW12, 7); - LW8 := LW8 xor TBitOperations.RotateLeft32(LW4 + LW0, 9); - LW12 := LW12 xor TBitOperations.RotateLeft32(LW8 + LW4, 13); - LW0 := LW0 xor TBitOperations.RotateLeft32(LW12 + LW8, 18); - LW9 := LW9 xor TBitOperations.RotateLeft32(LW5 + LW1, 7); - LW13 := LW13 xor TBitOperations.RotateLeft32(LW9 + LW5, 9); - LW1 := LW1 xor TBitOperations.RotateLeft32(LW13 + LW9, 13); - LW5 := LW5 xor TBitOperations.RotateLeft32(LW1 + LW13, 18); - LW14 := LW14 xor TBitOperations.RotateLeft32(LW10 + LW6, 7); - LW2 := LW2 xor TBitOperations.RotateLeft32(LW14 + LW10, 9); - LW6 := LW6 xor TBitOperations.RotateLeft32(LW2 + LW14, 13); - LW10 := LW10 xor TBitOperations.RotateLeft32(LW6 + LW2, 18); - LW3 := LW3 xor TBitOperations.RotateLeft32(LW15 + LW11, 7); - LW7 := LW7 xor TBitOperations.RotateLeft32(LW3 + LW15, 9); - LW11 := LW11 xor TBitOperations.RotateLeft32(LW7 + LW3, 13); - LW15 := LW15 xor TBitOperations.RotateLeft32(LW11 + LW7, 18); - - LW1 := LW1 xor TBitOperations.RotateLeft32(LW0 + LW3, 7); - LW2 := LW2 xor TBitOperations.RotateLeft32(LW1 + LW0, 9); - LW3 := LW3 xor TBitOperations.RotateLeft32(LW2 + LW1, 13); - LW0 := LW0 xor TBitOperations.RotateLeft32(LW3 + LW2, 18); - LW6 := LW6 xor TBitOperations.RotateLeft32(LW5 + LW4, 7); - LW7 := LW7 xor TBitOperations.RotateLeft32(LW6 + LW5, 9); - LW4 := LW4 xor TBitOperations.RotateLeft32(LW7 + LW6, 13); - LW5 := LW5 xor TBitOperations.RotateLeft32(LW4 + LW7, 18); - LW11 := LW11 xor TBitOperations.RotateLeft32(LW10 + LW9, 7); - LW8 := LW8 xor TBitOperations.RotateLeft32(LW11 + LW10, 9); - LW9 := LW9 xor TBitOperations.RotateLeft32(LW8 + LW11, 13); - LW10 := LW10 xor TBitOperations.RotateLeft32(LW9 + LW8, 18); - LW12 := LW12 xor TBitOperations.RotateLeft32(LW15 + LW14, 7); - LW13 := LW13 xor TBitOperations.RotateLeft32(LW12 + LW15, 9); - LW14 := LW14 xor TBitOperations.RotateLeft32(LW13 + LW12, 13); - LW15 := LW15 xor TBitOperations.RotateLeft32(LW14 + LW13, 18); - - System.Dec(LIdx); - end; - - LPS[0] := LPS[0] + LW0; - LPS[1] := LPS[1] + LW5; - LPS[2] := LPS[2] + LW10; - LPS[3] := LPS[3] + LW15; - LPS[4] := LPS[4] + LW4; - LPS[5] := LPS[5] + LW9; - LPS[6] := LPS[6] + LW14; - LPS[7] := LPS[7] + LW3; - LPS[8] := LPS[8] + LW8; - LPS[9] := LPS[9] + LW13; - LPS[10] := LPS[10] + LW2; - LPS[11] := LPS[11] + LW7; - LPS[12] := LPS[12] + LW12; - LPS[13] := LPS[13] + LW1; - LPS[14] := LPS[14] + LW6; - LPS[15] := LPS[15] + LW11; -end; - -// ============================================================================= -// SIMD implementations -// -// i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: NEON (not registered) -// ============================================================================= - -{$IFDEF HASHLIB_I386_ASM} - -procedure Scrypt_SalsaXor_Sse2(AState, AInput: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} - {$I ..\Include\Simd\Scrypt\ScryptSalsa8Sse2_i386.inc} -end; - -{$ENDIF HASHLIB_I386_ASM} - -{$IFDEF HASHLIB_X86_64_ASM} - -procedure Scrypt_SalsaXor_Sse2(AState, AInput: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} - {$I ..\Include\Simd\Scrypt\ScryptSalsa8Sse2_x86_64.inc} -end; - -procedure Scrypt_SalsaXor_Avx2(AState, AInput: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} - {$I ..\Include\Simd\Scrypt\ScryptSalsa8Avx2_x86_64.inc} -end; - -{$ENDIF HASHLIB_X86_64_ASM} - -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure Scrypt_SalsaXor_Neon(AState, AInput: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} - {$I ..\Include\Simd\Scrypt\ScryptSalsa8Neon_aarch64.inc} -end; - -{$ENDIF HASHLIB_AARCH64_ASM} - -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); -begin - Scrypt_SalsaXor := @Scrypt_SalsaXor_Scalar; -{$IFDEF HASHLIB_I386_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of - TX86SimdLevel.SSE2: - begin - Scrypt_SalsaXor := @Scrypt_SalsaXor_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of - TX86SimdLevel.AVX2: - begin - Scrypt_SalsaXor := @Scrypt_SalsaXor_Avx2; - end; - TX86SimdLevel.SSE2: - begin - Scrypt_SalsaXor := @Scrypt_SalsaXor_Sse2; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - // NEON intentionally NOT registered for scrypt: the scalar path is faster on - // AArch64. scrypt's Salsa20/8 is a single 64-byte block with a strictly serial - // dependency chain (BlockMix feeds each Salsa call from the previous one, and - // SMix is sequential by design), so there is no block-level parallelism for - // SIMD lanes to exploit at p=1. Meanwhile AArch64's 31 general-purpose - // registers let the scalar kernel hold the whole 16-word state with no spills - // and rotate via a single-cycle 'ror', whereas NEON adds a lane-shuffle (ext) - // tax and a 2-instruction (shl+sri) rotate. Benchmarks (Apple Silicon, FPC - // 3.2.2) confirm scalar wins at every N (e.g. N=16384: ~39 ms scalar vs - // ~56 ms NEON). This mirrors OpenSSL/libsodium/Tarsnap, which ship an x86 SSE2 - // scrypt (to relieve x86's 16-register pressure) but no NEON variant. The - // Scrypt_SalsaXor_Neon kernel is kept (verified, correct) for reference and in - // case a future p>1 / multi-block batched path makes SIMD worthwhile. - // - // case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of - // TArmSimdLevel.NEON: - // begin - // Scrypt_SalsaXor := @Scrypt_SalsaXor_Neon; - // end; - // end; -{$ENDIF} -end; - -initialization - InitDispatch(); - -end. diff --git a/HashLib/src/Packages/Delphi/HashLib4PascalPackage.dpk b/HashLib/src/Packages/Delphi/HashLib4PascalPackage.dpk index 535d6c41..96126128 100644 --- a/HashLib/src/Packages/Delphi/HashLib4PascalPackage.dpk +++ b/HashLib/src/Packages/Delphi/HashLib4PascalPackage.dpk @@ -46,12 +46,14 @@ contains HlpHMACNotBuildInAdapter in '..\..\Base\HlpHMACNotBuildInAdapter.pas', HlpMultipleTransformNonBlock in '..\..\Base\HlpMultipleTransformNonBlock.pas', HlpAdler32 in '..\..\Checksum\HlpAdler32.pas', - HlpAdler32Dispatch in '..\..\Checksum\HlpAdler32Dispatch.pas', + HlpAdler32Simd in '..\..\Simd\Facade\HlpAdler32Simd.pas', + HlpAdler32X86Backend in '..\..\Simd\Backend\HlpAdler32X86Backend.pas', + HlpAdler32ArmBackend in '..\..\Simd\Backend\HlpAdler32ArmBackend.pas', HlpCRC in '..\..\Checksum\HlpCRC.pas', - HlpCRCDispatch in '..\..\Checksum\HlpCRCDispatch.pas', - HlpCRCFoldConstants in '..\..\Checksum\HlpCRCFoldConstants.pas', - HlpCRCStandard in '..\..\Checksum\HlpCRCStandard.pas', - HlpCRC32Fast in '..\..\Checksum\HlpCRC32Fast.pas', + HlpCRCCore in '..\..\Checksum\HlpCRCCore.pas', + HlpCRCSimd in '..\..\Simd\Facade\HlpCRCSimd.pas', + HlpCRCX86Backend in '..\..\Simd\Backend\HlpCRCX86Backend.pas', + HlpCRCArmBackend in '..\..\Simd\Backend\HlpCRCArmBackend.pas', HlpGost in '..\..\Crypto\HlpGost.pas', HlpGrindahl256 in '..\..\Crypto\HlpGrindahl256.pas', HlpGrindahl512 in '..\..\Crypto\HlpGrindahl512.pas', @@ -71,27 +73,39 @@ contains HlpRIPEMD320 in '..\..\Crypto\HlpRIPEMD320.pas', HlpSHA0 in '..\..\Crypto\HlpSHA0.pas', HlpSHA1 in '..\..\Crypto\HlpSHA1.pas', - HlpSHA1Dispatch in '..\..\Crypto\HlpSHA1Dispatch.pas', + HlpSHA1Simd in '..\..\Simd\Facade\HlpSHA1Simd.pas', + HlpSHA1X86Backend in '..\..\Simd\Backend\HlpSHA1X86Backend.pas', + HlpSHA1ArmBackend in '..\..\Simd\Backend\HlpSHA1ArmBackend.pas', HlpSHA2_224 in '..\..\Crypto\HlpSHA2_224.pas', HlpSHA2_256 in '..\..\Crypto\HlpSHA2_256.pas', - HlpSHA2_256Dispatch in '..\..\Crypto\HlpSHA2_256Dispatch.pas', - HlpSHA2_512Dispatch in '..\..\Crypto\HlpSHA2_512Dispatch.pas', + HlpSHA2_256Simd in '..\..\Simd\Facade\HlpSHA2_256Simd.pas', + HlpSHA2_256X86Backend in '..\..\Simd\Backend\HlpSHA2_256X86Backend.pas', + HlpSHA2_256ArmBackend in '..\..\Simd\Backend\HlpSHA2_256ArmBackend.pas', + HlpSHA2_512Simd in '..\..\Simd\Facade\HlpSHA2_512Simd.pas', + HlpSHA2_512X86Backend in '..\..\Simd\Backend\HlpSHA2_512X86Backend.pas', + HlpSHA2_512ArmBackend in '..\..\Simd\Backend\HlpSHA2_512ArmBackend.pas', HlpSHA2_256Base in '..\..\Crypto\HlpSHA2_256Base.pas', HlpSHA2_384 in '..\..\Crypto\HlpSHA2_384.pas', HlpSHA2_512 in '..\..\Crypto\HlpSHA2_512.pas', HlpSHA2_512_224 in '..\..\Crypto\HlpSHA2_512_224.pas', HlpSHA2_512_256 in '..\..\Crypto\HlpSHA2_512_256.pas', HlpSHA2_512Base in '..\..\Crypto\HlpSHA2_512Base.pas', - HlpSHA3Dispatch in '..\..\Crypto\HlpSHA3Dispatch.pas', + HlpSHA3Simd in '..\..\Simd\Facade\HlpSHA3Simd.pas', + HlpSHA3X86Backend in '..\..\Simd\Backend\HlpSHA3X86Backend.pas', + HlpSHA3ArmBackend in '..\..\Simd\Backend\HlpSHA3ArmBackend.pas', HlpSHA3 in '..\..\Crypto\HlpSHA3.pas', HlpSnefru in '..\..\Crypto\HlpSnefru.pas', HlpTiger in '..\..\Crypto\HlpTiger.pas', HlpTiger2 in '..\..\Crypto\HlpTiger2.pas', HlpWhirlPool in '..\..\Crypto\HlpWhirlPool.pas', HlpGOST3411_2012 in '..\..\Crypto\HlpGOST3411_2012.pas', - HlpBlake2BDispatch in '..\..\Crypto\HlpBlake2BDispatch.pas', + HlpBlake2BSimd in '..\..\Simd\Facade\HlpBlake2BSimd.pas', + HlpBlake2BX86Backend in '..\..\Simd\Backend\HlpBlake2BX86Backend.pas', + HlpBlake2BArmBackend in '..\..\Simd\Backend\HlpBlake2BArmBackend.pas', HlpBlake2B in '..\..\Crypto\HlpBlake2B.pas', - HlpBlake2SDispatch in '..\..\Crypto\HlpBlake2SDispatch.pas', + HlpBlake2SSimd in '..\..\Simd\Facade\HlpBlake2SSimd.pas', + HlpBlake2SX86Backend in '..\..\Simd\Backend\HlpBlake2SX86Backend.pas', + HlpBlake2SArmBackend in '..\..\Simd\Backend\HlpBlake2SArmBackend.pas', HlpBlake2S in '..\..\Crypto\HlpBlake2S.pas', HlpBlake2BParams in '..\..\Crypto\Blake2BParams\HlpBlake2BParams.pas', HlpBlake2SParams in '..\..\Crypto\Blake2SParams\HlpBlake2SParams.pas', @@ -122,7 +136,9 @@ contains HlpMurmur2_64 in '..\..\Hash64\HlpMurmur2_64.pas', HlpSipHash in '..\..\Hash64\HlpSipHash.pas', HlpXXHash64 in '..\..\Hash64\HlpXXHash64.pas', - HlpXXHash3Dispatch in '..\..\Hash64\HlpXXHash3Dispatch.pas', + HlpXXHash3Simd in '..\..\Simd\Facade\HlpXXHash3Simd.pas', + HlpXXHash3X86Backend in '..\..\Simd\Backend\HlpXXHash3X86Backend.pas', + HlpXXHash3ArmBackend in '..\..\Simd\Backend\HlpXXHash3ArmBackend.pas', HlpXXHash3 in '..\..\Hash64\HlpXXHash3.pas', HlpMurmurHash3_x86_128 in '..\..\Hash128\HlpMurmurHash3_x86_128.pas', HlpMurmurHash3_x64_128 in '..\..\Hash128\HlpMurmurHash3_x64_128.pas', @@ -137,14 +153,20 @@ contains HlpIBlake2SParams in '..\..\Interfaces\IBlake2SParams\HlpIBlake2SParams.pas', HlpBlake2BP in '..\..\Crypto\HlpBlake2BP.pas', HlpBlake2SP in '..\..\Crypto\HlpBlake2SP.pas', - HlpBlake3Dispatch in '..\..\Crypto\HlpBlake3Dispatch.pas', + HlpBlake3Simd in '..\..\Simd\Facade\HlpBlake3Simd.pas', + HlpBlake3X86Backend in '..\..\Simd\Backend\HlpBlake3X86Backend.pas', + HlpBlake3ArmBackend in '..\..\Simd\Backend\HlpBlake3ArmBackend.pas', HlpBlake3 in '..\..\Crypto\HlpBlake3.pas', HlpPBKDF2_HMACNotBuildInAdapter in '..\..\KDF\HlpPBKDF2_HMACNotBuildInAdapter.pas', HlpPBKDF_Argon2NotBuildInAdapter in '..\..\KDF\HlpPBKDF_Argon2NotBuildInAdapter.pas', - HlpArgon2Dispatch in '..\..\KDF\HlpArgon2Dispatch.pas', + HlpArgon2Simd in '..\..\Simd\Facade\HlpArgon2Simd.pas', + HlpArgon2X86Backend in '..\..\Simd\Backend\HlpArgon2X86Backend.pas', + HlpArgon2ArmBackend in '..\..\Simd\Backend\HlpArgon2ArmBackend.pas', HlpArgon2TypeAndVersion in '..\..\KDF\HlpArgon2TypeAndVersion.pas', HlpPBKDF_ScryptNotBuildInAdapter in '..\..\KDF\HlpPBKDF_ScryptNotBuildInAdapter.pas', - HlpScryptDispatch in '..\..\KDF\HlpScryptDispatch.pas', + HlpScryptSimd in '..\..\Simd\Facade\HlpScryptSimd.pas', + HlpScryptX86Backend in '..\..\Simd\Backend\HlpScryptX86Backend.pas', + HlpScryptArmBackend in '..\..\Simd\Backend\HlpScryptArmBackend.pas', HlpConverters in '..\..\Utils\HlpConverters.pas', HlpBitOperations in '..\..\Utils\HlpBitOperations.pas', HlpBinaryPrimitives in '..\..\Utils\HlpBinaryPrimitives.pas', diff --git a/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk b/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk index d3300a50..4b6d6e78 100644 --- a/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk +++ b/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk @@ -10,7 +10,7 @@ - + @@ -29,7 +29,7 @@ "/> - + @@ -75,453 +75,541 @@ - - - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + - + - - - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/HashLib/src/Packages/FPC/HashLib4PascalPackage.pas b/HashLib/src/Packages/FPC/HashLib4PascalPackage.pas index 08e2ee37..26608075 100644 --- a/HashLib/src/Packages/FPC/HashLib4PascalPackage.pas +++ b/HashLib/src/Packages/FPC/HashLib4PascalPackage.pas @@ -10,31 +10,38 @@ interface uses HlpHash, HlpHashBuffer, HlpHashCryptoNotBuildIn, HlpHashFactory, HlpHashResult, HlpHashRounds, HlpHashSize, HlpHMACNotBuildInAdapter, - HlpMultipleTransformNonBlock, HlpAdler32, HlpCRC, HlpCRCStandard, - HlpGost, HlpGrindahl256, HlpGrindahl512, HlpHAS160, HlpHaval, - HlpMD2, HlpMD4, HlpMD5, HlpMDBase, HlpPanama, HlpRadioGatun32, - HlpRadioGatun64, HlpRIPEMD, HlpRIPEMD128, HlpRIPEMD160, HlpRIPEMD256, - HlpRIPEMD320, HlpSHA0, HlpSHA1, HlpSHA2_224, HlpSHA2_256, HlpSHA2_256Base, - HlpSHA2_384, HlpSHA2_512, HlpSHA2_512Base, HlpSHA2_512_224, HlpSHA2_512_256, - HlpSHA3, HlpSnefru, HlpTiger, HlpTiger2, HlpWhirlPool, - HlpMurmurHash3_x64_128, HlpNullDigest, HlpAP, HlpBernstein, HlpBernstein1, - HlpBKDR, HlpDEK, HlpDJB, HlpELF, HlpFNV, HlpFNV1a, HlpJenkins3, HlpJS, - HlpMurmur2, HlpMurmurHash3_x86_32, HlpOneAtTime, HlpPJW, HlpRotating, HlpRS, - HlpSDBM, HlpShiftAndXor, HlpSuperFast, HlpXXHash32, HlpFNV1a64, HlpFNV64, - HlpMurmur2_64, HlpSipHash, HlpXXHash64, HlpIHash, HlpIHashInfo, - HlpIHashResult, HlpConverters, HlpBitOperations, HlpBinaryPrimitives, HlpHashLibTypes, + HlpMultipleTransformNonBlock, HlpAdler32, HlpCRC, HlpGost, + HlpGrindahl256, HlpGrindahl512, HlpHAS160, HlpHaval, HlpMD2, HlpMD4, HlpMD5, + HlpMDBase, HlpPanama, HlpRadioGatun32, HlpRadioGatun64, HlpRIPEMD, + HlpRIPEMD128, HlpRIPEMD160, HlpRIPEMD256, HlpRIPEMD320, HlpSHA0, HlpSHA1, + HlpSHA2_224, HlpSHA2_256, HlpSHA2_256Base, HlpSHA2_384, HlpSHA2_512, + HlpSHA2_512Base, HlpSHA2_512_224, HlpSHA2_512_256, HlpSHA3, HlpSnefru, + HlpTiger, HlpTiger2, HlpWhirlPool, HlpMurmurHash3_x64_128, HlpNullDigest, + HlpAP, HlpBernstein, HlpBernstein1, HlpBKDR, HlpDEK, HlpDJB, HlpELF, HlpFNV, + HlpFNV1a, HlpJenkins3, HlpJS, HlpMurmur2, HlpMurmurHash3_x86_32, + HlpOneAtTime, HlpPJW, HlpRotating, HlpRS, HlpSDBM, HlpShiftAndXor, + HlpSuperFast, HlpXXHash32, HlpFNV1a64, HlpFNV64, HlpMurmur2_64, HlpSipHash, + HlpXXHash64, HlpIHash, HlpIHashInfo, HlpIHashResult, HlpConverters, + HlpBitOperations, HlpBinaryPrimitives, HlpHashLibTypes, HlpMurmurHash3_x86_128, HlpPBKDF2_HMACNotBuildInAdapter, HlpIKDF, HlpKDF, - HlpICRC, HlpBlake2B, HlpBlake2S, HlpGOST3411_2012, HlpCRC32Fast, + HlpICRC, HlpBlake2B, HlpBlake2S, HlpGOST3411_2012, HlpArgon2TypeAndVersion, HlpPBKDF_Argon2NotBuildInAdapter, HlpPBKDF_ScryptNotBuildInAdapter, HlpArrayUtils, HlpBlake2BP, HlpBlake2SP, HlpSipHash128, HlpBlake2SParams, HlpBlake2BParams, HlpIBlake2SParams, HlpIBlake2BParams, HlpBlake3, HlpXXHash3, HlpXXHash128, HlpCpuFeatures, - HlpXXHash3Dispatch, HlpBlake2BDispatch, HlpBlake2SDispatch, - HlpArgon2Dispatch, HlpScryptDispatch, HlpBlake3Dispatch, - HlpSHA2_256Dispatch, HlpSHA2_512Dispatch, HlpSHA1Dispatch, - HlpAdler32Dispatch, HlpCRCFoldConstants, HlpCRCDispatch, HlpSHA3Dispatch, + HlpXXHash3Simd, HlpBlake2BSimd, HlpBlake2SSimd, HlpArgon2Simd, + HlpScryptSimd, HlpBlake3Simd, HlpSHA2_256Simd, HlpSHA2_512Simd, HlpSHA1Simd, + HlpAdler32Simd, HlpCRCCore, HlpSHA3Simd, HlpArmSimdFeatures, HlpSimdLevels, HlpX86SimdFeatures, HlpArmHwCapProvider, - HlpDarwinSysCtl; + HlpDarwinSysCtl, HlpSHA2_256X86Backend, HlpSHA2_256ArmBackend, + HlpSHA1X86Backend, HlpSHA1ArmBackend, HlpSHA2_512X86Backend, + HlpSHA2_512ArmBackend, HlpSHA3X86Backend, HlpSHA3ArmBackend, + HlpBlake2BX86Backend, HlpBlake2BArmBackend, HlpBlake2SX86Backend, + HlpBlake2SArmBackend, HlpBlake3X86Backend, HlpBlake3ArmBackend, + HlpXXHash3X86Backend, HlpXXHash3ArmBackend, HlpAdler32X86Backend, + HlpAdler32ArmBackend, HlpArgon2X86Backend, HlpArgon2ArmBackend, + HlpScryptX86Backend, HlpScryptArmBackend, HlpCRCSimd, HlpCRCX86Backend, + HlpCRCArmBackend; implementation diff --git a/HashLib/src/Simd/Backend/HlpAdler32ArmBackend.pas b/HashLib/src/Simd/Backend/HlpAdler32ArmBackend.pas new file mode 100644 index 00000000..67f8012b --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpAdler32ArmBackend.pas @@ -0,0 +1,60 @@ +unit HlpAdler32ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpAdler32; + +type + /// + /// Arm SIMD backend for Adler-32: owns the AArch64 NEON block-processing kernel + /// (body in Include\Simd\Adler32\) and the runtime tier selection via + /// TCpuFeatures.Arm. Compiles on every target - without Arm SIMD, + /// Select just returns the scalar routine. + /// + TAdler32ArmBackend = class sealed + public + class function Select(AScalar: TAdler32UpdateProc): TAdler32UpdateProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: NEON +// ============================================================================= + +procedure Adler32_ProcessBlocks_Neon(AData: PByte; ANumBlocks: UInt32; + ASums, AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\Adler32\Adler32BlocksNeon_aarch64.inc} +end; + +procedure Adler32_Update_Neon(AData: PByte; ALength: UInt32; ASums: Pointer); +begin + Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Neon); +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TAdler32ArmBackend } + +class function TAdler32ArmBackend.Select(AScalar: TAdler32UpdateProc): TAdler32UpdateProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: Exit(@Adler32_Update_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpAdler32X86Backend.pas b/HashLib/src/Simd/Backend/HlpAdler32X86Backend.pas new file mode 100644 index 00000000..4e41cba9 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpAdler32X86Backend.pas @@ -0,0 +1,108 @@ +unit HlpAdler32X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpAdler32; + +type + /// + /// x86 SIMD backend for Adler-32: owns the SSE2 / SSSE3 / AVX2 block-processing + /// kernels (bodies in Include\Simd\Adler32\) and the runtime tier + /// selection via TCpuFeatures.X86. The SSSE3 tier is kept (its + /// pmaddubsw is materially faster than the SSE2 emulation). Compiles on every + /// target - without x86 SIMD, Select just returns the scalar routine. + /// + TAdler32X86Backend = class sealed + public + class function Select(AScalar: TAdler32UpdateProc): TAdler32UpdateProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// i386: AVX2, SSSE3, SSE2 +// x86_64: AVX2, SSSE3, SSE2 +// ============================================================================= + +procedure Adler32_ProcessBlocks_Sse2(AData: PByte; ANumBlocks: UInt32; + ASums, AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Adler32\Adler32BlocksSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Adler32\Adler32BlocksSse2_i386.inc} +{$ENDIF} +end; + +procedure Adler32_ProcessBlocks_Ssse3(AData: PByte; ANumBlocks: UInt32; + ASums, AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Adler32\Adler32BlocksSsse3_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Adler32\Adler32BlocksSsse3_i386.inc} +{$ENDIF} +end; + +procedure Adler32_ProcessBlocks_Avx2(AData: PByte; ANumBlocks: UInt32; + ASums, AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Adler32\Adler32BlocksAvx2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Adler32\Adler32BlocksAvx2_i386.inc} +{$ENDIF} +end; + +procedure Adler32_Update_Sse2(AData: PByte; ALength: UInt32; ASums: Pointer); +begin + Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Sse2); +end; + +procedure Adler32_Update_Ssse3(AData: PByte; ALength: UInt32; ASums: Pointer); +begin + Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Ssse3); +end; + +procedure Adler32_Update_Avx2(AData: PByte; ALength: UInt32; ASums: Pointer); +begin + Adler32_Update_Simd(AData, ALength, ASums, @Adler32_ProcessBlocks_Avx2); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TAdler32X86Backend } + +class function TAdler32X86Backend.Select(AScalar: TAdler32UpdateProc): TAdler32UpdateProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Adler32_Update_Avx2); + TX86SimdLevel.SSSE3: + Exit(@Adler32_Update_Ssse3); + TX86SimdLevel.SSE2: + Exit(@Adler32_Update_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpArgon2ArmBackend.pas b/HashLib/src/Simd/Backend/HlpArgon2ArmBackend.pas new file mode 100644 index 00000000..3b89e7f0 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpArgon2ArmBackend.pas @@ -0,0 +1,54 @@ +unit HlpArgon2ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpPBKDF_Argon2NotBuildInAdapter; + +type + /// + /// Arm SIMD backend for Argon2's fill-block: owns the AArch64 NEON kernel (body + /// in Include\Simd\Argon2\) and the runtime tier selection via + /// TCpuFeatures.Arm. Compiles on every target - built without Arm SIMD, + /// Select just returns the scalar routine. + /// + TArgon2ArmBackend = class sealed + public + class function Select(AScalar: TArgon2FillBlockProc): TArgon2FillBlockProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: NEON +// ============================================================================= + +procedure Argon2_FillBlock_Neon(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\Argon2\Argon2FillBlockNeon_aarch64.inc} +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TArgon2ArmBackend } + +class function TArgon2ArmBackend.Select(AScalar: TArgon2FillBlockProc): TArgon2FillBlockProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: Exit(@Argon2_FillBlock_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpArgon2X86Backend.pas b/HashLib/src/Simd/Backend/HlpArgon2X86Backend.pas new file mode 100644 index 00000000..e7733e6c --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpArgon2X86Backend.pas @@ -0,0 +1,124 @@ +unit HlpArgon2X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpPBKDF_Argon2NotBuildInAdapter; + +type + /// + /// x86 SIMD backend for Argon2's fill-block: owns the SSE2 / AVX2 kernels + /// (bodies in Include\Simd\Argon2\) and the runtime tier selection via + /// TCpuFeatures.X86. Compiles on every target - built without x86 SIMD, + /// Select just returns the scalar routine. + /// + TArgon2X86Backend = class sealed + public + class function Select(AScalar: TArgon2FillBlockProc): TArgon2FillBlockProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // vpshufb byte-rotation masks for the AVX2 and SSSE3 kernels: rotr64 by 24 + // (at +0) and by 16 (at +32) as single byte shuffles - Argon2's G uses the + // BLAKE2b rotations, so these match the official BLAKE2 AVX2 r24/r16 + // constants. Each 128-bit pattern is stored twice so one table serves both + // the 256-bit AVX2 loads and the 128-bit SSSE3 loads. + ARGON2_ROT_MASKS: array [0 .. 7] of UInt64 = ( + UInt64($0201000706050403), UInt64($0A09080F0E0D0C0B), + UInt64($0201000706050403), UInt64($0A09080F0E0D0C0B), + UInt64($0100070605040302), UInt64($09080F0E0D0C0B0A), + UInt64($0100070605040302), UInt64($09080F0E0D0C0B0A) + ); + +// ============================================================================= +// SIMD kernels +// i386: AVX2, SSE2 +// x86_64: AVX2, SSSE3, SSE2 +// ============================================================================= + +procedure Argon2_FillBlock_Sse2(ALeft, ARight, ACurrent: Pointer; AWithXor: Int32); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Argon2\Argon2FillBlockSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Argon2\Argon2FillBlockSse2_i386.inc} +{$ENDIF} +end; + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure Argon2_FillBlock_Ssse3(ALeft, ARight, ACurrent: Pointer; + AWithXor: Int32; AMasks: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\Argon2\Argon2FillBlockSsse3_x86_64.inc} +end; + +procedure Argon2_FillBlock_Ssse3_Wrap(ALeft, ARight, ACurrent: Pointer; + AWithXor: Int32); +begin + Argon2_FillBlock_Ssse3(ALeft, ARight, ACurrent, AWithXor, + @ARGON2_ROT_MASKS); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +procedure Argon2_FillBlock_Avx2(ALeft, ARight, ACurrent: Pointer; + AWithXor: Int32; AMasks: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} +{$I ..\..\Include\Simd\Argon2\Argon2FillBlockAvx2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\Argon2\Argon2FillBlockAvx2_i386.inc} +{$ENDIF} +end; + +procedure Argon2_FillBlock_Avx2_Wrap(ALeft, ARight, ACurrent: Pointer; + AWithXor: Int32); +begin + Argon2_FillBlock_Avx2(ALeft, ARight, ACurrent, AWithXor, + @ARGON2_ROT_MASKS); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TArgon2X86Backend } + +class function TArgon2X86Backend.Select(AScalar: TArgon2FillBlockProc): TArgon2FillBlockProc; +begin +{$IFDEF HASHLIB_I386_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Argon2_FillBlock_Avx2_Wrap); + TX86SimdLevel.SSE2: + Exit(@Argon2_FillBlock_Sse2); + end; +{$ENDIF} +{$IFDEF HASHLIB_X86_64_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Argon2_FillBlock_Avx2_Wrap); + TX86SimdLevel.SSSE3: + Exit(@Argon2_FillBlock_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@Argon2_FillBlock_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpBlake2BArmBackend.pas b/HashLib/src/Simd/Backend/HlpBlake2BArmBackend.pas new file mode 100644 index 00000000..9b667591 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpBlake2BArmBackend.pas @@ -0,0 +1,55 @@ +unit HlpBlake2BArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake2B; + +type + /// + /// Arm SIMD backend for Blake2B: owns the AArch64 NEON compression kernel + /// (body in Include\Simd\Blake2B\) and the runtime tier selection via + /// TCpuFeatures.Arm. Compiles on every target - built without Arm SIMD, + /// Select just returns the scalar routine. + /// + TBlake2BArmBackend = class sealed + public + class function Select(AScalar: TBlake2BCompressProc): TBlake2BCompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: NEON +// ============================================================================= + +procedure Blake2B_Compress_Neon(AState, AMsg, ACounterFlags, AIV: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\Blake2B\Blake2BCompressNeon_aarch64.inc} +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TBlake2BArmBackend } + +class function TBlake2BArmBackend.Select(AScalar: TBlake2BCompressProc): TBlake2BCompressProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@Blake2B_Compress_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpBlake2BX86Backend.pas b/HashLib/src/Simd/Backend/HlpBlake2BX86Backend.pas new file mode 100644 index 00000000..13687b4a --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpBlake2BX86Backend.pas @@ -0,0 +1,117 @@ +unit HlpBlake2BX86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake2B; + +type + /// + /// x86 SIMD backend for Blake2B: owns the SSE2 / AVX2 compression kernels + /// (bodies in Include\Simd\Blake2B\) and the runtime tier selection via + /// TCpuFeatures.X86. Compiles on every target - built without x86 SIMD, + /// Select just returns the scalar routine. + /// + TBlake2BX86Backend = class sealed + public + class function Select(AScalar: TBlake2BCompressProc): TBlake2BCompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // vpshufb byte-rotation masks for the AVX2 kernel: rotr64 by 24 (at +0) and + // by 16 (at +32) as single byte shuffles. Each 128-bit pattern is stored + // twice to fill a 256-bit register. Matches the official BLAKE2 AVX2 + // implementation's r24/r16 shuffle constants. + BLAKE2B_ROT_MASKS: array [0 .. 7] of UInt64 = ( + UInt64($0201000706050403), UInt64($0A09080F0E0D0C0B), + UInt64($0201000706050403), UInt64($0A09080F0E0D0C0B), + UInt64($0100070605040302), UInt64($09080F0E0D0C0B0A), + UInt64($0100070605040302), UInt64($09080F0E0D0C0B0A) + ); + +// ============================================================================= +// SIMD kernels +// i386: AVX2, SSSE3, SSE2 +// x86_64: AVX2, SSSE3, SSE2 +// ============================================================================= + +procedure Blake2B_Compress_Sse2(AState, AMsg, ACounterFlags, AIV: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake2B\Blake2BCompressSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Blake2B\Blake2BCompressSse2_i386.inc} +{$ENDIF} +end; + +procedure Blake2B_Compress_Ssse3(AState, AMsg, ACounterFlags, AIV, + AMasks: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake2B\Blake2BCompressSsse3_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\Blake2B\Blake2BCompressSsse3_i386.inc} +{$ENDIF} +end; + +procedure Blake2B_Compress_Ssse3_Wrap(AState, AMsg, ACounterFlags, + AIV: Pointer); +begin + Blake2B_Compress_Ssse3(AState, AMsg, ACounterFlags, AIV, + @BLAKE2B_ROT_MASKS); +end; + +procedure Blake2B_Compress_Avx2(AState, AMsg, ACounterFlags, AIV, + AMasks: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake2B\Blake2BCompressAvx2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\Blake2B\Blake2BCompressAvx2_i386.inc} +{$ENDIF} +end; + +procedure Blake2B_Compress_Avx2_Wrap(AState, AMsg, ACounterFlags, + AIV: Pointer); +begin + Blake2B_Compress_Avx2(AState, AMsg, ACounterFlags, AIV, + @BLAKE2B_ROT_MASKS); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TBlake2BX86Backend } + +class function TBlake2BX86Backend.Select(AScalar: TBlake2BCompressProc): TBlake2BCompressProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Blake2B_Compress_Avx2_Wrap); + TX86SimdLevel.SSSE3: + Exit(@Blake2B_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@Blake2B_Compress_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpBlake2SArmBackend.pas b/HashLib/src/Simd/Backend/HlpBlake2SArmBackend.pas new file mode 100644 index 00000000..706cc779 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpBlake2SArmBackend.pas @@ -0,0 +1,55 @@ +unit HlpBlake2SArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake2S; + +type + /// + /// Arm SIMD backend for Blake2S: owns the AArch64 NEON compression kernel + /// (body in Include\Simd\Blake2S\) and the runtime tier selection via + /// TCpuFeatures.Arm. Compiles on every target - built without Arm SIMD, + /// Select just returns the scalar routine. + /// + TBlake2SArmBackend = class sealed + public + class function Select(AScalar: TBlake2SCompressProc): TBlake2SCompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: NEON +// ============================================================================= + +procedure Blake2S_Compress_Neon(AState, AMsg, ACounterFlags, AIV: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\Blake2S\Blake2SCompressNeon_aarch64.inc} +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TBlake2SArmBackend } + +class function TBlake2SArmBackend.Select(AScalar: TBlake2SCompressProc): TBlake2SCompressProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@Blake2S_Compress_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpBlake2SX86Backend.pas b/HashLib/src/Simd/Backend/HlpBlake2SX86Backend.pas new file mode 100644 index 00000000..5072787c --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpBlake2SX86Backend.pas @@ -0,0 +1,114 @@ +unit HlpBlake2SX86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake2S; + +type + /// + /// x86 SIMD backend for Blake2S: owns the SSE2 / AVX2 compression kernels + /// (bodies in Include\Simd\Blake2S\) and the runtime tier selection via + /// TCpuFeatures.X86. Compiles on every target - built without x86 SIMD, + /// Select just returns the scalar routine. + /// + TBlake2SX86Backend = class sealed + public + class function Select(AScalar: TBlake2SCompressProc): TBlake2SCompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // pshufb byte-rotation masks for the AVX and SSSE3 kernels: rotr32 by 16 (at +0) and + // by 8 (at +16) as single byte shuffles. Matches the official BLAKE2 SSSE3+ + // implementation's r16/r8 shuffle constants. + BLAKE2S_ROT_MASKS: array [0 .. 3] of UInt64 = ( + UInt64($0504070601000302), UInt64($0D0C0F0E09080B0A), + UInt64($0407060500030201), UInt64($0C0F0E0D080B0A09) + ); + +// ============================================================================= +// SIMD kernels +// i386: AVX2, SSSE3, SSE2 +// x86_64: AVX2, SSSE3, SSE2 +// ============================================================================= + +procedure Blake2S_Compress_Sse2(AState, AMsg, ACounterFlags, AIV: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake2S\Blake2SCompressSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Blake2S\Blake2SCompressSse2_i386.inc} +{$ENDIF} +end; + +procedure Blake2S_Compress_Ssse3(AState, AMsg, ACounterFlags, AIV, + AMasks: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake2S\Blake2SCompressSsse3_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\Blake2S\Blake2SCompressSsse3_i386.inc} +{$ENDIF} +end; + +procedure Blake2S_Compress_Ssse3_Wrap(AState, AMsg, ACounterFlags, + AIV: Pointer); +begin + Blake2S_Compress_Ssse3(AState, AMsg, ACounterFlags, AIV, + @BLAKE2S_ROT_MASKS); +end; + +procedure Blake2S_Compress_Avx(AState, AMsg, ACounterFlags, AIV, + AMasks: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake2S\Blake2SCompressAvx_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_i386.inc} +{$I ..\..\Include\Simd\Blake2S\Blake2SCompressAvx_i386.inc} +{$ENDIF} +end; + +procedure Blake2S_Compress_Avx_Wrap(AState, AMsg, ACounterFlags, + AIV: Pointer); +begin + Blake2S_Compress_Avx(AState, AMsg, ACounterFlags, AIV, + @BLAKE2S_ROT_MASKS); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TBlake2SX86Backend } + +class function TBlake2SX86Backend.Select(AScalar: TBlake2SCompressProc): TBlake2SCompressProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Blake2S_Compress_Avx_Wrap); + TX86SimdLevel.SSSE3: + Exit(@Blake2S_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@Blake2S_Compress_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpBlake3ArmBackend.pas b/HashLib/src/Simd/Backend/HlpBlake3ArmBackend.pas new file mode 100644 index 00000000..bf91bd52 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpBlake3ArmBackend.pas @@ -0,0 +1,107 @@ +unit HlpBlake3ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake3; + +type + /// + /// Arm SIMD backend for Blake3: owns the AArch64 NEON compression and 4-way + /// hash-many kernels (bodies in Include\Simd\Blake3\) and the runtime + /// tier selection via TCpuFeatures.Arm. The hash-many wrapper processes + /// full 4-way groups in asm and delegates the remainder to the scalar + /// hash-many. Compiles on every target - without Arm SIMD the selectors return + /// the scalar routines / degree 1. + /// + TBlake3ArmBackend = class sealed + public + class function SelectCompress(AScalar: TBlake3CompressProc): TBlake3CompressProc; static; + class function SelectHashMany(AScalar: TBlake3HashManyProc): TBlake3HashManyProc; static; + class function SelectParallelDegree(ADefault: Int32): Int32; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: NEON +// ============================================================================= + +procedure Blake3_Compress_Neon(AState, AMsg, ACV, ACounterFlags: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\Blake3\Blake3CompressNeon_aarch64.inc} +end; + +procedure Blake3_Hash4_Neon(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); + {$I ..\..\Include\Simd\Common\HlpSimdProc6Begin_aarch64.inc} + {$I ..\..\Include\Simd\Blake3\Blake3Hash4Neon_aarch64.inc} +end; + +// NEON hash_many: hash4 -> delegate remainder to scalar hash_many +procedure Blake3_HashMany_Neon(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); +var + LPInput, LPOut: PByte; +begin + LPInput := PByte(AInput); + LPOut := PByte(AOut); + while ANumChunks >= 4 do + begin + Blake3_Hash4_Neon(LPInput, AKey, LPOut, 4, ACounter, AFlags); + System.Inc(LPInput, 4 * 1024); + System.Inc(LPOut, 4 * 32); + System.Inc(ACounter, 4); + System.Dec(ANumChunks, 4); + end; + if ANumChunks > 0 then + Blake3_HashMany_Scalar(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TBlake3ArmBackend } + +class function TBlake3ArmBackend.SelectCompress(AScalar: TBlake3CompressProc): TBlake3CompressProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@Blake3_Compress_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TBlake3ArmBackend.SelectHashMany(AScalar: TBlake3HashManyProc): TBlake3HashManyProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@Blake3_HashMany_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TBlake3ArmBackend.SelectParallelDegree(ADefault: Int32): Int32; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(4); + end; +{$ENDIF} + Result := ADefault; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpBlake3X86Backend.pas b/HashLib/src/Simd/Backend/HlpBlake3X86Backend.pas new file mode 100644 index 00000000..efde87e3 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpBlake3X86Backend.pas @@ -0,0 +1,256 @@ +unit HlpBlake3X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake3; + +type + /// + /// x86 SIMD backend for Blake3: owns the SSE2 / AVX2 compression and 4-/8-way + /// hash-many kernels (bodies in Include\Simd\Blake3\) and the runtime + /// tier selection via TCpuFeatures.X86. The hash-many wrappers process + /// full parallel groups in asm and delegate the remainder to the scalar + /// hash-many. Compiles on every target - without x86 SIMD the selectors return + /// the scalar routines / degree 1. + /// + TBlake3X86Backend = class sealed + public + class function SelectCompress(AScalar: TBlake3CompressProc): TBlake3CompressProc; static; + class function SelectHashMany(AScalar: TBlake3HashManyProc): TBlake3HashManyProc; static; + class function SelectParallelDegree(ADefault: Int32): Int32; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // vpshufb byte-rotation masks for the AVX2 kernels: rotr32 by 16 (at +0) + // and by 8 (at +32) as single byte shuffles. Each 128-bit pattern is stored + // twice so one table serves both the 256-bit Hash8 loads and the 128-bit + // Compress loads. Matches the official BLAKE3 AVX2 implementation's + // ROT16/ROT8 shuffle constants. + BLAKE3_ROT_MASKS: array [0 .. 7] of UInt64 = ( + UInt64($0504070601000302), UInt64($0D0C0F0E09080B0A), + UInt64($0504070601000302), UInt64($0D0C0F0E09080B0A), + UInt64($0407060500030201), UInt64($0C0F0E0D080B0A09), + UInt64($0407060500030201), UInt64($0C0F0E0D080B0A09) + ); + +// ============================================================================= +// SIMD kernels +// i386: SSE2 +// x86_64: AVX2, SSSE3, SSE2 +// ============================================================================= + +procedure Blake3_Compress_Sse2(AState, AMsg, ACV, ACounterFlags: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake3\Blake3CompressSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\Blake3\Blake3CompressSse2_i386.inc} +{$ENDIF} +end; + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure Blake3_Compress_Ssse3(AState, AMsg, ACV, ACounterFlags, + AMasks: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\Blake3\Blake3CompressSsse3_x86_64.inc} +end; + +procedure Blake3_Compress_Ssse3_Wrap(AState, AMsg, ACV, + ACounterFlags: Pointer); +begin + Blake3_Compress_Ssse3(AState, AMsg, ACV, ACounterFlags, + @BLAKE3_ROT_MASKS); +end; + +procedure Blake3_Compress_Avx(AState, AMsg, ACV, ACounterFlags, + AMasks: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\Blake3\Blake3CompressAvx_x86_64.inc} +end; + +procedure Blake3_Compress_Avx_Wrap(AState, AMsg, ACV, + ACounterFlags: Pointer); +begin + Blake3_Compress_Avx(AState, AMsg, ACV, ACounterFlags, + @BLAKE3_ROT_MASKS); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +procedure Blake3_Hash4_Sse2(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc6Begin_x86_64.inc} +{$I ..\..\Include\Simd\Blake3\Blake3Hash4Sse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc6Begin_i386.inc} +{$I ..\..\Include\Simd\Blake3\Blake3Hash4Sse2_i386.inc} +{$ENDIF} +end; + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure Blake3_Hash4_Ssse3(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32; AMasks: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc7Begin_x86_64.inc} + {$I ..\..\Include\Simd\Blake3\Blake3Hash4Ssse3_x86_64.inc} +end; + +procedure Blake3_Hash8_Avx2(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32; AMasks: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc7Begin_x86_64.inc} + {$I ..\..\Include\Simd\Blake3\Blake3Hash8Avx2_x86_64.inc} +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +// SSE2 hash_many: hash4 -> delegate remainder to scalar hash_many +procedure Blake3_HashMany_Sse2(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); +var + LPInput, LPOut: PByte; +begin + LPInput := PByte(AInput); + LPOut := PByte(AOut); + while ANumChunks >= 4 do + begin + Blake3_Hash4_Sse2(LPInput, AKey, LPOut, 4, ACounter, AFlags); + System.Inc(LPInput, 4 * 1024); + System.Inc(LPOut, 4 * 32); + System.Inc(ACounter, 4); + System.Dec(ANumChunks, 4); + end; + if ANumChunks > 0 then + Blake3_HashMany_Scalar(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); +end; + +{$IFDEF HASHLIB_X86_64_ASM} + +// SSSE3 hash_many: hash4 -> delegate remainder to scalar hash_many +procedure Blake3_HashMany_Ssse3(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); +var + LPInput, LPOut: PByte; +begin + LPInput := PByte(AInput); + LPOut := PByte(AOut); + while ANumChunks >= 4 do + begin + Blake3_Hash4_Ssse3(LPInput, AKey, LPOut, 4, ACounter, AFlags, + @BLAKE3_ROT_MASKS); + System.Inc(LPInput, 4 * 1024); + System.Inc(LPOut, 4 * 32); + System.Inc(ACounter, 4); + System.Dec(ANumChunks, 4); + end; + if ANumChunks > 0 then + Blake3_HashMany_Scalar(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); +end; + +// AVX2 hash_many: hash8 -> delegate remainder to SSSE3 hash_many +procedure Blake3_HashMany_Avx2(AInput, AKey, AOut: Pointer; + ANumChunks: Int32; ACounter: UInt64; AFlags: UInt32); +var + LPInput, LPOut: PByte; +begin + LPInput := PByte(AInput); + LPOut := PByte(AOut); + while ANumChunks >= 8 do + begin + Blake3_Hash8_Avx2(LPInput, AKey, LPOut, 8, ACounter, AFlags, + @BLAKE3_ROT_MASKS); + System.Inc(LPInput, 8 * 1024); + System.Inc(LPOut, 8 * 32); + System.Inc(ACounter, 8); + System.Dec(ANumChunks, 8); + end; + if ANumChunks > 0 then + Blake3_HashMany_Ssse3(LPInput, AKey, LPOut, ANumChunks, ACounter, AFlags); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +{$ENDIF HASHLIB_X86_SIMD} + +{ TBlake3X86Backend } + +class function TBlake3X86Backend.SelectCompress(AScalar: TBlake3CompressProc): TBlake3CompressProc; +begin +{$IFDEF HASHLIB_I386_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + Exit(@Blake3_Compress_Sse2); + end; +{$ENDIF} +{$IFDEF HASHLIB_X86_64_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Blake3_Compress_Avx_Wrap); + TX86SimdLevel.SSSE3: + Exit(@Blake3_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@Blake3_Compress_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TBlake3X86Backend.SelectHashMany(AScalar: TBlake3HashManyProc): TBlake3HashManyProc; +begin +{$IFDEF HASHLIB_I386_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + Exit(@Blake3_HashMany_Sse2); + end; +{$ENDIF} +{$IFDEF HASHLIB_X86_64_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Blake3_HashMany_Avx2); + TX86SimdLevel.SSSE3: + Exit(@Blake3_HashMany_Ssse3); + TX86SimdLevel.SSE2: + Exit(@Blake3_HashMany_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TBlake3X86Backend.SelectParallelDegree(ADefault: Int32): Int32; +begin +{$IFDEF HASHLIB_I386_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + Exit(4); + end; +{$ENDIF} +{$IFDEF HASHLIB_X86_64_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(8); + TX86SimdLevel.SSSE3, TX86SimdLevel.SSE2: + Exit(4); + end; +{$ENDIF} + Result := ADefault; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpCRCArmBackend.pas b/HashLib/src/Simd/Backend/HlpCRCArmBackend.pas new file mode 100644 index 00000000..09815fe7 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpCRCArmBackend.pas @@ -0,0 +1,68 @@ +unit HlpCRCArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpCRCCore; + +type + /// + /// Arm SIMD backend for CRC's fold: owns the PMULL kernels (bodies in + /// Include\Simd\CRC\) and the runtime tier selection via + /// TCpuFeatures.Arm. Compiles on every target - built without Arm SIMD, + /// Select just returns the scalar routines. + /// + TCRCArmBackend = class sealed + public + class function Select(AReflectedScalar, AForwardScalar: TCRCFoldFunc) + : TCRCFoldSelection; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures; + +// ============================================================================= +// SIMD kernels +// aarch64: PMULL +// ============================================================================= + +function CRC_Fold_Reflected_Pmull(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\CRC\CRCFoldReflectedPmull_aarch64.inc} +end; + +function CRC_Fold_Forward_Pmull(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\CRC\CRCFoldForwardPmull_aarch64.inc} +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TCRCArmBackend } + +class function TCRCArmBackend.Select(AReflectedScalar, + AForwardScalar: TCRCFoldFunc): TCRCFoldSelection; +begin + Result.Reflected := AReflectedScalar; + Result.Fwd := AForwardScalar; + Result.UsesCarrylessMul := False; + +{$IFDEF HASHLIB_AARCH64_ASM} + if TCpuFeatures.Arm.HasPMULL() then + begin + Result.Reflected := @CRC_Fold_Reflected_Pmull; + Result.Fwd := @CRC_Fold_Forward_Pmull; + Result.UsesCarrylessMul := True; + end; +{$ENDIF HASHLIB_AARCH64_ASM} +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpCRCX86Backend.pas b/HashLib/src/Simd/Backend/HlpCRCX86Backend.pas new file mode 100644 index 00000000..554b0764 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpCRCX86Backend.pas @@ -0,0 +1,147 @@ +unit HlpCRCX86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpCRCCore; + +type + /// + /// x86 SIMD backend for CRC's fold: owns the SSE2 / PCLMULQDQ / VPCLMULQDQ + /// kernels (bodies in Include\Simd\CRC\) and the runtime tier selection + /// via TCpuFeatures.X86. Compiles on every target - built without x86 + /// SIMD, Select just returns the scalar routines. + /// + TCRCX86Backend = class sealed + public + class function Select(AReflectedScalar, AForwardScalar: TCRCFoldFunc) + : TCRCFoldSelection; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// i386: VPCLMULQDQ, PCLMULQDQ, SSE2 +// x86_64: VPCLMULQDQ, PCLMULQDQ, SSE2 +// ============================================================================= + +function CRC_Fold_Reflected_Sse2(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldReflectedSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldReflectedSse2_i386.inc} +{$ENDIF} +end; + +function CRC_Fold_Forward_Gpr(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldForwardGpr_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldForwardGpr_i386.inc} +{$ENDIF} +end; + +function CRC_Fold_Reflected_Pclmul(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldReflectedPclmul_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldReflectedPclmul_i386.inc} +{$ENDIF} +end; + +function CRC_Fold_Reflected_Vpclmul(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldReflectedVpclmul_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldReflectedVpclmul_i386.inc} +{$ENDIF} +end; + +function CRC_Fold_Forward_Pclmul(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldForwardPclmul_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldForwardPclmul_i386.inc} +{$ENDIF} +end; + +function CRC_Fold_Forward_Vpclmul(AData: PByte; ALength: UInt32; + AState: Pointer; AConstants: Pointer): UInt64; +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldForwardVpclmul_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\CRC\CRCFoldForwardVpclmul_i386.inc} +{$ENDIF} +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TCRCX86Backend } + +class function TCRCX86Backend.Select(AReflectedScalar, + AForwardScalar: TCRCFoldFunc): TCRCFoldSelection; +begin + Result.Reflected := AReflectedScalar; + Result.Fwd := AForwardScalar; + Result.UsesCarrylessMul := False; + +{$IFDEF HASHLIB_X86_SIMD} + if TCpuFeatures.X86.HasVPCLMULQDQ() then + begin + Result.Reflected := @CRC_Fold_Reflected_Vpclmul; + Result.Fwd := @CRC_Fold_Forward_Vpclmul; + Result.UsesCarrylessMul := True; + Exit; + end; + + if TCpuFeatures.X86.HasPCLMULQDQ() then + begin + Result.Reflected := @CRC_Fold_Reflected_Pclmul; + Result.Fwd := @CRC_Fold_Forward_Pclmul; + Result.UsesCarrylessMul := True; + Exit; + end; + + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + begin + Result.Reflected := @CRC_Fold_Reflected_Sse2; + Result.Fwd := @CRC_Fold_Forward_Gpr; + end; + end; +{$ENDIF HASHLIB_X86_SIMD} +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpSHA1ArmBackend.pas b/HashLib/src/Simd/Backend/HlpSHA1ArmBackend.pas new file mode 100644 index 00000000..cf40f216 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA1ArmBackend.pas @@ -0,0 +1,68 @@ +unit HlpSHA1ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA1; + +type + /// + /// Arm SIMD backend for SHA-1: owns the AArch64 FEAT_SHA1 crypto-extension + /// kernel (body in Include\Simd\SHA1\) and the runtime feature check via + /// TCpuFeatures.Arm. Compiles on every target - built without Arm SIMD, + /// Select just returns the scalar routine. + /// + TSHA1ArmBackend = class sealed + public + class function Select(AScalar: TSHA1CompressProc): TSHA1CompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: SHA1 Crypto Extensions, NEON +// ============================================================================= + +procedure SHA1_Compress_CryptoExt(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA1\SHA1CompressCryptoExt_aarch64.inc} +end; + +procedure SHA1_Compress_CryptoExt_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_CryptoExt(AState, AData, ANumBlocks, @K_SHA1); +end; + +procedure SHA1_Compress_Gpr(AState, AData: Pointer; ANumBlocks: UInt32); + {$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA1\SHA1CompressGpr_aarch64.inc} +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TSHA1ArmBackend } + +class function TSHA1ArmBackend.Select(AScalar: TSHA1CompressProc): TSHA1CompressProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + if TCpuFeatures.Arm.HasSHA1() then + Exit(@SHA1_Compress_CryptoExt_Wrap); + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@SHA1_Compress_Gpr); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpSHA1X86Backend.pas b/HashLib/src/Simd/Backend/HlpSHA1X86Backend.pas new file mode 100644 index 00000000..4728b777 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA1X86Backend.pas @@ -0,0 +1,194 @@ +unit HlpSHA1X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA1; + +type + /// + /// x86 SIMD backend for SHA-1: owns the SSE2 / AVX2 / SHA-NI keystream kernels + /// (bodies in Include\Simd\SHA1\) and the runtime tier selection via + /// TCpuFeatures.X86. Compiles on every target - built without x86 SIMD, + /// Select just returns the scalar routine. + /// + TSHA1X86Backend = class sealed + public + class function Select(AScalar: TSHA1CompressProc): TSHA1CompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // BSWAP32 shuffle mask for pshufb (x86 SIMD only): byte-swaps and reverses + // dword order in one shuffle (sha1rnds4 reads its four words in reverse). Not a + // SHA-1 constant; used only by the SHA-NI kernel. + BSWAP32_MASK: array [0 .. 3] of UInt32 = ( + $0C0D0E0F, $08090A0B, $04050607, $00010203 + ); + + // Doubled SHA-1 round constants plus the AVX2 byte-swap masks, shared by the + // AVX2 and SSE2 SIMD-schedule SHA-1 kernels. Each round constant fills a 128-bit + // lane (its four dwords) and is stored twice so one table feeds both the 256-bit + // AVX2 read and the 128-bit SSE2 reads (both read at a 32-byte stride, skipping + // the duplicate halves). Only the AVX2 kernel uses the appended masks: the + // byte-swap mask (BSWAP32 pattern, twice) then a whole-vector reverse mask; the + // SSE2 kernel computes its byte-swap and needs no mask. + K_SHA1_Doubled: array [0 .. 43] of UInt32 = ( + $5A827999, $5A827999, $5A827999, $5A827999, + $5A827999, $5A827999, $5A827999, $5A827999, + $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, + $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, $6ED9EBA1, + $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, + $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, $8F1BBCDC, + $CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6, + $CA62C1D6, $CA62C1D6, $CA62C1D6, $CA62C1D6, + $00010203, $04050607, $08090A0B, $0C0D0E0F, + $00010203, $04050607, $08090A0B, $0C0D0E0F, + $0C0D0E0F, $08090A0B, $04050607, $00010203 + ); + +// ============================================================================= +// SIMD kernels +// i386: ShaNi, AVX2, SSSE3, SSE2 +// x86_64: ShaNi, AVX2, SSSE3, SSE2 +// ============================================================================= + +{$IFDEF HASHLIB_I386_ASM} + +procedure SHA1_Compress_ShaNi(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} + {$I ..\..\Include\Simd\SHA1\SHA1CompressShaNi_i386.inc} +end; + +procedure SHA1_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_ShaNi(AState, AData, ANumBlocks, @K_SHA1_Doubled); +end; + +{$ENDIF HASHLIB_I386_ASM} + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure SHA1_Compress_ShaNi(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants, AMask: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA1\SHA1CompressShaNi_x86_64.inc} +end; + +procedure SHA1_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_ShaNi(AState, AData, ANumBlocks, @K_SHA1, @BSWAP32_MASK); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +procedure SHA1_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\SHA1\SHA1CompressSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\SHA1\SHA1CompressSse2_i386.inc} +{$ENDIF} +end; + +procedure SHA1_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\SHA1\SHA1CompressSsse3_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\SHA1\SHA1CompressSsse3_i386.inc} +{$ENDIF} +end; + +{$IFDEF HASHLIB_I386_ASM} + +procedure SHA1_Compress_Avx(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} + {$I ..\..\Include\Simd\SHA1\SHA1CompressAvx_i386.inc} +end; + +procedure SHA1_Compress_Avx_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_Avx(AState, AData, ANumBlocks, @K_SHA1_Doubled); +end; + +{$ENDIF HASHLIB_I386_ASM} + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure SHA1_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA1\SHA1CompressAvx2_x86_64.inc} +end; + +procedure SHA1_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_Avx2(AState, AData, ANumBlocks, @K_SHA1_Doubled); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +procedure SHA1_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_Sse2(AState, AData, ANumBlocks, @K_SHA1_Doubled); +end; + +procedure SHA1_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA1_Compress_Ssse3(AState, AData, ANumBlocks, @K_SHA1_Doubled); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TSHA1X86Backend } + +class function TSHA1X86Backend.Select(AScalar: TSHA1CompressProc): TSHA1CompressProc; +begin +{$IFDEF HASHLIB_I386_ASM} + if TCpuFeatures.X86.HasSHANI() then + Exit(@SHA1_Compress_ShaNi_Wrap); + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@SHA1_Compress_Avx_Wrap); + TX86SimdLevel.SSSE3: + Exit(@SHA1_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@SHA1_Compress_Sse2_Wrap); + end; +{$ENDIF} +{$IFDEF HASHLIB_X86_64_ASM} + if TCpuFeatures.X86.HasSHANI() then + Exit(@SHA1_Compress_ShaNi_Wrap); + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@SHA1_Compress_Avx2_Wrap); + TX86SimdLevel.SSSE3: + Exit(@SHA1_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@SHA1_Compress_Sse2_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpSHA2_256ArmBackend.pas b/HashLib/src/Simd/Backend/HlpSHA2_256ArmBackend.pas new file mode 100644 index 00000000..6b2c4074 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA2_256ArmBackend.pas @@ -0,0 +1,99 @@ +unit HlpSHA2_256ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA2_256Base; + +type + /// + /// Arm SIMD backend for SHA-256: owns the AArch64 FEAT_SHA256 crypto-extension + /// kernel (body in Include\Simd\SHA256\) and the runtime feature check + /// via TCpuFeatures.Arm. Compiles on every target - built without Arm + /// SIMD, Select just returns the scalar routine. + /// + TSHA2_256ArmBackend = class sealed + public + class function Select(AScalar: TSHA256CompressProc): TSHA256CompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // K256 with a trailing zero terminator for the pure-GPR kernel: its + // message-schedule loop ends when the loaded K word is zero (cbnz), like + // the original's sentinel-terminated table - the plain K256 const cannot + // drive that loop. The crypto-extension kernel keeps using K256. + K256_Gpr: array [0 .. 64] of UInt32 = ( + $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, + $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, + $D807AA98, $12835B01, $243185BE, $550C7DC3, + $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, + $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, + $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, + $983E5152, $A831C66D, $B00327C8, $BF597FC7, + $C6E00BF3, $D5A79147, $06CA6351, $14292967, + $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, + $650A7354, $766A0ABB, $81C2C92E, $92722C85, + $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, + $D192E819, $D6990624, $F40E3585, $106AA070, + $19A4C116, $1E376C08, $2748774C, $34B0BCB5, + $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, + $748F82EE, $78A5636F, $84C87814, $8CC70208, + $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2, + 0 // terminator - the schedule loop ends on a zero K word + ); + +// ============================================================================= +// SIMD kernels +// aarch64: SHA256 Crypto Extensions, NEON +// ============================================================================= + +procedure SHA256_Compress_CryptoExt(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA256\SHA256CompressCryptoExt_aarch64.inc} +end; + +procedure SHA256_Compress_CryptoExt_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_CryptoExt(AState, AData, ANumBlocks, @K256); +end; + +procedure SHA256_Compress_Gpr(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA256\SHA256CompressGpr_aarch64.inc} +end; + +procedure SHA256_Compress_Gpr_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_Gpr(AState, AData, ANumBlocks, @K256_Gpr); +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TSHA2_256ArmBackend } + +class function TSHA2_256ArmBackend.Select(AScalar: TSHA256CompressProc): TSHA256CompressProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + if TCpuFeatures.Arm.HasSHA256() then + Exit(@SHA256_Compress_CryptoExt_Wrap); + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@SHA256_Compress_Gpr_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpSHA2_256X86Backend.pas b/HashLib/src/Simd/Backend/HlpSHA2_256X86Backend.pas new file mode 100644 index 00000000..ad0ee0cf --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA2_256X86Backend.pas @@ -0,0 +1,221 @@ +unit HlpSHA2_256X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA2_256Base; + +type + /// + /// x86 SIMD backend for SHA-256: owns the SSE2 / AVX2 / SHA-NI keystream + /// kernels (bodies in Include\Simd\SHA256\) and the runtime tier + /// selection via TCpuFeatures.X86. Compiles on every target - built + /// without x86 SIMD, Select just returns the scalar routine. + /// + TSHA2_256X86Backend = class sealed + public + class function Select(AScalar: TSHA256CompressProc): TSHA256CompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // BSWAP32 shuffle mask for pshufb (x86 SIMD only): reverses bytes within each + // dword. Not a SHA-256 constant; used only by the SHA-NI kernel. + BSWAP32_MASK: array [0 .. 3] of UInt32 = ( + $00010203, $04050607, $08090A0B, $0C0D0E0F + ); + + // Doubled K256 round constants plus the three AVX2 message-schedule masks, + // shared by the AVX2 and SSE2 SIMD-schedule SHA-256 kernels. Each 128-bit K256 + // quadruple is stored twice so one table feeds both the 256-bit AVX2 lanes and + // the 128-bit SSE2 reads (both read at a 32-byte stride, skipping the duplicate + // halves). Only the AVX2 kernel uses the appended masks: the byte-swap mask + // (BSWAP32 pattern, twice) occupies [128..135] and the two schedule shuffle + // masks follow at [136..143] and [144..151]; the SSE2 kernel computes its + // byte-swap and needs no mask. Derived from K256. + K256_Doubled: array [0 .. 151] of UInt32 = ( + $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, + $428A2F98, $71374491, $B5C0FBCF, $E9B5DBA5, + $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, + $3956C25B, $59F111F1, $923F82A4, $AB1C5ED5, + $D807AA98, $12835B01, $243185BE, $550C7DC3, + $D807AA98, $12835B01, $243185BE, $550C7DC3, + $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, + $72BE5D74, $80DEB1FE, $9BDC06A7, $C19BF174, + $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, + $E49B69C1, $EFBE4786, $0FC19DC6, $240CA1CC, + $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, + $2DE92C6F, $4A7484AA, $5CB0A9DC, $76F988DA, + $983E5152, $A831C66D, $B00327C8, $BF597FC7, + $983E5152, $A831C66D, $B00327C8, $BF597FC7, + $C6E00BF3, $D5A79147, $06CA6351, $14292967, + $C6E00BF3, $D5A79147, $06CA6351, $14292967, + $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, + $27B70A85, $2E1B2138, $4D2C6DFC, $53380D13, + $650A7354, $766A0ABB, $81C2C92E, $92722C85, + $650A7354, $766A0ABB, $81C2C92E, $92722C85, + $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, + $A2BFE8A1, $A81A664B, $C24B8B70, $C76C51A3, + $D192E819, $D6990624, $F40E3585, $106AA070, + $D192E819, $D6990624, $F40E3585, $106AA070, + $19A4C116, $1E376C08, $2748774C, $34B0BCB5, + $19A4C116, $1E376C08, $2748774C, $34B0BCB5, + $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, + $391C0CB3, $4ED8AA4A, $5B9CCA4F, $682E6FF3, + $748F82EE, $78A5636F, $84C87814, $8CC70208, + $748F82EE, $78A5636F, $84C87814, $8CC70208, + $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2, + $90BEFFFA, $A4506CEB, $BEF9A3F7, $C67178F2, + $00010203, $04050607, $08090A0B, $0C0D0E0F, + $00010203, $04050607, $08090A0B, $0C0D0E0F, + $03020100, $0B0A0908, $FFFFFFFF, $FFFFFFFF, + $03020100, $0B0A0908, $FFFFFFFF, $FFFFFFFF, + $FFFFFFFF, $FFFFFFFF, $03020100, $0B0A0908, + $FFFFFFFF, $FFFFFFFF, $03020100, $0B0A0908 + ); + +// ============================================================================= +// SIMD kernels +// i386: ShaNi, AVX2, SSSE3, SSE2 +// x86_64: ShaNi, AVX2, SSSE3, SSE2 +// ============================================================================= + +{$IFDEF HASHLIB_I386_ASM} + +procedure SHA256_Compress_ShaNi(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} + {$I ..\..\Include\Simd\SHA256\SHA256CompressShaNi_i386.inc} +end; + +procedure SHA256_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_ShaNi(AState, AData, ANumBlocks, @K256_Doubled); +end; + +{$ENDIF HASHLIB_I386_ASM} + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure SHA256_Compress_ShaNi(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants, AMask: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA256\SHA256CompressShaNi_x86_64.inc} +end; + +procedure SHA256_Compress_ShaNi_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_ShaNi(AState, AData, ANumBlocks, @K256, @BSWAP32_MASK); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +procedure SHA256_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\SHA256\SHA256CompressSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\SHA256\SHA256CompressSse2_i386.inc} +{$ENDIF} +end; + +procedure SHA256_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\SHA256\SHA256CompressSsse3_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\SHA256\SHA256CompressSsse3_i386.inc} +{$ENDIF} +end; + +{$IFDEF HASHLIB_I386_ASM} + +procedure SHA256_Compress_Avx(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} + {$I ..\..\Include\Simd\SHA256\SHA256CompressAvx_i386.inc} +end; + +procedure SHA256_Compress_Avx_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_Avx(AState, AData, ANumBlocks, @K256_Doubled); +end; + +{$ENDIF HASHLIB_I386_ASM} + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure SHA256_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA256\SHA256CompressAvx2_x86_64.inc} +end; + +procedure SHA256_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_Avx2(AState, AData, ANumBlocks, @K256_Doubled); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +procedure SHA256_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_Ssse3(AState, AData, ANumBlocks, @K256_Doubled); +end; + +procedure SHA256_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA256_Compress_Sse2(AState, AData, ANumBlocks, @K256_Doubled); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TSHA2_256X86Backend } + +class function TSHA2_256X86Backend.Select(AScalar: TSHA256CompressProc): TSHA256CompressProc; +begin +{$IFDEF HASHLIB_I386_ASM} + if TCpuFeatures.X86.HasSHANI() then + Exit(@SHA256_Compress_ShaNi_Wrap); + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@SHA256_Compress_Avx_Wrap); + TX86SimdLevel.SSSE3: + Exit(@SHA256_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@SHA256_Compress_Sse2_Wrap); + end; +{$ENDIF} +{$IFDEF HASHLIB_X86_64_ASM} + if TCpuFeatures.X86.HasSHANI() then + Exit(@SHA256_Compress_ShaNi_Wrap); + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@SHA256_Compress_Avx2_Wrap); + TX86SimdLevel.SSSE3: + Exit(@SHA256_Compress_Ssse3_Wrap); + TX86SimdLevel.SSE2: + Exit(@SHA256_Compress_Sse2_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpSHA2_512ArmBackend.pas b/HashLib/src/Simd/Backend/HlpSHA2_512ArmBackend.pas new file mode 100644 index 00000000..1c64277a --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA2_512ArmBackend.pas @@ -0,0 +1,123 @@ +unit HlpSHA2_512ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA2_512Base; + +type + /// + /// Arm SIMD backend for SHA-512: owns the AArch64 FEAT_SHA512 crypto-extension + /// kernel (body in Include\Simd\SHA512\) and the runtime feature check + /// via TCpuFeatures.Arm. Compiles on every target - built without Arm + /// SIMD, Select just returns the scalar routine. + /// + TSHA2_512ArmBackend = class sealed + public + class function Select(AScalar: TSHA512CompressProc): TSHA512CompressProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // K512 with a trailing zero terminator for the plain-GPR kernel: its + // message-schedule loop ends when the loaded K word is zero (cbnz), like + // the original's sentinel-terminated table - the plain K512 const cannot + // drive that loop. The crypto-extension kernel keeps using K512. + K512_Gpr: array [0 .. 80] of UInt64 = ( + UInt64($428A2F98D728AE22), UInt64($7137449123EF65CD), + UInt64($B5C0FBCFEC4D3B2F), UInt64($E9B5DBA58189DBBC), + UInt64($3956C25BF348B538), UInt64($59F111F1B605D019), + UInt64($923F82A4AF194F9B), UInt64($AB1C5ED5DA6D8118), + UInt64($D807AA98A3030242), UInt64($12835B0145706FBE), + UInt64($243185BE4EE4B28C), UInt64($550C7DC3D5FFB4E2), + UInt64($72BE5D74F27B896F), UInt64($80DEB1FE3B1696B1), + UInt64($9BDC06A725C71235), UInt64($C19BF174CF692694), + UInt64($E49B69C19EF14AD2), UInt64($EFBE4786384F25E3), + UInt64($0FC19DC68B8CD5B5), UInt64($240CA1CC77AC9C65), + UInt64($2DE92C6F592B0275), UInt64($4A7484AA6EA6E483), + UInt64($5CB0A9DCBD41FBD4), UInt64($76F988DA831153B5), + UInt64($983E5152EE66DFAB), UInt64($A831C66D2DB43210), + UInt64($B00327C898FB213F), UInt64($BF597FC7BEEF0EE4), + UInt64($C6E00BF33DA88FC2), UInt64($D5A79147930AA725), + UInt64($06CA6351E003826F), UInt64($142929670A0E6E70), + UInt64($27B70A8546D22FFC), UInt64($2E1B21385C26C926), + UInt64($4D2C6DFC5AC42AED), UInt64($53380D139D95B3DF), + UInt64($650A73548BAF63DE), UInt64($766A0ABB3C77B2A8), + UInt64($81C2C92E47EDAEE6), UInt64($92722C851482353B), + UInt64($A2BFE8A14CF10364), UInt64($A81A664BBC423001), + UInt64($C24B8B70D0F89791), UInt64($C76C51A30654BE30), + UInt64($D192E819D6EF5218), UInt64($D69906245565A910), + UInt64($F40E35855771202A), UInt64($106AA07032BBD1B8), + UInt64($19A4C116B8D2D0C8), UInt64($1E376C085141AB53), + UInt64($2748774CDF8EEB99), UInt64($34B0BCB5E19B48A8), + UInt64($391C0CB3C5C95A63), UInt64($4ED8AA4AE3418ACB), + UInt64($5B9CCA4F7763E373), UInt64($682E6FF3D6B2B8A3), + UInt64($748F82EE5DEFB2FC), UInt64($78A5636F43172F60), + UInt64($84C87814A1F0AB72), UInt64($8CC702081A6439EC), + UInt64($90BEFFFA23631E28), UInt64($A4506CEBDE82BDE9), + UInt64($BEF9A3F7B2C67915), UInt64($C67178F2E372532B), + UInt64($CA273ECEEA26619C), UInt64($D186B8C721C0C207), + UInt64($EADA7DD6CDE0EB1E), UInt64($F57D4F7FEE6ED178), + UInt64($06F067AA72176FBA), UInt64($0A637DC5A2C898A6), + UInt64($113F9804BEF90DAE), UInt64($1B710B35131C471B), + UInt64($28DB77F523047D84), UInt64($32CAAB7B40C72493), + UInt64($3C9EBE0A15C9BEBC), UInt64($431D67C49C100D4C), + UInt64($4CC5D4BECB3E42B6), UInt64($597F299CFC657E2A), + UInt64($5FCB6FAB3AD6FAEC), UInt64($6C44198C4A475817), + UInt64(0) // terminator - the schedule loop ends on a zero K word + ); + +// ============================================================================= +// SIMD kernels +// aarch64: SHA512 Crypto Extensions, NEON +// ============================================================================= + +procedure SHA512_Compress_CryptoExt(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA512\SHA512CompressCryptoExt_aarch64.inc} +end; + +procedure SHA512_Compress_CryptoExt_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA512_Compress_CryptoExt(AState, AData, ANumBlocks, @K512); +end; + +procedure SHA512_Compress_Gpr(AState, AData: Pointer; ANumBlocks: UInt32; + AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA512\SHA512CompressGpr_aarch64.inc} +end; + +procedure SHA512_Compress_Gpr_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA512_Compress_Gpr(AState, AData, ANumBlocks, @K512_Gpr); +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TSHA2_512ArmBackend } + +class function TSHA2_512ArmBackend.Select(AScalar: TSHA512CompressProc): TSHA512CompressProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + if TCpuFeatures.Arm.HasSHA512() then + Exit(@SHA512_Compress_CryptoExt_Wrap); + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@SHA512_Compress_Gpr_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Crypto/HlpSHA2_512Dispatch.pas b/HashLib/src/Simd/Backend/HlpSHA2_512X86Backend.pas similarity index 50% rename from HashLib/src/Crypto/HlpSHA2_512Dispatch.pas rename to HashLib/src/Simd/Backend/HlpSHA2_512X86Backend.pas index dfec4e61..6ce14d9b 100644 --- a/HashLib/src/Crypto/HlpSHA2_512Dispatch.pas +++ b/HashLib/src/Simd/Backend/HlpSHA2_512X86Backend.pas @@ -1,67 +1,39 @@ -unit HlpSHA2_512Dispatch; +unit HlpSHA2_512X86Backend; -{$I ..\Include\HashLib.inc} +{$I ..\..\Include\HashLib.inc} interface -type - TSHA512CompressProc = procedure(AState, AData: Pointer; ANumBlocks: UInt32); +uses + HlpSHA2_512Base; -var - SHA512_Compress: TSHA512CompressProc; +type + /// + /// x86 SIMD backend for SHA-512: owns the SSE2 / AVX2 keystream kernels (bodies + /// in Include\Simd\SHA512\) and the runtime tier selection via + /// TCpuFeatures.X86. Compiles on every target - built without x86 SIMD, + /// Select just returns the scalar routine. + /// + TSHA2_512X86Backend = class sealed + public + class function Select(AScalar: TSHA512CompressProc): TSHA512CompressProc; static; + end; -const - // K512 round constants (80 UInt64 = 640 bytes) - K512: array [0 .. 79] of UInt64 = ( - UInt64($428A2F98D728AE22), UInt64($7137449123EF65CD), - UInt64($B5C0FBCFEC4D3B2F), UInt64($E9B5DBA58189DBBC), - UInt64($3956C25BF348B538), UInt64($59F111F1B605D019), - UInt64($923F82A4AF194F9B), UInt64($AB1C5ED5DA6D8118), - UInt64($D807AA98A3030242), UInt64($12835B0145706FBE), - UInt64($243185BE4EE4B28C), UInt64($550C7DC3D5FFB4E2), - UInt64($72BE5D74F27B896F), UInt64($80DEB1FE3B1696B1), - UInt64($9BDC06A725C71235), UInt64($C19BF174CF692694), - UInt64($E49B69C19EF14AD2), UInt64($EFBE4786384F25E3), - UInt64($0FC19DC68B8CD5B5), UInt64($240CA1CC77AC9C65), - UInt64($2DE92C6F592B0275), UInt64($4A7484AA6EA6E483), - UInt64($5CB0A9DCBD41FBD4), UInt64($76F988DA831153B5), - UInt64($983E5152EE66DFAB), UInt64($A831C66D2DB43210), - UInt64($B00327C898FB213F), UInt64($BF597FC7BEEF0EE4), - UInt64($C6E00BF33DA88FC2), UInt64($D5A79147930AA725), - UInt64($06CA6351E003826F), UInt64($142929670A0E6E70), - UInt64($27B70A8546D22FFC), UInt64($2E1B21385C26C926), - UInt64($4D2C6DFC5AC42AED), UInt64($53380D139D95B3DF), - UInt64($650A73548BAF63DE), UInt64($766A0ABB3C77B2A8), - UInt64($81C2C92E47EDAEE6), UInt64($92722C851482353B), - UInt64($A2BFE8A14CF10364), UInt64($A81A664BBC423001), - UInt64($C24B8B70D0F89791), UInt64($C76C51A30654BE30), - UInt64($D192E819D6EF5218), UInt64($D69906245565A910), - UInt64($F40E35855771202A), UInt64($106AA07032BBD1B8), - UInt64($19A4C116B8D2D0C8), UInt64($1E376C085141AB53), - UInt64($2748774CDF8EEB99), UInt64($34B0BCB5E19B48A8), - UInt64($391C0CB3C5C95A63), UInt64($4ED8AA4AE3418ACB), - UInt64($5B9CCA4F7763E373), UInt64($682E6FF3D6B2B8A3), - UInt64($748F82EE5DEFB2FC), UInt64($78A5636F43172F60), - UInt64($84C87814A1F0AB72), UInt64($8CC702081A6439EC), - UInt64($90BEFFFA23631E28), UInt64($A4506CEBDE82BDE9), - UInt64($BEF9A3F7B2C67915), UInt64($C67178F2E372532B), - UInt64($CA273ECEEA26619C), UInt64($D186B8C721C0C207), - UInt64($EADA7DD6CDE0EB1E), UInt64($F57D4F7FEE6ED178), - UInt64($06F067AA72176FBA), UInt64($0A637DC5A2C898A6), - UInt64($113F9804BEF90DAE), UInt64($1B710B35131C471B), - UInt64($28DB77F523047D84), UInt64($32CAAB7B40C72493), - UInt64($3C9EBE0A15C9BEBC), UInt64($431D67C49C100D4C), - UInt64($4CC5D4BECB3E42B6), UInt64($597F299CFC657E2A), - UInt64($5FCB6FAB3AD6FAEC), UInt64($6C44198C4A475817) - ); +implementation {$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const // Doubled K512 round constants plus the BSWAP64 mask, shared by the AVX2 and // SSE2 SIMD-schedule SHA-512 kernels. Each 128-bit K512 constant pair is stored // twice so one table feeds both the 256-bit AVX2 lanes and the 128-bit SSE2 // reads (both read at a 32-byte stride). Only the AVX2 kernel uses the appended // byte-swap mask (BSWAP64 pattern, twice) at [160..163]; the SSE2 kernel computes - // its byte-swap and needs no mask. Derived from K512. + // its byte-swap and needs no mask. K512_Doubled: array [0 .. 163] of UInt64 = ( UInt64($428A2F98D728AE22), UInt64($7137449123EF65CD), UInt64($428A2F98D728AE22), UInt64($7137449123EF65CD), @@ -146,101 +118,42 @@ interface UInt64($0001020304050607), UInt64($08090A0B0C0D0E0F), UInt64($0001020304050607), UInt64($08090A0B0C0D0E0F) ); -{$ENDIF HASHLIB_X86_SIMD} - -implementation - -uses - HlpBinaryPrimitives, - HlpBitOperations, - HlpCpuFeatures, - HlpSimdLevels; // ============================================================================= -// Scalar fallback implementation -// ============================================================================= - -procedure SHA512_Compress_Scalar(AState, AData: Pointer; ANumBlocks: UInt32); -var - LPState: PUInt64; - LPData: PByte; - LA, LB, LC, LD, LE, LF, LG, LH, LT1, LT2: UInt64; - LW: array [0 .. 79] of UInt64; - LRound: Int32; -begin - LPState := PUInt64(AState); - LPData := PByte(AData); - - while ANumBlocks > 0 do - begin - TBinaryPrimitives.CopyUInt64BigEndian(LPData, 0, @LW[0], 0, 128); - - for LRound := 16 to 79 do - begin - LT1 := LW[LRound - 2]; - LT2 := LW[LRound - 15]; - LW[LRound] := (TBitOperations.RotateRight64(LT1, 19) xor TBitOperations.RotateRight64(LT1, 61) - xor (LT1 shr 6)) + LW[LRound - 7] + - (TBitOperations.RotateRight64(LT2, 1) xor TBitOperations.RotateRight64(LT2, 8) - xor (LT2 shr 7)) + LW[LRound - 16]; - end; - - LA := LPState[0]; LB := LPState[1]; LC := LPState[2]; LD := LPState[3]; - LE := LPState[4]; LF := LPState[5]; LG := LPState[6]; LH := LPState[7]; - - for LRound := 0 to 79 do - begin - LT1 := LH + (TBitOperations.RotateRight64(LE, 14) xor TBitOperations.RotateRight64(LE, 18) - xor TBitOperations.RotateRight64(LE, 41)) + ((LE and LF) xor (not LE and LG)) - + K512[LRound] + LW[LRound]; - LT2 := (TBitOperations.RotateRight64(LA, 28) xor TBitOperations.RotateRight64(LA, 34) - xor TBitOperations.RotateRight64(LA, 39)) + - ((LA and LB) xor (LA and LC) xor (LB and LC)); - LH := LG; LG := LF; LF := LE; LE := LD + LT1; - LD := LC; LC := LB; LB := LA; LA := LT1 + LT2; - end; - - LPState[0] := LPState[0] + LA; LPState[1] := LPState[1] + LB; - LPState[2] := LPState[2] + LC; LPState[3] := LPState[3] + LD; - LPState[4] := LPState[4] + LE; LPState[5] := LPState[5] + LF; - LPState[6] := LPState[6] + LG; LPState[7] := LPState[7] + LH; - - System.FillChar(LW, System.SizeOf(LW), 0); - System.Inc(LPData, 128); - System.Dec(ANumBlocks); - end; -end; - -// ============================================================================= -// SIMD implementations -// +// SIMD kernels // i386: SSE2 -// x86_64: AVX2, SSE2 -// aarch64: SHA512 Crypto Extensions +// x86_64: AVX2, SSSE3, SSE2 // ============================================================================= -{$IFDEF HASHLIB_I386_ASM} - procedure SHA512_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} - {$I ..\Include\Simd\SHA512\SHA512CompressSse2_i386.inc} +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} +{$I ..\..\Include\Simd\SHA512\SHA512CompressSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_i386.inc} +{$I ..\..\Include\Simd\SHA512\SHA512CompressSse2_i386.inc} +{$ENDIF} end; -{$ENDIF HASHLIB_I386_ASM} - {$IFDEF HASHLIB_X86_64_ASM} -procedure SHA512_Compress_Sse2(AState, AData: Pointer; ANumBlocks: UInt32; +procedure SHA512_Compress_Ssse3(AState, AData: Pointer; ANumBlocks: UInt32; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\SHA512\SHA512CompressSse2_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA512\SHA512CompressSsse3_x86_64.inc} end; procedure SHA512_Compress_Avx2(AState, AData: Pointer; ANumBlocks: UInt32; AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} - {$I ..\Include\Simd\SHA512\SHA512CompressAvx2_x86_64.inc} + {$I ..\..\Include\Simd\Common\HlpSimdProc4Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA512\SHA512CompressAvx2_x86_64.inc} +end; + +procedure SHA512_Compress_Ssse3_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); +begin + SHA512_Compress_Ssse3(AState, AData, ANumBlocks, @K512_Doubled); end; procedure SHA512_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); @@ -250,8 +163,6 @@ procedure SHA512_Compress_Avx2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); {$ENDIF HASHLIB_X86_64_ASM} -{$IFDEF HASHLIB_X86_SIMD} - procedure SHA512_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); begin {$IFDEF HASHLIB_X86_64_ASM} @@ -263,57 +174,28 @@ procedure SHA512_Compress_Sse2_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); {$ENDIF HASHLIB_X86_SIMD} -{$IFDEF HASHLIB_AARCH64_ASM} - -procedure SHA512_Compress_CryptoExt(AState, AData: Pointer; ANumBlocks: UInt32; - AConstants: Pointer); - {$I ..\Include\Simd\Common\HlpSimdProc4Begin_aarch64.inc} - {$I ..\Include\Simd\SHA512\SHA512CompressCryptoExt_aarch64.inc} -end; - -procedure SHA512_Compress_CryptoExt_Wrap(AState, AData: Pointer; ANumBlocks: UInt32); -begin - SHA512_Compress_CryptoExt(AState, AData, ANumBlocks, @K512); -end; - -{$ENDIF HASHLIB_AARCH64_ASM} +{ TSHA2_512X86Backend } -// ============================================================================= -// Dispatch initialization -// ============================================================================= - -procedure InitDispatch(); +class function TSHA2_512X86Backend.Select(AScalar: TSHA512CompressProc): TSHA512CompressProc; begin - SHA512_Compress := @SHA512_Compress_Scalar; {$IFDEF HASHLIB_I386_ASM} case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of TX86SimdLevel.SSE2: - begin - SHA512_Compress := @SHA512_Compress_Sse2_Wrap; - end; + Exit(@SHA512_Compress_Sse2_Wrap); end; {$ENDIF} {$IFDEF HASHLIB_X86_64_ASM} - case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSSE3, + TX86SimdLevel.SSE2]) of TX86SimdLevel.AVX2: - begin - SHA512_Compress := @SHA512_Compress_Avx2_Wrap; - end; + Exit(@SHA512_Compress_Avx2_Wrap); + TX86SimdLevel.SSSE3: + Exit(@SHA512_Compress_Ssse3_Wrap); TX86SimdLevel.SSE2: - begin - SHA512_Compress := @SHA512_Compress_Sse2_Wrap; - end; - end; -{$ENDIF} -{$IFDEF HASHLIB_AARCH64_ASM} - if TCpuFeatures.Arm.HasSHA512() then - begin - SHA512_Compress := @SHA512_Compress_CryptoExt_Wrap; + Exit(@SHA512_Compress_Sse2_Wrap); end; {$ENDIF} + Result := AScalar; end; -initialization - InitDispatch(); - end. diff --git a/HashLib/src/Simd/Backend/HlpSHA3ArmBackend.pas b/HashLib/src/Simd/Backend/HlpSHA3ArmBackend.pas new file mode 100644 index 00000000..84f6f4bf --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA3ArmBackend.pas @@ -0,0 +1,114 @@ +unit HlpSHA3ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA3; + +type + /// + /// Arm SIMD backend for Keccak-F1600: owns the AArch64 FEAT_SHA3 crypto- + /// extension permute / absorb kernels (bodies in Include\Simd\SHA3\) and + /// the runtime feature check via TCpuFeatures.Arm. Compiles on every + /// target - without Arm SIMD the selectors return the scalar routines. + /// + TSHA3ArmBackend = class sealed + public + class function SelectPermute(AScalar: TKeccakF1600Proc): TKeccakF1600Proc; static; + class function SelectAbsorb(AScalar: TKeccakF1600AbsorbProc): TKeccakF1600AbsorbProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: SHA3 Crypto Extensions, NEON +// +// Both kernels reuse the plain RC round-constant table directly (the +// FEAT_SHA3 asm broadcasts each 64-bit iota with ld1r; the GPR permute walks +// it behind an end-pointer), so no packed constant block is needed. +// ============================================================================= + +procedure KeccakF1600_CryptoExt(AState: Pointer; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600CryptoExt_aarch64.inc} +end; + +procedure KeccakF1600_CryptoExt_Wrap(AState: Pointer); +begin + KeccakF1600_CryptoExt(AState, @RC); +end; + +procedure KeccakF1600_CryptoExt_Absorb(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600CryptoExtAbsorb_aarch64.inc} +end; + +procedure KeccakF1600_CryptoExt_Absorb_Wrap(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); +begin + KeccakF1600_CryptoExt_Absorb(AState, AData, ABlockCount, ABlockSize, @RC); +end; + +procedure KeccakF1600_Gpr(AState: Pointer; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600Gpr_aarch64.inc} +end; + +procedure KeccakF1600_Gpr_Wrap(AState: Pointer); +begin + KeccakF1600_Gpr(AState, @RC); +end; + +procedure KeccakF1600_Gpr_Absorb(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_aarch64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600GprAbsorb_aarch64.inc} +end; + +procedure KeccakF1600_Gpr_Absorb_Wrap(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); +begin + KeccakF1600_Gpr_Absorb(AState, AData, ABlockCount, ABlockSize, @RC); +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TSHA3ArmBackend } + +class function TSHA3ArmBackend.SelectPermute(AScalar: TKeccakF1600Proc): TKeccakF1600Proc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + if TCpuFeatures.Arm.HasSHA3() then + Exit(@KeccakF1600_CryptoExt_Wrap); + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@KeccakF1600_Gpr_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TSHA3ArmBackend.SelectAbsorb(AScalar: TKeccakF1600AbsorbProc): TKeccakF1600AbsorbProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + if TCpuFeatures.Arm.HasSHA3() then + Exit(@KeccakF1600_CryptoExt_Absorb_Wrap); + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: + Exit(@KeccakF1600_Gpr_Absorb_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpSHA3X86Backend.pas b/HashLib/src/Simd/Backend/HlpSHA3X86Backend.pas new file mode 100644 index 00000000..d7c8492f --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpSHA3X86Backend.pas @@ -0,0 +1,225 @@ +unit HlpSHA3X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA3; + +type + /// + /// x86 SIMD backend for Keccak-F1600: owns the x86_64 AVX2 permute / absorb + /// kernels and the i386 SSE2 permute kernel (bodies in + /// Include\Simd\SHA3\) plus the runtime tier selection via + /// TCpuFeatures.X86. Compiles on every target - without a matching + /// tier the selectors return the scalar routines. + /// + TSHA3X86Backend = class sealed + public + class function SelectPermute(AScalar: TKeccakF1600Proc): TKeccakF1600Proc; static; + class function SelectAbsorb(AScalar: TKeccakF1600AbsorbProc): TKeccakF1600AbsorbProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +const + // Plain 24-entry iota round constants (stride 8) for the scalar-layout + // kernels (i386 SSE2 + x86_64 GPR; the AVX2 kernel uses the x4-broadcast + // table below). + K_KECCAK_IOTAS: array [0 .. 23] of UInt64 = ( + UInt64($0000000000000001), UInt64($0000000000008082), + UInt64($800000000000808A), UInt64($8000000080008000), + UInt64($000000000000808B), UInt64($0000000080000001), + UInt64($8000000080008081), UInt64($8000000000008009), + UInt64($000000000000008A), UInt64($0000000000000088), + UInt64($0000000080008009), UInt64($000000008000000A), + UInt64($000000008000808B), UInt64($800000000000008B), + UInt64($8000000000008089), UInt64($8000000000008003), + UInt64($8000000000008002), UInt64($8000000000000080), + UInt64($000000000000800A), UInt64($800000008000000A), + UInt64($8000000080008081), UInt64($8000000000008080), + UInt64($0000000080000001), UInt64($8000000080008008)); + + K_KECCAK: packed record + RhotatesLeft: array [0..23] of UInt64; + RhotatesRight: array [0..23] of UInt64; + Iotas: array [0..95] of UInt64; + Jagged: array [0..24] of Int32; + end = ( + RhotatesLeft: ( + 3, 18, 36, 41, // ymm2: [2][0] [4][0] [1][0] [3][0] + 1, 62, 28, 27, // ymm1: [0][1] [0][2] [0][3] [0][4] + 45, 6, 56, 39, // ymm3: [3][1] [1][2] [4][3] [2][4] + 10, 61, 55, 8, // ymm4: [2][1] [4][2] [1][3] [3][4] + 2, 15, 25, 20, // ymm5: [4][1] [3][2] [2][3] [1][4] + 44, 43, 21, 14); // ymm6: [1][1] [2][2] [3][3] [4][4] + RhotatesRight: ( + 61, 46, 28, 23, + 63, 2, 36, 37, + 19, 58, 8, 25, + 54, 3, 9, 56, + 62, 49, 39, 44, + 20, 21, 43, 50); + Iotas: ( + UInt64($0000000000000001), UInt64($0000000000000001), UInt64($0000000000000001), UInt64($0000000000000001), + UInt64($0000000000008082), UInt64($0000000000008082), UInt64($0000000000008082), UInt64($0000000000008082), + UInt64($800000000000808A), UInt64($800000000000808A), UInt64($800000000000808A), UInt64($800000000000808A), + UInt64($8000000080008000), UInt64($8000000080008000), UInt64($8000000080008000), UInt64($8000000080008000), + UInt64($000000000000808B), UInt64($000000000000808B), UInt64($000000000000808B), UInt64($000000000000808B), + UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), + UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), + UInt64($8000000000008009), UInt64($8000000000008009), UInt64($8000000000008009), UInt64($8000000000008009), + UInt64($000000000000008A), UInt64($000000000000008A), UInt64($000000000000008A), UInt64($000000000000008A), + UInt64($0000000000000088), UInt64($0000000000000088), UInt64($0000000000000088), UInt64($0000000000000088), + UInt64($0000000080008009), UInt64($0000000080008009), UInt64($0000000080008009), UInt64($0000000080008009), + UInt64($000000008000000A), UInt64($000000008000000A), UInt64($000000008000000A), UInt64($000000008000000A), + UInt64($000000008000808B), UInt64($000000008000808B), UInt64($000000008000808B), UInt64($000000008000808B), + UInt64($800000000000008B), UInt64($800000000000008B), UInt64($800000000000008B), UInt64($800000000000008B), + UInt64($8000000000008089), UInt64($8000000000008089), UInt64($8000000000008089), UInt64($8000000000008089), + UInt64($8000000000008003), UInt64($8000000000008003), UInt64($8000000000008003), UInt64($8000000000008003), + UInt64($8000000000008002), UInt64($8000000000008002), UInt64($8000000000008002), UInt64($8000000000008002), + UInt64($8000000000000080), UInt64($8000000000000080), UInt64($8000000000000080), UInt64($8000000000000080), + UInt64($000000000000800A), UInt64($000000000000800A), UInt64($000000000000800A), UInt64($000000000000800A), + UInt64($800000008000000A), UInt64($800000008000000A), UInt64($800000008000000A), UInt64($800000008000000A), + UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), UInt64($8000000080008081), + UInt64($8000000000008080), UInt64($8000000000008080), UInt64($8000000000008080), UInt64($8000000000008080), + UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), UInt64($0000000080000001), + UInt64($8000000080008008), UInt64($8000000080008008), UInt64($8000000080008008), UInt64($8000000080008008)); + Jagged: ( + 0, 32, 40, 48, 56, 80, 192, 104, 144, 184, + 64, 128, 200, 176, 120, 88, 96, 168, 208, 152, + 72, 160, 136, 112, 216) + ); + +// ============================================================================= +// SIMD kernels +// i386: SSE2 +// x86_64: AVX2, SSE2 +// ============================================================================= + +{$IFDEF HASHLIB_I386_ASM} + +procedure KeccakF1600_Sse2(AState: Pointer; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600Sse2_i386.inc} +end; + +procedure KeccakF1600_Sse2_Absorb(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_i386.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600Sse2Absorb_i386.inc} +end; + +procedure KeccakF1600_Sse2_Wrap(AState: Pointer); +begin + KeccakF1600_Sse2(AState, @K_KECCAK_IOTAS); +end; + +procedure KeccakF1600_Sse2_Absorb_Wrap(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); +begin + KeccakF1600_Sse2_Absorb(AState, AData, ABlockCount, ABlockSize, + @K_KECCAK_IOTAS); +end; + +{$ENDIF HASHLIB_I386_ASM} + +{$IFDEF HASHLIB_X86_64_ASM} + +procedure KeccakF1600_Avx2(AState: Pointer; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600Avx2_x86_64.inc} +end; + +procedure KeccakF1600_Avx2_Absorb(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600Avx2Absorb_x86_64.inc} +end; + +procedure KeccakF1600_Gpr(AState: Pointer; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600Gpr_x86_64.inc} +end; + +procedure KeccakF1600_Gpr_Absorb(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32; AConstants: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc5Begin_x86_64.inc} + {$I ..\..\Include\Simd\SHA3\KeccakF1600GprAbsorb_x86_64.inc} +end; + +procedure KeccakF1600_Avx2_Wrap(AState: Pointer); +begin + KeccakF1600_Avx2(AState, @K_KECCAK); +end; + +procedure KeccakF1600_Avx2_Absorb_Wrap(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); +begin + KeccakF1600_Avx2_Absorb(AState, AData, ABlockCount, ABlockSize, @K_KECCAK); +end; + +procedure KeccakF1600_Gpr_Wrap(AState: Pointer); +begin + KeccakF1600_Gpr(AState, @K_KECCAK_IOTAS); +end; + +procedure KeccakF1600_Gpr_Absorb_Wrap(AState: Pointer; AData: PByte; + ABlockCount: Int32; ABlockSize: Int32); +begin + KeccakF1600_Gpr_Absorb(AState, AData, ABlockCount, ABlockSize, + @K_KECCAK_IOTAS); +end; + +{$ENDIF HASHLIB_X86_64_ASM} + +{$ENDIF HASHLIB_X86_SIMD} + +{ TSHA3X86Backend } + +class function TSHA3X86Backend.SelectPermute(AScalar: TKeccakF1600Proc): TKeccakF1600Proc; +begin +{$IFDEF HASHLIB_X86_64_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@KeccakF1600_Avx2_Wrap); + TX86SimdLevel.SSE2: + Exit(@KeccakF1600_Gpr_Wrap); + end; +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + Exit(@KeccakF1600_Sse2_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TSHA3X86Backend.SelectAbsorb(AScalar: TKeccakF1600AbsorbProc): TKeccakF1600AbsorbProc; +begin +{$IFDEF HASHLIB_X86_64_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@KeccakF1600_Avx2_Absorb_Wrap); + TX86SimdLevel.SSE2: + Exit(@KeccakF1600_Gpr_Absorb_Wrap); + end; +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.SSE2]) of + TX86SimdLevel.SSE2: + Exit(@KeccakF1600_Sse2_Absorb_Wrap); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpScryptArmBackend.pas b/HashLib/src/Simd/Backend/HlpScryptArmBackend.pas new file mode 100644 index 00000000..51112edb --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpScryptArmBackend.pas @@ -0,0 +1,49 @@ +unit HlpScryptArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpPBKDF_ScryptNotBuildInAdapter; + +type + /// + /// Arm SIMD backend for scrypt's Salsa20/8 XOR core. The NEON kernel (body in + /// Include\Simd\Scrypt\) is compiled and verified but intentionally NOT + /// selected: at p=1 scrypt's Salsa20/8 is a single 64-byte block on a strictly + /// serial chain, and AArch64's 31 GP registers + single-cycle 'ror' let the + /// scalar kernel win at every N (this mirrors OpenSSL/libsodium/Tarsnap, which + /// ship x86 SSE2 scrypt but no NEON). Kept for reference / a future p>1 path. + /// So Select always returns the scalar routine. + /// + TScryptArmBackend = class sealed + public + class function Select(AScalar: TScryptSalsaXorProc): TScryptSalsaXorProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +// ============================================================================= +// SIMD kernels +// aarch64: NEON (compiled/verified but not registered - see the class comment) +// ============================================================================= + +procedure Scrypt_SalsaXor_Neon(AState, AInput: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} + {$I ..\..\Include\Simd\Scrypt\ScryptSalsa8Neon_aarch64.inc} +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TScryptArmBackend } + +class function TScryptArmBackend.Select(AScalar: TScryptSalsaXorProc): TScryptSalsaXorProc; +begin + // NEON not registered for scrypt on AArch64 (scalar is faster at p=1). + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpScryptX86Backend.pas b/HashLib/src/Simd/Backend/HlpScryptX86Backend.pas new file mode 100644 index 00000000..e2e26b9f --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpScryptX86Backend.pas @@ -0,0 +1,75 @@ +unit HlpScryptX86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpPBKDF_ScryptNotBuildInAdapter; + +type + /// + /// x86 SIMD backend for scrypt's Salsa20/8 XOR core: owns the SSE2 / AVX2 + /// kernels (bodies in Include\Simd\Scrypt\) and the runtime tier + /// selection via TCpuFeatures.X86. Compiles on every target - built + /// without x86 SIMD, Select just returns the scalar routine. + /// + TScryptX86Backend = class sealed + public + class function Select(AScalar: TScryptSalsaXorProc): TScryptSalsaXorProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// i386: AVX2, SSE2 +// x86_64: AVX2, SSE2 +// ============================================================================= + +procedure Scrypt_SalsaXor_Sse2(AState, AInput: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} +{$I ..\..\Include\Simd\Scrypt\ScryptSalsa8Sse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} +{$I ..\..\Include\Simd\Scrypt\ScryptSalsa8Sse2_i386.inc} +{$ENDIF} +end; + +procedure Scrypt_SalsaXor_Avx(AState, AInput: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} +{$I ..\..\Include\Simd\Scrypt\ScryptSalsa8Avx_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} +{$I ..\..\Include\Simd\Scrypt\ScryptSalsa8Avx_i386.inc} +{$ENDIF} +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TScryptX86Backend } + +class function TScryptX86Backend.Select(AScalar: TScryptSalsaXorProc): TScryptSalsaXorProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@Scrypt_SalsaXor_Avx); + TX86SimdLevel.SSE2: + Exit(@Scrypt_SalsaXor_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpXXHash3ArmBackend.pas b/HashLib/src/Simd/Backend/HlpXXHash3ArmBackend.pas new file mode 100644 index 00000000..4ee54372 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpXXHash3ArmBackend.pas @@ -0,0 +1,105 @@ +unit HlpXXHash3ArmBackend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpXXHash3; + +type + /// + /// Arm SIMD backend for XXH3: owns the AArch64 NEON accumulate / scramble / + /// init-secret kernels (bodies in Include\Simd\XXH3\) and the runtime + /// tier selection via TCpuFeatures.Arm. Compiles on every target - + /// without Arm SIMD the selectors return the scalar routines. + /// + TXXHash3ArmBackend = class sealed + public + class function SelectAccumulate512(AScalar: TXXH3Accumulate512Proc): TXXH3Accumulate512Proc; static; + class function SelectAccumulate(AScalar: TXXH3AccumulateProc): TXXH3AccumulateProc; static; + class function SelectScrambleAcc(AScalar: TXXH3ScrambleAccProc): TXXH3ScrambleAccProc; static; + class function SelectInitSecret(AScalar: TXXH3InitSecretProc): TXXH3InitSecretProc; static; + end; + +implementation + +{$IFDEF HASHLIB_AARCH64_ASM} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// aarch64: NEON +// ============================================================================= + +procedure XXH3_Accumulate512_Neon(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_aarch64.inc} + {$I ..\..\Include\Simd\XXH3\XXH3Acc512Neon_aarch64.inc} +end; + +procedure XXH3_ScrambleAcc_Neon(AAcc: Pointer; ASecret: Pointer); + {$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_aarch64.inc} + {$I ..\..\Include\Simd\XXH3\XXH3ScrambleNeon_aarch64.inc} +end; + +procedure XXH3_InitSecret_Neon(ACustomSecret: Pointer; + ADefaultSecret: Pointer; ASeedPtr: PUInt64); + {$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_aarch64.inc} + {$I ..\..\Include\Simd\XXH3\XXH3InitSecretNeon_aarch64.inc} +end; + +procedure XXH3_Accumulate_Neon(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32); +begin + XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Neon); +end; + +{$ENDIF HASHLIB_AARCH64_ASM} + +{ TXXHash3ArmBackend } + +class function TXXHash3ArmBackend.SelectAccumulate512(AScalar: TXXH3Accumulate512Proc): TXXH3Accumulate512Proc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: Exit(@XXH3_Accumulate512_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TXXHash3ArmBackend.SelectAccumulate(AScalar: TXXH3AccumulateProc): TXXH3AccumulateProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: Exit(@XXH3_Accumulate_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TXXHash3ArmBackend.SelectScrambleAcc(AScalar: TXXH3ScrambleAccProc): TXXH3ScrambleAccProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: Exit(@XXH3_ScrambleAcc_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TXXHash3ArmBackend.SelectInitSecret(AScalar: TXXH3InitSecretProc): TXXH3InitSecretProc; +begin +{$IFDEF HASHLIB_AARCH64_ASM} + case TCpuFeatures.Arm.SelectSlot([TArmSimdLevel.NEON]) of + TArmSimdLevel.NEON: Exit(@XXH3_InitSecret_Neon); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Backend/HlpXXHash3X86Backend.pas b/HashLib/src/Simd/Backend/HlpXXHash3X86Backend.pas new file mode 100644 index 00000000..b31d3180 --- /dev/null +++ b/HashLib/src/Simd/Backend/HlpXXHash3X86Backend.pas @@ -0,0 +1,177 @@ +unit HlpXXHash3X86Backend; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpXXHash3; + +type + /// + /// x86 SIMD backend for XXH3: owns the SSE2 / AVX2 accumulate / scramble / + /// init-secret kernels (bodies in Include\Simd\XXH3\) and the runtime + /// tier selection via TCpuFeatures.X86. Compiles on every target - + /// without x86 SIMD the selectors return the scalar routines. + /// + TXXHash3X86Backend = class sealed + public + class function SelectAccumulate512(AScalar: TXXH3Accumulate512Proc): TXXH3Accumulate512Proc; static; + class function SelectAccumulate(AScalar: TXXH3AccumulateProc): TXXH3AccumulateProc; static; + class function SelectScrambleAcc(AScalar: TXXH3ScrambleAccProc): TXXH3ScrambleAccProc; static; + class function SelectInitSecret(AScalar: TXXH3InitSecretProc): TXXH3InitSecretProc; static; + end; + +implementation + +{$IFDEF HASHLIB_X86_SIMD} + +uses + HlpCpuFeatures, + HlpSimdLevels; + +// ============================================================================= +// SIMD kernels +// i386: AVX2, SSE2 +// x86_64: AVX2, SSE2 +// ============================================================================= + +procedure XXH3_Accumulate512_Sse2(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} +{$I ..\..\Include\Simd\XXH3\XXH3Acc512Sse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_i386.inc} +{$I ..\..\Include\Simd\XXH3\XXH3Acc512Sse2_i386.inc} +{$ENDIF} +end; + +procedure XXH3_ScrambleAcc_Sse2(AAcc: Pointer; ASecret: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} +{$I ..\..\Include\Simd\XXH3\XXH3ScrambleSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} +{$I ..\..\Include\Simd\XXH3\XXH3ScrambleSse2_i386.inc} +{$ENDIF} +end; + +procedure XXH3_InitSecret_Sse2(ACustomSecret: Pointer; + ADefaultSecret: Pointer; ASeedPtr: PUInt64); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} +{$I ..\..\Include\Simd\XXH3\XXH3InitSecretSse2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_i386.inc} +{$I ..\..\Include\Simd\XXH3\XXH3InitSecretSse2_i386.inc} +{$ENDIF} +end; + +procedure XXH3_Accumulate512_Avx2(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} +{$I ..\..\Include\Simd\XXH3\XXH3Acc512Avx2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_i386.inc} +{$I ..\..\Include\Simd\XXH3\XXH3Acc512Avx2_i386.inc} +{$ENDIF} +end; + +procedure XXH3_ScrambleAcc_Avx2(AAcc: Pointer; ASecret: Pointer); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_x86_64.inc} +{$I ..\..\Include\Simd\XXH3\XXH3ScrambleAvx2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc2Begin_i386.inc} +{$I ..\..\Include\Simd\XXH3\XXH3ScrambleAvx2_i386.inc} +{$ENDIF} +end; + +procedure XXH3_InitSecret_Avx2(ACustomSecret: Pointer; + ADefaultSecret: Pointer; ASeedPtr: PUInt64); +{$IFDEF HASHLIB_X86_64_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_x86_64.inc} +{$I ..\..\Include\Simd\XXH3\XXH3InitSecretAvx2_x86_64.inc} +{$ENDIF} +{$IFDEF HASHLIB_I386_ASM} +{$I ..\..\Include\Simd\Common\HlpSimdProc3Begin_i386.inc} +{$I ..\..\Include\Simd\XXH3\XXH3InitSecretAvx2_i386.inc} +{$ENDIF} +end; + +procedure XXH3_Accumulate_Sse2(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32); +begin + XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Sse2); +end; + +procedure XXH3_Accumulate_Avx2(AAcc: Pointer; AInput: Pointer; + ASecret: Pointer; ANbStripes: Int32); +begin + XXH3_Accumulate_Loop(AAcc, AInput, ASecret, ANbStripes, @XXH3_Accumulate512_Avx2); +end; + +{$ENDIF HASHLIB_X86_SIMD} + +{ TXXHash3X86Backend } + +class function TXXHash3X86Backend.SelectAccumulate512(AScalar: TXXH3Accumulate512Proc): TXXH3Accumulate512Proc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@XXH3_Accumulate512_Avx2); + TX86SimdLevel.SSE2: + Exit(@XXH3_Accumulate512_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TXXHash3X86Backend.SelectAccumulate(AScalar: TXXH3AccumulateProc): TXXH3AccumulateProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@XXH3_Accumulate_Avx2); + TX86SimdLevel.SSE2: + Exit(@XXH3_Accumulate_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TXXHash3X86Backend.SelectScrambleAcc(AScalar: TXXH3ScrambleAccProc): TXXH3ScrambleAccProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@XXH3_ScrambleAcc_Avx2); + TX86SimdLevel.SSE2: + Exit(@XXH3_ScrambleAcc_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +class function TXXHash3X86Backend.SelectInitSecret(AScalar: TXXH3InitSecretProc): TXXH3InitSecretProc; +begin +{$IFDEF HASHLIB_X86_SIMD} + case TCpuFeatures.X86.SelectSlot([TX86SimdLevel.AVX2, TX86SimdLevel.SSE2]) of + TX86SimdLevel.AVX2: + Exit(@XXH3_InitSecret_Avx2); + TX86SimdLevel.SSE2: + Exit(@XXH3_InitSecret_Sse2); + end; +{$ENDIF} + Result := AScalar; +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpAdler32Simd.pas b/HashLib/src/Simd/Facade/HlpAdler32Simd.pas new file mode 100644 index 00000000..fead6def --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpAdler32Simd.pas @@ -0,0 +1,46 @@ +unit HlpAdler32Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpAdler32; + +type + /// + /// Arch-neutral SIMD facade for Adler-32. Picks the per-arch backend at compile + /// time and returns the best update routine the running CPU supports, or + /// AScalar when no SIMD backend is built/available. The hash unit calls + /// only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TAdler32Simd = class sealed + public + class function Select(AScalar: TAdler32UpdateProc): TAdler32UpdateProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpAdler32X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpAdler32ArmBackend; +{$IFEND} + +{ TAdler32Simd } + +class function TAdler32Simd.Select(AScalar: TAdler32UpdateProc): TAdler32UpdateProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TAdler32X86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TAdler32ArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpArgon2Simd.pas b/HashLib/src/Simd/Facade/HlpArgon2Simd.pas new file mode 100644 index 00000000..7ae267bf --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpArgon2Simd.pas @@ -0,0 +1,46 @@ +unit HlpArgon2Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpPBKDF_Argon2NotBuildInAdapter; + +type + /// + /// Arch-neutral SIMD facade for Argon2's fill-block. Picks the per-arch backend + /// at compile time and returns the best routine the running CPU supports, or + /// AScalar when no SIMD backend is built/available. The KDF unit calls + /// only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TArgon2Simd = class sealed + public + class function Select(AScalar: TArgon2FillBlockProc): TArgon2FillBlockProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpArgon2X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpArgon2ArmBackend; +{$IFEND} + +{ TArgon2Simd } + +class function TArgon2Simd.Select(AScalar: TArgon2FillBlockProc): TArgon2FillBlockProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TArgon2X86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TArgon2ArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpBlake2BSimd.pas b/HashLib/src/Simd/Facade/HlpBlake2BSimd.pas new file mode 100644 index 00000000..3e7a00a0 --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpBlake2BSimd.pas @@ -0,0 +1,46 @@ +unit HlpBlake2BSimd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake2B; + +type + /// + /// Arch-neutral SIMD facade for Blake2B. Picks the per-arch backend at compile + /// time and returns the best compression routine the running CPU supports, or + /// AScalar when no SIMD backend is built/available. The hash unit calls + /// only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TBlake2BSimd = class sealed + public + class function Select(AScalar: TBlake2BCompressProc): TBlake2BCompressProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpBlake2BX86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpBlake2BArmBackend; +{$IFEND} + +{ TBlake2BSimd } + +class function TBlake2BSimd.Select(AScalar: TBlake2BCompressProc): TBlake2BCompressProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TBlake2BX86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TBlake2BArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpBlake2SSimd.pas b/HashLib/src/Simd/Facade/HlpBlake2SSimd.pas new file mode 100644 index 00000000..7dfc2648 --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpBlake2SSimd.pas @@ -0,0 +1,46 @@ +unit HlpBlake2SSimd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake2S; + +type + /// + /// Arch-neutral SIMD facade for Blake2S. Picks the per-arch backend at compile + /// time and returns the best compression routine the running CPU supports, or + /// AScalar when no SIMD backend is built/available. The hash unit calls + /// only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TBlake2SSimd = class sealed + public + class function Select(AScalar: TBlake2SCompressProc): TBlake2SCompressProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpBlake2SX86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpBlake2SArmBackend; +{$IFEND} + +{ TBlake2SSimd } + +class function TBlake2SSimd.Select(AScalar: TBlake2SCompressProc): TBlake2SCompressProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TBlake2SX86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TBlake2SArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpBlake3Simd.pas b/HashLib/src/Simd/Facade/HlpBlake3Simd.pas new file mode 100644 index 00000000..20bcaf09 --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpBlake3Simd.pas @@ -0,0 +1,70 @@ +unit HlpBlake3Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpBlake3; + +type + /// + /// Arch-neutral SIMD facade for Blake3. Picks the per-arch backend at compile + /// time and returns the best compression / hash-many routines (and the parallel + /// degree) the running CPU supports, or the scalar reference when no SIMD + /// backend is built/available. The hash unit calls only this facade and stays + /// free of any TCpuFeatures / HASHLIB_*_ASM knowledge. + /// + TBlake3Simd = class sealed + public + class function SelectCompress(AScalar: TBlake3CompressProc): TBlake3CompressProc; static; + class function SelectHashMany(AScalar: TBlake3HashManyProc): TBlake3HashManyProc; static; + class function SelectParallelDegree(ADefault: Int32): Int32; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpBlake3X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpBlake3ArmBackend; +{$IFEND} + +{ TBlake3Simd } + +class function TBlake3Simd.SelectCompress(AScalar: TBlake3CompressProc): TBlake3CompressProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TBlake3X86Backend.SelectCompress(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TBlake3ArmBackend.SelectCompress(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +class function TBlake3Simd.SelectHashMany(AScalar: TBlake3HashManyProc): TBlake3HashManyProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TBlake3X86Backend.SelectHashMany(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TBlake3ArmBackend.SelectHashMany(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +class function TBlake3Simd.SelectParallelDegree(ADefault: Int32): Int32; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TBlake3X86Backend.SelectParallelDegree(ADefault); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TBlake3ArmBackend.SelectParallelDegree(ADefault); +{$ELSE} + Result := ADefault; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpCRCSimd.pas b/HashLib/src/Simd/Facade/HlpCRCSimd.pas new file mode 100644 index 00000000..40d00039 --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpCRCSimd.pas @@ -0,0 +1,51 @@ +unit HlpCRCSimd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpCRCCore; + +type + /// + /// Arch-neutral SIMD facade for CRC's carry-less-multiply fold. Picks the + /// per-arch backend at compile time and returns the reflected / forward fold + /// entry points the running CPU supports (plus whether they use carry-less + /// multiply), or the scalar routines when no SIMD backend is built/available. + /// The CRC core calls only this facade and stays free of any + /// TCpuFeatures / HASHLIB_*_ASM knowledge. + /// + TCRCSimd = class sealed + public + class function Select(AReflectedScalar, AForwardScalar: TCRCFoldFunc) + : TCRCFoldSelection; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpCRCX86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpCRCArmBackend; +{$IFEND} + +{ TCRCSimd } + +class function TCRCSimd.Select(AReflectedScalar, AForwardScalar: TCRCFoldFunc) + : TCRCFoldSelection; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TCRCX86Backend.Select(AReflectedScalar, AForwardScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TCRCArmBackend.Select(AReflectedScalar, AForwardScalar); +{$ELSE} + Result.Reflected := AReflectedScalar; + Result.Fwd := AForwardScalar; + Result.UsesCarrylessMul := False; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpSHA1Simd.pas b/HashLib/src/Simd/Facade/HlpSHA1Simd.pas new file mode 100644 index 00000000..ea184972 --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpSHA1Simd.pas @@ -0,0 +1,48 @@ +unit HlpSHA1Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA1; + +type + /// + /// Arch-neutral SIMD facade for SHA-1. Picks the per-arch backend at compile + /// time and returns the best block-compression routine the running CPU + /// supports, or AScalar when no SIMD backend is built/available. The + /// hash unit calls only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TSHA1Simd = class sealed + public + /// Return the fastest available SHA-1 compression routine for the + /// running CPU, falling back to AScalar. Call once at init. + class function Select(AScalar: TSHA1CompressProc): TSHA1CompressProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpSHA1X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpSHA1ArmBackend; +{$IFEND} + +{ TSHA1Simd } + +class function TSHA1Simd.Select(AScalar: TSHA1CompressProc): TSHA1CompressProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TSHA1X86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TSHA1ArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpSHA2_256Simd.pas b/HashLib/src/Simd/Facade/HlpSHA2_256Simd.pas new file mode 100644 index 00000000..da7f5ecc --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpSHA2_256Simd.pas @@ -0,0 +1,48 @@ +unit HlpSHA2_256Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA2_256Base; + +type + /// + /// Arch-neutral SIMD facade for SHA-256. Picks the per-arch backend at compile + /// time and returns the best block-compression routine the running CPU + /// supports, or AScalar when no SIMD backend is built/available. The + /// hash unit calls only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TSHA2_256Simd = class sealed + public + /// Return the fastest available SHA-256 compression routine for the + /// running CPU, falling back to AScalar. Call once at init. + class function Select(AScalar: TSHA256CompressProc): TSHA256CompressProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpSHA2_256X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpSHA2_256ArmBackend; +{$IFEND} + +{ TSHA2_256Simd } + +class function TSHA2_256Simd.Select(AScalar: TSHA256CompressProc): TSHA256CompressProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TSHA2_256X86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TSHA2_256ArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpSHA2_512Simd.pas b/HashLib/src/Simd/Facade/HlpSHA2_512Simd.pas new file mode 100644 index 00000000..d9a6f9ff --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpSHA2_512Simd.pas @@ -0,0 +1,48 @@ +unit HlpSHA2_512Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA2_512Base; + +type + /// + /// Arch-neutral SIMD facade for SHA-512. Picks the per-arch backend at compile + /// time and returns the best block-compression routine the running CPU + /// supports, or AScalar when no SIMD backend is built/available. The + /// hash unit calls only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TSHA2_512Simd = class sealed + public + /// Return the fastest available SHA-512 compression routine for the + /// running CPU, falling back to AScalar. Call once at init. + class function Select(AScalar: TSHA512CompressProc): TSHA512CompressProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpSHA2_512X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpSHA2_512ArmBackend; +{$IFEND} + +{ TSHA2_512Simd } + +class function TSHA2_512Simd.Select(AScalar: TSHA512CompressProc): TSHA512CompressProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TSHA2_512X86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TSHA2_512ArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpSHA3Simd.pas b/HashLib/src/Simd/Facade/HlpSHA3Simd.pas new file mode 100644 index 00000000..63a7c7da --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpSHA3Simd.pas @@ -0,0 +1,58 @@ +unit HlpSHA3Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpSHA3; + +type + /// + /// Arch-neutral SIMD facade for Keccak-F1600 (SHA-3 / SHAKE). Picks the + /// per-arch backend at compile time and returns the best permute / absorb + /// routines the running CPU supports, or the scalar reference when no SIMD + /// backend is built/available. The hash unit calls only this facade and stays + /// free of any TCpuFeatures / HASHLIB_*_ASM knowledge. + /// + TSHA3Simd = class sealed + public + class function SelectPermute(AScalar: TKeccakF1600Proc): TKeccakF1600Proc; static; + class function SelectAbsorb(AScalar: TKeccakF1600AbsorbProc): TKeccakF1600AbsorbProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpSHA3X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpSHA3ArmBackend; +{$IFEND} + +{ TSHA3Simd } + +class function TSHA3Simd.SelectPermute(AScalar: TKeccakF1600Proc): TKeccakF1600Proc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TSHA3X86Backend.SelectPermute(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TSHA3ArmBackend.SelectPermute(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +class function TSHA3Simd.SelectAbsorb(AScalar: TKeccakF1600AbsorbProc): TKeccakF1600AbsorbProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TSHA3X86Backend.SelectAbsorb(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TSHA3ArmBackend.SelectAbsorb(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpScryptSimd.pas b/HashLib/src/Simd/Facade/HlpScryptSimd.pas new file mode 100644 index 00000000..704ed600 --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpScryptSimd.pas @@ -0,0 +1,46 @@ +unit HlpScryptSimd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpPBKDF_ScryptNotBuildInAdapter; + +type + /// + /// Arch-neutral SIMD facade for scrypt's Salsa20/8 XOR core. Picks the per-arch + /// backend at compile time and returns the best routine the running CPU + /// supports, or AScalar when no SIMD backend is built/available. The KDF + /// unit calls only this facade and stays free of any TCpuFeatures / + /// HASHLIB_*_ASM knowledge. + /// + TScryptSimd = class sealed + public + class function Select(AScalar: TScryptSalsaXorProc): TScryptSalsaXorProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpScryptX86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpScryptArmBackend; +{$IFEND} + +{ TScryptSimd } + +class function TScryptSimd.Select(AScalar: TScryptSalsaXorProc): TScryptSalsaXorProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TScryptX86Backend.Select(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TScryptArmBackend.Select(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/HashLib/src/Simd/Facade/HlpXXHash3Simd.pas b/HashLib/src/Simd/Facade/HlpXXHash3Simd.pas new file mode 100644 index 00000000..75ab030b --- /dev/null +++ b/HashLib/src/Simd/Facade/HlpXXHash3Simd.pas @@ -0,0 +1,82 @@ +unit HlpXXHash3Simd; + +{$I ..\..\Include\HashLib.inc} + +interface + +uses + HlpXXHash3; + +type + /// + /// Arch-neutral SIMD facade for XXH3. Picks the per-arch backend at compile + /// time and returns the best accumulate / scramble / init-secret routines the + /// running CPU supports, or the scalar references when no SIMD backend is + /// built/available. The hash unit calls only this facade and stays free of any + /// TCpuFeatures / HASHLIB_*_ASM knowledge. + /// + TXXHash3Simd = class sealed + public + class function SelectAccumulate512(AScalar: TXXH3Accumulate512Proc): TXXH3Accumulate512Proc; static; + class function SelectAccumulate(AScalar: TXXH3AccumulateProc): TXXH3AccumulateProc; static; + class function SelectScrambleAcc(AScalar: TXXH3ScrambleAccProc): TXXH3ScrambleAccProc; static; + class function SelectInitSecret(AScalar: TXXH3InitSecretProc): TXXH3InitSecretProc; static; + end; + +implementation + +{$IF DEFINED(HASHLIB_X86_SIMD)} +uses + HlpXXHash3X86Backend; +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} +uses + HlpXXHash3ArmBackend; +{$IFEND} + +{ TXXHash3Simd } + +class function TXXHash3Simd.SelectAccumulate512(AScalar: TXXH3Accumulate512Proc): TXXH3Accumulate512Proc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TXXHash3X86Backend.SelectAccumulate512(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TXXHash3ArmBackend.SelectAccumulate512(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +class function TXXHash3Simd.SelectAccumulate(AScalar: TXXH3AccumulateProc): TXXH3AccumulateProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TXXHash3X86Backend.SelectAccumulate(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TXXHash3ArmBackend.SelectAccumulate(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +class function TXXHash3Simd.SelectScrambleAcc(AScalar: TXXH3ScrambleAccProc): TXXH3ScrambleAccProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TXXHash3X86Backend.SelectScrambleAcc(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TXXHash3ArmBackend.SelectScrambleAcc(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +class function TXXHash3Simd.SelectInitSecret(AScalar: TXXH3InitSecretProc): TXXH3InitSecretProc; +begin +{$IF DEFINED(HASHLIB_X86_SIMD)} + Result := TXXHash3X86Backend.SelectInitSecret(AScalar); +{$ELSEIF DEFINED(HASHLIB_AARCH64_ASM)} + Result := TXXHash3ArmBackend.SelectInitSecret(AScalar); +{$ELSE} + Result := AScalar; +{$IFEND} +end; + +end. diff --git a/scripts/check-duplicate-guids.ps1 b/scripts/maintenance/check-duplicate-guids.ps1 similarity index 90% rename from scripts/check-duplicate-guids.ps1 rename to scripts/maintenance/check-duplicate-guids.ps1 index 13ac5a28..1f8ad072 100644 --- a/scripts/check-duplicate-guids.ps1 +++ b/scripts/maintenance/check-duplicate-guids.ps1 @@ -10,8 +10,8 @@ $ErrorActionPreference = 'Stop' -# Repo root: script lives in /scripts/check-duplicate-guids.ps1 -$root = if ($PSScriptRoot) { Split-Path $PSScriptRoot -Parent } else { Get-Location } +# Repo root: script lives in /scripts/maintenance/check-duplicate-guids.ps1 +$root = if ($PSScriptRoot) { Split-Path (Split-Path $PSScriptRoot -Parent) -Parent } else { Get-Location } $pasFiles = Get-ChildItem -Path $root -Filter '*.pas' -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.FullName -notmatch '\.git\\' } diff --git a/scripts/maintenance/check-file-format.ps1 b/scripts/maintenance/check-file-format.ps1 new file mode 100644 index 00000000..6c96b84a --- /dev/null +++ b/scripts/maintenance/check-file-format.ps1 @@ -0,0 +1,68 @@ +<# +.SYNOPSIS + Checks repo file-format rules: CRLF line endings, valid UTF-8, no BOM. +.DESCRIPTION + Scans tracked source/text files (*.pas, *.inc, *.md, *.py, *.ps1, *.lpr) + under HashLib, HashLib.Tests, scripts and docs. Fails (non-zero exit) if a + file starts with a UTF-8 BOM, contains bytes that are not valid UTF-8, or + contains any line ending other than CRLF (bare LF or bare CR). +.EXAMPLE + .\scripts\maintenance\check-file-format.ps1 +#> + +$ErrorActionPreference = 'Stop' + +# Repo root: script lives in /scripts/maintenance/check-file-format.ps1 +$root = if ($PSScriptRoot) { Split-Path (Split-Path $PSScriptRoot -Parent) -Parent } else { Get-Location } + +$patterns = @('*.pas', '*.inc', '*.md', '*.py', '*.ps1', '*.lpr') +$dirs = @('HashLib\src', 'HashLib.Tests\src', 'scripts', 'docs') + +$files = foreach ($d in $dirs) { + $p = Join-Path $root $d + if (Test-Path $p) { + Get-ChildItem -Path $p -Include $patterns -Recurse -File -ErrorAction SilentlyContinue + } +} + +$failures = @() +$utf8Strict = New-Object System.Text.UTF8Encoding($false, $true) + +foreach ($f in $files) { + $bytes = [System.IO.File]::ReadAllBytes($f.FullName) + if ($bytes.Length -eq 0) { continue } + + if ($bytes.Length -ge 3 -and $bytes[0] -eq 0xEF -and $bytes[1] -eq 0xBB -and $bytes[2] -eq 0xBF) { + $failures += "$($f.FullName): UTF-8 BOM present" + } + + try { + [void]$utf8Strict.GetString($bytes) + } catch { + $failures += "$($f.FullName): not valid UTF-8" + } + + for ($i = 0; $i -lt $bytes.Length; $i++) { + $b = $bytes[$i] + if ($b -eq 10) { + if ($i -eq 0 -or $bytes[$i - 1] -ne 13) { + $failures += "$($f.FullName): bare LF at offset $i" + break + } + } elseif ($b -eq 13) { + if ($i -eq $bytes.Length - 1 -or $bytes[$i + 1] -ne 10) { + $failures += "$($f.FullName): bare CR at offset $i" + break + } + } + } +} + +if ($failures.Count -gt 0) { + Write-Host 'File-format check FAILED (rules: CRLF, valid UTF-8, no BOM):' + $failures | ForEach-Object { Write-Host " $_" } + exit 1 +} + +Write-Host "File-format check passed ($(@($files).Count) files)." +exit 0 diff --git a/scripts/sync-mobile-testdata.ps1 b/scripts/maintenance/sync-mobile-testdata.ps1 similarity index 99% rename from scripts/sync-mobile-testdata.ps1 rename to scripts/maintenance/sync-mobile-testdata.ps1 index 9abd6e4b..f8cbf792 100644 --- a/scripts/sync-mobile-testdata.ps1 +++ b/scripts/maintenance/sync-mobile-testdata.ps1 @@ -5,7 +5,7 @@ # Re-run when files under HashLib.Tests/Data change (or AppDeployFolderName changes), then rebuild the mobile test host. param( - [string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path, + [string]$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path, [string]$DeployProjName = "HashLib.Tests.Mobile.TestData.deployproj", [string]$AppDeployFolderName = "HashLib.Tests.Mobile" )